Merge branch 'hide-insular-instances' into 'develop'

Hide insular instances

Closes #69

See merge request taobojlen/fediverse.space!55

backend/lib/backend/api.ex

@@ -17,51 +17,27 @@ defmodule Backend.Api do
 
   @doc """
   Returns a list of instances that
-  * have at least one successful crawl
   * have a user count (required to give the instance a size on the graph)
+  * have x and y coordinates
   """
   @spec list_nodes() :: [Instance.t()]
   def list_nodes() do
-    crawl_subquery =
-      Crawl
-      |> select([c], %{
-        instance_domain: c.instance_domain,
-        crawl_count: count(c.id)
-      })
-      |> where([c], is_nil(c.error))
-      |> group_by([c], c.instance_domain)
-
     Instance
-    |> join(:inner, [i], c in subquery(crawl_subquery), on: i.domain == c.instance_domain)
+    |> where([i], not is_nil(i.x) and not is_nil(i.y) and not is_nil(i.user_count))
-    |> where(
-      [i, c],
-      c.crawl_count > 0 and not is_nil(i.user_count) and not is_nil(i.x) and not is_nil(i.y)
-    )
     |> select([c], [:domain, :user_count, :x, :y])
     |> Repo.all()
   end
 
   @spec list_edges() :: [Edge.t()]
   def list_edges() do
-    crawl_subquery =
-      Crawl
-      |> select([c], %{
-        instance_domain: c.instance_domain,
-        crawl_count: count(c.id)
-      })
-      |> where([c], is_nil(c.error))
-      |> group_by([c], c.instance_domain)
-
     Edge
-    |> join(:inner, [e], c1 in subquery(crawl_subquery), on: e.source_domain == c1.instance_domain)
-    |> join(:inner, [e], c2 in subquery(crawl_subquery), on: e.target_domain == c2.instance_domain)
     |> join(:inner, [e], i1 in Instance, on: e.source_domain == i1.domain)
     |> join(:inner, [e], i2 in Instance, on: e.target_domain == i2.domain)
     |> select([e], [:id, :source_domain, :target_domain, :weight])
     |> where(
-      [e, c1, c2, i1, i2],
+      [e, i1, i2],
-      c1.crawl_count > 0 and c2.crawl_count > 0 and not is_nil(i1.x) and not is_nil(i1.y) and
+      not is_nil(i1.x) and not is_nil(i1.y) and
-        not is_nil(i2.x) and not is_nil(i2.y) and e.source_domain != e.target_domain
+        not is_nil(i2.x) and not is_nil(i2.y)
     )
     |> Repo.all()
   end

backend/lib/backend/scheduler.ex

@@ -60,10 +60,6 @@ defmodule Backend.Scheduler do
         interactions: min(c.interactions_seen)
       })
       |> Repo.all()
-      |> (fn o ->
-            Logger.info(inspect(o))
-            o
-          end).()
       |> Enum.map(fn %{domain: domain, mentions: mentions, interactions: interactions} ->
         %{
           domain: domain,
@@ -82,7 +78,8 @@ defmodule Backend.Scheduler do
 
   @doc """
   This function aggregates statistics from the interactions in the database.
-  It calculates the strength of edges between nodes.
+  It calculates the strength of edges between nodes. Self-edges are not generated.
+  Edges are only generated if both instances have been succesfully crawled.
   """
   def generate_edges() do
     now = get_now()
@@ -98,12 +95,13 @@ defmodule Backend.Scheduler do
 
     interactions =
       CrawlInteraction
-      |> join(:left, [ci], c_source in subquery(crawls_subquery),
+      |> join(:inner, [ci], c_source in subquery(crawls_subquery),
         on: ci.source_domain == c_source.instance_domain
       )
-      |> join(:left, [ci], c_target in subquery(crawls_subquery),
+      |> join(:inner, [ci], c_target in subquery(crawls_subquery),
         on: ci.target_domain == c_target.instance_domain
       )
+      |> where([ci], ci.source_domain != ci.target_domain)
       |> group_by([ci], [ci.source_domain, ci.target_domain])
       |> select([ci, c_source, c_target], %{
         source_domain: ci.source_domain,
@@ -130,9 +128,8 @@ defmodule Backend.Scheduler do
             mentions: mentions,
             source_statuses_seen: source_statuses_seen,
             target_statuses_seen: target_statuses_seen
-          } = x,
+          },
           acc ->
-            Logger.info(inspect(x))
             key = get_interaction_key(source_domain, target_domain)
 
             # target_statuses_seen might be nil if that instance was never crawled. default to 0.

backend/lib/backend_web/views/instance_view.ex

@@ -16,8 +16,6 @@ defmodule BackendWeb.InstanceView do
   end
 
   def render("instance_detail.json", %{instance: instance, crawl: crawl}) do
-    Logger.info("keys: #{inspect(instance)}")
-
     [status, last_updated] =
       case crawl do
         nil ->

frontend/src/components/CytoscapeGraph.tsx

@@ -128,7 +128,23 @@ class GraphImpl extends React.Component<IGraphProps, IGraphState> {
     this.cy = cytoscape({
       autoungrabify: false,
       container: this.cytoscapeDiv.current,
-      elements: graph,
+      elements: {
+        edges: graph.edges.map(e => ({
+          ...e,
+          data: {
+            ...e.data,
+            weight: Math.min(Math.max(e.data.weight * 100, 2), 10)
+          },
+          selectable: false
+        })),
+        nodes: graph.nodes.map(n => ({
+          ...n,
+          data: {
+            ...n.data,
+            size: Math.min(Math.max(n.data.size * 10, 10), 80)
+          }
+        }))
+      },
       layout: {
         name: "preset"
       },
@@ -144,7 +160,22 @@ class GraphImpl extends React.Component<IGraphProps, IGraphState> {
         {
           selector: "node",
           style: {
-            "background-color": DEFAULT_NODE_COLOR
+            "background-color": DEFAULT_NODE_COLOR,
+            height: "data(size)",
+            label: "data(id)",
+            width: "data(size)"
+          }
+        },
+        {
+          selector: "edge",
+          style: {
+            width: "data(weight)"
+          }
+        },
+        {
+          selector: "label",
+          style: {
+            color: DEFAULT_NODE_COLOR
           }
         }
       ]

frontend/src/components/Sidebar.tsx

@@ -5,10 +5,12 @@ import * as React from "react";
 import { connect } from "react-redux";
 import { Dispatch } from "redux";
 import sanitize from "sanitize-html";
+import styled from "styled-components";
 
 import {
   AnchorButton,
   Button,
+  Callout,
   Card,
   Classes,
   Code,
@@ -40,12 +42,24 @@ interface ISidebarProps {
 }
 interface ISidebarState {
   isOpen: boolean;
+  neighbors?: string[];
+  isProcessingNeighbors: boolean;
 }
 class SidebarImpl extends React.Component<ISidebarProps, ISidebarState> {
   constructor(props: ISidebarProps) {
     super(props);
     const isOpen = window.innerWidth >= 900 ? true : false;
-    this.state = { isOpen };
+    this.state = { isOpen, isProcessingNeighbors: false };
+  }
+
+  public componentDidMount() {
+    this.processEdgesToFindNeighbors();
+  }
+
+  public componentDidUpdate(prevProps: ISidebarProps, prevState: ISidebarState) {
+    if (prevProps.instanceName !== this.props.instanceName) {
+      this.processEdgesToFindNeighbors();
+    }
   }
 
   public render() {
@@ -71,8 +85,26 @@ class SidebarImpl extends React.Component<ISidebarProps, ISidebarState> {
     this.setState({ isOpen: !this.state.isOpen });
   };
 
+  private processEdgesToFindNeighbors = () => {
+    const { graph, instanceName } = this.props;
+    if (!graph || !instanceName) {
+      return;
+    }
+    this.setState({ isProcessingNeighbors: true });
+    const edges = graph.edges.filter(e => [e.data.source, e.data.target].indexOf(instanceName!) > -1);
+    const neighbors: any[] = [];
+    edges.forEach(e => {
+      if (e.data.source === instanceName) {
+        neighbors.push({ neighbor: e.data.target, weight: e.data.weight });
+      } else {
+        neighbors.push({ neighbor: e.data.source, weight: e.data.weight });
+      }
+    });
+    this.setState({ neighbors, isProcessingNeighbors: false });
+  };
+
   private renderSidebarContents = () => {
-    if (this.props.isLoadingInstanceDetails) {
+    if (this.props.isLoadingInstanceDetails || this.state.isProcessingNeighbors) {
       return this.renderLoadingState();
     } else if (!this.props.instanceDetails) {
       return this.renderEmptyState();
@@ -82,12 +114,6 @@ class SidebarImpl extends React.Component<ISidebarProps, ISidebarState> {
       return this.renderMissingDataState();
     } else if (this.props.instanceLoadError) {
       return <ErrorState />;
-    } else if (
-      this.props.graph &&
-      this.props.instanceName &&
-      this.props.graph.nodes.map(n => n.data.id).indexOf(this.props.instanceName) < 0
-    ) {
-      return this.renderQuietInstanceState();
     }
     return (
       <div>
@@ -123,9 +149,20 @@ class SidebarImpl extends React.Component<ISidebarProps, ISidebarState> {
         </span>
       );
     }
+
+    const hasNeighbors = this.state.neighbors && this.state.neighbors.length > 0;
+
+    const insularCallout = hasNeighbors ? (
+      undefined
+    ) : (
+      <Callout icon={IconNames.INFO_SIGN} title="Insular instance">
+        <p>This instance doesn't have any neighbors that we know of, so it's hidden from the graph.</p>
+      </Callout>
+    );
     return (
       <div>
         <H2>{content}</H2>
+        {insularCallout}
         <Divider />
       </div>
     );
@@ -318,22 +355,21 @@ class SidebarImpl extends React.Component<ISidebarProps, ISidebarState> {
   };
 
   private renderMissingDataState = () => {
+    const FillDiv = styled.div`
+      width: 100%;
+      height: 100%;
+    `;
     return (
+      <FillDiv>
         <NonIdealState
           icon={IconNames.ERROR}
           title="No data"
           description="This instance could not be crawled. Either it was down or it's an instance type we don't support yet."
         />
-    );
+        <span className="sidebar-hidden-instance-status" style={{ display: "none" }}>
-  };
+          {this.props.instanceDetails && this.props.instanceDetails.status}
-
+        </span>
-  private renderQuietInstanceState = () => {
+      </FillDiv>
-    return (
-      <NonIdealState
-        icon={IconNames.CLEAN}
-        title="No interactions"
-        description="Users on this instance have not publicly interacted with any other instances recently. "
-      />
     );
   };
 

gephi/src/main/java/space/fediverse/graph/GraphBuilder.java

@@ -27,18 +27,16 @@ import java.sql.SQLException;
 import java.util.concurrent.TimeUnit;
 
 public class GraphBuilder {
+    private static final String nodeQuery = new StringBuilder().append("SELECT i.domain as id, i.domain as label")
+            .append(" FROM instances i INNER JOIN edges e ON i.domain = e.source_domain OR i.domain = e.target_domain")
+            .append(" WHERE i.user_count IS NOT NULL").toString();
 
-    private static final String nodeQuery = String.join("", "WITH successful_crawls AS (", " SELECT",
+    private static final String edgeQuery = new StringBuilder().append("SELECT e.source_domain AS source,")
-            "   c.instance_domain AS instance_domain,", "   COUNT(c.id) AS crawl_count",
+            .append(" e.target_domain AS target, e.weight AS weight FROM edges e").toString();
-            " FROM crawls c WHERE c.error IS NULL", " GROUP BY c.instance_domain)", " SELECT", "   i.domain AS id,",
-            "   i.domain AS label", " FROM instances i",
-            " INNER JOIN successful_crawls c ON i.domain = c.instance_domain",
-            " WHERE c.crawl_count > 0 AND i.user_count IS NOT NULL");
-
-    private static final String edgeQuery = String.join("", "SELECT", "   e.source_domain AS source,",
-            "   e.target_domain AS target,", "   e.weight AS weight", " FROM edges e");
 
     public static void main(String[] args) {
+        // System.out.println("Node query: " + nodeQuery);
+        // System.out.println("Edge query: " + edgeQuery);
 
         // Init project & workspace; required to do things w/ gephi
         ProjectController pc = Lookup.getDefault().lookup(ProjectController.class);
@@ -114,7 +112,14 @@ public class GraphBuilder {
             }
             throw new RuntimeException(e);
         }
-        // Update
+        // Remove all x and y
+        try {
+            PreparedStatement delStatement = conn.prepareStatement("UPDATE instances SET x=NULL, y=NULL");
+            delStatement.executeUpdate();
+        } catch (SQLException e) {
+            throw new RuntimeException(e);
+        }
+        // Update to new x's and y's
         UndirectedGraph graph = graphModel.getUndirectedGraph();
         for (Node node : graph.getNodes()) {
             String id = node.getId().toString();

netlify.toml

@@ -7,3 +7,6 @@
 
 [context.branch-deploy.environment]
   REACT_APP_STAGING = "true"
+
+[context.deploy-preview.environment]
+  REACT_APP_STAGING = "true"