diff --git a/backend/lib/backend/api.ex b/backend/lib/backend/api.ex index be436d3..6e5c217 100644 --- a/backend/lib/backend/api.ex +++ b/backend/lib/backend/api.ex @@ -17,51 +17,27 @@ defmodule Backend.Api do @doc """ Returns a list of instances that - * have at least one successful crawl * have a user count (required to give the instance a size on the graph) + * have x and y coordinates """ @spec list_nodes() :: [Instance.t()] def list_nodes() do - crawl_subquery = - Crawl - |> select([c], %{ - instance_domain: c.instance_domain, - crawl_count: count(c.id) - }) - |> where([c], is_nil(c.error)) - |> group_by([c], c.instance_domain) - Instance - |> join(:inner, [i], c in subquery(crawl_subquery), on: i.domain == c.instance_domain) - |> where( - [i, c], - c.crawl_count > 0 and not is_nil(i.user_count) and not is_nil(i.x) and not is_nil(i.y) - ) + |> where([i], not is_nil(i.x) and not is_nil(i.y) and not is_nil(i.user_count)) |> select([c], [:domain, :user_count, :x, :y]) |> Repo.all() end @spec list_edges() :: [Edge.t()] def list_edges() do - crawl_subquery = - Crawl - |> select([c], %{ - instance_domain: c.instance_domain, - crawl_count: count(c.id) - }) - |> where([c], is_nil(c.error)) - |> group_by([c], c.instance_domain) - Edge - |> join(:inner, [e], c1 in subquery(crawl_subquery), on: e.source_domain == c1.instance_domain) - |> join(:inner, [e], c2 in subquery(crawl_subquery), on: e.target_domain == c2.instance_domain) |> join(:inner, [e], i1 in Instance, on: e.source_domain == i1.domain) |> join(:inner, [e], i2 in Instance, on: e.target_domain == i2.domain) |> select([e], [:id, :source_domain, :target_domain, :weight]) |> where( - [e, c1, c2, i1, i2], - c1.crawl_count > 0 and c2.crawl_count > 0 and not is_nil(i1.x) and not is_nil(i1.y) and - not is_nil(i2.x) and not is_nil(i2.y) and e.source_domain != e.target_domain + [e, i1, i2], + not is_nil(i1.x) and not is_nil(i1.y) and + not is_nil(i2.x) and not is_nil(i2.y) ) |> Repo.all() end diff --git a/backend/lib/backend/scheduler.ex b/backend/lib/backend/scheduler.ex index ea88070..d19e5f6 100644 --- a/backend/lib/backend/scheduler.ex +++ b/backend/lib/backend/scheduler.ex @@ -60,10 +60,6 @@ defmodule Backend.Scheduler do interactions: min(c.interactions_seen) }) |> Repo.all() - |> (fn o -> - Logger.info(inspect(o)) - o - end).() |> Enum.map(fn %{domain: domain, mentions: mentions, interactions: interactions} -> %{ domain: domain, @@ -82,7 +78,8 @@ defmodule Backend.Scheduler do @doc """ This function aggregates statistics from the interactions in the database. - It calculates the strength of edges between nodes. + It calculates the strength of edges between nodes. Self-edges are not generated. + Edges are only generated if both instances have been succesfully crawled. """ def generate_edges() do now = get_now() @@ -98,12 +95,13 @@ defmodule Backend.Scheduler do interactions = CrawlInteraction - |> join(:left, [ci], c_source in subquery(crawls_subquery), + |> join(:inner, [ci], c_source in subquery(crawls_subquery), on: ci.source_domain == c_source.instance_domain ) - |> join(:left, [ci], c_target in subquery(crawls_subquery), + |> join(:inner, [ci], c_target in subquery(crawls_subquery), on: ci.target_domain == c_target.instance_domain ) + |> where([ci], ci.source_domain != ci.target_domain) |> group_by([ci], [ci.source_domain, ci.target_domain]) |> select([ci, c_source, c_target], %{ source_domain: ci.source_domain, @@ -130,9 +128,8 @@ defmodule Backend.Scheduler do mentions: mentions, source_statuses_seen: source_statuses_seen, target_statuses_seen: target_statuses_seen - } = x, + }, acc -> - Logger.info(inspect(x)) key = get_interaction_key(source_domain, target_domain) # target_statuses_seen might be nil if that instance was never crawled. default to 0. diff --git a/backend/lib/backend_web/views/instance_view.ex b/backend/lib/backend_web/views/instance_view.ex index 108ca69..dd808e5 100644 --- a/backend/lib/backend_web/views/instance_view.ex +++ b/backend/lib/backend_web/views/instance_view.ex @@ -16,8 +16,6 @@ defmodule BackendWeb.InstanceView do end def render("instance_detail.json", %{instance: instance, crawl: crawl}) do - Logger.info("keys: #{inspect(instance)}") - [status, last_updated] = case crawl do nil -> diff --git a/frontend/src/components/CytoscapeGraph.tsx b/frontend/src/components/CytoscapeGraph.tsx index 06a225c..0be272c 100644 --- a/frontend/src/components/CytoscapeGraph.tsx +++ b/frontend/src/components/CytoscapeGraph.tsx @@ -128,7 +128,23 @@ class GraphImpl extends React.Component { this.cy = cytoscape({ autoungrabify: false, container: this.cytoscapeDiv.current, - elements: graph, + elements: { + edges: graph.edges.map(e => ({ + ...e, + data: { + ...e.data, + weight: Math.min(Math.max(e.data.weight * 100, 2), 10) + }, + selectable: false + })), + nodes: graph.nodes.map(n => ({ + ...n, + data: { + ...n.data, + size: Math.min(Math.max(n.data.size * 10, 10), 80) + } + })) + }, layout: { name: "preset" }, @@ -144,7 +160,22 @@ class GraphImpl extends React.Component { { selector: "node", style: { - "background-color": DEFAULT_NODE_COLOR + "background-color": DEFAULT_NODE_COLOR, + height: "data(size)", + label: "data(id)", + width: "data(size)" + } + }, + { + selector: "edge", + style: { + width: "data(weight)" + } + }, + { + selector: "label", + style: { + color: DEFAULT_NODE_COLOR } } ] diff --git a/frontend/src/components/Sidebar.tsx b/frontend/src/components/Sidebar.tsx index 0b52abe..61af731 100644 --- a/frontend/src/components/Sidebar.tsx +++ b/frontend/src/components/Sidebar.tsx @@ -5,10 +5,12 @@ import * as React from "react"; import { connect } from "react-redux"; import { Dispatch } from "redux"; import sanitize from "sanitize-html"; +import styled from "styled-components"; import { AnchorButton, Button, + Callout, Card, Classes, Code, @@ -40,12 +42,24 @@ interface ISidebarProps { } interface ISidebarState { isOpen: boolean; + neighbors?: string[]; + isProcessingNeighbors: boolean; } class SidebarImpl extends React.Component { constructor(props: ISidebarProps) { super(props); const isOpen = window.innerWidth >= 900 ? true : false; - this.state = { isOpen }; + this.state = { isOpen, isProcessingNeighbors: false }; + } + + public componentDidMount() { + this.processEdgesToFindNeighbors(); + } + + public componentDidUpdate(prevProps: ISidebarProps, prevState: ISidebarState) { + if (prevProps.instanceName !== this.props.instanceName) { + this.processEdgesToFindNeighbors(); + } } public render() { @@ -71,8 +85,26 @@ class SidebarImpl extends React.Component { this.setState({ isOpen: !this.state.isOpen }); }; + private processEdgesToFindNeighbors = () => { + const { graph, instanceName } = this.props; + if (!graph || !instanceName) { + return; + } + this.setState({ isProcessingNeighbors: true }); + const edges = graph.edges.filter(e => [e.data.source, e.data.target].indexOf(instanceName!) > -1); + const neighbors: any[] = []; + edges.forEach(e => { + if (e.data.source === instanceName) { + neighbors.push({ neighbor: e.data.target, weight: e.data.weight }); + } else { + neighbors.push({ neighbor: e.data.source, weight: e.data.weight }); + } + }); + this.setState({ neighbors, isProcessingNeighbors: false }); + }; + private renderSidebarContents = () => { - if (this.props.isLoadingInstanceDetails) { + if (this.props.isLoadingInstanceDetails || this.state.isProcessingNeighbors) { return this.renderLoadingState(); } else if (!this.props.instanceDetails) { return this.renderEmptyState(); @@ -82,12 +114,6 @@ class SidebarImpl extends React.Component { return this.renderMissingDataState(); } else if (this.props.instanceLoadError) { return ; - } else if ( - this.props.graph && - this.props.instanceName && - this.props.graph.nodes.map(n => n.data.id).indexOf(this.props.instanceName) < 0 - ) { - return this.renderQuietInstanceState(); } return (
@@ -123,9 +149,20 @@ class SidebarImpl extends React.Component { ); } + + const hasNeighbors = this.state.neighbors && this.state.neighbors.length > 0; + + const insularCallout = hasNeighbors ? ( + undefined + ) : ( + +

This instance doesn't have any neighbors that we know of, so it's hidden from the graph.

+ + ); return (

{content}

+ {insularCallout}
); @@ -318,22 +355,21 @@ class SidebarImpl extends React.Component { }; private renderMissingDataState = () => { + const FillDiv = styled.div` + width: 100%; + height: 100%; + `; return ( - - ); - }; - - private renderQuietInstanceState = () => { - return ( - + + + + {this.props.instanceDetails && this.props.instanceDetails.status} + + ); }; diff --git a/gephi/bin/main/space/fediverse/graph/GraphBuilder.class b/gephi/bin/main/space/fediverse/graph/GraphBuilder.class index b5339a6..94f5019 100644 Binary files a/gephi/bin/main/space/fediverse/graph/GraphBuilder.class and b/gephi/bin/main/space/fediverse/graph/GraphBuilder.class differ diff --git a/gephi/src/main/java/space/fediverse/graph/GraphBuilder.java b/gephi/src/main/java/space/fediverse/graph/GraphBuilder.java index b7de6e1..0bed87c 100644 --- a/gephi/src/main/java/space/fediverse/graph/GraphBuilder.java +++ b/gephi/src/main/java/space/fediverse/graph/GraphBuilder.java @@ -27,18 +27,16 @@ import java.sql.SQLException; import java.util.concurrent.TimeUnit; public class GraphBuilder { + private static final String nodeQuery = new StringBuilder().append("SELECT i.domain as id, i.domain as label") + .append(" FROM instances i INNER JOIN edges e ON i.domain = e.source_domain OR i.domain = e.target_domain") + .append(" WHERE i.user_count IS NOT NULL").toString(); - private static final String nodeQuery = String.join("", "WITH successful_crawls AS (", " SELECT", - " c.instance_domain AS instance_domain,", " COUNT(c.id) AS crawl_count", - " FROM crawls c WHERE c.error IS NULL", " GROUP BY c.instance_domain)", " SELECT", " i.domain AS id,", - " i.domain AS label", " FROM instances i", - " INNER JOIN successful_crawls c ON i.domain = c.instance_domain", - " WHERE c.crawl_count > 0 AND i.user_count IS NOT NULL"); - - private static final String edgeQuery = String.join("", "SELECT", " e.source_domain AS source,", - " e.target_domain AS target,", " e.weight AS weight", " FROM edges e"); + private static final String edgeQuery = new StringBuilder().append("SELECT e.source_domain AS source,") + .append(" e.target_domain AS target, e.weight AS weight FROM edges e").toString(); public static void main(String[] args) { + // System.out.println("Node query: " + nodeQuery); + // System.out.println("Edge query: " + edgeQuery); // Init project & workspace; required to do things w/ gephi ProjectController pc = Lookup.getDefault().lookup(ProjectController.class); @@ -114,7 +112,14 @@ public class GraphBuilder { } throw new RuntimeException(e); } - // Update + // Remove all x and y + try { + PreparedStatement delStatement = conn.prepareStatement("UPDATE instances SET x=NULL, y=NULL"); + delStatement.executeUpdate(); + } catch (SQLException e) { + throw new RuntimeException(e); + } + // Update to new x's and y's UndirectedGraph graph = graphModel.getUndirectedGraph(); for (Node node : graph.getNodes()) { String id = node.getId().toString(); diff --git a/netlify.toml b/netlify.toml index 4438afa..635f69d 100644 --- a/netlify.toml +++ b/netlify.toml @@ -7,3 +7,6 @@ [context.branch-deploy.environment] REACT_APP_STAGING = "true" + +[context.deploy-preview.environment] + REACT_APP_STAGING = "true" \ No newline at end of file