inex/index.community
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

Merge branch 'hide-insular-instances' into 'develop'

Browse source 
Hide insular instances

Closes #69

See merge request taobojlen/fediverse.space!55
This commit is contained in:
Tao Bojlén 2019-07-18 20:05:17 +00:00
parent b145ac7160 2e61b70602
commit 725604c7e0
8 changed files with 121 additions and 75 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

34
backend/lib/backend/api.ex
View file

@ -17,51 +17,27 @@ defmodule Backend.Api do
@doc """
Returns a list of instances that
* have at least one successful crawl
* have a user count (required to give the instance a size on the graph)
* have x and y coordinates
"""
@spec list_nodes() :: [Instance.t()]
def list_nodes() do
crawl_subquery =
Crawl
|> select([c], %{
instance_domain: c.instance_domain,
crawl_count: count(c.id)
})
|> where([c], is_nil(c.error))
|> group_by([c], c.instance_domain)
Instance
|> join(:inner, [i], c in subquery(crawl_subquery), on: i.domain == c.instance_domain)
|> where(
[i, c],
c.crawl_count > 0 and not is_nil(i.user_count) and not is_nil(i.x) and not is_nil(i.y)
)
|> where([i], not is_nil(i.x) and not is_nil(i.y) and not is_nil(i.user_count))
|> select([c], [:domain, :user_count, :x, :y])
|> Repo.all()
end
@spec list_edges() :: [Edge.t()]
def list_edges() do
crawl_subquery =
Crawl
|> select([c], %{
instance_domain: c.instance_domain,
crawl_count: count(c.id)
})
|> where([c], is_nil(c.error))
|> group_by([c], c.instance_domain)
Edge
|> join(:inner, [e], c1 in subquery(crawl_subquery), on: e.source_domain == c1.instance_domain)
|> join(:inner, [e], c2 in subquery(crawl_subquery), on: e.target_domain == c2.instance_domain)
|> join(:inner, [e], i1 in Instance, on: e.source_domain == i1.domain)
|> join(:inner, [e], i2 in Instance, on: e.target_domain == i2.domain)
|> select([e], [:id, :source_domain, :target_domain, :weight])
|> where(
[e, c1, c2, i1, i2],
c1.crawl_count > 0 and c2.crawl_count > 0 and not is_nil(i1.x) and not is_nil(i1.y) and
not is_nil(i2.x) and not is_nil(i2.y) and e.source_domain != e.target_domain
[e, i1, i2],
not is_nil(i1.x) and not is_nil(i1.y) and
not is_nil(i2.x) and not is_nil(i2.y)
)
|> Repo.all()
end

15
backend/lib/backend/scheduler.ex
View file

@ -60,10 +60,6 @@ defmodule Backend.Scheduler do
interactions: min(c.interactions_seen)
})
|> Repo.all()
|> (fn o ->
Logger.info(inspect(o))
o
end).()
|> Enum.map(fn %{domain: domain, mentions: mentions, interactions: interactions} ->
%{
domain: domain,
@ -82,7 +78,8 @@ defmodule Backend.Scheduler do
@doc """
This function aggregates statistics from the interactions in the database.
It calculates the strength of edges between nodes.
It calculates the strength of edges between nodes. Self-edges are not generated.
Edges are only generated if both instances have been succesfully crawled.
"""
def generate_edges() do
now = get_now()
@ -98,12 +95,13 @@ defmodule Backend.Scheduler do
interactions =
CrawlInteraction
|> join(:left, [ci], c_source in subquery(crawls_subquery),
|> join(:inner, [ci], c_source in subquery(crawls_subquery),
on: ci.source_domain == c_source.instance_domain
)
|> join(:left, [ci], c_target in subquery(crawls_subquery),
|> join(:inner, [ci], c_target in subquery(crawls_subquery),
on: ci.target_domain == c_target.instance_domain
)
|> where([ci], ci.source_domain != ci.target_domain)
|> group_by([ci], [ci.source_domain, ci.target_domain])
|> select([ci, c_source, c_target], %{
source_domain: ci.source_domain,
@ -130,9 +128,8 @@ defmodule Backend.Scheduler do
mentions: mentions,
source_statuses_seen: source_statuses_seen,
target_statuses_seen: target_statuses_seen
} = x,
},
acc ->
Logger.info(inspect(x))
key = get_interaction_key(source_domain, target_domain)
# target_statuses_seen might be nil if that instance was never crawled. default to 0.

2
backend/lib/backend_web/views/instance_view.ex
View file

@ -16,8 +16,6 @@ defmodule BackendWeb.InstanceView do
end
def render("instance_detail.json", %{instance: instance, crawl: crawl}) do
Logger.info("keys: #{inspect(instance)}")
[status, last_updated] =
case crawl do
nil ->

35
frontend/src/components/CytoscapeGraph.tsx
View file

@ -128,7 +128,23 @@ class GraphImpl extends React.Component<IGraphProps, IGraphState> {
this.cy = cytoscape({
autoungrabify: false,
container: this.cytoscapeDiv.current,
elements: graph,
elements: {
edges: graph.edges.map(e => ({
...e,
data: {
...e.data,
weight: Math.min(Math.max(e.data.weight * 100, 2), 10)
},
selectable: false
})),
nodes: graph.nodes.map(n => ({
...n,
data: {
...n.data,
size: Math.min(Math.max(n.data.size * 10, 10), 80)
}
}))
},
layout: {
name: "preset"
},
@ -144,7 +160,22 @@ class GraphImpl extends React.Component<IGraphProps, IGraphState> {
{
selector: "node",
style: {
"background-color": DEFAULT_NODE_COLOR
"background-color": DEFAULT_NODE_COLOR,
height: "data(size)",
label: "data(id)",
width: "data(size)"
}
},
{
selector: "edge",
style: {
width: "data(weight)"
}
},
{
selector: "label",
style: {
color: DEFAULT_NODE_COLOR
}
}
]

82
frontend/src/components/Sidebar.tsx
View file

@ -5,10 +5,12 @@ import * as React from "react";
import { connect } from "react-redux";
import { Dispatch } from "redux";
import sanitize from "sanitize-html";
import styled from "styled-components";
import {
AnchorButton,
Button,
Callout,
Card,
Classes,
Code,
@ -40,12 +42,24 @@ interface ISidebarProps {
}
interface ISidebarState {
isOpen: boolean;
neighbors?: string[];
isProcessingNeighbors: boolean;
}
class SidebarImpl extends React.Component<ISidebarProps, ISidebarState> {
constructor(props: ISidebarProps) {
super(props);
const isOpen = window.innerWidth >= 900 ? true : false;
this.state = { isOpen };
this.state = { isOpen, isProcessingNeighbors: false };
}
public componentDidMount() {
this.processEdgesToFindNeighbors();
}
public componentDidUpdate(prevProps: ISidebarProps, prevState: ISidebarState) {
if (prevProps.instanceName !== this.props.instanceName) {
this.processEdgesToFindNeighbors();
}
}
public render() {
@ -71,8 +85,26 @@ class SidebarImpl extends React.Component<ISidebarProps, ISidebarState> {
this.setState({ isOpen: !this.state.isOpen });
};
private processEdgesToFindNeighbors = () => {
const { graph, instanceName } = this.props;
if (!graph || !instanceName) {
return;
}
this.setState({ isProcessingNeighbors: true });
const edges = graph.edges.filter(e => [e.data.source, e.data.target].indexOf(instanceName!) > -1);
const neighbors: any[] = [];
edges.forEach(e => {
if (e.data.source === instanceName) {
neighbors.push({ neighbor: e.data.target, weight: e.data.weight });
} else {
neighbors.push({ neighbor: e.data.source, weight: e.data.weight });
}
});
this.setState({ neighbors, isProcessingNeighbors: false });
};
private renderSidebarContents = () => {
if (this.props.isLoadingInstanceDetails) {
if (this.props.isLoadingInstanceDetails || this.state.isProcessingNeighbors) {
return this.renderLoadingState();
} else if (!this.props.instanceDetails) {
return this.renderEmptyState();
@ -82,12 +114,6 @@ class SidebarImpl extends React.Component<ISidebarProps, ISidebarState> {
return this.renderMissingDataState();
} else if (this.props.instanceLoadError) {
return <ErrorState />;
} else if (
this.props.graph &&
this.props.instanceName &&
this.props.graph.nodes.map(n => n.data.id).indexOf(this.props.instanceName) < 0
) {
return this.renderQuietInstanceState();
}
return (
<div>
@ -123,9 +149,20 @@ class SidebarImpl extends React.Component<ISidebarProps, ISidebarState> {
</span>
);
}
const hasNeighbors = this.state.neighbors && this.state.neighbors.length > 0;
const insularCallout = hasNeighbors ? (
undefined
) : (
<Callout icon={IconNames.INFO_SIGN} title="Insular instance">
<p>This instance doesn't have any neighbors that we know of, so it's hidden from the graph.</p>
</Callout>
);
return (
<div>
<H2>{content}</H2>
{insularCallout}
<Divider />
</div>
);
@ -318,22 +355,21 @@ class SidebarImpl extends React.Component<ISidebarProps, ISidebarState> {
};
private renderMissingDataState = () => {
const FillDiv = styled.div`
width: 100%;
height: 100%;
`;
return (
<NonIdealState
icon={IconNames.ERROR}
title="No data"
description="This instance could not be crawled. Either it was down or it's an instance type we don't support yet."
/>
);
};
private renderQuietInstanceState = () => {
return (
<NonIdealState
icon={IconNames.CLEAN}
title="No interactions"
description="Users on this instance have not publicly interacted with any other instances recently. "
/>
<FillDiv>
<NonIdealState
icon={IconNames.ERROR}
title="No data"
description="This instance could not be crawled. Either it was down or it's an instance type we don't support yet."
/>
<span className="sidebar-hidden-instance-status" style={{ display: "none" }}>
{this.props.instanceDetails && this.props.instanceDetails.status}
</span>
</FillDiv>
);
};

BIN
gephi/bin/main/space/fediverse/graph/GraphBuilder.class
View file

Binary file not shown.

25
gephi/src/main/java/space/fediverse/graph/GraphBuilder.java
View file

@ -27,18 +27,16 @@ import java.sql.SQLException;
import java.util.concurrent.TimeUnit;
public class GraphBuilder {
private static final String nodeQuery = new StringBuilder().append("SELECT i.domain as id, i.domain as label")
.append(" FROM instances i INNER JOIN edges e ON i.domain = e.source_domain OR i.domain = e.target_domain")
.append(" WHERE i.user_count IS NOT NULL").toString();
private static final String nodeQuery = String.join("", "WITH successful_crawls AS (", " SELECT",
" c.instance_domain AS instance_domain,", " COUNT(c.id) AS crawl_count",
" FROM crawls c WHERE c.error IS NULL", " GROUP BY c.instance_domain)", " SELECT", " i.domain AS id,",
" i.domain AS label", " FROM instances i",
" INNER JOIN successful_crawls c ON i.domain = c.instance_domain",
" WHERE c.crawl_count > 0 AND i.user_count IS NOT NULL");
private static final String edgeQuery = String.join("", "SELECT", " e.source_domain AS source,",
" e.target_domain AS target,", " e.weight AS weight", " FROM edges e");
private static final String edgeQuery = new StringBuilder().append("SELECT e.source_domain AS source,")
.append(" e.target_domain AS target, e.weight AS weight FROM edges e").toString();
public static void main(String[] args) {
// System.out.println("Node query: " + nodeQuery);
// System.out.println("Edge query: " + edgeQuery);
// Init project & workspace; required to do things w/ gephi
ProjectController pc = Lookup.getDefault().lookup(ProjectController.class);
@ -114,7 +112,14 @@ public class GraphBuilder {
}
throw new RuntimeException(e);
}
// Update
// Remove all x and y
try {
PreparedStatement delStatement = conn.prepareStatement("UPDATE instances SET x=NULL, y=NULL");
delStatement.executeUpdate();
} catch (SQLException e) {
throw new RuntimeException(e);
}
// Update to new x's and y's
UndirectedGraph graph = graphModel.getUndirectedGraph();
for (Node node : graph.getNodes()) {
String id = node.getId().toString();

3
netlify.toml
View file

@ -7,3 +7,6 @@
[context.branch-deploy.environment]
REACT_APP_STAGING = "true"
[context.deploy-preview.environment]
REACT_APP_STAGING = "true"