various UI improvements

apiv1/serializers.py

@@ -1,6 +1,7 @@
 from rest_framework import serializers
+import math
 from collections import OrderedDict
-from scraper.models import Instance, PeerRelationship
+from scraper.models import Instance, Edge
 
 
 class InstanceListSerializer(serializers.ModelSerializer):
@@ -39,19 +40,23 @@ class InstanceDetailSerializer(serializers.ModelSerializer):
 
     class Meta:
         model = Instance
-        fields = ('name', 'description', 'version', 'userCount', 'statusCount', 'domainCount', 'peers', 'lastUpdated')
+        fields = ('name', 'description', 'version', 'userCount', 'statusCount', 'domainCount', 'peers', 'lastUpdated', 'status')
 
 
 class EdgeSerializer(serializers.ModelSerializer):
     id = serializers.SerializerMethodField('get_pk')
+    size = serializers.SerializerMethodField('get_weight')
 
     class Meta:
-        model = PeerRelationship
-        fields = ('source', 'target', 'id')
+        model = Edge
+        fields = ('source', 'target', 'id', 'size')
 
     def get_pk(self, obj):
         return obj.pk
 
+    def get_weight(self, obj):
+        return obj.weight
+
 
 class NodeSerializer(serializers.ModelSerializer):
     id = serializers.SerializerMethodField('get_name')
@@ -68,7 +73,7 @@ class NodeSerializer(serializers.ModelSerializer):
         return obj.name
 
     def get_size(self, obj):
-        return obj.user_count or 1
+        return math.log(obj.user_count) if obj.user_count else 1
 
     def get_x(self, obj):
         return obj.x_coord

frontend/src/components/Graph.jsx

@@ -1,6 +1,6 @@
 import * as React from 'react';
 import { connect } from 'react-redux';
-import { RandomizeNodePositions, RelativeSize, Sigma, SigmaEnableWebGL, Filter } from 'react-sigma';
+import { Sigma, SigmaEnableWebGL, Filter, ForceAtlas2 } from 'react-sigma';
 
 import { selectAndLoadInstance } from '../redux/actions';
 
@@ -19,6 +19,9 @@ const SETTINGS = {
     drawLabels: true,
     edgeColor: "default",
     labelColor: "default",
+    labelThreshold: 10,
+    maxEdgeSize: 1,
+    minEdgeSize: 0.3,
 }
 
 class GraphImpl extends React.Component {
@@ -37,6 +40,7 @@ class GraphImpl extends React.Component {
                 onClickStage={(e) => this.props.selectAndLoadInstance(null)}
             >
                 <Filter neighborsOf={this.props.currentInstanceName} />
+                <ForceAtlas2 iterationsPerRender={1} timeout={6000}/>
             </Sigma>
         )
     }

frontend/src/components/Sidebar.tsx

@@ -4,7 +4,9 @@ import { connect } from 'react-redux';
 import { Dispatch } from 'redux';
 import * as sanitize from 'sanitize-html';
 
-import { Card, Classes, Divider, Elevation, HTMLTable, NonIdealState } from '@blueprintjs/core';
+import {
+    AnchorButton, Card, Classes, Divider, Elevation, HTMLTable, NonIdealState, Position, Tooltip
+} from '@blueprintjs/core';
 import { IconNames } from '@blueprintjs/icons';
 
 import { selectAndLoadInstance } from '../redux/actions';
@@ -30,11 +32,14 @@ class SidebarImpl extends React.Component<ISidebarProps> {
             return this.renderLoadingState();
         } else if (!this.props.instanceDetails) {
             return this.renderEmptyState();
+        } else if (this.props.instanceDetails.status.toLowerCase().indexOf('personalinstance') > -1) {
+            return this.renderPersonalInstanceErrorState();
+        } else if (this.props.instanceDetails.status !== 'success') {
+            return this.renderMissingDataState();
         }
         return (
             <div>
-                <h2>{this.props.instanceName || "No instance selected"}</h2>
-                <Divider />
+                {this.renderHeading()}
                 {this.renderDescription()}
                 {this.renderVersion()}
                 {this.renderCounts()}
@@ -43,6 +48,32 @@ class SidebarImpl extends React.Component<ISidebarProps> {
         );
     }
 
+    private renderHeading = () => {
+        let content: JSX.Element;
+        if (!this.props.instanceName) {
+            content = <span>{"No instance selected"}</span>;
+        } else {
+            content = (
+                <span>
+                    {this.props.instanceName + '  '}
+                    <Tooltip
+                        content="Open link in new tab"
+                        position={Position.TOP}
+                        className={Classes.DARK}
+                    >
+                        <AnchorButton icon={IconNames.LINK} minimal={true} onClick={this.openInstanceLink} />
+                    </Tooltip>
+                </span>
+            );
+        }
+        return (
+            <div>
+                <h2>{content}</h2>
+                <Divider />
+            </div>
+        );
+    }
+
     private renderDescription = () => {
         const description = this.props.instanceDetails!.description;
         if (!description) {
@@ -166,6 +197,30 @@ class SidebarImpl extends React.Component<ISidebarProps> {
         );
     }
 
+    private renderPersonalInstanceErrorState = () => {
+        return (
+            <NonIdealState
+                icon={IconNames.BLOCKED_PERSON}
+                title="No data"
+                description="This instance has fewer than 5 users and was not crawled."
+            />
+        )
+    }
+
+    private renderMissingDataState = () => {
+        return (
+            <NonIdealState
+                icon={IconNames.ERROR}
+                title="No data"
+                description="This instance could not be crawled. Either it was down or it's an instance type we don't support yet."
+            />
+        )
+    }
+
+    private openInstanceLink = () => {
+        window.open("https://" + this.props.instanceName, "_blank");
+    }
+
     private selectInstance = (e: any)=> {
         this.props.selectAndLoadInstance(e.target.innerText);
     }

frontend/src/redux/types.ts

@@ -27,6 +27,7 @@ export interface IInstanceDetails {
     userCount?: number;
     version?: string;
     lastUpdated?: string;
+    status: string;
 }
 
 interface IGraphNode {
@@ -42,6 +43,7 @@ interface IGraphEdge {
     source: string;
     target: string;
     id?: string;
+    size?: number;
 }
 
 export interface IGraph {

gephi/src/main/java/space/fediverse/graph/GraphBuilder.java

@@ -83,7 +83,7 @@ public class GraphBuilder {
         importController.process(container, new DefaultProcessor(), workspace);
 
         // Layout
-        AutoLayout autoLayout = new AutoLayout(1, TimeUnit.MINUTES);
+        AutoLayout autoLayout = new AutoLayout(2, TimeUnit.MINUTES);
         autoLayout.setGraphModel(graphModel);
 //        YifanHuLayout firstLayout = new YifanHuLayout(null, new StepDisplacement(1f));
         ForceAtlas2 forceAtlas2Layout = new ForceAtlas2(null);

scraper/management/commands/scrape.py

@@ -31,7 +31,7 @@ from scraper.management.commands._util import require_lock, InvalidResponseExcep
 
 SEED = 'mastodon.social'
 TIMEOUT = 20  # seconds
-NUM_THREADS = 64
+NUM_THREADS = 64  # roughly 40MB each
 PERSONAL_INSTANCE_THRESHOLD = 5  # instances with <= this many users won't be scraped
 STATUS_SCRAPE_LIMIT = 5000