Merge branch 'insularity-score' into 'develop'

add Insularity score

See merge request taobojlen/fediverse.space!54

README.md

@@ -1,9 +1,13 @@
 # fediverse.space 🌐
+
 The map of the fediverse that you always wanted.
 
+Read the latest updates on Mastodon: [@fediversespace](https://cursed.technology/@fediversespace)
+
 ![A screenshot of fediverse.space](screenshot.png)
 
 ## Requirements
+
 - For the scraper + API:
   - Elixir
   - Postgres
@@ -16,28 +20,35 @@ The map of the fediverse that you always wanted.
 All of the above can also be run through Docker with `docker-compose`.
 
 ## Running it
+
 ### Backend
+
 - `cp example.env .env` and modify environment variables as required
 - `docker-compose build`
 - `docker-compose up -d phoenix`
   - if you don't specify `phoenix`, it'll also start `gephi` which should only be run as a regular one-off job
+
 ### Frontend
+
 - `cd frontend && yarn install`
 - `yarn start`
 
 ## Commands
+
 ### Backend
 
 After running the backend in Docker:
+
 - `docker-compose run gephi java -Xmx1g -jar build/libs/graphBuilder.jar` lays out the graph
 
 `./gradlew shadowJar` compiles the graph layout program. `java -Xmx1g -jar build/libs/graphBuilder.jar` runs it.
 
 ### Frontend
+
 - `yarn build` to create an optimized build for deployment
 
 ### Acknowledgements
 
 [![NLnet logo](https://i.imgur.com/huV3rvo.png)](https://nlnet.nl/project/fediverse_space/)
 
-Many thanks to [NLnet](https://nlnet.nl/project/fediverse_space/) for their support and guidance of this project.
+Many thanks to [NLnet](https://nlnet.nl/project/fediverse_space/) for their support and guidance of this project.

backend/config/config.exs

@@ -44,7 +44,9 @@ config :backend, Backend.Scheduler,
     # At midnight every day
     {"@daily", {Backend.Scheduler, :prune_crawls, [1, "month"]}},
     # 00.15 daily
-    {"15 0 * * *", {Backend.Scheduler, :generate_edges, []}}
+    {"15 0 * * *", {Backend.Scheduler, :generate_edges, []}},
+    # 00.30 every night
+    {"30 0 * * *", {Backend.Scheduler, :generate_insularity_scores, []}}
   ]
 
 # Import environment specific config. This must remain at the bottom

backend/lib/backend_web/views/instance_view.ex

@@ -35,6 +35,7 @@ defmodule BackendWeb.InstanceView do
       description: instance.description,
       version: instance.version,
       userCount: instance.user_count,
+      insularity: instance.insularity,
       statusCount: instance.status_count,
       domainCount: length(instance.peers),
       peers: render_many(instance.peers, InstanceView, "instance.json"),

frontend/package.json

@@ -38,6 +38,7 @@
     "lodash": "^4.17.14",
     "moment": "^2.22.2",
     "normalize.css": "^8.0.0",
+    "numeral": "^2.0.6",
     "react": "^16.4.2",
     "react-dom": "^16.4.2",
     "react-redux": "^7.1.0",
@@ -57,6 +58,7 @@
     "@types/jest": "^24.0.15",
     "@types/lodash": "^4.14.136",
     "@types/node": "^12.6.2",
+    "@types/numeral": "^0.0.25",
     "@types/react": "^16.8.23",
     "@types/react-dom": "^16.8.4",
     "@types/react-redux": "^7.1.1",

frontend/src/components/Sidebar.tsx

@@ -1,5 +1,6 @@
 import { orderBy } from "lodash";
 import moment from "moment";
+import * as numeral from "numeral";
 import * as React from "react";
 import { connect } from "react-redux";
 import { Dispatch } from "redux";
@@ -16,6 +17,7 @@ import {
   H2,
   H4,
   HTMLTable,
+  Icon,
   NonIdealState,
   Position,
   Tab,
@@ -138,11 +140,10 @@ class SidebarImpl extends React.Component<ISidebarProps, ISidebarState> {
   };
 
   private renderVersionAndCounts = () => {
-    const version = this.props.instanceDetails!.version;
+    if (!this.props.instanceDetails) {
-    const userCount = this.props.instanceDetails!.userCount;
+      throw new Error("Did not receive instance details as expected!");
-    const statusCount = this.props.instanceDetails!.statusCount;
+    }
-    const domainCount = this.props.instanceDetails!.domainCount;
+    const { version, userCount, statusCount, domainCount, lastUpdated, insularity } = this.props.instanceDetails;
-    const lastUpdated = this.props.instanceDetails!.lastUpdated;
     return (
       <div>
         <HTMLTable small={true} striped={true} className="fediverse-sidebar-table">
@@ -153,15 +154,34 @@ class SidebarImpl extends React.Component<ISidebarProps, ISidebarState> {
             </tr>
             <tr>
               <td>Users</td>
-              <td>{userCount || "Unknown"}</td>
+              <td>{(userCount && numeral.default(userCount).format("0,0")) || "Unknown"}</td>
             </tr>
             <tr>
               <td>Statuses</td>
-              <td>{statusCount || "Unknown"}</td>
+              <td>{(statusCount && numeral.default(statusCount).format("0,0")) || "Unknown"}</td>
+            </tr>
+            <tr>
+              <td>
+                Insularity{" "}
+                <Tooltip
+                  content={
+                    <span>
+                      The percentage of mentions that are directed
+                      <br />
+                      toward users on the same instance.
+                    </span>
+                  }
+                  position={Position.TOP}
+                  className={Classes.DARK}
+                >
+                  <Icon icon={IconNames.HELP} iconSize={Icon.SIZE_STANDARD} />
+                </Tooltip>
+              </td>
+              <td>{(insularity && numeral.default(insularity).format("0.0%")) || "Unknown"}</td>
             </tr>
             <tr>
               <td>Known peers</td>
-              <td>{domainCount || "Unknown"}</td>
+              <td>{(domainCount && numeral.default(domainCount).format("0,0")) || "Unknown"}</td>
             </tr>
             <tr>
               <td>Last updated</td>

frontend/src/components/screens/AboutScreen.tsx

@@ -7,7 +7,7 @@ export const AboutScreen: React.FC = () => (
     <H1>About</H1>
     <p className={Classes.RUNNING_TEXT}>
       fediverse.space is a tool to visualize networks and communities on the{" "}
-      <a href="https://en.wikipedia.org/wiki/Fediverse" target="_blank">
+      <a href="https://en.wikipedia.org/wiki/Fediverse" target="_blank" rel="noopener noreferrer">
         fediverse
       </a>
       . It works by crawling every instance it can find and aggregating statistics on communication between these.
@@ -26,7 +26,7 @@ export const AboutScreen: React.FC = () => (
     </H4>
     <p className={Classes.RUNNING_TEXT}>
       Check out{" "}
-      <a href="https://gitlab.com/taobojlen/fediverse.space/issues/24" target="_blank">
+      <a href="https://gitlab.com/taobojlen/fediverse.space/issues/24" target="_blank" rel="noopener noreferrer">
         this GitLab issue
       </a>
       .
@@ -35,7 +35,7 @@ export const AboutScreen: React.FC = () => (
     <H4>How do I add my personal instance?</H4>
     <p className={Classes.RUNNING_TEXT}>
       Send a DM to{" "}
-      <a href="https://cursed.technology/@fediversespace" target="_blank">
+      <a href="https://cursed.technology/@fediversespace" target="_blank" rel="noopener noreferrer">
         @fediversespace
       </a>{" "}
       on Mastodon. Make sure to send it from the account that's listed as the instance admin.
@@ -54,27 +54,31 @@ export const AboutScreen: React.FC = () => (
       This site is inspired by several other sites in the same vein:
       <ul className={Classes.LIST}>
         <li>
-          <a href="https://the-federation.info/" target="_blank">
+          <a href="https://the-federation.info/" target="_blank" rel="noopener noreferrer">
             the-federation.info
           </a>
         </li>
         <li>
-          <a href="http://fediverse.network/" target="_blank">
+          <a href="http://fediverse.network/" target="_blank" rel="noopener noreferrer">
             fediverse.network
           </a>
         </li>
         <li>
-          <a href="https://lucahammer.at/vis/fediverse/2018-08-30-mastoverse_hashtags/" target="_blank">
+          <a
+            href="https://lucahammer.at/vis/fediverse/2018-08-30-mastoverse_hashtags/"
+            target="_blank"
+            rel="noopener noreferrer"
+          >
             Mastodon hashtag network
           </a>
           {" by "}
-          <a href="https://vis.social/web/statuses/100634284168959187" target="_blank">
+          <a href="https://vis.social/web/statuses/100634284168959187" target="_blank" rel="noopener noreferrer">
             @Luca@vis.social
           </a>
         </li>
       </ul>
       The source code for fediverse.space is available on{" "}
-      <a href="https://gitlab.com/taobojlen/fediverse.space" target="_blank">
+      <a href="https://gitlab.com/taobojlen/fediverse.space" target="_blank" rel="noopener noreferrer">
         GitLab
       </a>
       ; issues and pull requests are welcome!

frontend/src/redux/types.ts

@@ -17,17 +17,17 @@ export interface IAction {
 
 export interface IInstance {
   name: string;
-  numUsers?: number;
 }
 
 export interface IInstanceDetails {
   name: string;
-  peers?: IInstance[];
   description?: string;
-  domainCount?: number;
-  statusCount?: number;
-  userCount?: number;
   version?: string;
+  userCount?: number;
+  insularity?: number;
+  statusCount?: number;
+  domainCount?: number;
+  peers?: IInstance[];
   lastUpdated?: string;
   status: string;
 }

frontend/yarn.lock

@@ -1433,6 +1433,11 @@
   resolved "https://registry.yarnpkg.com/@types/normalize-package-data/-/normalize-package-data-2.4.0.tgz#e486d0d97396d79beedd0a6e33f4534ff6b4973e"
   integrity sha512-f5j5b/Gf71L+dbqxIpQ4Z2WlmI/mPJ0fOkGGmFgtb6sAu97EPczzbS3/tJKxmcYDj55OX6ssqwDAWOHIYDRDGA==
 
+"@types/numeral@^0.0.25":
+  version "0.0.25"
+  resolved "https://registry.yarnpkg.com/@types/numeral/-/numeral-0.0.25.tgz#b6f55062827a4787fe4ab151cf3412a468e65271"
+  integrity sha512-ShHzHkYD+Ldw3eyttptCpUhF1/mkInWwasQkCNXZHOsJMJ/UMa8wXrxSrTJaVk0r4pLK/VnESVM0wFsfQzNEKQ==
+
 "@types/prop-types@*":
   version "15.7.0"
   resolved "https://registry.yarnpkg.com/@types/prop-types/-/prop-types-15.7.0.tgz#4c48fed958d6dcf9487195a0ef6456d5f6e0163a"
@@ -7336,6 +7341,11 @@ number-is-nan@^1.0.0:
   resolved "https://registry.yarnpkg.com/number-is-nan/-/number-is-nan-1.0.1.tgz#097b602b53422a522c1afb8790318336941a011d"
   integrity sha1-CXtgK1NCKlIsGvuHkDGDNpQaAR0=
 
+numeral@^2.0.6:
+  version "2.0.6"
+  resolved "https://registry.yarnpkg.com/numeral/-/numeral-2.0.6.tgz#4ad080936d443c2561aed9f2197efffe25f4e506"
+  integrity sha1-StCAk21EPCVhrtnyGX7//iX05QY=
+
 nwsapi@^2.0.7:
   version "2.1.3"
   resolved "https://registry.yarnpkg.com/nwsapi/-/nwsapi-2.1.3.tgz#25f3a5cec26c654f7376df6659cdf84b99df9558"