diff --git a/.gitignore b/.gitignore index 6d687ca..10a1db2 100644 --- a/.gitignore +++ b/.gitignore @@ -2,7 +2,7 @@ .idea/ *.gexf data/ -.vscode/ +*.class backend/.sobelow @@ -80,6 +80,7 @@ yarn-error.log* /gephi/.gradle/ /gephi/build/ +/gephi/bin/ /gephi/lib/* /gephi/!lib/.gitkeep # 64MB file but I don't have much faith that it'll remain available... diff --git a/.vscode/extensions.json b/.vscode/extensions.json new file mode 100644 index 0000000..df38b8d --- /dev/null +++ b/.vscode/extensions.json @@ -0,0 +1,8 @@ +{ + "recommendations": [ + "jakebecker.elixir-ls", + "ms-vscode.vscode-typescript-tslint-plugin", + "kevinmcgowan.typescriptimport", + "msjsdiag.debugger-for-chrome" + ] +} diff --git a/.vscode/launch.json b/.vscode/launch.json new file mode 100644 index 0000000..e351800 --- /dev/null +++ b/.vscode/launch.json @@ -0,0 +1,16 @@ +{ + // Use IntelliSense to learn about possible attributes. + // Hover to view descriptions of existing attributes. + // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387 + "version": "0.2.0", + "configurations": [ + { + "type": "chrome", + "request": "launch", + "name": "Launch Chrome", + "url": "http://localhost:3000", + "webRoot": "${workspaceFolder}/frontend/src", + "runtimeExecutable": "/usr/bin/chromium-browser" + } + ] +} diff --git a/README.md b/README.md index bfdcfe8..062fe71 100644 --- a/README.md +++ b/README.md @@ -6,8 +6,16 @@ Read the latest updates on Mastodon: [@fediversespace](https://cursed.technology ![A screenshot of fediverse.space](screenshot.png) +1. [Requirements](#requirements) +2. [Running it](#running-it) +3. [Commands](#commands) +4. [Privacy](#privacy) +5. [Acknowledgements](#acknowledgements) + ## Requirements +Though dockerized, backend development is easiest if you have the following installed. + - For the scraper + API: - Elixir - Postgres @@ -17,8 +25,6 @@ Read the latest updates on Mastodon: [@fediversespace](https://cursed.technology - Node.js - Yarn -All of the above can also be run through Docker with `docker-compose`. - ## Running it ### Backend @@ -37,17 +43,21 @@ All of the above can also be run through Docker with `docker-compose`. ### Backend -After running the backend in Docker: - -- `docker-compose run gephi java -Xmx1g -jar build/libs/graphBuilder.jar` lays out the graph - `./gradlew shadowJar` compiles the graph layout program. `java -Xmx1g -jar build/libs/graphBuilder.jar` runs it. +If running in docker, this means you run + +- `docker-compose build gephi` +- `docker-compose run gephi java -Xmx1g -jar build/libs/graphBuilder.jar` lays out the graph ### Frontend -- `yarn build` to create an optimized build for deployment +- `yarn build` creates an optimized build for deployment -### Acknowledgements +## Privacy + +This project doesn't crawl personal instances: the goal is to understand communities, not individuals. The threshold for what makes an instance "personal" is defined in the [backend config](backend/config/config.exs) and the [graph builder SQL](gephi/src/main/java/space/fediverse/graph/GraphBuilder.java). + +## Acknowledgements [![NLnet logo](https://i.imgur.com/huV3rvo.png)](https://nlnet.nl/project/fediverse_space/) diff --git a/backend/lib/backend/api.ex b/backend/lib/backend/api.ex index 6e5c217..c1d86df 100644 --- a/backend/lib/backend/api.ex +++ b/backend/lib/backend/api.ex @@ -1,5 +1,6 @@ defmodule Backend.Api do alias Backend.{Crawl, Edge, Instance, Repo} + import Backend.Util import Ecto.Query @spec list_instances() :: [Instance.t()] @@ -18,18 +19,27 @@ defmodule Backend.Api do @doc """ Returns a list of instances that * have a user count (required to give the instance a size on the graph) + * the user count is > the threshold * have x and y coordinates """ @spec list_nodes() :: [Instance.t()] def list_nodes() do + user_threshold = get_config(:personal_instance_threshold) + Instance - |> where([i], not is_nil(i.x) and not is_nil(i.y) and not is_nil(i.user_count)) + |> where( + [i], + not is_nil(i.x) and not is_nil(i.y) and not is_nil(i.user_count) and + i.user_count >= ^user_threshold + ) |> select([c], [:domain, :user_count, :x, :y]) |> Repo.all() end @spec list_edges() :: [Edge.t()] def list_edges() do + user_threshold = get_config(:personal_instance_threshold) + Edge |> join(:inner, [e], i1 in Instance, on: e.source_domain == i1.domain) |> join(:inner, [e], i2 in Instance, on: e.target_domain == i2.domain) @@ -37,7 +47,8 @@ defmodule Backend.Api do |> where( [e, i1, i2], not is_nil(i1.x) and not is_nil(i1.y) and - not is_nil(i2.x) and not is_nil(i2.y) + not is_nil(i2.x) and not is_nil(i2.y) and + i1.user_count >= ^user_threshold and i2.user_count >= ^user_threshold ) |> Repo.all() end diff --git a/backend/lib/backend/crawler/crawler.ex b/backend/lib/backend/crawler/crawler.ex index 534e4c3..846acee 100644 --- a/backend/lib/backend/crawler/crawler.ex +++ b/backend/lib/backend/crawler/crawler.ex @@ -188,6 +188,10 @@ defmodule Backend.Crawler do end defp save(%{domain: domain, error: error}) do + if error == nil do + error = "no api found" + end + Repo.insert!(%Crawl{ instance_domain: domain, error: error diff --git a/backend/lib/backend_web/controllers/instance_controller.ex b/backend/lib/backend_web/controllers/instance_controller.ex index cbf292a..850ccae 100644 --- a/backend/lib/backend_web/controllers/instance_controller.ex +++ b/backend/lib/backend_web/controllers/instance_controller.ex @@ -4,7 +4,7 @@ defmodule BackendWeb.InstanceController do import Backend.Util alias Backend.Api - action_fallback BackendWeb.FallbackController + action_fallback(BackendWeb.FallbackController) def index(conn, _params) do instances = Api.list_instances() @@ -13,7 +13,7 @@ defmodule BackendWeb.InstanceController do def show(conn, %{"id" => domain}) do instance = Api.get_instance!(domain) - last_crawl = get_last_crawl(domain) + last_crawl = get_last_successful_crawl(domain) render(conn, "show.json", instance: instance, crawl: last_crawl) end diff --git a/backend/lib/backend_web/views/instance_view.ex b/backend/lib/backend_web/views/instance_view.ex index dd808e5..8cda1f0 100644 --- a/backend/lib/backend_web/views/instance_view.ex +++ b/backend/lib/backend_web/views/instance_view.ex @@ -1,6 +1,7 @@ defmodule BackendWeb.InstanceView do use BackendWeb, :view alias BackendWeb.InstanceView + import Backend.Util require Logger def render("index.json", %{instances: instances}) do @@ -16,6 +17,8 @@ defmodule BackendWeb.InstanceView do end def render("instance_detail.json", %{instance: instance, crawl: crawl}) do + user_threshold = get_config(:personal_instance_threshold) + [status, last_updated] = case crawl do nil -> @@ -28,17 +31,26 @@ defmodule BackendWeb.InstanceView do end end - %{ - name: instance.domain, - description: instance.description, - version: instance.version, - userCount: instance.user_count, - insularity: instance.insularity, - statusCount: instance.status_count, - domainCount: length(instance.peers), - peers: render_many(instance.peers, InstanceView, "instance.json"), - lastUpdated: last_updated, - status: status - } + cond do + instance.user_count < user_threshold -> + %{ + name: instance.domain, + status: "personal instance" + } + + true -> + %{ + name: instance.domain, + description: instance.description, + version: instance.version, + userCount: instance.user_count, + insularity: instance.insularity, + statusCount: instance.status_count, + domainCount: length(instance.peers), + peers: render_many(instance.peers, InstanceView, "instance.json"), + lastUpdated: last_updated, + status: status + } + end end end diff --git a/frontend/package.json b/frontend/package.json index e4db79b..b44341c 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -34,7 +34,7 @@ "classnames": "^2.2.6", "cross-fetch": "^3.0.4", "cytoscape": "^3.8.1", - "cytoscape-cola": "^2.3.0", + "cytoscape-popper": "^1.0.4", "lodash": "^4.17.14", "moment": "^2.22.2", "normalize.css": "^8.0.0", @@ -49,7 +49,8 @@ "redux": "^4.0.4", "redux-thunk": "^2.3.0", "sanitize-html": "^1.20.1", - "styled-components": "^4.3.2" + "styled-components": "^4.3.2", + "tippy.js": "^4.3.4" }, "devDependencies": { "@blueprintjs/tslint-config": "^1.8.1", diff --git a/frontend/src/components/Cytoscape.tsx b/frontend/src/components/Cytoscape.tsx new file mode 100644 index 0000000..f787838 --- /dev/null +++ b/frontend/src/components/Cytoscape.tsx @@ -0,0 +1,152 @@ +import cytoscape from "cytoscape"; +import popper from "cytoscape-popper"; +import * as React from "react"; +import ReactDOM from "react-dom"; +import styled from "styled-components"; +import tippy, { Instance } from "tippy.js"; +import { DEFAULT_NODE_COLOR, SELECTED_NODE_COLOR } from "../constants"; + +const EntireWindowDiv = styled.div` + position: absolute; + top: 50px; + bottom: 0; + right: 0; + left: 0; +`; + +interface ICytoscapeProps { + elements: cytoscape.ElementsDefinition; + onInstanceSelect: (domain: string) => void; + onInstanceDeselect: () => void; +} +class Cytoscape extends React.Component { + public cy?: cytoscape.Core; + + public componentDidMount() { + const container = ReactDOM.findDOMNode(this); + cytoscape.use(popper as any); + this.cy = cytoscape({ + autoungrabify: true, + container: container as any, + elements: this.props.elements, + hideEdgesOnViewport: true, + hideLabelsOnViewport: true, + layout: { + name: "preset" + }, + maxZoom: 2, + minZoom: 0.03, + pixelRatio: 1.0, + selectionType: "single" + }); + + // Setup node tooltip on hover + this.cy.nodes().forEach(n => { + const domain = n.data("id"); + const ref = (n as any).popperRef(); + const t = tippy(ref, { + animateFill: false, + animation: "fade", + content: domain, + duration: 100, + trigger: "manual" + }); + n.on("mouseover", e => { + (t as Instance).show(); + }); + n.on("mouseout", e => { + (t as Instance).hide(); + }); + }); + + const style = this.cy.style() as any; + + style + .clear() + .selector("node") + .style({ + "background-color": DEFAULT_NODE_COLOR, + // The size from the backend is log_10(userCount), which from 10 <= userCount <= 1,000,000 gives us the range + // 1-6. We map this to the range of sizes we want. + // TODO: I should probably check that that the backend is actually using log_10 and not log_e, but it look + // quite good as it is, so... + height: "mapData(size, 1, 6, 20, 200)", + label: "data(id)", + width: "mapData(size, 1, 6, 20, 200)" + }) + .selector("node:selected") + .style({ + "background-color": SELECTED_NODE_COLOR + }) + .selector("edge") + .style({ + "curve-style": "haystack", // fast edges + "line-color": DEFAULT_NODE_COLOR, + width: "mapData(weight, 0, 0.5, 1, 20)" + }) + .selector("node[label]") + .style({ + color: DEFAULT_NODE_COLOR, + "font-size": 50, + "min-zoomed-font-size": 16 + }) + .selector(".hidden") + .style({ + display: "none" + }) + .selector(".thickEdge") + .style({ + width: 2 + }) + .update(); + + this.cy.nodes().on("select", e => { + const instanceId = e.target.data("id"); + if (instanceId) { + this.props.onInstanceSelect(instanceId); + } + + const neighborhood = this.cy!.$id(instanceId).closedNeighborhood(); + // Reset graph visibility + this.cy!.batch(() => { + this.cy!.nodes().removeClass("hidden"); + this.cy!.edges().removeClass("thickEdge"); + // Then hide everything except neighborhood + this.cy!.nodes() + .diff(neighborhood) + .left.addClass("hidden"); + neighborhood.connectedEdges().addClass("thickEdge"); + }); + }); + this.cy.nodes().on("unselect", e => { + this.props.onInstanceDeselect(); + this.cy!.batch(() => { + this.cy!.nodes().removeClass("hidden"); + this.cy!.edges().removeClass("thickEdge"); + }); + }); + this.cy.on("click", e => { + // Clicking on the background should also deselect + const target = e.target; + if (!target) { + this.props.onInstanceDeselect(); + } + this.cy!.batch(() => { + this.cy!.nodes().removeClass("hidden"); + this.cy!.edges().removeClass("thickEdge"); + }); + }); + } + + public componentWillUnmount() { + if (this.cy) { + this.cy.destroy(); + } + } + + public render() { + return ; + } +} + +export default Cytoscape; diff --git a/frontend/src/components/CytoscapeGraph.tsx b/frontend/src/components/CytoscapeGraph.tsx deleted file mode 100644 index 0be272c..0000000 --- a/frontend/src/components/CytoscapeGraph.tsx +++ /dev/null @@ -1,218 +0,0 @@ -import cytoscape from "cytoscape"; -// import cola from "cytoscape-cola"; -import * as React from "react"; -import { connect } from "react-redux"; - -import { Dispatch } from "redux"; -import styled from "styled-components"; -import { DEFAULT_NODE_COLOR, SELECTED_NODE_COLOR } from "../constants"; -import { selectAndLoadInstance } from "../redux/actions"; -import { IAppState, IGraph } from "../redux/types"; -import { ErrorState } from "./ErrorState"; -// import { FloatingLayoutSelect } from "./FloatingLayoutSelect"; -import { FloatingResetButton } from "./FloatingResetButton"; - -interface IGraphProps { - graph?: IGraph; - currentInstanceName: string | null; - selectAndLoadInstance: (name: string) => void; -} -interface IGraphState { - layoutAlgorithm: string; - isLayouting: boolean; - didError: boolean; -} -class GraphImpl extends React.Component { - private cy?: cytoscape.Core; - // private layout?: cytoscape.Layouts; - private cytoscapeDiv: React.RefObject; - - public constructor(props: IGraphProps) { - super(props); - this.cytoscapeDiv = React.createRef(); - this.state = { layoutAlgorithm: "cola", isLayouting: false, didError: false }; - } - - public render() { - if (this.state.didError) { - return ; - } - - const FullDiv = styled.div` - position: absolute; - top: 50px; - bottom: 0; - right: 0; - left: 0; - `; - - return ( -
- - {/* */} - -
- ); - } - - public componentDidMount() { - let { graph } = this.props; - if (!graph) { - this.setState({ didError: true }); - return; - } - - // Check that all nodes have size & coordinates; otherwise the graph will look messed up - const lengthBeforeFilter = graph.nodes.length; - graph = { ...graph, nodes: graph.nodes.filter(n => n.data.size && n.position.x && n.position.y) }; - if (graph.nodes.length !== lengthBeforeFilter) { - // tslint:disable-next-line:no-console - console.error( - "Some nodes were missing details: " + - graph.nodes.filter(n => !n.data.size || !n.position.x || !n.position.y).map(n => n.data.label) - ); - this.setState({ didError: true }); - } - - // cytoscape.use(cola as any); - this.initGraph(); - } - - public componentDidUpdate() { - this.initGraph(); - } - - // private handleLayoutSelect = (layout: string) => { - // this.setState({ layoutAlgorithm: layout }); - // }; - - // private startLayout = () => { - // if (!this.cy) { - // return; - // } - // const options = { - // cola: { - // animate: true, - // convergenceThreshold: 0.1, - // edgeLength: (edge: any) => 1 / edge.data("weight"), - // name: "cola" - // }, - // cose: { - // animate: false, - // idealEdgeLength: (edge: any) => 1 / edge.data("weight"), - // name: "cose", - // numIter: 100 - // } - // }; - // this.layout = this.cy.layout(options[this.state.layoutAlgorithm] as any); - // this.layout.run(); - // }; - - // private stopLayout = () => { - // if (!this.layout) { - // return; - // } - // this.layout.stop(); - // }; - - private initGraph = () => { - const { graph } = this.props; - if (this.state.didError || !graph) { - return; - } - this.cy = cytoscape({ - autoungrabify: false, - container: this.cytoscapeDiv.current, - elements: { - edges: graph.edges.map(e => ({ - ...e, - data: { - ...e.data, - weight: Math.min(Math.max(e.data.weight * 100, 2), 10) - }, - selectable: false - })), - nodes: graph.nodes.map(n => ({ - ...n, - data: { - ...n.data, - size: Math.min(Math.max(n.data.size * 10, 10), 80) - } - })) - }, - layout: { - name: "preset" - }, - selectionType: "single", - style: [ - { - selector: "node:selected", - style: { - "background-color": SELECTED_NODE_COLOR, - label: "data(id)" - } - }, - { - selector: "node", - style: { - "background-color": DEFAULT_NODE_COLOR, - height: "data(size)", - label: "data(id)", - width: "data(size)" - } - }, - { - selector: "edge", - style: { - width: "data(weight)" - } - }, - { - selector: "label", - style: { - color: DEFAULT_NODE_COLOR - } - } - ] - }); - this.cy.nodes().on("select", e => { - const instanceId = e.target.data("id"); - if (instanceId) { - // console.log(`selecting ${instanceId}`); - // console.log(`now selected: ${this.cy && this.cy.$(":selected")}`); - this.props.selectAndLoadInstance(instanceId); - } - }); - this.cy.nodes().on("unselect", e => { - const instanceId = e.target.data("id"); - if (instanceId) { - // console.log(`unselecting ${instanceId}`); - this.props.selectAndLoadInstance(""); - } - }); - }; - - private resetGraph = () => { - if (!this.cy) { - return; - } - this.cy.reset(); - }; -} - -const mapStateToProps = (state: IAppState) => ({ - currentInstanceName: state.currentInstance.currentInstanceName, - graph: state.data.graph -}); -const mapDispatchToProps = (dispatch: Dispatch) => ({ - selectAndLoadInstance: (instanceName: string) => dispatch(selectAndLoadInstance(instanceName) as any) -}); -export const CytoscapeGraph = connect( - mapStateToProps, - mapDispatchToProps -)(GraphImpl); diff --git a/frontend/src/components/FloatingLayoutSelect.tsx b/frontend/src/components/FloatingLayoutSelect.tsx deleted file mode 100644 index f59f37d..0000000 --- a/frontend/src/components/FloatingLayoutSelect.tsx +++ /dev/null @@ -1,53 +0,0 @@ -import { Button, H6, MenuItem } from "@blueprintjs/core"; -import { IconNames } from "@blueprintjs/icons"; -import { ItemRenderer, Select } from "@blueprintjs/select"; -import * as React from "react"; -import FloatingCard from "./FloatingCard"; - -interface ILayoutToDisplayName { - [key: string]: string; -} -const layouts: ILayoutToDisplayName = { - cola: "COLA", - cose: "CoSE" -}; -const LayoutSelect = Select.ofType(); - -const LayoutItemRenderer: ItemRenderer = (layout, { handleClick, modifiers }) => ( - -); - -interface IFloatingLayoutSelectProps { - currentLayoutKey: string; - onItemSelect: (layout: string) => void; - startLayout: () => void; - stopLayout: () => void; -} -export const FloatingLayoutSelect: React.FC = ({ - currentLayoutKey, - onItemSelect, - startLayout, - stopLayout -}) => { - return ( - -
Layout
- -