diff --git a/CHANGELOG.md b/CHANGELOG.md index 3325ea3..f7499c7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,6 +13,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Added ElasticSearch full-text search over instance domains and descriptions. - Search results are now highlighted on the graph. - When you hover a search result, it is now highlighted on the graph. +- Instance details now show activity rate (average number of statuses posted per day). +- It's now possible to color code by activity rate. ### Changed diff --git a/README.md b/README.md index 3eddbef..8c152c9 100644 --- a/README.md +++ b/README.md @@ -59,22 +59,31 @@ If running in docker, this means you run This project doesn't crawl personal instances: the goal is to understand communities, not individuals. The threshold for what makes an instance "personal" is defined in the [backend config](backend/config/config.exs) and the [graph builder SQL](gephi/src/main/java/space/fediverse/graph/GraphBuilder.java). ## Deployment + You don't have to follow these instructions, but it's one way to set up a continuous deployment pipeline. The following are for the backend; the frontend is just a static HTML/JS site that can be deployed anywhere. + 1. Install [Dokku](http://dokku.viewdocs.io/dokku/) on your web server. 2. Install [dokku-postgres](https://github.com/dokku/dokku-postgres), [dokku-monorepo](https://github.com/notpushkin/dokku-monorepo), and [dokku-letsencrypt](https://github.com/dokku/dokku-letsencrypt). 3. Create the apps - * `dokku apps:create phoenix` - * `dokku apps:create gephi` + +- `dokku apps:create phoenix` +- `dokku apps:create gephi` + 4. Create the backing database - * `dokku postgres:create fediversedb` - * `dokku postgres:link fediversedb phoenix` - * `dokku postgres:link fediversedb gephi` + +- `dokku postgres:create fediversedb` +- `dokku postgres:link fediversedb phoenix` +- `dokku postgres:link fediversedb gephi` + 5. Update the backend configuration. In particular, change the `user_agent` in [config.exs](/backend/config/config.exs) to something descriptive. 6. Push the apps, e.g. `git push dokku@:phoenix` (note that the first push cannot be from the CD pipeline). 7. Set up SSL for the Phoenix app - * `dokku letsencrypt phoenix` - * `dokku letsencrypt:cron-job --add` + +- `dokku letsencrypt phoenix` +- `dokku letsencrypt:cron-job --add` + 8. Set up a cron job for the graph layout (use the `dokku` user). E.g. + ``` SHELL=/bin/bash 0 2 * * * /usr/bin/dokku run gephi java -Xmx1g -jar build/libs/graphBuilder.jar @@ -82,6 +91,6 @@ SHELL=/bin/bash ## Acknowledgements -[![NLnet logo](https://i.imgur.com/huV3rvo.png)](https://nlnet.nl/project/fediverse_space/) +[![NLnet logo](/nlnet-logo.png)](https://nlnet.nl/project/fediverse_space/) Many thanks to [NLnet](https://nlnet.nl/project/fediverse_space/) for their support and guidance of this project. diff --git a/backend/config/config.exs b/backend/config/config.exs index ecef9a5..18eb2c0 100644 --- a/backend/config/config.exs +++ b/backend/config/config.exs @@ -74,6 +74,8 @@ config :backend, Backend.Scheduler, {"15 0 * * *", {Backend.Scheduler, :generate_edges, []}}, # 00.30 every night {"30 0 * * *", {Backend.Scheduler, :generate_insularity_scores, []}}, + # 00.45 every night + {"45 0 * * *", {Backend.Scheduler, :generate_status_rate, []}}, # Every 3 hours {"0 */3 * * *", {Backend.Scheduler, :check_for_spam_instances, []}} ] diff --git a/backend/lib/backend/api.ex b/backend/lib/backend/api.ex index 4dcc230..ec935ce 100644 --- a/backend/lib/backend/api.ex +++ b/backend/lib/backend/api.ex @@ -37,7 +37,7 @@ defmodule Backend.Api do i.user_count >= ^user_threshold and not i.opt_out ) |> maybe_filter_nodes_to_neighborhood(domain) - |> select([c], [:domain, :user_count, :x, :y, :type]) + |> select([c], [:domain, :user_count, :x, :y, :type, :statuses_per_day]) |> Repo.all() end diff --git a/backend/lib/backend/instance.ex b/backend/lib/backend/instance.ex index e0d6891..d497cf4 100644 --- a/backend/lib/backend/instance.ex +++ b/backend/lib/backend/instance.ex @@ -10,6 +10,7 @@ defmodule Backend.Instance do field :version, :string field :insularity, :float field :type, :string + field :statuses_per_day, :float field :base_domain, :string field :opt_in, :boolean field :opt_out, :boolean @@ -39,6 +40,7 @@ defmodule Backend.Instance do :insularity, :updated_at, :type, + :statuses_per_day, :base_domain, :opt_in, :opt_out diff --git a/backend/lib/backend/scheduler.ex b/backend/lib/backend/scheduler.ex index 5ca1eaa..d2899d9 100644 --- a/backend/lib/backend/scheduler.ex +++ b/backend/lib/backend/scheduler.ex @@ -76,6 +76,61 @@ defmodule Backend.Scheduler do ) end + @doc """ + This function calculates the average number of statuses per hour over the last month. + """ + def generate_status_rate() do + now = get_now() + # We want the earliest sucessful crawl so that we can exclude it from the statistics. + # This is because the first crawl goes up to one month into the past -- this would mess up the counts! + # The statistics from here assume that all statuses were written at exactly the crawl's inserted_at timestamp. + earliest_successful_crawl_subquery = + Crawl + |> group_by([c], c.instance_domain) + |> select([c], %{ + instance_domain: c.instance_domain, + earliest_crawl: min(c.inserted_at) + }) + + instances = + Crawl + |> join(:inner, [c], c2 in subquery(earliest_successful_crawl_subquery), + on: c.instance_domain == c2.instance_domain + ) + |> where( + [c, c2], + c.inserted_at > c2.earliest_crawl and not is_nil(c.statuses_seen) and is_nil(c.error) + ) + |> select([c], %{ + instance_domain: c.instance_domain, + status_count: sum(c.statuses_seen), + second_earliest_crawl: min(c.inserted_at) + }) + |> group_by([c], c.instance_domain) + |> Repo.all() + |> Enum.map(fn %{ + instance_domain: domain, + status_count: status_count, + second_earliest_crawl: oldest_timestamp + } -> + time_diff_days = NaiveDateTime.diff(now, oldest_timestamp, :second) / (3600 * 24) + + # (we're actually only ever updating, not inserting, so inserted_at will always be ignored...) + %{ + domain: domain, + statuses_per_day: status_count / time_diff_days, + updated_at: now, + inserted_at: now + } + end) + + Instance + |> Repo.insert_all(instances, + on_conflict: {:replace, [:statuses_per_day, :updated_at]}, + conflict_target: :domain + ) + end + @doc """ This function aggregates statistics from the interactions in the database. It calculates the strength of edges between nodes. Self-edges are not generated. diff --git a/backend/lib/backend_web/views/graph_view.ex b/backend/lib/backend_web/views/graph_view.ex index 04afab0..b231177 100644 --- a/backend/lib/backend_web/views/graph_view.ex +++ b/backend/lib/backend_web/views/graph_view.ex @@ -3,9 +3,24 @@ defmodule BackendWeb.GraphView do alias BackendWeb.GraphView def render("index.json", %{nodes: nodes, edges: edges}) do + statuses_per_day = + nodes + |> Enum.map(fn %{statuses_per_day: statuses_per_day} -> statuses_per_day end) + |> Enum.filter(fn s -> s != nil end) + %{ - nodes: render_many(nodes, GraphView, "node.json", as: :node), - edges: render_many(edges, GraphView, "edge.json", as: :edge) + graph: %{ + nodes: render_many(nodes, GraphView, "node.json", as: :node), + edges: render_many(edges, GraphView, "edge.json", as: :edge) + }, + metadata: %{ + ranges: %{ + statusesPerDay: [ + Enum.min(statuses_per_day), + Enum.max(statuses_per_day) + ] + } + } } end @@ -22,7 +37,8 @@ defmodule BackendWeb.GraphView do id: node.domain, label: node.domain, size: size, - type: node.type + type: node.type, + statusesPerDay: node.statuses_per_day }, position: %{ x: node.x, diff --git a/backend/lib/backend_web/views/instance_view.ex b/backend/lib/backend_web/views/instance_view.ex index 8391062..3c04b8e 100644 --- a/backend/lib/backend_web/views/instance_view.ex +++ b/backend/lib/backend_web/views/instance_view.ex @@ -42,7 +42,8 @@ defmodule BackendWeb.InstanceView do peers: render_many(filtered_peers, InstanceView, "instance.json"), lastUpdated: last_updated, status: status, - type: instance.type + type: instance.type, + statusesPerDay: instance.statuses_per_day } end end diff --git a/backend/priv/repo/migrations/20190727140717_add_statuses_per_day.exs b/backend/priv/repo/migrations/20190727140717_add_statuses_per_day.exs new file mode 100644 index 0000000..0598d22 --- /dev/null +++ b/backend/priv/repo/migrations/20190727140717_add_statuses_per_day.exs @@ -0,0 +1,9 @@ +defmodule Backend.Repo.Migrations.AddStatusesPerHour do + use Ecto.Migration + + def change do + alter table(:instances) do + add :statuses_per_day, :float + end + end +end diff --git a/frontend/src/assets/nlnet.png b/frontend/src/assets/nlnet.png new file mode 100644 index 0000000..27fad98 Binary files /dev/null and b/frontend/src/assets/nlnet.png differ diff --git a/frontend/src/components/atoms/FloatingCard.tsx b/frontend/src/components/atoms/FloatingCard.tsx index 87b63cd..32b6a0d 100644 --- a/frontend/src/components/atoms/FloatingCard.tsx +++ b/frontend/src/components/atoms/FloatingCard.tsx @@ -4,6 +4,7 @@ import styled from "styled-components"; const FloatingCardRow = styled.div` display: flex; + max-width: 250px; `; const FloatingCardElement = styled(Card)` margin: 0 0 10px 10px; diff --git a/frontend/src/components/atoms/GraphKey.tsx b/frontend/src/components/atoms/GraphKey.tsx index 83f6908..a978d2b 100644 --- a/frontend/src/components/atoms/GraphKey.tsx +++ b/frontend/src/components/atoms/GraphKey.tsx @@ -1,12 +1,14 @@ -import { Button, Classes, H5, H6, MenuItem } from "@blueprintjs/core"; +import { Button, Classes, H5, MenuItem } from "@blueprintjs/core"; import { IconNames } from "@blueprintjs/icons"; import { ItemRenderer, Select } from "@blueprintjs/select"; +import * as numeral from "numeral"; import React from "react"; import styled from "styled-components"; import { FloatingCard, InstanceType } from "."; -import { IColorSchemeType } from "../../types"; +import { QUANTITATIVE_COLOR_SCHEME } from "../../constants"; +import { IColorScheme } from "../../types"; -const ColorSchemeSelect = Select.ofType(); +const ColorSchemeSelect = Select.ofType(); const StyledLi = styled.li` margin-top: 2px; @@ -14,16 +16,49 @@ const StyledLi = styled.li` const StyledKeyContainer = styled.div` margin-top: 10px; `; +const ColorKeyContainer = styled.div` + display: flex; + flex-direction: row; + height: 100px; +`; +const ColorBarContainer = styled.div` + width: 10px; + display: flex; + flex-direction: column; + margin-right: 10px; +`; +interface IColorBarProps { + color: string; +} +const ColorBar = styled.div` + width: 10px; + background-color: ${props => props.color}; + flex: 1; +`; +const TextContainer = styled.div` + display: flex; + flex-direction: column; + justify-content: space-between; +`; interface IGraphKeyProps { - current?: IColorSchemeType; - colorSchemes: IColorSchemeType[]; - onItemSelect: (colorScheme?: IColorSchemeType) => void; + current?: IColorScheme; + colorSchemes: IColorScheme[]; + ranges?: { [key: string]: [number, number] }; + onItemSelect: (colorScheme?: IColorScheme) => void; } -const GraphKey: React.FC = ({ current, colorSchemes, onItemSelect }) => { +const GraphKey: React.FC = ({ current, colorSchemes, ranges, onItemSelect }) => { const unsetColorScheme = () => { onItemSelect(undefined); }; + let key; + if (current) { + if (current.type === "qualitative") { + key = renderQualitativeKey(current.values); + } else if (current.type === "quantitative") { + key = renderQuantitativeKey(ranges![current.cytoscapeDataKey]); + } + } return (
Color coding
@@ -42,27 +77,49 @@ const GraphKey: React.FC = ({ current, colorSchemes, onItemSelec />