diff --git a/CHANGELOG.md b/CHANGELOG.md index ed06886..de33c30 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added +- It is now possible to color code the graph by instance type (e.g. Mastodon, Pleroma, etc.) + ### Changed ### Deprecated diff --git a/backend/lib/backend/api.ex b/backend/lib/backend/api.ex index 8e14fc6..ce812cc 100644 --- a/backend/lib/backend/api.ex +++ b/backend/lib/backend/api.ex @@ -35,7 +35,7 @@ defmodule Backend.Api do i.user_count >= ^user_threshold ) |> maybe_filter_nodes_to_neighborhood(domain) - |> select([c], [:domain, :user_count, :x, :y]) + |> select([c], [:domain, :user_count, :x, :y, :type]) |> Repo.all() end diff --git a/backend/lib/backend/crawler/api_crawler.ex b/backend/lib/backend/crawler/api_crawler.ex index 1b9f45a..4147e2e 100644 --- a/backend/lib/backend/crawler/api_crawler.ex +++ b/backend/lib/backend/crawler/api_crawler.ex @@ -14,6 +14,8 @@ defmodule Backend.Crawler.ApiCrawler do # {domain_mentioned, count} @type instance_interactions :: %{String.t() => integer} + @type instance_type :: :mastodon | :pleroma | :gab + defstruct [ :version, :description, @@ -21,7 +23,8 @@ defmodule Backend.Crawler.ApiCrawler do :status_count, :peers, :interactions, - :statuses_seen + :statuses_seen, + :instance_type ] @type t() :: %__MODULE__{ @@ -31,7 +34,8 @@ defmodule Backend.Crawler.ApiCrawler do status_count: integer, peers: [String.t()], interactions: instance_interactions, - statuses_seen: integer + statuses_seen: integer, + instance_type: instance_type } @doc """ diff --git a/backend/lib/backend/crawler/crawler.ex b/backend/lib/backend/crawler/crawler.ex index cc26661..2c90368 100644 --- a/backend/lib/backend/crawler/crawler.ex +++ b/backend/lib/backend/crawler/crawler.ex @@ -103,6 +103,12 @@ defmodule Backend.Crawler do }) do now = get_now() + instance_type = + case result.instance_type do + nil -> nil + not_nil_type -> Atom.to_string(not_nil_type) + end + ## Update the instance we crawled ## Repo.insert!( %Instance{ @@ -110,7 +116,8 @@ defmodule Backend.Crawler do description: result.description, version: result.version, user_count: result.user_count, - status_count: result.status_count + status_count: result.status_count, + type: instance_type }, on_conflict: [ set: [ @@ -118,6 +125,7 @@ defmodule Backend.Crawler do version: result.version, user_count: result.user_count, status_count: result.status_count, + type: instance_type, updated_at: now ] ], @@ -140,7 +148,7 @@ defmodule Backend.Crawler do result.interactions |> Map.keys() |> list_union(result.peers) - |> Enum.filter(fn domain -> not is_blacklisted?(domain) end) + |> Enum.filter(fn domain -> domain != nil and not is_blacklisted?(domain) end) peers = peers_domains diff --git a/backend/lib/backend/crawler/crawlers/mastodon.ex b/backend/lib/backend/crawler/crawlers/mastodon.ex index edd6bcb..f1f4a91 100644 --- a/backend/lib/backend/crawler/crawlers/mastodon.ex +++ b/backend/lib/backend/crawler/crawlers/mastodon.ex @@ -38,12 +38,17 @@ defmodule Backend.Crawler.Crawlers.Mastodon do crawl_large_instance(domain, instance) else Map.merge( - Map.merge( - Map.take(instance, ["version", "description"]), - Map.take(instance["stats"], ["user_count", "status_count"]) - ) + Map.take(instance["stats"], ["user_count"]) |> Map.new(fn {k, v} -> {String.to_atom(k), v} end), - %{peers: [], interactions: %{}, statuses_seen: 0} + %{ + peers: [], + interactions: %{}, + statuses_seen: 0, + instance_type: nil, + description: nil, + version: nil, + status_count: nil + } ) end end @@ -68,13 +73,25 @@ defmodule Backend.Crawler.Crawlers.Mastodon do } mentions in #{statuses_seen} statuses." ) + instance_type = + cond do + Map.get(instance, "version") |> String.downcase() =~ "pleroma" -> :pleroma + is_gab?(instance) -> :gab + true -> :mastodon + end + Map.merge( Map.merge( Map.take(instance, ["version", "description"]), Map.take(instance["stats"], ["user_count", "status_count"]) ) |> Map.new(fn {k, v} -> {String.to_atom(k), v} end), - %{peers: peers, interactions: interactions, statuses_seen: statuses_seen} + %{ + peers: peers, + interactions: interactions, + statuses_seen: statuses_seen, + instance_type: instance_type + } ) end @@ -208,4 +225,18 @@ defmodule Backend.Crawler.Crawlers.Mastodon do Map.merge(acc, map) end) end + + defp is_gab?(instance) do + title_is_gab = Map.get(instance, "title") |> String.downcase() == "gab social" + + contact_account = Map.get(instance, "contact_account") + + if contact_account != nil do + gab_keys = ["is_pro", "is_verified", "is_donor", "is_investor"] + has_gab_keys = gab_keys |> Enum.any?(&Map.has_key?(contact_account, &1)) + title_is_gab or has_gab_keys + else + title_is_gab + end + end end diff --git a/backend/lib/backend/crawler/stale_instance_manager.ex b/backend/lib/backend/crawler/stale_instance_manager.ex index 79bf15f..b78fcca 100644 --- a/backend/lib/backend/crawler/stale_instance_manager.ex +++ b/backend/lib/backend/crawler/stale_instance_manager.ex @@ -47,6 +47,7 @@ defmodule Backend.Crawler.StaleInstanceManager do Process.send_after(self(), :queue_stale_domains, 60_000) end + # TODO: crawl instances with a blocking robots.txt less often (daily?) defp queue_stale_domains() do interval = -1 * get_config(:crawl_interval_mins) diff --git a/backend/lib/backend/instance.ex b/backend/lib/backend/instance.ex index 656a668..70c1fe4 100644 --- a/backend/lib/backend/instance.ex +++ b/backend/lib/backend/instance.ex @@ -9,6 +9,7 @@ defmodule Backend.Instance do field :status_count, :integer field :version, :string field :insularity, :float + field :type, :string many_to_many :peers, Backend.Instance, join_through: Backend.InstancePeer, @@ -33,7 +34,8 @@ defmodule Backend.Instance do :status_count, :version, :insularity, - :updated_at + :updated_at, + :type ]) |> validate_required([:domain]) |> put_assoc(:peers, attrs.peers) diff --git a/backend/lib/backend_web/views/graph_view.ex b/backend/lib/backend_web/views/graph_view.ex index a8915c1..04afab0 100644 --- a/backend/lib/backend_web/views/graph_view.ex +++ b/backend/lib/backend_web/views/graph_view.ex @@ -4,12 +4,12 @@ defmodule BackendWeb.GraphView do def render("index.json", %{nodes: nodes, edges: edges}) do %{ - nodes: render_many(nodes, GraphView, "node.json"), - edges: render_many(edges, GraphView, "edge.json") + nodes: render_many(nodes, GraphView, "node.json", as: :node), + edges: render_many(edges, GraphView, "edge.json", as: :edge) } end - def render("node.json", %{graph: node}) do + def render("node.json", %{node: node}) do size = case node.user_count > 1 do true -> :math.log(node.user_count) @@ -21,7 +21,8 @@ defmodule BackendWeb.GraphView do data: %{ id: node.domain, label: node.domain, - size: size + size: size, + type: node.type }, position: %{ x: node.x, @@ -30,7 +31,7 @@ defmodule BackendWeb.GraphView do } end - def render("edge.json", %{graph: edge}) do + def render("edge.json", %{edge: edge}) do %{ data: %{ id: edge.id, diff --git a/backend/lib/backend_web/views/search_view.ex b/backend/lib/backend_web/views/search_view.ex index 0be3fe7..87d23c4 100644 --- a/backend/lib/backend_web/views/search_view.ex +++ b/backend/lib/backend_web/views/search_view.ex @@ -23,7 +23,8 @@ defmodule BackendWeb.SearchView do %{ name: instance.domain, description: description, - userCount: instance.user_count + userCount: instance.user_count, + type: instance.type } end end diff --git a/backend/priv/repo/migrations/20190724114437_add_instance_type.exs b/backend/priv/repo/migrations/20190724114437_add_instance_type.exs new file mode 100644 index 0000000..0a6be99 --- /dev/null +++ b/backend/priv/repo/migrations/20190724114437_add_instance_type.exs @@ -0,0 +1,9 @@ +defmodule Backend.Repo.Migrations.AddInstanceType do + use Ecto.Migration + + def change do + alter table(:instances) do + add :type, :string + end + end +end diff --git a/frontend/src/components/atoms/FloatingCard.tsx b/frontend/src/components/atoms/FloatingCard.tsx index 1b4fa80..87b63cd 100644 --- a/frontend/src/components/atoms/FloatingCard.tsx +++ b/frontend/src/components/atoms/FloatingCard.tsx @@ -2,13 +2,18 @@ import { Card, Elevation, ICardProps } from "@blueprintjs/core"; import * as React from "react"; import styled from "styled-components"; +const FloatingCardRow = styled.div` + display: flex; +`; const FloatingCardElement = styled(Card)` - position: absolute; - bottom: 10px; - left: 10px; + margin: 0 0 10px 10px; z-index: 20; `; -const FloatingCard: React.FC = props => ; +const FloatingCard: React.FC = props => ( + + + +); export default FloatingCard; diff --git a/frontend/src/components/atoms/GraphKey.tsx b/frontend/src/components/atoms/GraphKey.tsx new file mode 100644 index 0000000..0461678 --- /dev/null +++ b/frontend/src/components/atoms/GraphKey.tsx @@ -0,0 +1,74 @@ +import { Button, Classes, H5, H6, Icon, MenuItem } from "@blueprintjs/core"; +import { IconNames } from "@blueprintjs/icons"; +import { ItemRenderer, Select } from "@blueprintjs/select"; +import React from "react"; +import styled from "styled-components"; +import { FloatingCard } from "."; +import { QUALITATIVE_COLOR_SCHEME } from "../../constants"; +import { IColorSchemeType } from "../../types"; +import { capitalize } from "../../util"; + +const ColorSchemeSelect = Select.ofType(); + +const StyledLi = styled.li` + margin-top: 2px; +`; +const StyledIcon = styled(Icon)` + margin-right: 5px; +`; +const StyledKeyContainer = styled.div` + margin-top: 10px; +`; + +interface IGraphKeyProps { + current?: IColorSchemeType; + colorSchemes: IColorSchemeType[]; + onItemSelect: (colorScheme?: IColorSchemeType) => void; +} +const GraphKey: React.FC = ({ current, colorSchemes, onItemSelect }) => { + const unsetColorScheme = () => { + onItemSelect(undefined); + }; + return ( + +
Color coding
+ +