improved edges

This commit is contained in:
Tao Bojlén 2019-08-27 13:50:16 +00:00
parent 4d333dd14c
commit 693cf2b2d9
31 changed files with 622 additions and 166 deletions

View file

@ -63,7 +63,9 @@ config :backend, :crawler,
crawl_workers: 20,
blacklist: [
"gab.best",
"4chan.icu"
"4chan.icu",
"pleroma.site",
"pleroma.online"
],
user_agent: "fediverse.space crawler",
admin_phone: System.get_env("ADMIN_PHONE"),

View file

@ -62,8 +62,4 @@ config :backend, :crawler,
personal_instance_threshold: 5,
crawl_interval_mins: 60,
crawl_workers: 10,
blacklist: [
"gab.best",
"4chan.icu"
],
frontend_domain: "localhost:3000"

View file

@ -6,6 +6,13 @@ defmodule Backend.Api do
import Backend.Util
import Ecto.Query
@spec get_instances(Integer.t() | nil) :: Scrivener.Page.t()
def get_instances(page \\ nil) do
Instance
|> where([i], not is_nil(i.type))
|> Repo.paginate(page: page)
end
@spec get_instance(String.t()) :: Instance.t() | nil
def get_instance(domain) do
Instance

View file

@ -26,25 +26,39 @@ defmodule Backend.Crawler.ApiCrawler do
:peers,
:interactions,
:statuses_seen,
:instance_type
:instance_type,
:blocked_domains
]
@type t() :: %__MODULE__{
version: String.t(),
description: String.t(),
version: String.t() | nil,
description: String.t() | nil,
user_count: integer | nil,
status_count: integer | nil,
peers: [String.t()],
interactions: instance_interactions,
statuses_seen: integer,
instance_type: instance_type
instance_type: instance_type | nil,
blocked_domains: [String.t()]
}
@empty_result %{
version: nil,
description: nil,
user_count: nil,
status_count: nil,
peers: [],
interactions: %{},
statuses_seen: 0,
instance_type: nil,
blocked_domains: []
}
@doc """
Check whether the instance at the given domain is of the type that this ApiCrawler implements.
Arguments are the instance domain and the nodeinfo results.
"""
@callback is_instance_type?(String.t(), Nodeinfo.t()) :: boolean()
@callback is_instance_type?(String.t(), ApiCrawler.t()) :: boolean()
@doc """
Check whether the instance allows crawling according to its robots.txt or otherwise.
@ -56,4 +70,11 @@ defmodule Backend.Crawler.ApiCrawler do
Takes two arguments: the domain to crawl and the existing results (from nodeinfo).
"""
@callback crawl(String.t(), Nodeinfo.t()) :: t()
@doc """
Returns the default, empty state
"""
def get_default do
@empty_result
end
end

View file

@ -4,7 +4,17 @@ defmodule Backend.Crawler do
"""
alias __MODULE__
alias Backend.{Crawl, CrawlInteraction, Instance, InstancePeer, MostRecentCrawl, Repo}
alias Backend.{
Crawl,
CrawlInteraction,
FederationRestriction,
Instance,
InstancePeer,
MostRecentCrawl,
Repo
}
alias Backend.Crawler.ApiCrawler
alias Backend.Crawler.Crawlers.{Friendica, GnuSocial, Mastodon, Misskey, Nodeinfo}
@ -75,15 +85,25 @@ defmodule Backend.Crawler do
# a) it should always be run first
# b) it passes the results on to the next crawlers (e.g. user_count)
defp crawl(%Crawler{api_crawlers: [Nodeinfo | remaining_crawlers], domain: domain} = state) do
with true <- Nodeinfo.allows_crawling?(domain), {:ok, nodeinfo} <- Nodeinfo.crawl(domain) do
if Nodeinfo.allows_crawling?(domain) do
nodeinfo = Nodeinfo.crawl(domain, nil)
if nodeinfo != nil do
Logger.debug("Found nodeinfo for #{domain}.")
result = Map.merge(nodeinfo, %{peers: [], interactions: %{}, statuses_seen: 0})
crawl(%Crawler{state | result: result, found_api?: true, api_crawlers: remaining_crawlers})
crawl(%Crawler{
state
| result: nodeinfo,
found_api?: true,
api_crawlers: remaining_crawlers
})
else
_ ->
Logger.debug("Did not find nodeinfo for #{domain}.")
crawl(%Crawler{state | api_crawlers: remaining_crawlers})
end
else
crawl(%Crawler{state | api_crawlers: remaining_crawlers, allows_crawling?: false})
end
end
defp crawl(
@ -165,7 +185,7 @@ defmodule Backend.Crawler do
Elasticsearch.put_document!(Backend.Elasticsearch.Cluster, instance, "instances/_doc")
# Save details of a new crawl
## Save details of a new crawl ##
curr_crawl =
Repo.insert!(%Crawl{
instance_domain: domain,
@ -196,18 +216,22 @@ defmodule Backend.Crawler do
|> list_union(result.peers)
|> Enum.filter(fn domain -> domain != nil and not is_blacklisted?(domain) end)
|> Enum.map(&clean_domain(&1))
if not Enum.all?(peers_domains, &is_valid_domain?(&1)) do
invalid_peers = Enum.filter(peers_domains, fn d -> not is_valid_domain?(d) end)
raise "#{domain} has invalid peers: #{Enum.join(invalid_peers, ", ")}"
|> Enum.filter(fn peer_domain ->
if is_valid_domain?(peer_domain) do
true
else
Logger.info("Found invalid peer domain from #{domain}: #{peer_domain}")
false
end
end)
peers =
new_instances =
peers_domains
|> list_union(result.blocked_domains)
|> Enum.map(&%{domain: &1, inserted_at: now, updated_at: now, next_crawl: now})
Instance
|> Repo.insert_all(peers, on_conflict: :nothing, conflict_target: :domain)
|> Repo.insert_all(new_instances, on_conflict: :nothing, conflict_target: :domain)
Repo.transaction(fn ->
## Save peer relationships ##
@ -249,6 +273,56 @@ defmodule Backend.Crawler do
|> Repo.insert_all(new_instance_peers)
end)
## Save federation restrictions ##
Repo.transaction(fn ->
current_restrictions =
FederationRestriction
|> select([fr], {fr.target_domain, fr.type})
|> where(source_domain: ^domain)
|> Repo.all()
wanted_restrictions_set =
result.blocked_domains
|> Enum.map(&{&1, "reject"})
|> MapSet.new()
current_restrictions_set = MapSet.new(current_restrictions)
# Delete the ones we don't want
restrictions_to_delete =
current_restrictions_set
|> MapSet.difference(wanted_restrictions_set)
|> MapSet.to_list()
|> Enum.map(fn {target_domain, _type} -> target_domain end)
if length(restrictions_to_delete) > 0 do
FederationRestriction
|> where(
[fr],
fr.source_domain == ^domain and fr.target_domain in ^restrictions_to_delete
)
|> Repo.delete_all()
end
# Save the new ones
new_restrictions =
wanted_restrictions_set
|> MapSet.difference(current_restrictions_set)
|> MapSet.to_list()
|> Enum.map(fn {target_domain, type} ->
%{
source_domain: domain,
target_domain: target_domain,
type: type,
inserted_at: now,
updated_at: now
}
end)
FederationRestriction
|> Repo.insert_all(new_restrictions)
end)
## Save interactions ##
interactions =
result.interactions

View file

@ -62,12 +62,11 @@ defmodule Backend.Crawler.Crawlers.Friendica do
end)
if details |> Map.get(:user_count, 0) |> is_above_user_threshold?() do
Map.merge(
%{peers: peers, interactions: %{}, statuses_seen: 0, instance_type: :friendica},
Map.take(details, [:description, :version, :user_count, :status_count])
)
ApiCrawler.get_default()
|> Map.merge(%{peers: peers, instance_type: :friendica})
|> Map.merge(Map.take(details, [:description, :version, :user_count, :status_count]))
else
nodeinfo_result
Map.merge(ApiCrawler.get_default(), nodeinfo_result)
end
end

View file

@ -3,7 +3,6 @@ defmodule Backend.Crawler.Crawlers.GnuSocial do
Crawler for GNU Social servers.
"""
alias Backend.Crawler.ApiCrawler
alias Backend.Crawler.Crawlers.Nodeinfo
import Backend.Crawler.Util
import Backend.Util
require Logger
@ -32,17 +31,17 @@ defmodule Backend.Crawler.Crawlers.GnuSocial do
end
@impl ApiCrawler
def crawl(domain, nodeinfo_result) do
if nodeinfo_result == nil or
nodeinfo_result |> Map.get(:user_count) |> is_above_user_threshold?() do
crawl_large_instance(domain, nodeinfo_result)
def crawl(domain, nodeinfo) do
if nodeinfo == nil or
nodeinfo |> Map.get(:user_count) |> is_above_user_threshold?() do
Map.merge(crawl_large_instance(domain), nodeinfo)
else
nodeinfo_result
Map.merge(ApiCrawler.get_default(), nodeinfo)
end
end
@spec crawl_large_instance(String.t(), Nodeinfo.t()) :: ApiCrawler.t()
defp crawl_large_instance(domain, nodeinfo_result) do
@spec crawl_large_instance(String.t()) :: ApiCrawler.t()
defp crawl_large_instance(domain) do
status_datetime_threshold =
NaiveDateTime.utc_now()
|> NaiveDateTime.add(get_config(:status_age_limit_days) * 24 * 3600 * -1, :second)
@ -52,24 +51,14 @@ defmodule Backend.Crawler.Crawlers.GnuSocial do
{interactions, statuses_seen} = get_interactions(domain, min_timestamp)
if nodeinfo_result != nil do
Map.merge(nodeinfo_result, %{
interactions: interactions,
statuses_seen: statuses_seen,
peers: []
})
else
Map.merge(
ApiCrawler.get_default(),
%{
version: nil,
description: nil,
user_count: nil,
status_count: nil,
peers: [],
interactions: interactions,
statuses_seen: statuses_seen,
instance_type: :gnusocial
}
end
)
end
@spec get_interactions(

View file

@ -34,26 +34,19 @@ defmodule Backend.Crawler.Crawlers.Mastodon do
end
@impl ApiCrawler
def crawl(domain, _current_result) do
def crawl(domain, nodeinfo) do
instance = get_and_decode!("https://#{domain}/api/v1/instance")
user_count = get_in(instance, ["stats", "user_count"])
if is_above_user_threshold?(user_count) or has_opted_in?(domain) do
crawl_large_instance(domain, instance)
Map.merge(nodeinfo, crawl_large_instance(domain, instance))
else
Map.merge(
Map.take(instance["stats"], ["user_count"])
|> convert_keys_to_atoms(),
%{
ApiCrawler.get_default()
|> Map.merge(nodeinfo)
|> Map.merge(%{
instance_type: get_instance_type(instance),
peers: [],
interactions: %{},
statuses_seen: 0,
description: nil,
version: nil,
status_count: nil
}
)
user_count: get_in(instance, ["stats", "user_count"])
})
end
end

View file

@ -35,22 +35,18 @@ defmodule Backend.Crawler.Crawlers.Misskey do
end
@impl ApiCrawler
def crawl(domain, _result) do
def crawl(domain, nodeinfo) do
with {:ok, %{"originalUsersCount" => user_count, "originalNotesCount" => status_count}} <-
post_and_decode("https://#{domain}/api/stats") do
if is_above_user_threshold?(user_count) or has_opted_in?(domain) do
crawl_large_instance(domain, user_count, status_count)
Map.merge(nodeinfo, crawl_large_instance(domain, user_count, status_count))
else
%{
instance_type: :misskey,
version: nil,
description: nil,
ApiCrawler.get_default()
|> Map.merge(nodeinfo)
|> Map.merge(%{
user_count: user_count,
status_count: nil,
peers: [],
interactions: %{},
statuses_seen: 0
}
type: :misskey
})
end
end
end

View file

@ -1,34 +1,16 @@
defmodule Backend.Crawler.Crawlers.Nodeinfo do
@moduledoc """
This module is slightly different from the other crawlers.
It doesn't implement the ApiCrawler spec because it isn't run as a self-contained crawler.
Instead, it's run before all the other crawlers.
This is to get the user count. Some servers don't publish this in other places (e.g. GNU Social, PeerTube) so we need
nodeinfo to know whether it's a personal instance or not.
This module is slightly different from the other crawlers. It's run before all the others and its
result is included in theirs.
"""
alias Backend.Crawler.ApiCrawler
require Logger
import Backend.Util
import Backend.Crawler.Util
@behaviour ApiCrawler
defstruct [
:description,
:user_count,
:status_count,
:instance_type,
:version
]
@type t() :: %__MODULE__{
description: String.t(),
user_count: integer,
status_count: integer,
instance_type: ApiCrawler.instance_type(),
version: String.t()
}
@spec allows_crawling?(String.t()) :: boolean()
@impl ApiCrawler
def allows_crawling?(domain) do
[
".well-known/nodeinfo"
@ -37,13 +19,19 @@ defmodule Backend.Crawler.Crawlers.Nodeinfo do
|> urls_are_crawlable?()
end
@spec crawl(String.t()) :: {:ok, t()} | {:error, nil}
def crawl(domain) do
@impl ApiCrawler
def is_instance_type?(_domain, _nodeinfo) do
# This crawler is used slightly differently from the others -- we always check for nodeinfo.
true
end
@impl ApiCrawler
def crawl(domain, _curr_result) do
with {:ok, nodeinfo_url} <- get_nodeinfo_url(domain),
{:ok, nodeinfo} <- get_nodeinfo(nodeinfo_url) do
{:ok, nodeinfo}
nodeinfo
else
_other -> {:error, nil}
_other -> ApiCrawler.get_default()
end
end
@ -65,8 +53,7 @@ defmodule Backend.Crawler.Crawlers.Nodeinfo do
|> Map.get("href")
end
@spec get_nodeinfo(String.t()) ::
{:ok, t()} | {:error, Jason.DecodeError.t() | HTTPoison.Error.t()}
@spec get_nodeinfo(String.t()) :: ApiCrawler.t()
defp get_nodeinfo(nodeinfo_url) do
case get_and_decode(nodeinfo_url) do
{:ok, nodeinfo} -> {:ok, process_nodeinfo(nodeinfo)}
@ -74,7 +61,7 @@ defmodule Backend.Crawler.Crawlers.Nodeinfo do
end
end
@spec process_nodeinfo(any()) :: t()
@spec process_nodeinfo(any()) :: ApiCrawler.t()
defp process_nodeinfo(nodeinfo) do
user_count = get_in(nodeinfo, ["usage", "users", "total"])
@ -90,21 +77,33 @@ defmodule Backend.Crawler.Crawlers.Nodeinfo do
type = nodeinfo |> get_in(["software", "name"]) |> String.downcase() |> String.to_atom()
%__MODULE__{
Map.merge(
ApiCrawler.get_default(),
%{
description: description,
user_count: user_count,
status_count: get_in(nodeinfo, ["usage", "localPosts"]),
instance_type: type,
version: get_in(nodeinfo, ["software", "version"])
}
version: get_in(nodeinfo, ["software", "version"]),
blocked_domains:
get_in(nodeinfo, ["metadata", "federation", "mrf_simple", "reject"])
|> (fn b ->
if b == nil do
[]
else
%{
description: nil,
user_count: user_count,
status_count: nil,
instance_type: nil,
version: nil
b
end
end).()
|> Enum.map(&clean_domain(&1))
}
)
else
Map.merge(
ApiCrawler.get_default(),
%{
user_count: user_count
}
)
end
end

View file

@ -0,0 +1,28 @@
defmodule Backend.FederationRestriction do
@moduledoc false
use Ecto.Schema
import Ecto.Changeset
schema "federation_restrictions" do
belongs_to :source, Backend.Instance,
references: :domain,
type: :string,
foreign_key: :source_domain
belongs_to :target, Backend.Instance,
references: :domain,
type: :string,
foreign_key: :target_domain
field :type, :string
timestamps()
end
@doc false
def changeset(federation_restriction, attrs) do
federation_restriction
|> cast(attrs, [:source, :target, :type])
|> validate_required([:source, :target, :type])
end
end

View file

@ -4,7 +4,7 @@ defmodule Backend.Repo do
adapter: Ecto.Adapters.Postgres,
timeout: 25_000
use Paginator
use Scrivener, page_size: 20
def init(_type, config) do
{:ok, Keyword.put(config, :url, System.get_env("DATABASE_URL"))}

View file

@ -3,10 +3,9 @@ defmodule Backend.Scheduler do
This module runs recurring tasks.
"""
use Appsignal.Instrumentation.Decorators
use Quantum.Scheduler, otp_app: :backend
alias Backend.{Crawl, CrawlInteraction, Edge, Instance, Repo}
alias Backend.{Crawl, CrawlInteraction, Edge, FederationRestriction, Instance, Repo}
alias Backend.Mailer.AdminEmail
import Backend.Util
@ -21,7 +20,6 @@ defmodule Backend.Scheduler do
`unit` must singular, e.g. "second", "minute", "hour", "month", "year", etc...
"""
@spec prune_crawls(integer, String.t()) :: any
@decorate transaction()
def prune_crawls(amount, unit) do
{deleted_num, _} =
Crawl
@ -39,7 +37,6 @@ defmodule Backend.Scheduler do
Calculates every instance's "insularity score" -- that is, the percentage of mentions that are among users on the
instance, rather than at other instances.
"""
@decorate transaction()
def generate_insularity_scores do
now = get_now()
@ -85,7 +82,6 @@ defmodule Backend.Scheduler do
@doc """
This function calculates the average number of statuses per hour over the last month.
"""
@decorate transaction()
def generate_status_rate do
now = get_now()
# We want the earliest sucessful crawl so that we can exclude it from the statistics.
@ -143,9 +139,11 @@ defmodule Backend.Scheduler do
@doc """
This function aggregates statistics from the interactions in the database.
It calculates the strength of edges between nodes. Self-edges are not generated.
Edges are only generated if both instances have been succesfully crawled.
Edges are only generated if
* both instances have been succesfully crawled
* neither of the instances have blocked each other
* there are interactions in each direction
"""
@decorate transaction()
def generate_edges do
now = get_now()
@ -177,15 +175,30 @@ defmodule Backend.Scheduler do
})
|> Repo.all(timeout: :infinity)
federation_blocks =
FederationRestriction
|> select([fr], {fr.source_domain, fr.target_domain})
|> where([fr], fr.type == "reject")
|> Repo.all()
|> MapSet.new()
# Get edges and their weights
Repo.transaction(
fn ->
Edge
|> Repo.delete_all(timeout: :infinity)
edges =
mentions =
interactions
|> reduce_mention_count()
|> reduce_mention_count(federation_blocks)
# Filter down to mentions where there are interactions in both directions
filtered_mentions =
mentions
|> Enum.filter(&has_opposite_mention?(&1, mentions))
edges =
filtered_mentions
|> Enum.map(fn {{source_domain, target_domain}, {mention_count, statuses_seen}} ->
%{
source_domain: source_domain,
@ -207,7 +220,6 @@ defmodule Backend.Scheduler do
This function checks to see if a lot of instances on the same base domain have been created recently. If so,
notifies the server admin over SMS.
"""
@decorate transaction()
def check_for_spam_instances do
hour_range = 3
@ -254,10 +266,9 @@ defmodule Backend.Scheduler do
end
end
# Takes a list of Interactions
# Takes a list of Interactions and a MapSet of blocks in the form {source_domain, target_domain}
# Returns a map of %{{source, target} => {total_mention_count, total_statuses_seen}}
@decorate transaction_event()
defp reduce_mention_count(interactions) do
defp reduce_mention_count(interactions, federation_blocks) do
Enum.reduce(interactions, %{}, fn
%{
source_domain: source_domain,
@ -278,9 +289,46 @@ defmodule Backend.Scheduler do
statuses_seen = source_statuses_seen + target_statuses_seen
maybe_update_map(
acc,
key,
source_domain,
target_domain,
mentions,
statuses_seen,
federation_blocks
)
end)
end
defp maybe_update_map(
acc,
key,
source_domain,
target_domain,
mentions,
statuses_seen,
federation_blocks
) do
if not MapSet.member?(federation_blocks, {source_domain, target_domain}) and
not MapSet.member?(federation_blocks, {target_domain, source_domain}) do
Map.update(acc, key, {mentions, statuses_seen}, fn {curr_mentions, curr_statuses_seen} ->
{curr_mentions + mentions, curr_statuses_seen}
end)
end)
end
end
defp has_opposite_mention?(mention, all_mentions) do
{{source_domain, target_domain}, {mention_count, _statuses_seen}} = mention
other_direction_key = {target_domain, source_domain}
if mention_count > 0 and Map.has_key?(all_mentions, other_direction_key) do
{other_direction_mentions, _other_statuses_seen} =
Map.get(all_mentions, other_direction_key)
other_direction_mentions > 0
else
false
end
end
end

View file

@ -128,6 +128,7 @@ defmodule Backend.Util do
end
end
@spec clean_domain(String.t()) :: String.t()
def clean_domain(domain) do
cleaned =
domain
@ -136,7 +137,7 @@ defmodule Backend.Util do
|> String.trim()
|> String.downcase()
Regex.replace(~r/:\d+/, cleaned, "")
Regex.replace(~r/(:\d+|\.)$/, cleaned, "")
end
def get_account(username, domain) do
@ -209,6 +210,6 @@ defmodule Backend.Util do
@spec is_valid_domain?(String.t()) :: boolean
def is_valid_domain?(domain) do
Regex.match?(~r/^[\w\.\-_]+$/, domain)
Regex.match?(~r/^[\pL\d\.\-_]+\.[a-zA-Z]+$/, domain)
end
end

View file

@ -1,9 +1,30 @@
defmodule BackendWeb.InstanceController do
use BackendWeb, :controller
alias Backend.Api
alias Graph.Cache
action_fallback(BackendWeb.FallbackController)
def index(conn, params) do
page = Map.get(params, "page")
%{
entries: instances,
total_pages: total_pages,
page_number: page_number,
total_entries: total_entries,
page_size: page_size
} = Api.get_instances(page)
render(conn, "index.json",
instances: instances,
total_pages: total_pages,
page_number: page_number,
total_entries: total_entries,
page_size: page_size
)
end
def show(conn, %{"id" => domain}) do
instance = Cache.get_instance_with_peers(domain)

View file

@ -8,7 +8,7 @@ defmodule BackendWeb.Router do
scope "/api", BackendWeb do
pipe_through(:api)
resources("/instances", InstanceController, only: [:show])
resources("/instances", InstanceController, only: [:index, :show])
resources("/graph", GraphController, only: [:index, :show])
resources("/search", SearchController, only: [:index])

View file

@ -3,6 +3,40 @@ defmodule BackendWeb.InstanceView do
alias BackendWeb.InstanceView
import Backend.Util
def render("index.json", %{
instances: instances,
total_pages: total_pages,
page_number: page_number,
total_entries: total_entries,
page_size: page_size
}) do
%{
instances: render_many(instances, InstanceView, "index_instance.json"),
pageNumber: page_number,
totalPages: total_pages,
totalEntries: total_entries,
pageSize: page_size
}
end
@doc """
Used when rendering the index of all instances (the different from show.json is primarily that it does not
include peers).
"""
def render("index_instance.json", %{instance: instance}) do
%{
name: instance.domain,
description: instance.description,
version: instance.version,
userCount: instance.user_count,
insularity: instance.insularity,
statusCount: instance.status_count,
type: instance.type,
statusesPerDay: instance.statuses_per_day,
statusesPerUserPerDay: get_statuses_per_user_per_day(instance)
}
end
def render("show.json", %{instance: instance, crawl: crawl}) do
user_threshold = get_config(:personal_instance_threshold)
@ -21,7 +55,7 @@ defmodule BackendWeb.InstanceView do
end
end
def render("instance.json", %{instance: instance}) do
def render("peer.json", %{instance: instance}) do
%{name: instance.domain}
end
@ -46,14 +80,6 @@ defmodule BackendWeb.InstanceView do
instance.peers
|> Enum.filter(fn peer -> not peer.opt_out end)
statuses_per_user_per_day =
if instance.statuses_per_day != nil and instance.user_count != nil and
instance.user_count > 0 do
instance.statuses_per_day / instance.user_count
else
nil
end
%{
name: instance.domain,
description: instance.description,
@ -62,12 +88,21 @@ defmodule BackendWeb.InstanceView do
insularity: instance.insularity,
statusCount: instance.status_count,
domainCount: length(instance.peers),
peers: render_many(filtered_peers, InstanceView, "instance.json"),
peers: render_many(filtered_peers, InstanceView, "peer.json"),
lastUpdated: last_updated,
status: "success",
type: instance.type,
statusesPerDay: instance.statuses_per_day,
statusesPerUserPerDay: statuses_per_user_per_day
statusesPerUserPerDay: get_statuses_per_user_per_day(instance)
}
end
defp get_statuses_per_user_per_day(instance) do
if instance.statuses_per_day != nil and instance.user_count != nil and
instance.user_count > 0 do
instance.statuses_per_day / instance.user_count
else
nil
end
end
end

View file

@ -56,7 +56,6 @@ defmodule Backend.MixProject do
{:corsica, "~> 1.1.2"},
{:sobelow, "~> 0.8", only: [:dev, :test]},
{:gollum, "~> 0.3.2"},
{:paginator, "~> 0.6.0"},
{:public_suffix, "~> 0.6.0"},
{:idna, "~> 5.1.2", override: true},
{:swoosh, "~> 0.23.3"},
@ -66,7 +65,8 @@ defmodule Backend.MixProject do
{:credo, "~> 1.1", only: [:dev, :test], runtime: false},
{:nebulex, "~> 1.1"},
{:hunter, "~> 0.5.1"},
{:poison, "~> 4.0", override: true}
{:poison, "~> 4.0", override: true},
{:scrivener_ecto, "~> 2.2"}
]
end

View file

@ -50,6 +50,8 @@
"public_suffix": {:hex, :public_suffix, "0.6.0", "100cfe86f13f9f6f0cf67e743b1b83c78dd1223a2c422fa03ebf4adff514cbc3", [:mix], [{:idna, ">= 1.2.0 and < 6.0.0", [hex: :idna, repo: "hexpm", optional: false]}], "hexpm"},
"quantum": {:hex, :quantum, "2.3.4", "72a0e8855e2adc101459eac8454787cb74ab4169de6ca50f670e72142d4960e9", [:mix], [{:calendar, "~> 0.17", [hex: :calendar, repo: "hexpm", optional: true]}, {:crontab, "~> 1.1", [hex: :crontab, repo: "hexpm", optional: false]}, {:gen_stage, "~> 0.12", [hex: :gen_stage, repo: "hexpm", optional: false]}, {:swarm, "~> 3.3", [hex: :swarm, repo: "hexpm", optional: false]}, {:timex, "~> 3.1", [hex: :timex, repo: "hexpm", optional: true]}], "hexpm"},
"ranch": {:hex, :ranch, "1.7.1", "6b1fab51b49196860b733a49c07604465a47bdb78aa10c1c16a3d199f7f8c881", [:rebar3], [], "hexpm"},
"scrivener": {:hex, :scrivener, "2.7.0", "fa94cdea21fad0649921d8066b1833d18d296217bfdf4a5389a2f45ee857b773", [:mix], [], "hexpm"},
"scrivener_ecto": {:hex, :scrivener_ecto, "2.2.0", "53d5f1ba28f35f17891cf526ee102f8f225b7024d1cdaf8984875467158c9c5e", [:mix], [{:ecto, "~> 3.0", [hex: :ecto, repo: "hexpm", optional: false]}, {:scrivener, "~> 2.4", [hex: :scrivener, repo: "hexpm", optional: false]}], "hexpm"},
"shards": {:hex, :shards, "0.6.0", "678d292ad74a4598a872930f9b12251f43e97f6050287f1fb712fbfd3d282f75", [:make, :rebar3], [], "hexpm"},
"sobelow": {:hex, :sobelow, "0.8.0", "a3ec73e546dfde19f14818e5000c418e3f305d9edb070e79dd391de0ae1cd1ea", [:mix], [], "hexpm"},
"ssl_verify_fun": {:hex, :ssl_verify_fun, "1.1.4", "f0eafff810d2041e93f915ef59899c923f4568f4585904d010387ed74988e77b", [:make, :mix, :rebar3], [], "hexpm"},

View file

@ -0,0 +1,22 @@
defmodule Backend.Repo.Migrations.CreateFederationRestrictions do
use Ecto.Migration
def change do
create table(:federation_restrictions) do
add :source_domain,
references(:instances, column: :domain, type: :string, on_delete: :delete_all),
null: false
add :target_domain,
references(:instances, column: :domain, type: :string, on_delete: :delete_all),
null: false
add :type, :string, null: false
timestamps()
end
create index(:federation_restrictions, [:source_domain])
create index(:federation_restrictions, [:target_domain])
end
end

View file

@ -5,13 +5,21 @@ import { Classes } from "@blueprintjs/core";
import { ConnectedRouter } from "connected-react-router";
import { Route } from "react-router-dom";
import { Nav } from "./components/organisms/";
import { AboutScreen, AdminScreen, GraphScreen, LoginScreen, VerifyLoginScreen } from "./components/screens/";
import {
AboutScreen,
AdminScreen,
GraphScreen,
LoginScreen,
TableScreen,
VerifyLoginScreen
} from "./components/screens/";
import { history } from "./index";
const AppRouter: React.FC = () => (
<ConnectedRouter history={history}>
<div className={`${Classes.DARK} App`}>
<Nav />
<Route path="/instances" exact={true} component={TableScreen} />
<Route path="/about" exact={true} component={AboutScreen} />
<Route path="/admin/login" exact={true} component={LoginScreen} />
<Route path="/admin/verify" exact={true} component={VerifyLoginScreen} />

View file

@ -111,7 +111,7 @@ const mapStateToProps = (state: IAppState) => {
const match = domainMatchSelector(state);
return {
currentInstanceName: match && match.params.domain,
graphLoadError: state.data.error,
graphLoadError: state.data.graphLoadError,
graphResponse: state.data.graphResponse,
hoveringOverResult: state.search.hoveringOverResult,
isLoadingGraph: state.data.isLoadingGraph,

View file

@ -0,0 +1,127 @@
import { Button, ButtonGroup, Code, HTMLTable, Intent, NonIdealState, Spinner } from "@blueprintjs/core";
import { push } from "connected-react-router";
import { range } from "lodash";
import * as numeral from "numeral";
import React from "react";
import { connect } from "react-redux";
import { Dispatch } from "redux";
import styled from "styled-components";
import { loadInstanceList } from "../../redux/actions";
import { IAppState, IInstanceListResponse } from "../../redux/types";
import { InstanceType } from "../atoms";
import { ErrorState } from "../molecules";
const StyledTable = styled(HTMLTable)`
width: 100%;
`;
const PaginationContainer = styled.div`
margin-top: 20px;
display: flex;
flex-direction: column;
flex: 1;
align-items: center;
`;
interface IInstanceTableProps {
loadError: boolean;
instancesResponse?: IInstanceListResponse;
isLoading: boolean;
fetchInstances: (page?: number) => void;
navigate: (path: string) => void;
}
class InstanceTable extends React.PureComponent<IInstanceTableProps> {
public componentDidMount() {
const { isLoading, instancesResponse, loadError } = this.props;
if (!isLoading && !instancesResponse && !loadError) {
this.props.fetchInstances();
}
}
public render() {
const { isLoading, instancesResponse, loadError } = this.props;
if (loadError) {
return <ErrorState />;
} else if (isLoading || !instancesResponse) {
return <NonIdealState icon={<Spinner />} />;
}
const { instances, pageNumber, totalPages, totalEntries, pageSize } = instancesResponse!;
return (
<>
<StyledTable striped={true} bordered={true} interactive={true}>
<thead>
<tr>
<th>Instance</th>
<th>Server type</th>
<th>Version</th>
<th>Users</th>
<th>Statuses</th>
<th>Insularity</th>
</tr>
</thead>
<tbody>
{instances.map(i => (
<tr key={i.name} onClick={this.goToInstanceFactory(i.name)}>
<td>{i.name}</td>
<td>{i.type && <InstanceType type={i.type} />}</td>
<td>{i.version && <Code>{i.version}</Code>}</td>
<td>{i.userCount}</td>
<td>{i.statusCount}</td>
<td>{i.insularity && numeral.default(i.insularity).format("0.0%")}</td>
</tr>
))}
</tbody>
</StyledTable>
<PaginationContainer>
<p>
Showing {(pageNumber - 1) * pageSize + 1}-{Math.min(pageNumber * pageSize, totalEntries)} of {totalEntries}{" "}
known instances
</p>
<ButtonGroup>
{range(totalPages).map(n => {
const isCurrentPage = pageNumber === n + 1;
return (
<Button
key={n}
onClick={this.loadPageFactory(n + 1)}
disabled={isCurrentPage}
intent={isCurrentPage ? Intent.PRIMARY : undefined}
>
{n + 1}
</Button>
);
})}
</ButtonGroup>
</PaginationContainer>
</>
);
}
private loadPageFactory = (page: number) => () => {
this.props.fetchInstances(page);
};
private goToInstanceFactory = (domain: string) => () => {
this.props.navigate(`/instance/${domain}`);
};
}
const mapStateToProps = (state: IAppState) => {
return {
instancesResponse: state.data.instancesResponse,
isLoading: state.data.isLoadingInstanceList,
loadError: state.data.instanceListLoadError
};
};
const mapDispatchToProps = (dispatch: Dispatch) => ({
fetchInstances: (page?: number) => dispatch(loadInstanceList(page) as any),
navigate: (path: string) => dispatch(push(path))
});
export default connect(
mapStateToProps,
mapDispatchToProps
)(InstanceTable);

View file

@ -11,7 +11,7 @@ interface INavState {
aboutIsOpen: boolean;
}
const linkIsActive = (currMatch: match<IInstanceDomainPath>, location: Location) => {
const graphIsActive = (currMatch: match<IInstanceDomainPath>, location: Location) => {
return location.pathname === "/" || location.pathname.startsWith("/instance/");
};
@ -31,10 +31,17 @@ class Nav extends React.Component<{}, INavState> {
to="/"
className={`${Classes.BUTTON} ${Classes.MINIMAL} bp3-icon-${IconNames.GLOBE_NETWORK}`}
activeClassName={Classes.INTENT_PRIMARY}
isActive={linkIsActive as any}
isActive={graphIsActive as any}
>
Home
</NavLink>
<NavLink
to="/instances"
className={`${Classes.BUTTON} ${Classes.MINIMAL} bp3-icon-${IconNames.TH}`}
activeClassName={Classes.INTENT_PRIMARY}
>
Instances
</NavLink>
<NavLink
to="/about"
className={`${Classes.BUTTON} ${Classes.MINIMAL} bp3-icon-${IconNames.INFO_SIGN}`}

View file

@ -2,3 +2,4 @@ export { default as Graph } from "./Graph";
export { default as Nav } from "./Nav";
export { default as SidebarContainer } from "./SidebarContainer";
export { default as SearchFilters } from "./SearchFilters";
export { default as InstanceTable } from "./InstanceTable";

View file

@ -98,7 +98,7 @@ const mapStateToProps = (state: IAppState) => {
const match = domainMatchSelector(state);
return {
currentInstanceName: match && match.params.domain,
graphLoadError: state.data.error,
graphLoadError: state.data.graphLoadError,
pathname: state.router.location.pathname
};
};

View file

@ -0,0 +1,17 @@
import { H1 } from "@blueprintjs/core";
import React from "react";
import { Page } from "../atoms";
import { InstanceTable } from "../organisms";
class TableScreen extends React.PureComponent {
public render() {
return (
<Page>
<H1>{"Instances"}</H1>
<InstanceTable />
</Page>
);
}
}
export default TableScreen;

View file

@ -5,3 +5,4 @@ export { default as InstanceScreen } from "./InstanceScreen";
export { default as AdminScreen } from "./AdminScreen";
export { default as LoginScreen } from "./LoginScreen";
export { default as VerifyLoginScreen } from "./VerifyLoginScreen";
export { default as TableScreen } from "./TableScreen";

View file

@ -48,6 +48,18 @@ const graphLoadFailed = () => {
};
};
// Instance list
const requestInstanceList = () => ({
type: ActionType.REQUEST_INSTANCES
});
const receiveInstanceList = (instances: IInstanceDetails[]) => ({
payload: instances,
type: ActionType.RECEIVE_INSTANCES
});
const instanceListLoadFailed = () => ({
type: ActionType.INSTANCE_LIST_LOAD_ERROR
});
// Search
const requestSearchResult = (query: string, filters: ISearchFilter[]) => {
return {
@ -138,3 +150,17 @@ export const fetchGraph = () => {
.catch(() => dispatch(graphLoadFailed()));
};
};
export const loadInstanceList = (page?: number) => {
return (dispatch: Dispatch) => {
dispatch(requestInstanceList());
let params = "";
if (!!page) {
params += `page=${page}`;
}
const path = !!params ? `instances?${params}` : "instances";
return getFromApi(path)
.then(instancesListResponse => dispatch(receiveInstanceList(instancesListResponse)))
.catch(() => dispatch(instanceListLoadFailed()));
};
};

View file

@ -5,9 +5,11 @@ import { combineReducers } from "redux";
import { History } from "history";
import { ActionType, IAction, ICurrentInstanceState, IDataState, ISearchState } from "./types";
const initialDataState = {
error: false,
isLoadingGraph: false
const initialDataState: IDataState = {
graphLoadError: false,
instanceListLoadError: false,
isLoadingGraph: false,
isLoadingInstanceList: false
};
const data = (state: IDataState = initialDataState, action: IAction): IDataState => {
switch (action.type) {
@ -26,9 +28,28 @@ const data = (state: IDataState = initialDataState, action: IAction): IDataState
case ActionType.GRAPH_LOAD_ERROR:
return {
...state,
error: true,
graphLoadError: true,
isLoadingGraph: false
};
case ActionType.REQUEST_INSTANCES:
return {
...state,
instanceListLoadError: false,
instancesResponse: undefined,
isLoadingInstanceList: true
};
case ActionType.RECEIVE_INSTANCES:
return {
...state,
instancesResponse: action.payload,
isLoadingInstanceList: false
};
case ActionType.INSTANCE_LIST_LOAD_ERROR:
return {
...state,
instanceListLoadError: true,
isLoadingInstanceList: false
};
default:
return state;
}

View file

@ -10,6 +10,10 @@ export enum ActionType {
REQUEST_GRAPH = "REQUEST_GRAPH",
RECEIVE_GRAPH = "RECEIVE_GRAPH",
GRAPH_LOAD_ERROR = "GRAPH_LOAD_ERROR",
// Instance list
REQUEST_INSTANCES = "REQUEST_INSTANCES",
RECEIVE_INSTANCES = "RECEIVE_INSTANCES",
INSTANCE_LIST_LOAD_ERROR = "INSTANCE_LIST_LOAD_ERROR",
// Nav
DESELECT_INSTANCE = "DESELECT_INSTANCE",
// Search
@ -26,7 +30,7 @@ export interface IAction {
payload: any;
}
export interface IInstance {
export interface IPeer {
name: string;
}
@ -45,7 +49,7 @@ export interface IInstanceDetails {
insularity?: number;
statusCount?: number;
domainCount?: number;
peers?: IInstance[];
peers?: IPeer[];
lastUpdated?: string;
status: string;
type?: string;
@ -93,6 +97,14 @@ export interface ISearchResponse {
next: string | null;
}
export interface IInstanceListResponse {
pageNumber: number;
totalPages: number;
totalEntries: number;
pageSize: number;
instances: IInstanceDetails[];
}
// Redux state
// The current instance name is stored in the URL. See state -> router -> location
@ -104,8 +116,11 @@ export interface ICurrentInstanceState {
export interface IDataState {
graphResponse?: IGraphResponse;
instancesResponse?: IInstanceListResponse;
isLoadingGraph: boolean;
error: boolean;
isLoadingInstanceList: boolean;
graphLoadError: boolean;
instanceListLoadError: boolean;
}
export interface ISearchState {