feature/administration

CHANGELOG.md

@@ -9,6 +9,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ### Added
 
+- Instance administrators can now log in to opt in or out of crawling.
+
 ### Changed
 
 - Instances are now crawled hourly instead of every 30 minutes.

backend/README.md

@@ -6,6 +6,25 @@
 - Run with `SKIP_CRAWL=true` to just run the server (useful for working on the API without also crawling)
 - This project is automatically scanned for potential vulnerabilities with [Sobelow](https://sobelow.io/).
 
+## Configuration
+
+There are several environment variables you can set to configure how the crawler behaves.
+
+- `DATABASE_URL` (required) . The URL of the Postgres db.
+- `POOL_SIZE`. The size of the database pool. Default: 10
+- `PORT`. Default: 4000
+- `BACKEND_HOSTNAME` (required). The url the backend is running on.
+- `SECRET_KEY_BASE` (required). Used for signing tokens.
+- `TWILIO_ACCOUNT_SID`. Used for sending SMS alerts to the admin.
+- `TWILIO_AUTH_TOKEN`. As above.
+- `ADMIN_PHONE`. The phone number to receive alerts at.
+  - At the moment, the only alert is when there are potential new spam domains.
+- `TWILIO_PHONE`. The phone number to send alerts from.
+- `ADMIN_EMAIL`. Used for receiving alerts.
+- `FRONTEND_DOMAIN` (required). Used to generate login links for instance admins.
+  - Don't enter `https://`, this is added automatically.
+- `SENDGRID_API_KEY`. Needed to send emails to the admin, or to instance admins who want to opt in/out.
+
 ## Deployment
 
 Deployment with Docker is handled as per the [Distillery docs](https://hexdocs.pm/distillery/guides/working_with_docker.html).

backend/config/config.exs

@@ -48,7 +48,8 @@ config :backend, :crawler,
   user_agent: "fediverse.space crawler",
   admin_phone: System.get_env("ADMIN_PHONE"),
   twilio_phone: System.get_env("TWILIO_PHONE"),
-  admin_email: System.get_env("ADMIN_EMAIL")
+  admin_email: System.get_env("ADMIN_EMAIL"),
+  frontend_domain: "https://www.fediverse.space"
 
 config :backend, Backend.Scheduler,
   jobs: [

backend/config/releases.exs

@@ -32,7 +32,8 @@ config :ex_twilio,
 config :backend, :crawler,
   admin_phone: System.get_env("ADMIN_PHONE"),
   twilio_phone: System.get_env("TWILIO_PHONE"),
-  admin_email: System.get_env("ADMIN_EMAIL")
+  admin_email: System.get_env("ADMIN_EMAIL"),
+  frontend_domain: System.get_env("FRONTEND_DOMAIN")
 
 config :backend, Backend.Mailer,
   adapter: Swoosh.Adapters.Sendgrid,

backend/lib/backend/api.ex

@@ -3,12 +3,6 @@ defmodule Backend.Api do
   import Backend.Util
   import Ecto.Query
 
-  @spec list_instances() :: [Instance.t()]
-  def list_instances() do
-    Instance
-    |> Repo.all()
-  end
-
   @spec get_instance!(String.t()) :: Instance.t()
   def get_instance!(domain) do
     Instance
@@ -16,6 +10,14 @@ defmodule Backend.Api do
     |> Repo.get_by!(domain: domain)
   end
 
+  def update_instance(instance) do
+    Repo.insert(
+      instance,
+      on_conflict: {:replace, [:opt_in, :opt_out]},
+      conflict_target: :domain
+    )
+  end
+
   @doc """
   Returns a list of instances that
   * have a user count (required to give the instance a size on the graph)
@@ -32,7 +34,7 @@ defmodule Backend.Api do
     |> where(
       [i],
       not is_nil(i.x) and not is_nil(i.y) and not is_nil(i.user_count) and
-        i.user_count >= ^user_threshold
+        i.user_count >= ^user_threshold and not i.opt_out
     )
     |> maybe_filter_nodes_to_neighborhood(domain)
     |> select([c], [:domain, :user_count, :x, :y, :type])
@@ -71,7 +73,8 @@ defmodule Backend.Api do
       [e, i1, i2],
       not is_nil(i1.x) and not is_nil(i1.y) and
         not is_nil(i2.x) and not is_nil(i2.y) and
-        i1.user_count >= ^user_threshold and i2.user_count >= ^user_threshold
+        i1.user_count >= ^user_threshold and i2.user_count >= ^user_threshold and
+        not i1.opt_out and not i2.opt_out
     )
     |> Repo.all()
   end
@@ -103,7 +106,7 @@ defmodule Backend.Api do
 
     %{entries: instances, metadata: metadata} =
       Instance
-      |> where([i], ilike(i.domain, ^ilike_query))
+      |> where([i], ilike(i.domain, ^ilike_query) and not i.opt_out)
       |> order_by(asc: :id)
       |> Repo.paginate(after: cursor_after, cursor_fields: [:id], limit: 50)
 

backend/lib/backend/application.ex

@@ -4,7 +4,6 @@ defmodule Backend.Application do
   @moduledoc false
 
   use Application
-  require Logger
   import Backend.Util
 
   def start(_type, _args) do

backend/lib/backend/auth.ex

@@ -0,0 +1,17 @@
+defmodule Backend.Auth do
+  alias Phoenix.Token
+  import Backend.Util
+
+  @salt "fedi auth salt"
+
+  def get_login_link(domain) do
+    token = Token.sign(BackendWeb.Endpoint, @salt, domain)
+    frontend_domain = get_config(:frontend_domain)
+    "https://#{frontend_domain}/admin/verify?token=#{URI.encode(token)}"
+  end
+
+  def verify_token(token) do
+    # tokens are valid for 12 hours
+    Token.verify(BackendWeb.Endpoint, @salt, token, max_age: 43200)
+  end
+end

backend/lib/backend/crawler/api_crawler.ex

@@ -6,7 +6,7 @@ defmodule Backend.Crawler.ApiCrawler do
   * You must adhere to the following configuration values:
     * `:status_age_limit_days` specifies that you must only crawl statuses from the most recent N days
     * `:status_count_limit` specifies the max number of statuses to crawl in one go
-    * `:personal_instance_threshold` specifies that instances with fewer than this number of users should not be crawled
+    * `:personal_instance_threshold` specifies that instances with fewer than this number of users should not be crawled (unless :opt_in is true)
   * profiles with the string "nobot" (case insensitive) in their profile must not be included in any stats
   * Make sure to check the most recent crawl of the instance so you don't re-crawl old statuses
   """

backend/lib/backend/crawler/crawlers/mastodon.ex

@@ -2,7 +2,9 @@ defmodule Backend.Crawler.Crawlers.Mastodon do
   require Logger
   import Backend.Crawler.Util
   import Backend.Util
+  import Ecto.Query
   alias Backend.Crawler.ApiCrawler
+  alias Backend.{Instance, Repo}
 
   @behaviour ApiCrawler
 
@@ -34,7 +36,14 @@ defmodule Backend.Crawler.Crawlers.Mastodon do
   def crawl(domain) do
     instance = Jason.decode!(get!("https://#{domain}/api/v1/instance").body)
 
-    if get_in(instance, ["stats", "user_count"]) > get_config(:personal_instance_threshold) do
+    has_opted_in =
+      case Instance |> select([:opt_in]) |> Repo.get_by(domain: domain) do
+        %{opt_in: true} -> true
+        _ -> false
+      end
+
+    if get_in(instance, ["stats", "user_count"]) > get_config(:personal_instance_threshold) or
+         has_opted_in do
       crawl_large_instance(domain, instance)
     else
       Map.merge(

backend/lib/backend/crawler/stale_instance_manager.ex

@@ -67,8 +67,8 @@ defmodule Backend.Crawler.StaleInstanceManager do
       |> join(:left, [i], c in subquery(crawls_subquery), on: i.domain == c.instance_domain)
       |> where(
         [i, c],
-        c.most_recent_crawl < datetime_add(^NaiveDateTime.utc_now(), ^interval, "minute") or
-          is_nil(c.crawl_count)
+        (c.most_recent_crawl < datetime_add(^NaiveDateTime.utc_now(), ^interval, "minute") or
+           is_nil(c.crawl_count)) and not i.opt_out
       )
       |> select([i], i.domain)
       |> Repo.all()

backend/lib/backend/instance.ex

@@ -11,6 +11,8 @@ defmodule Backend.Instance do
     field :insularity, :float
     field :type, :string
     field :base_domain, :string
+    field :opt_in, :boolean
+    field :opt_out, :boolean
 
     many_to_many :peers, Backend.Instance,
       join_through: Backend.InstancePeer,
@@ -37,7 +39,9 @@ defmodule Backend.Instance do
       :insularity,
       :updated_at,
       :type,
-      :base_domain
+      :base_domain,
+      :opt_in,
+      :opt_out
     ])
     |> validate_required([:domain])
     |> put_assoc(:peers, attrs.peers)

backend/lib/backend/util.ex

@@ -142,4 +142,19 @@ defmodule Backend.Util do
       Logger.info("Could not send SMS to admin; not configured.")
     end
   end
+
+  def clean_domain(domain) do
+    domain
+    |> String.replace_prefix("https://", "")
+    |> String.trim_trailing("/")
+    |> String.downcase()
+  end
+
+  def get_account(username, domain) do
+    if username == nil or domain == nil do
+      nil
+    else
+      "#{String.downcase(username)}@#{clean_domain(domain)}"
+    end
+  end
 end

backend/lib/backend_web/controllers/admin_controller.ex

@@ -0,0 +1,34 @@
+defmodule BackendWeb.AdminController do
+  use BackendWeb, :controller
+  alias Backend.{Auth, Api, Instance}
+  require Logger
+
+  action_fallback BackendWeb.FallbackController
+
+  def show(conn, _params) do
+    [token] = get_req_header(conn, "token")
+
+    with {:ok, domain} <- Auth.verify_token(token) do
+      instance = Api.get_instance!(domain)
+      render(conn, "show.json", instance: instance)
+    end
+  end
+
+  def update(conn, params) do
+    [token] = get_req_header(conn, "token")
+
+    with {:ok, domain} <- Auth.verify_token(token) do
+      %{"optIn" => opt_in, "optOut" => opt_out} = params
+
+      instance = %Instance{
+        domain: domain,
+        opt_in: opt_in,
+        opt_out: opt_out
+      }
+
+      with {:ok, updated_instance} <- Api.update_instance(instance) do
+        render(conn, "show.json", instance: updated_instance)
+      end
+    end
+  end
+end

backend/lib/backend_web/controllers/admin_login_controller.ex

@@ -0,0 +1,52 @@
+defmodule BackendWeb.AdminLoginController do
+  use BackendWeb, :controller
+  import Backend.Util
+  alias Backend.Mailer.UserEmail
+
+  action_fallback BackendWeb.FallbackController
+
+  @doc """
+  Given an instance, looks up the login types (email or admin account) and returns them. The user can then
+  choose one or the other by POSTing back.
+  """
+  def show(conn, %{"id" => domain}) do
+    # TODO: this should really be handled in a more async manner
+    # TODO: this assumes mastodon/pleroma API
+    cleaned_domain = clean_domain(domain)
+
+    instance_data =
+      HTTPoison.get!("https://#{cleaned_domain}/api/v1/instance")
+      |> Map.get(:body)
+      |> Jason.decode!()
+
+    render(conn, "show.json", instance_data: instance_data, cleaned_domain: cleaned_domain)
+  end
+
+  def create(conn, %{"domain" => domain, "type" => type}) do
+    cleaned_domain = clean_domain(domain)
+
+    instance_data =
+      HTTPoison.get!("https://#{cleaned_domain}/api/v1/instance")
+      |> Map.get(:body)
+      |> Jason.decode!()
+
+    error =
+      cond do
+        type == "email" ->
+          email = Map.get(instance_data, "email")
+
+          case UserEmail.send_login_email(email, cleaned_domain) do
+            {:ok, _} -> nil
+            {:error, _} -> "Failed to send email."
+          end
+
+        # type == "fediverseAccount" ->
+        #   account = nil
+
+        true ->
+          "Invalid account type. Must be 'email' or 'fediverseAccount'."
+      end
+
+    render(conn, "create.json", error: error)
+  end
+end

backend/lib/backend_web/controllers/fallback_controller.ex

@@ -12,4 +12,10 @@ defmodule BackendWeb.FallbackController do
     |> put_view(BackendWeb.ErrorView)
     |> render(:"404")
   end
+
+  def call(conn, {:error, _}) do
+    conn
+    |> put_status(500)
+    |> render(:"500")
+  end
 end

backend/lib/backend_web/controllers/instance_controller.ex

@@ -6,11 +6,6 @@ defmodule BackendWeb.InstanceController do
 
   action_fallback(BackendWeb.FallbackController)
 
-  def index(conn, _params) do
-    instances = Api.list_instances()
-    render(conn, "index.json", instances: instances)
-  end
-
   def show(conn, %{"id" => domain}) do
     instance = Api.get_instance!(domain)
     last_crawl = get_last_crawl(domain)

backend/lib/backend_web/endpoint.ex

@@ -44,8 +44,10 @@ defmodule BackendWeb.Endpoint do
     signing_salt: "HJa1j4FI"
   )
 
-  # TODO
-  plug(Corsica, origins: "*")
+  plug(Corsica,
+    origins: ["http://localhost:3000", ~r{^https?://(.*\.?)fediverse\.space$}],
+    allow_headers: ["content-type", "token"]
+  )
 
   plug(BackendWeb.Router)
 end

backend/lib/backend_web/router.ex

@@ -8,8 +8,12 @@ defmodule BackendWeb.Router do
   scope "/api", BackendWeb do
     pipe_through(:api)
 
-    resources("/instances", InstanceController, only: [:index, :show])
+    resources("/instances", InstanceController, only: [:show])
     resources("/graph", GraphController, only: [:index, :show])
     resources("/search", SearchController, only: [:index])
+
+    resources("/admin/login", AdminLoginController, only: [:show, :create])
+    get "/admin", AdminController, :show
+    post "/admin", AdminController, :update
   end
 end

backend/lib/backend_web/views/admin_login_view.ex

@@ -0,0 +1,28 @@
+defmodule BackendWeb.AdminLoginView do
+  use BackendWeb, :view
+  import Backend.Util
+
+  def render("show.json", %{instance_data: instance_data, cleaned_domain: cleaned_domain}) do
+    username = get_in(instance_data, ["contact_account", "username"])
+
+    fedi_account = get_account(username, cleaned_domain)
+
+    %{
+      domain: cleaned_domain,
+      email: Map.get(instance_data, "email"),
+      fediverseAccount: fedi_account
+    }
+  end
+
+  def render("create.json", %{error: error}) do
+    if error != nil do
+      %{
+        error: error
+      }
+    else
+      %{
+        data: "success"
+      }
+    end
+  end
+end

backend/lib/backend_web/views/admin_view.ex

@@ -0,0 +1,25 @@
+defmodule BackendWeb.AdminView do
+  use BackendWeb, :view
+  import Backend.Util
+  require Logger
+
+  def render("show.json", %{instance: instance}) do
+    Logger.info(inspect(instance))
+
+    %{
+      domain: domain,
+      opt_in: opt_in,
+      opt_out: opt_out,
+      user_count: user_count,
+      status_count: status_count
+    } = instance
+
+    %{
+      domain: domain,
+      optIn: opt_in,
+      optOut: opt_out,
+      userCount: user_count,
+      statusCount: status_count
+    }
+  end
+end

backend/lib/backend_web/views/instance_view.ex

@@ -4,19 +4,7 @@ defmodule BackendWeb.InstanceView do
   import Backend.Util
   require Logger
 
-  def render("index.json", %{instances: instances}) do
-    render_many(instances, InstanceView, "instance.json")
-  end
-
   def render("show.json", %{instance: instance, crawl: crawl}) do
-    render_one(instance, InstanceView, "instance_detail.json", crawl: crawl)
-  end
-
-  def render("instance.json", %{instance: instance}) do
-    %{name: instance.domain}
-  end
-
-  def render("instance_detail.json", %{instance: instance, crawl: crawl}) do
     user_threshold = get_config(:personal_instance_threshold)
 
     [status, last_updated] =
@@ -39,6 +27,10 @@ defmodule BackendWeb.InstanceView do
         }
 
       true ->
+        filtered_peers =
+          instance.peers
+          |> Enum.filter(fn peer -> not peer.opt_out end)
+
         %{
           name: instance.domain,
           description: instance.description,
@@ -47,11 +39,15 @@ defmodule BackendWeb.InstanceView do
           insularity: instance.insularity,
           statusCount: instance.status_count,
           domainCount: length(instance.peers),
-          peers: render_many(instance.peers, InstanceView, "instance.json"),
+          peers: render_many(filtered_peers, InstanceView, "instance.json"),
           lastUpdated: last_updated,
           status: status,
           type: instance.type
         }
     end
   end
+
+  def render("instance.json", %{instance: instance}) do
+    %{name: instance.domain}
+  end
 end

backend/lib/mailer/user_email.ex

@@ -0,0 +1,21 @@
+defmodule Backend.Mailer.UserEmail do
+  import Swoosh.Email
+  import Backend.Auth
+  require Logger
+
+  @spec send_login_email(String.t(), String.t()) :: {:ok | :error, term}
+  def send_login_email(address, domain) do
+    frontend_domain = get_config(:frontend_domain)
+
+    body =
+      "Someone tried to log in to #{domain} on https://#{frontend_domain}.\n\nIf it was you, click here to confirm:\n\n" <>
+        get_login_link(domain)
+
+    new()
+    |> to(address)
+    |> from("noreply@fediverse.space")
+    |> subject("Login to fediverse.space")
+    |> text_body(body)
+    |> Backend.Mailer.deliver()
+  end
+end

backend/priv/repo/migrations/20190726104004_add_instance_settings.exs

@@ -0,0 +1,12 @@
+defmodule Backend.Repo.Migrations.AddInstanceSettings do
+  use Ecto.Migration
+
+  def change do
+    alter table(:instances) do
+      add :opt_in, :boolean, default: false, null: false
+      add :opt_out, :boolean, default: false, null: false
+    end
+
+    create index(:instances, [:opt_out])
+  end
+end

frontend/src/AppRouter.tsx

@@ -1,18 +1,21 @@
-import * as React from "react";
+import React from "react";
 
 import { Classes } from "@blueprintjs/core";
 
 import { ConnectedRouter } from "connected-react-router";
 import { Route } from "react-router-dom";
 import { Nav } from "./components/organisms/";
-import { AboutScreen, GraphScreen } from "./components/screens/";
+import { AboutScreen, AdminScreen, GraphScreen, LoginScreen, VerifyLoginScreen } from "./components/screens/";
 import { history } from "./index";
 
 const AppRouter: React.FC = () => (
   <ConnectedRouter history={history}>
     <div className={`${Classes.DARK} App`}>
       <Nav />
-      <Route path="/about" component={AboutScreen} />
+      <Route path="/about" exact={true} component={AboutScreen} />
+      <Route path="/admin/login" exact={true} component={LoginScreen} />
+      <Route path="/admin/verify" exact={true} component={VerifyLoginScreen} />
+      <Route path="/admin" exact={true} component={AdminScreen} />
       {/* We always want the GraphScreen to be rendered (since un- and re-mounting it is expensive */}
       <GraphScreen />
     </div>

frontend/src/components/molecules/ErrorState.tsx

@@ -2,6 +2,11 @@ import { NonIdealState } from "@blueprintjs/core";
 import { IconNames } from "@blueprintjs/icons";
 import * as React from "react";
 
-const ErrorState: React.SFC = () => <NonIdealState icon={IconNames.ERROR} title={"Something went wrong."} />;
+interface IErrorStateProps {
+  description?: string;
+}
+const ErrorState: React.FC<IErrorStateProps> = ({ description }) => (
+  <NonIdealState icon={IconNames.ERROR} title={"Something went wrong."} description={description} />
+);
 
 export default ErrorState;

frontend/src/components/organisms/Nav.tsx

@@ -44,6 +44,15 @@ class Nav extends React.Component<{}, INavState> {
             About
           </NavLink>
         </Navbar.Group>
+        <Navbar.Group align={Alignment.RIGHT}>
+          <NavLink
+            to="/admin"
+            className={`${Classes.BUTTON} ${Classes.MINIMAL} bp3-icon-${IconNames.COG}`}
+            activeClassName={Classes.INTENT_PRIMARY}
+          >
+            Administration
+          </NavLink>
+        </Navbar.Group>
       </Navbar>
     );
   }

frontend/src/components/screens/AboutScreen.tsx

@@ -50,33 +50,33 @@ const AboutScreen: React.FC = () => (
     </p>
 
     <H2>Credits</H2>
-    <p className={Classes.RUNNING_TEXT}>
-      This site is inspired by several other sites in the same vein:
-      <ul className={Classes.LIST}>
-        <li>
-          <a href="https://the-federation.info/" target="_blank" rel="noopener noreferrer">
-            the-federation.info
-          </a>
-        </li>
-        <li>
-          <a href="http://fediverse.network/" target="_blank" rel="noopener noreferrer">
-            fediverse.network
-          </a>
-        </li>
-        <li>
-          <a
-            href="https://lucahammer.at/vis/fediverse/2018-08-30-mastoverse_hashtags/"
-            target="_blank"
-            rel="noopener noreferrer"
-          >
-            Mastodon hashtag network
-          </a>
-          {" by "}
-          <a href="https://vis.social/web/statuses/100634284168959187" target="_blank" rel="noopener noreferrer">
-            @Luca@vis.social
-          </a>
-        </li>
-      </ul>
+    <p className={Classes.RUNNING_TEXT}>This site is inspired by several other sites in the same vein:</p>
+    <ul className={Classes.LIST}>
+      <li>
+        <a href="https://the-federation.info/" target="_blank" rel="noopener noreferrer">
+          the-federation.info
+        </a>
+      </li>
+      <li>
+        <a href="http://fediverse.network/" target="_blank" rel="noopener noreferrer">
+          fediverse.network
+        </a>
+      </li>
+      <li>
+        <a
+          href="https://lucahammer.at/vis/fediverse/2018-08-30-mastoverse_hashtags/"
+          target="_blank"
+          rel="noopener noreferrer"
+        >
+          Mastodon hashtag network
+        </a>
+        {" by "}
+        <a href="https://vis.social/web/statuses/100634284168959187" target="_blank" rel="noopener noreferrer">
+          @Luca@vis.social
+        </a>
+      </li>
+    </ul>
+    <p>
       The source code for fediverse.space is available on{" "}
       <a href="https://gitlab.com/taobojlen/fediverse.space" target="_blank" rel="noopener noreferrer">
         GitLab

frontend/src/components/screens/AdminScreen.tsx

@@ -0,0 +1,169 @@
+import { Button, FormGroup, H1, H2, Intent, NonIdealState, Spinner, Switch } from "@blueprintjs/core";
+import { IconNames } from "@blueprintjs/icons";
+import { push } from "connected-react-router";
+import React from "react";
+import { connect } from "react-redux";
+import { Redirect } from "react-router";
+import { Dispatch } from "redux";
+import styled from "styled-components";
+import AppToaster from "../../toaster";
+import { getAuthToken, getFromApi, postToApi, unsetAuthToken } from "../../util";
+import { Page } from "../atoms";
+
+const ButtonContainer = styled.div`
+  display: flex;
+  flex-direction: row;
+  width: 100%;
+  justify-content: space-between;
+`;
+
+interface IAdminSettings {
+  domain: string;
+  optIn: boolean;
+  optOut: boolean;
+  userCount: number;
+  statusCount: number;
+}
+
+interface IAdminScreenProps {
+  navigate: (path: string) => void;
+}
+interface IAdminScreenState {
+  settings?: IAdminSettings;
+  isUpdating: boolean;
+}
+class AdminScreen extends React.PureComponent<IAdminScreenProps, IAdminScreenState> {
+  private authToken = getAuthToken();
+
+  public constructor(props: IAdminScreenProps) {
+    super(props);
+    this.state = { isUpdating: false };
+  }
+
+  public componentDidMount() {
+    // Load instance settings from server
+    getFromApi(`admin`, this.authToken!)
+      .then(response => {
+        this.setState({ settings: response });
+      })
+      .catch(() => {
+        AppToaster.show({
+          icon: IconNames.ERROR,
+          intent: Intent.DANGER,
+          message: "Failed to load settings.",
+          timeout: 0
+        });
+      });
+  }
+
+  public render() {
+    if (!this.authToken) {
+      return <Redirect to="/admin/login" />;
+    }
+    const { settings, isUpdating } = this.state;
+    let content;
+    if (!settings) {
+      content = <NonIdealState icon={<Spinner />} />;
+    } else {
+      content = (
+        <>
+          <H2>{settings.domain}</H2>
+          <p>{`${settings.userCount} users with ${settings.statusCount || "(unknown)"} statuses.`}</p>
+          <form onSubmit={this.updateSettings}>
+            {settings.userCount < 10 && (
+              <FormGroup helperText="Check this if you'd like your personal instance to be crawled by fediverse.space.">
+                <Switch
+                  id="opt-in-switch"
+                  checked={!!settings.optIn}
+                  large={true}
+                  label="Opt in"
+                  disabled={!!isUpdating}
+                  onChange={this.updateOptIn}
+                />
+              </FormGroup>
+            )}
+            <FormGroup helperText="Check this if you don't want to your instance to be crawled. You won't appear on fediverse.space.">
+              <Switch
+                id="opt-out-switch"
+                checked={!!settings.optOut}
+                large={true}
+                label="Opt out"
+                disabled={!!isUpdating}
+                onChange={this.updateOptOut}
+              />
+            </FormGroup>
+            <ButtonContainer>
+              <Button intent={Intent.PRIMARY} type="submit" loading={!!isUpdating}>
+                Update settings
+              </Button>
+              <Button intent={Intent.DANGER} onClick={this.logout} icon={IconNames.LOG_OUT}>
+                Log out
+              </Button>
+            </ButtonContainer>
+          </form>
+        </>
+      );
+    }
+    return (
+      <Page>
+        <H1>Instance administration</H1>
+        {content}
+      </Page>
+    );
+  }
+
+  private updateOptIn = (e: React.FormEvent<HTMLInputElement>) => {
+    const settings = this.state.settings as IAdminSettings;
+    const optIn = e.currentTarget.checked;
+    let optOut = settings.optOut;
+    if (optIn) {
+      optOut = false;
+    }
+    this.setState({ settings: { ...settings, optIn, optOut } });
+  };
+
+  private updateOptOut = (e: React.FormEvent<HTMLInputElement>) => {
+    const settings = this.state.settings as IAdminSettings;
+    const optOut = e.currentTarget.checked;
+    let optIn = settings.optIn;
+    if (optOut) {
+      optIn = false;
+    }
+    this.setState({ settings: { ...settings, optIn, optOut } });
+  };
+
+  private updateSettings = (e: React.FormEvent<HTMLFormElement>) => {
+    e.preventDefault();
+    this.setState({ isUpdating: true });
+    const body = {
+      optIn: this.state.settings!.optIn,
+      optOut: this.state.settings!.optOut
+    };
+    postToApi(`admin`, body, this.authToken!)
+      .then(response => {
+        this.setState({ settings: response, isUpdating: false });
+        AppToaster.show({
+          icon: IconNames.TICK,
+          intent: Intent.SUCCESS,
+          message: "Successfully updated settings."
+        });
+      })
+      .catch(() => {
+        this.setState({ isUpdating: false });
+        AppToaster.show({ intent: Intent.DANGER, icon: IconNames.ERROR, message: "Failed to update settings." });
+      });
+  };
+
+  private logout = () => {
+    unsetAuthToken();
+    this.props.navigate("/admin/login");
+  };
+}
+
+const mapDispatchToProps = (dispatch: Dispatch) => ({
+  navigate: (path: string) => dispatch(push(path))
+});
+export default connect(
+  undefined,
+  mapDispatchToProps
+)(AdminScreen);

frontend/src/components/screens/LoginScreen.tsx

@@ -0,0 +1,196 @@
+import { Button, Classes, FormGroup, H1, H4, Icon, InputGroup, Intent } from "@blueprintjs/core";
+import { IconNames } from "@blueprintjs/icons";
+import React from "react";
+import { Redirect } from "react-router";
+import styled from "styled-components";
+import { getAuthToken, getFromApi, postToApi } from "../../util";
+import { Page } from "../atoms";
+import { ErrorState } from "../molecules";
+
+interface IFormContainerProps {
+  error: boolean;
+}
+const FormContainer = styled.div<IFormContainerProps>`
+  ${props => (props.error ? "margin: 20px auto 0 auto;" : "margin-top: 20px;")}
+`;
+const LoginTypeContainer = styled.div`
+  display: flex;
+  width: 100%;
+`;
+const LoginTypeButton = styled(Button)`
+  flex: 1;
+  margin: 0 10px;
+`;
+const PostLoginContainer = styled.div`
+  align-self: center;
+  text-align: center;
+  margin-top: 20px;
+`;
+const StyledIcon = styled(Icon)`
+  margin-bottom: 10px;
+`;
+
+interface ILoginTypes {
+  domain: string;
+  email?: string;
+  fediverseAccount?: string;
+}
+interface ILoginScreenState {
+  domain: string;
+  isGettingLoginTypes: boolean;
+  isSendingLoginRequest: boolean;
+  loginTypes?: ILoginTypes;
+  selectedLoginType?: "email" | "fediverseAccount";
+  error: boolean;
+}
+class LoginScreen extends React.PureComponent<{}, ILoginScreenState> {
+  public constructor(props: any) {
+    super(props);
+    this.state = { domain: "", error: false, isGettingLoginTypes: false, isSendingLoginRequest: false };
+  }
+
+  public render() {
+    const authToken = getAuthToken();
+    if (authToken) {
+      return <Redirect to="/admin" />;
+    }
+
+    const { error, loginTypes, isSendingLoginRequest, selectedLoginType } = this.state;
+
+    let content;
+    if (!!error) {
+      content = (
+        <ErrorState description="This could be because the instance is down. If not, please reload the page and try again." />
+      );
+    } else if (!!selectedLoginType && !isSendingLoginRequest) {
+      content = this.renderPostLogin();
+    } else if (!!loginTypes) {
+      content = this.renderChooseLoginType();
+    } else {
+      content = this.renderChooseInstance();
+    }
+
+    return (
+      <Page>
+        <H1>Login</H1>
+        <p className={Classes.RUNNING_TEXT}>
+          To manage how fediverse.space interacts with your instance, you must be the instance admin.
+        </p>
+        <FormContainer error={this.state.error}>{content}</FormContainer>
+      </Page>
+    );
+  }
+
+  private renderChooseInstance = () => {
+    const { isGettingLoginTypes } = this.state;
+    return (
+      <form onSubmit={this.getLoginTypes}>
+        <FormGroup label="Instance domain" labelFor="domain-input" disabled={isGettingLoginTypes} inline={true}>
+          <InputGroup
+            disabled={isGettingLoginTypes}
+            id="domain-input"
+            value={this.state.domain}
+            onChange={this.updateDomainInState}
+            rightElement={
+              <Button
+                intent={Intent.PRIMARY}
+                minimal={true}
+                rightIcon={IconNames.ARROW_RIGHT}
+                title="submit"
+                loading={isGettingLoginTypes}
+              />
+            }
+            placeholder="mastodon.social"
+          />
+        </FormGroup>
+      </form>
+    );
+  };
+
+  private renderChooseLoginType = () => {
+    const { loginTypes, isSendingLoginRequest } = this.state;
+    if (!loginTypes) {
+      return;
+    }
+    const loginWithEmail = () => this.login("email");
+    // const loginWithDm = () => this.login("fediverseAccount");
+    return (
+      <>
+        <H4>Choose an authentication method</H4>
+        <LoginTypeContainer>
+          {loginTypes.email && (
+            <LoginTypeButton
+              large={true}
+              icon={IconNames.ENVELOPE}
+              onClick={loginWithEmail}
+              loading={!!isSendingLoginRequest}
+            >
+              {`Email ${loginTypes.email}`}
+            </LoginTypeButton>
+          )}
+          {/* {loginTypes.fediverseAccount && (
+            <LoginTypeButton
+              large={true}
+              icon={IconNames.GLOBE_NETWORK}
+              onClick={loginWithDm}
+              loading={!!isSendingLoginRequest}
+            >
+              {`DM ${loginTypes.fediverseAccount}`}
+            </LoginTypeButton>
+          )} */}
+        </LoginTypeContainer>
+      </>
+    );
+  };
+
+  private renderPostLogin = () => {
+    const { selectedLoginType, loginTypes } = this.state;
+    let message;
+    if (selectedLoginType === "email") {
+      message = `Check ${loginTypes!.email} for a login link`;
+    } else {
+      message = `Check ${loginTypes!.fediverseAccount}'s DMs for a login link.`;
+    }
+    return (
+      <PostLoginContainer>
+        <StyledIcon icon={IconNames.ENVELOPE} iconSize={80} />
+        <p className={Classes.TEXT_LARGE}>{message}</p>
+      </PostLoginContainer>
+    );
+  };
+
+  private updateDomainInState = (event: React.ChangeEvent<HTMLInputElement>) => {
+    this.setState({ domain: event.target.value });
+  };
+
+  private getLoginTypes = (e: React.FormEvent<HTMLFormElement>) => {
+    e.preventDefault();
+    this.setState({ isGettingLoginTypes: true });
+    let { domain } = this.state;
+    if (domain.startsWith("https://")) {
+      domain = domain.slice(8);
+    }
+    getFromApi(`admin/login/${domain}`)
+      .then(response => {
+        this.setState({ loginTypes: response, isGettingLoginTypes: false });
+      })
+      .catch(() => {
+        this.setState({ error: true });
+      });
+  };
+
+  private login = (type: "email" | "fediverseAccount") => {
+    this.setState({ isSendingLoginRequest: true, selectedLoginType: type });
+    postToApi("admin/login", { domain: this.state.loginTypes!.domain, type })
+      .then(response => {
+        if ("error" in response) {
+          this.setState({ isSendingLoginRequest: false, error: true });
+        } else {
+          this.setState({ isSendingLoginRequest: false });
+        }
+      })
+      .catch(() => this.setState({ isSendingLoginRequest: false, error: true }));
+  };
+}
+
+export default LoginScreen;

frontend/src/components/screens/VerifyLoginScreen.tsx

@@ -0,0 +1,36 @@
+import React, { useEffect, useState } from "react";
+import { connect } from "react-redux";
+import { Redirect } from "react-router";
+import { IAppState } from "../../redux/types";
+import { setAuthToken } from "../../util";
+import { Page } from "../atoms";
+
+interface IVerifyLoginScreenProps {
+  search: string;
+}
+const VerifyLoginScreen: React.FC<IVerifyLoginScreenProps> = ({ search }) => {
+  const [didSaveToken, setDidSaveToken] = useState(false);
+  const token = new URLSearchParams(search).get("token");
+
+  useEffect(() => {
+    // Save the auth token
+    if (!!token) {
+      setAuthToken(token);
+      setDidSaveToken(true);
+    }
+  }, [token]);
+
+  if (!token) {
+    return <Redirect to="/admin/login" />;
+  } else if (!didSaveToken) {
+    return <Page />;
+  }
+  return <Redirect to="/admin" />;
+};
+
+const mapStateToProps = (state: IAppState) => {
+  return {
+    search: state.router.location.search
+  };
+};
+export default connect(mapStateToProps)(VerifyLoginScreen);

frontend/src/components/screens/index.ts

@@ -2,3 +2,6 @@ export { default as AboutScreen } from "./AboutScreen";
 export { default as GraphScreen } from "./GraphScreen";
 export { default as SearchScreen } from "./SearchScreen";
 export { default as InstanceScreen } from "./InstanceScreen";
+export { default as AdminScreen } from "./AdminScreen";
+export { default as LoginScreen } from "./LoginScreen";
+export { default as VerifyLoginScreen } from "./VerifyLoginScreen";

frontend/src/index.css

@@ -9,8 +9,6 @@ body {
     Icons16, sans-serif;
 }
 
-.fediverse-instance-search-popover {
-  max-height: 350px;
-  min-width: 300px;
-  overflow-x: hidden;
+.app-toaster {
+  z-index: 1000;
 }

frontend/src/toaster.ts

@@ -0,0 +1,6 @@
+import { Position, Toaster } from "@blueprintjs/core";
+
+export default Toaster.create({
+  className: "app-toaster",
+  position: Position.TOP
+});

frontend/src/util.ts

@@ -10,9 +10,28 @@ if (["true", true, 1, "1"].indexOf(process.env.REACT_APP_STAGING || "") > -1) {
   API_ROOT = "https://phoenix.api.fediverse.space/api/";
 }
 
-export const getFromApi = (path: string): Promise<any> => {
+export const getFromApi = (path: string, token?: string): Promise<any> => {
   const domain = API_ROOT.endsWith("/") ? API_ROOT : API_ROOT + "/";
-  return fetch(encodeURI(domain + path)).then(response => response.json());
+  const headers = token ? { token } : undefined;
+  return fetch(encodeURI(domain + path), {
+    headers
+  }).then(response => response.json());
+};
+
+export const postToApi = (path: string, body: any, token?: string): Promise<any> => {
+  const domain = API_ROOT.endsWith("/") ? API_ROOT : API_ROOT + "/";
+  const defaultHeaders = {
+    Accept: "application/json",
+    "Content-Type": "application/json"
+  };
+  const headers = token ? { ...defaultHeaders, token } : defaultHeaders;
+  return fetch(encodeURI(domain + path), {
+    body: JSON.stringify(body),
+    headers,
+    method: "POST"
+  }).then(response => {
+    return response.json();
+  });
 };
 
 export const domainMatchSelector = createMatchSelector<IAppState, IInstanceDomainPath>(INSTANCE_DOMAIN_PATH);
@@ -20,3 +39,16 @@ export const domainMatchSelector = createMatchSelector<IAppState, IInstanceDomai
 export const isSmallScreen = window.innerWidth < DESKTOP_WIDTH_THRESHOLD;
 
 export const capitalize = (s: string): string => s.charAt(0).toUpperCase() + s.slice(1).toLowerCase();
+
+export const setAuthToken = (token: string) => {
+  sessionStorage.setItem("adminToken", token);
+};
+
+export const unsetAuthToken = () => {
+  sessionStorage.removeItem("adminToken");
+};
+
+export const getAuthToken = () => {
+  return sessionStorage.getItem("adminToken");
+  // TODO: check if it's expired, and if so a) delete it and b) return null
+};

gephi/src/main/java/space/fediverse/graph/GraphBuilder.java

@@ -29,7 +29,7 @@ import java.util.concurrent.TimeUnit;
 public class GraphBuilder {
     private static final String nodeQuery = new StringBuilder().append("SELECT i.domain as id, i.domain as label")
             .append(" FROM instances i INNER JOIN edges e ON i.domain = e.source_domain OR i.domain = e.target_domain")
-            .append(" WHERE i.user_count IS NOT NULL").toString();
+            .append(" WHERE i.user_count IS NOT NULL AND NOT i.opt_out").toString();
 
     private static final String edgeQuery = new StringBuilder().append("SELECT e.source_domain AS source,")
             .append(" e.target_domain AS target, e.weight AS weight FROM edges e").toString();