add rate limiting

README.md

@@ -20,11 +20,11 @@ Read the latest updates on Mastodon: [@fediversespace](https://mastodon.social/@
 
 ## Requirements
 
-Note: examples here use `podman`. In most cases you should be able to replace `podman` with `docker`, `podman-compose` with `docker-compose`, and so on.
+Note: examples here use `podman`. In most cases you should be able to replace `podman` with `docker`.
 
 Though containerized, backend development is easiest if you have the following installed.
 
-- For the scraper + API:
+- For the crawler + API:
   - Elixir
   - Postgres
 - For laying out the graph:
@@ -38,7 +38,10 @@ Though containerized, backend development is easiest if you have the following i
 ### Backend
 
 - `cp example.env .env` and modify environment variables as required
-- `podman-compose build`
+- `podman build gephi && podman build phoenix`
+- `podman run --name elasticsearch -p 9200:9200 -p 9300:9300 -e "discovery.type=single-node" elasticsearch:6.8.9`
+  - If you've `run` this container previously, use `podman start elasticsearch`
+- `podman run --name postgres -e "POSTGRES_USER=postgres" -e "POSTGRES_PASSWORD=postgres" -p 5432:5432 postgres:12`
 - `podman-compose -f compose.backend-services.yml -f compose.phoenix.yml`
 - Create the elasticsearch index:
   - `iex -S mix app.start`
@@ -54,10 +57,6 @@ Though containerized, backend development is easiest if you have the following i
 ### Backend
 
 `./gradlew shadowJar` compiles the graph layout program. `java -Xmx1g -jar build/libs/graphBuilder.jar` runs it.
-If running in docker, this means you run
-
-- `docker-compose build gephi`
-- `docker-compose run gephi java -Xmx1g -jar build/libs/graphBuilder.jar` lays out the graph
 
 ### Frontend
 
@@ -103,8 +102,6 @@ SHELL=/bin/bash
 0 2 * * * /usr/bin/dokku run gephi java -Xmx1g -jar build/libs/graphBuilder.jar
 ```
 
-10. (Optional) Set up caching with something like [dokku-nginx-cache](https://github.com/Aluxian/dokku-nginx-cache)
-
 Before the app starts running, make sure that the Elasticsearch index exists -- otherwise it'll create one called
 `instances`, which should be the name of the alias. Then it won't be able to hot swap if you reindex in the future.
 

backend/lib/backend_web/rate_limiter.ex

@@ -0,0 +1,48 @@
+defmodule BackendWeb.RateLimiter do
+  @moduledoc """
+  Functions used to rate limit:
+  * all endpoints by IP/endpoint
+  * authentication endpoints by domain
+  """
+
+  import Phoenix.Controller, only: [json: 2]
+  import Plug.Conn, only: [put_status: 2]
+  use Plug.Builder
+
+  def rate_limit(conn, options \\ []) do
+    case check_rate(conn, options) do
+      {:ok, _count}   -> conn # Do nothing, allow execution to continue
+      {:error, _count} -> render_error(conn)
+    end
+  end
+
+  def rate_limit_authentication(conn, options \\ []) do
+    %{"id" => domain} = conn.params
+    options = Keyword.put(options, :bucket_name, "authorization: #{domain}")
+    rate_limit(conn, options)
+  end
+
+  defp check_rate(conn, options) do
+    interval_milliseconds = options[:interval_seconds] * 1000
+    max_requests = options[:max_requests]
+    bucket_name = options[:bucket_name] || bucket_name(conn)
+
+    ExRated.check_rate(bucket_name, interval_milliseconds, max_requests)
+  end
+
+  # Bucket name should be a combination of ip address and request path, like so:
+  #
+  # "127.0.0.1:/api/v1/authorizations"
+  defp bucket_name(conn) do
+    path = Enum.join(conn.path_info, "/")
+    ip   = conn.remote_ip |> Tuple.to_list |> Enum.join(".")
+    "#{ip}:#{path}"
+  end
+
+  defp render_error(conn) do
+    conn
+    |> put_status(:forbidden)
+    |> json(%{error: "Rate limit exceeded."})
+    |> halt # Stop execution of further plugs, return response now
+  end
+end

backend/lib/backend_web/router.ex

@@ -1,8 +1,14 @@
 defmodule BackendWeb.Router do
   use BackendWeb, :router
+  import BackendWeb.RateLimiter
 
   pipeline :api do
     plug(:accepts, ["json"])
+    plug(:rate_limit, max_requests: 5, interval_seconds: 10) # requests to the same endpoint
+  end
+
+  pipeline :api_admin do
+    plug(:rate_limit_authentication, max_requests: 5, interval_seconds: 60)
   end
 
   scope "/api", BackendWeb do
@@ -12,8 +18,12 @@ defmodule BackendWeb.Router do
     resources("/graph", GraphController, only: [:index, :show])
     resources("/search", SearchController, only: [:index])
 
-    resources("/admin/login", AdminLoginController, only: [:show, :create])
+    scope "/admin" do
-    get "/admin", AdminController, :show
+      pipe_through :api_admin
-    post "/admin", AdminController, :update
+
+      resources("/login", AdminLoginController, only: [:show, :create])
+      get "/", AdminController, :show
+      post "/", AdminController, :update
+    end
   end
 end

backend/mix.exs

@@ -67,7 +67,8 @@ defmodule Backend.MixProject do
       {:hunter, "~> 0.5.1"},
       {:poison, "~> 4.0", override: true},
       {:scrivener_ecto, "~> 2.2"},
-      {:recase, "~> 0.6.0"}
+      {:recase, "~> 0.6.0"},
+      {:ex_rated, "~> 1.3"}
     ]
   end
 

backend/mix.lock

@@ -18,6 +18,8 @@
   "ecto": {:hex, :ecto, "3.4.4", "a2c881e80dc756d648197ae0d936216c0308370332c5e77a2325a10293eef845", [:mix], [{:decimal, "~> 1.6 or ~> 2.0", [hex: :decimal, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "cc4bd3ad62abc3b21fb629f0f7a3dab23a192fca837d257dd08449fba7373561"},
   "ecto_sql": {:hex, :ecto_sql, "3.4.3", "c552aa8a7ccff2b64024f835503b3155d8e73452c180298527fbdbcd6e79710b", [:mix], [{:db_connection, "~> 2.2", [hex: :db_connection, repo: "hexpm", optional: false]}, {:ecto, "~> 3.4.3", [hex: :ecto, repo: "hexpm", optional: false]}, {:myxql, "~> 0.3.0 or ~> 0.4.0", [hex: :myxql, repo: "hexpm", optional: true]}, {:postgrex, "~> 0.15.0", [hex: :postgrex, repo: "hexpm", optional: true]}, {:tds, "~> 2.1.0", [hex: :tds, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "ec9e59d6fa3f8cfda9963ada371e9e6659167c2338a997bd7ea23b10b245842b"},
   "elasticsearch": {:hex, :elasticsearch, "1.0.0", "626d3fb8e7554d9c93eb18817ae2a3d22c2a4191cc903c4644b1334469b15374", [:mix], [{:httpoison, ">= 0.0.0", [hex: :httpoison, repo: "hexpm", optional: false]}, {:poison, ">= 0.0.0", [hex: :poison, repo: "hexpm", optional: true]}, {:sigaws, "~> 0.7", [hex: :sigaws, repo: "hexpm", optional: true]}, {:vex, "~> 0.6.0", [hex: :vex, repo: "hexpm", optional: false]}], "hexpm", "9fa0b717ad57a54c28451b3eb10c5121211c29a7b33615d2bcc7e2f3c9418b2e"},
+  "ex2ms": {:hex, :ex2ms, "1.6.0", "f39bbd9ff1b0f27b3f707bab2d167066dd8965e7df1149b962d94c74615d0e09", [:mix], [], "hexpm", "0d1ab5e08421af5cd69146efb408dbb1ff77f38a2f4df5f086f2512dc8cf65bf"},
+  "ex_rated": {:hex, :ex_rated, "1.3.3", "30ecbdabe91f7eaa9d37fa4e81c85ba420f371babeb9d1910adbcd79ec798d27", [:mix], [{:ex2ms, "~> 1.5", [hex: :ex2ms, repo: "hexpm", optional: false]}], "hexpm", "26817cf0927b7a2e7bc2e14b4dab66a329fafa4520b513a8a4025532ac5a7cbf"},
   "ex_twilio": {:hex, :ex_twilio, "0.7.0", "d7ce624ef4661311ae28c3e3aa060ecb66a9f4843184d7400c29072f7d3f5a4a", [:mix], [{:httpoison, ">= 0.9.0", [hex: :httpoison, repo: "hexpm", optional: false]}, {:inflex, "~> 1.0", [hex: :inflex, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}, {:joken, "~> 2.0", [hex: :joken, repo: "hexpm", optional: false]}, {:poison, "~> 3.0", [hex: :poison, repo: "hexpm", optional: false]}], "hexpm", "6be84f1508ed47d443d18cdc4ea0561f8ad4095b69791dd9be5f2fe14b1dafc5"},
   "gen_stage": {:hex, :gen_stage, "1.0.0", "51c8ae56ff54f9a2a604ca583798c210ad245f415115453b773b621c49776df5", [:mix], [], "hexpm", "1d9fc978db5305ac54e6f5fec7adf80cd893b1000cf78271564c516aa2af7706"},
   "gen_state_machine": {:hex, :gen_state_machine, "2.1.0", "a38b0e53fad812d29ec149f0d354da5d1bc0d7222c3711f3a0bd5aa608b42992", [:mix], [], "hexpm", "ae367038808db25cee2f2c4b8d0531522ea587c4995eb6f96ee73410a60fa06b"},

frontend/src/components/screens/AboutScreen.tsx

@@ -39,7 +39,7 @@ const AboutScreen: React.FC = () => (
     <H4>Why can&apos;t I see details about my instance?</H4>
     <p className={Classes.RUNNING_TEXT}>
       fediverse.space only supports servers using the Mastodon API, the Misskey API, the GNU Social API, or Nodeinfo.
-      Instances with 10 or fewer users won&apos;t be scraped -- it&apos;s a tool for understanding communities, not
+      Instances with 10 or fewer users won&apos;t be crawled -- it&apos;s a tool for understanding communities, not
       individuals.
     </p>