diff --git a/README.md b/README.md index 0f0c3a1..8c8dbfc 100644 --- a/README.md +++ b/README.md @@ -20,11 +20,11 @@ Read the latest updates on Mastodon: [@fediversespace](https://mastodon.social/@ ## Requirements -Note: examples here use `podman`. In most cases you should be able to replace `podman` with `docker`, `podman-compose` with `docker-compose`, and so on. +Note: examples here use `podman`. In most cases you should be able to replace `podman` with `docker`. Though containerized, backend development is easiest if you have the following installed. -- For the scraper + API: +- For the crawler + API: - Elixir - Postgres - For laying out the graph: @@ -38,7 +38,10 @@ Though containerized, backend development is easiest if you have the following i ### Backend - `cp example.env .env` and modify environment variables as required -- `podman-compose build` +- `podman build gephi && podman build phoenix` +- `podman run --name elasticsearch -p 9200:9200 -p 9300:9300 -e "discovery.type=single-node" elasticsearch:6.8.9` + - If you've `run` this container previously, use `podman start elasticsearch` +- `podman run --name postgres -e "POSTGRES_USER=postgres" -e "POSTGRES_PASSWORD=postgres" -p 5432:5432 postgres:12` - `podman-compose -f compose.backend-services.yml -f compose.phoenix.yml` - Create the elasticsearch index: - `iex -S mix app.start` @@ -54,10 +57,6 @@ Though containerized, backend development is easiest if you have the following i ### Backend `./gradlew shadowJar` compiles the graph layout program. `java -Xmx1g -jar build/libs/graphBuilder.jar` runs it. -If running in docker, this means you run - -- `docker-compose build gephi` -- `docker-compose run gephi java -Xmx1g -jar build/libs/graphBuilder.jar` lays out the graph ### Frontend @@ -103,8 +102,6 @@ SHELL=/bin/bash 0 2 * * * /usr/bin/dokku run gephi java -Xmx1g -jar build/libs/graphBuilder.jar ``` -10. (Optional) Set up caching with something like [dokku-nginx-cache](https://github.com/Aluxian/dokku-nginx-cache) - Before the app starts running, make sure that the Elasticsearch index exists -- otherwise it'll create one called `instances`, which should be the name of the alias. Then it won't be able to hot swap if you reindex in the future. diff --git a/backend/lib/backend_web/rate_limiter.ex b/backend/lib/backend_web/rate_limiter.ex new file mode 100644 index 0000000..9ab8186 --- /dev/null +++ b/backend/lib/backend_web/rate_limiter.ex @@ -0,0 +1,48 @@ +defmodule BackendWeb.RateLimiter do + @moduledoc """ + Functions used to rate limit: + * all endpoints by IP/endpoint + * authentication endpoints by domain + """ + + import Phoenix.Controller, only: [json: 2] + import Plug.Conn, only: [put_status: 2] + use Plug.Builder + + def rate_limit(conn, options \\ []) do + case check_rate(conn, options) do + {:ok, _count} -> conn # Do nothing, allow execution to continue + {:error, _count} -> render_error(conn) + end + end + + def rate_limit_authentication(conn, options \\ []) do + %{"id" => domain} = conn.params + options = Keyword.put(options, :bucket_name, "authorization: #{domain}") + rate_limit(conn, options) + end + + defp check_rate(conn, options) do + interval_milliseconds = options[:interval_seconds] * 1000 + max_requests = options[:max_requests] + bucket_name = options[:bucket_name] || bucket_name(conn) + + ExRated.check_rate(bucket_name, interval_milliseconds, max_requests) + end + + # Bucket name should be a combination of ip address and request path, like so: + # + # "127.0.0.1:/api/v1/authorizations" + defp bucket_name(conn) do + path = Enum.join(conn.path_info, "/") + ip = conn.remote_ip |> Tuple.to_list |> Enum.join(".") + "#{ip}:#{path}" + end + + defp render_error(conn) do + conn + |> put_status(:forbidden) + |> json(%{error: "Rate limit exceeded."}) + |> halt # Stop execution of further plugs, return response now + end +end diff --git a/backend/lib/backend_web/router.ex b/backend/lib/backend_web/router.ex index c62c51d..571db5b 100644 --- a/backend/lib/backend_web/router.ex +++ b/backend/lib/backend_web/router.ex @@ -1,8 +1,14 @@ defmodule BackendWeb.Router do use BackendWeb, :router + import BackendWeb.RateLimiter pipeline :api do plug(:accepts, ["json"]) + plug(:rate_limit, max_requests: 5, interval_seconds: 10) # requests to the same endpoint + end + + pipeline :api_admin do + plug(:rate_limit_authentication, max_requests: 5, interval_seconds: 60) end scope "/api", BackendWeb do @@ -12,8 +18,12 @@ defmodule BackendWeb.Router do resources("/graph", GraphController, only: [:index, :show]) resources("/search", SearchController, only: [:index]) - resources("/admin/login", AdminLoginController, only: [:show, :create]) - get "/admin", AdminController, :show - post "/admin", AdminController, :update + scope "/admin" do + pipe_through :api_admin + + resources("/login", AdminLoginController, only: [:show, :create]) + get "/", AdminController, :show + post "/", AdminController, :update + end end end diff --git a/backend/mix.exs b/backend/mix.exs index fca1d71..ca62b35 100644 --- a/backend/mix.exs +++ b/backend/mix.exs @@ -67,7 +67,8 @@ defmodule Backend.MixProject do {:hunter, "~> 0.5.1"}, {:poison, "~> 4.0", override: true}, {:scrivener_ecto, "~> 2.2"}, - {:recase, "~> 0.6.0"} + {:recase, "~> 0.6.0"}, + {:ex_rated, "~> 1.3"} ] end diff --git a/backend/mix.lock b/backend/mix.lock index 51d8269..c768a3d 100644 --- a/backend/mix.lock +++ b/backend/mix.lock @@ -18,6 +18,8 @@ "ecto": {:hex, :ecto, "3.4.4", "a2c881e80dc756d648197ae0d936216c0308370332c5e77a2325a10293eef845", [:mix], [{:decimal, "~> 1.6 or ~> 2.0", [hex: :decimal, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "cc4bd3ad62abc3b21fb629f0f7a3dab23a192fca837d257dd08449fba7373561"}, "ecto_sql": {:hex, :ecto_sql, "3.4.3", "c552aa8a7ccff2b64024f835503b3155d8e73452c180298527fbdbcd6e79710b", [:mix], [{:db_connection, "~> 2.2", [hex: :db_connection, repo: "hexpm", optional: false]}, {:ecto, "~> 3.4.3", [hex: :ecto, repo: "hexpm", optional: false]}, {:myxql, "~> 0.3.0 or ~> 0.4.0", [hex: :myxql, repo: "hexpm", optional: true]}, {:postgrex, "~> 0.15.0", [hex: :postgrex, repo: "hexpm", optional: true]}, {:tds, "~> 2.1.0", [hex: :tds, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "ec9e59d6fa3f8cfda9963ada371e9e6659167c2338a997bd7ea23b10b245842b"}, "elasticsearch": {:hex, :elasticsearch, "1.0.0", "626d3fb8e7554d9c93eb18817ae2a3d22c2a4191cc903c4644b1334469b15374", [:mix], [{:httpoison, ">= 0.0.0", [hex: :httpoison, repo: "hexpm", optional: false]}, {:poison, ">= 0.0.0", [hex: :poison, repo: "hexpm", optional: true]}, {:sigaws, "~> 0.7", [hex: :sigaws, repo: "hexpm", optional: true]}, {:vex, "~> 0.6.0", [hex: :vex, repo: "hexpm", optional: false]}], "hexpm", "9fa0b717ad57a54c28451b3eb10c5121211c29a7b33615d2bcc7e2f3c9418b2e"}, + "ex2ms": {:hex, :ex2ms, "1.6.0", "f39bbd9ff1b0f27b3f707bab2d167066dd8965e7df1149b962d94c74615d0e09", [:mix], [], "hexpm", "0d1ab5e08421af5cd69146efb408dbb1ff77f38a2f4df5f086f2512dc8cf65bf"}, + "ex_rated": {:hex, :ex_rated, "1.3.3", "30ecbdabe91f7eaa9d37fa4e81c85ba420f371babeb9d1910adbcd79ec798d27", [:mix], [{:ex2ms, "~> 1.5", [hex: :ex2ms, repo: "hexpm", optional: false]}], "hexpm", "26817cf0927b7a2e7bc2e14b4dab66a329fafa4520b513a8a4025532ac5a7cbf"}, "ex_twilio": {:hex, :ex_twilio, "0.7.0", "d7ce624ef4661311ae28c3e3aa060ecb66a9f4843184d7400c29072f7d3f5a4a", [:mix], [{:httpoison, ">= 0.9.0", [hex: :httpoison, repo: "hexpm", optional: false]}, {:inflex, "~> 1.0", [hex: :inflex, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}, {:joken, "~> 2.0", [hex: :joken, repo: "hexpm", optional: false]}, {:poison, "~> 3.0", [hex: :poison, repo: "hexpm", optional: false]}], "hexpm", "6be84f1508ed47d443d18cdc4ea0561f8ad4095b69791dd9be5f2fe14b1dafc5"}, "gen_stage": {:hex, :gen_stage, "1.0.0", "51c8ae56ff54f9a2a604ca583798c210ad245f415115453b773b621c49776df5", [:mix], [], "hexpm", "1d9fc978db5305ac54e6f5fec7adf80cd893b1000cf78271564c516aa2af7706"}, "gen_state_machine": {:hex, :gen_state_machine, "2.1.0", "a38b0e53fad812d29ec149f0d354da5d1bc0d7222c3711f3a0bd5aa608b42992", [:mix], [], "hexpm", "ae367038808db25cee2f2c4b8d0531522ea587c4995eb6f96ee73410a60fa06b"}, diff --git a/frontend/src/components/screens/AboutScreen.tsx b/frontend/src/components/screens/AboutScreen.tsx index 319b167..6f5465b 100644 --- a/frontend/src/components/screens/AboutScreen.tsx +++ b/frontend/src/components/screens/AboutScreen.tsx @@ -39,7 +39,7 @@ const AboutScreen: React.FC = () => (

Why can't I see details about my instance?

fediverse.space only supports servers using the Mastodon API, the Misskey API, the GNU Social API, or Nodeinfo. - Instances with 10 or fewer users won't be scraped -- it's a tool for understanding communities, not + Instances with 10 or fewer users won't be crawled -- it's a tool for understanding communities, not individuals.