Browse Source

add rate limiting

environments/develop/deployments/314
Tao Bojlén 1 year ago
parent
commit
1c251866ff
No known key found for this signature in database GPG Key ID: C6EC7AAB905F9E6F
  1. 15
      README.md
  2. 48
      backend/lib/backend_web/rate_limiter.ex
  3. 16
      backend/lib/backend_web/router.ex
  4. 3
      backend/mix.exs
  5. 2
      backend/mix.lock
  6. 2
      frontend/src/components/screens/AboutScreen.tsx

15
README.md

@ -20,11 +20,11 @@ Read the latest updates on Mastodon: [@fediversespace](https://mastodon.social/@
## Requirements
Note: examples here use `podman`. In most cases you should be able to replace `podman` with `docker`, `podman-compose` with `docker-compose`, and so on.
Note: examples here use `podman`. In most cases you should be able to replace `podman` with `docker`.
Though containerized, backend development is easiest if you have the following installed.
- For the scraper + API:
- For the crawler + API:
- Elixir
- Postgres
- For laying out the graph:
@ -38,7 +38,10 @@ Though containerized, backend development is easiest if you have the following i
### Backend
- `cp example.env .env` and modify environment variables as required
- `podman-compose build`
- `podman build gephi && podman build phoenix`
- `podman run --name elasticsearch -p 9200:9200 -p 9300:9300 -e "discovery.type=single-node" elasticsearch:6.8.9`
- If you've `run` this container previously, use `podman start elasticsearch`
- `podman run --name postgres -e "POSTGRES_USER=postgres" -e "POSTGRES_PASSWORD=postgres" -p 5432:5432 postgres:12`
- `podman-compose -f compose.backend-services.yml -f compose.phoenix.yml`
- Create the elasticsearch index:
- `iex -S mix app.start`
@ -54,10 +57,6 @@ Though containerized, backend development is easiest if you have the following i
### Backend
`./gradlew shadowJar` compiles the graph layout program. `java -Xmx1g -jar build/libs/graphBuilder.jar` runs it.
If running in docker, this means you run
- `docker-compose build gephi`
- `docker-compose run gephi java -Xmx1g -jar build/libs/graphBuilder.jar` lays out the graph
### Frontend
@ -103,8 +102,6 @@ SHELL=/bin/bash
0 2 * * * /usr/bin/dokku run gephi java -Xmx1g -jar build/libs/graphBuilder.jar
```
10. (Optional) Set up caching with something like [dokku-nginx-cache](https://github.com/Aluxian/dokku-nginx-cache)
Before the app starts running, make sure that the Elasticsearch index exists -- otherwise it'll create one called
`instances`, which should be the name of the alias. Then it won't be able to hot swap if you reindex in the future.

48
backend/lib/backend_web/rate_limiter.ex

@ -0,0 +1,48 @@
defmodule BackendWeb.RateLimiter do
@moduledoc """
Functions used to rate limit:
* all endpoints by IP/endpoint
* authentication endpoints by domain
"""
import Phoenix.Controller, only: [json: 2]
import Plug.Conn, only: [put_status: 2]
use Plug.Builder
def rate_limit(conn, options \\ []) do
case check_rate(conn, options) do
{:ok, _count} -> conn # Do nothing, allow execution to continue
{:error, _count} -> render_error(conn)
end
end
def rate_limit_authentication(conn, options \\ []) do
%{"id" => domain} = conn.params
options = Keyword.put(options, :bucket_name, "authorization: #{domain}")
rate_limit(conn, options)
end
defp check_rate(conn, options) do
interval_milliseconds = options[:interval_seconds] * 1000
max_requests = options[:max_requests]
bucket_name = options[:bucket_name] || bucket_name(conn)
ExRated.check_rate(bucket_name, interval_milliseconds, max_requests)
end
# Bucket name should be a combination of ip address and request path, like so:
#
# "127.0.0.1:/api/v1/authorizations"
defp bucket_name(conn) do
path = Enum.join(conn.path_info, "/")
ip = conn.remote_ip |> Tuple.to_list |> Enum.join(".")
"#{ip}:#{path}"
end
defp render_error(conn) do
conn
|> put_status(:forbidden)
|> json(%{error: "Rate limit exceeded."})
|> halt # Stop execution of further plugs, return response now
end
end

16
backend/lib/backend_web/router.ex

@ -1,8 +1,14 @@
defmodule BackendWeb.Router do
use BackendWeb, :router
import BackendWeb.RateLimiter
pipeline :api do
plug(:accepts, ["json"])
plug(:rate_limit, max_requests: 5, interval_seconds: 10) # requests to the same endpoint
end
pipeline :api_admin do
plug(:rate_limit_authentication, max_requests: 5, interval_seconds: 60)
end
scope "/api", BackendWeb do
@ -12,8 +18,12 @@ defmodule BackendWeb.Router do
resources("/graph", GraphController, only: [:index, :show])
resources("/search", SearchController, only: [:index])
resources("/admin/login", AdminLoginController, only: [:show, :create])
get "/admin", AdminController, :show
post "/admin", AdminController, :update
scope "/admin" do
pipe_through :api_admin
resources("/login", AdminLoginController, only: [:show, :create])
get "/", AdminController, :show
post "/", AdminController, :update
end
end
end

3
backend/mix.exs

@ -67,7 +67,8 @@ defmodule Backend.MixProject do
{:hunter, "~> 0.5.1"},
{:poison, "~> 4.0", override: true},
{:scrivener_ecto, "~> 2.2"},
{:recase, "~> 0.6.0"}
{:recase, "~> 0.6.0"},
{:ex_rated, "~> 1.3"}
]
end

2
backend/mix.lock

@ -18,6 +18,8 @@
"ecto": {:hex, :ecto, "3.4.4", "a2c881e80dc756d648197ae0d936216c0308370332c5e77a2325a10293eef845", [:mix], [{:decimal, "~> 1.6 or ~> 2.0", [hex: :decimal, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "cc4bd3ad62abc3b21fb629f0f7a3dab23a192fca837d257dd08449fba7373561"},
"ecto_sql": {:hex, :ecto_sql, "3.4.3", "c552aa8a7ccff2b64024f835503b3155d8e73452c180298527fbdbcd6e79710b", [:mix], [{:db_connection, "~> 2.2", [hex: :db_connection, repo: "hexpm", optional: false]}, {:ecto, "~> 3.4.3", [hex: :ecto, repo: "hexpm", optional: false]}, {:myxql, "~> 0.3.0 or ~> 0.4.0", [hex: :myxql, repo: "hexpm", optional: true]}, {:postgrex, "~> 0.15.0", [hex: :postgrex, repo: "hexpm", optional: true]}, {:tds, "~> 2.1.0", [hex: :tds, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "ec9e59d6fa3f8cfda9963ada371e9e6659167c2338a997bd7ea23b10b245842b"},
"elasticsearch": {:hex, :elasticsearch, "1.0.0", "626d3fb8e7554d9c93eb18817ae2a3d22c2a4191cc903c4644b1334469b15374", [:mix], [{:httpoison, ">= 0.0.0", [hex: :httpoison, repo: "hexpm", optional: false]}, {:poison, ">= 0.0.0", [hex: :poison, repo: "hexpm", optional: true]}, {:sigaws, "~> 0.7", [hex: :sigaws, repo: "hexpm", optional: true]}, {:vex, "~> 0.6.0", [hex: :vex, repo: "hexpm", optional: false]}], "hexpm", "9fa0b717ad57a54c28451b3eb10c5121211c29a7b33615d2bcc7e2f3c9418b2e"},
"ex2ms": {:hex, :ex2ms, "1.6.0", "f39bbd9ff1b0f27b3f707bab2d167066dd8965e7df1149b962d94c74615d0e09", [:mix], [], "hexpm", "0d1ab5e08421af5cd69146efb408dbb1ff77f38a2f4df5f086f2512dc8cf65bf"},
"ex_rated": {:hex, :ex_rated, "1.3.3", "30ecbdabe91f7eaa9d37fa4e81c85ba420f371babeb9d1910adbcd79ec798d27", [:mix], [{:ex2ms, "~> 1.5", [hex: :ex2ms, repo: "hexpm", optional: false]}], "hexpm", "26817cf0927b7a2e7bc2e14b4dab66a329fafa4520b513a8a4025532ac5a7cbf"},
"ex_twilio": {:hex, :ex_twilio, "0.7.0", "d7ce624ef4661311ae28c3e3aa060ecb66a9f4843184d7400c29072f7d3f5a4a", [:mix], [{:httpoison, ">= 0.9.0", [hex: :httpoison, repo: "hexpm", optional: false]}, {:inflex, "~> 1.0", [hex: :inflex, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}, {:joken, "~> 2.0", [hex: :joken, repo: "hexpm", optional: false]}, {:poison, "~> 3.0", [hex: :poison, repo: "hexpm", optional: false]}], "hexpm", "6be84f1508ed47d443d18cdc4ea0561f8ad4095b69791dd9be5f2fe14b1dafc5"},
"gen_stage": {:hex, :gen_stage, "1.0.0", "51c8ae56ff54f9a2a604ca583798c210ad245f415115453b773b621c49776df5", [:mix], [], "hexpm", "1d9fc978db5305ac54e6f5fec7adf80cd893b1000cf78271564c516aa2af7706"},
"gen_state_machine": {:hex, :gen_state_machine, "2.1.0", "a38b0e53fad812d29ec149f0d354da5d1bc0d7222c3711f3a0bd5aa608b42992", [:mix], [], "hexpm", "ae367038808db25cee2f2c4b8d0531522ea587c4995eb6f96ee73410a60fa06b"},

2
frontend/src/components/screens/AboutScreen.tsx

@ -39,7 +39,7 @@ const AboutScreen: React.FC = () => (
<H4>Why can&apos;t I see details about my instance?</H4>
<p className={Classes.RUNNING_TEXT}>
fediverse.space only supports servers using the Mastodon API, the Misskey API, the GNU Social API, or Nodeinfo.
Instances with 10 or fewer users won&apos;t be scraped -- it&apos;s a tool for understanding communities, not
Instances with 10 or fewer users won&apos;t be crawled -- it&apos;s a tool for understanding communities, not
individuals.
</p>

Loading…
Cancel
Save