add /api/instances endpoint

backend/README.md

@@ -1,4 +1,9 @@
-# Backend
+# fediverse.space backend
+
+## Notes
+* Run with `SKIP_CRAWL=true` to just run the server (useful for working on the API without also crawling)
+
+### Default README
 
 To start your Phoenix server:
 

backend/lib/backend/api.ex

@@ -0,0 +1,17 @@
+defmodule Backend.Api do
+  alias Backend.{Instance, Repo}
+  import Ecto.Query
+
+  @spec list_instances() :: [Instance.t()]
+  def list_instances() do
+    Instance
+    |> Repo.all()
+  end
+
+  @spec get_instance!(String.t()) :: Instance.t()
+  def get_instance!(domain) do
+    Instance
+    |> preload(:peers)
+    |> Repo.get_by!(domain: domain)
+  end
+end

backend/lib/backend/application.ex

@@ -22,10 +22,15 @@ defmodule Backend.Application do
          Honeydew.start_queue(:crawl_queue, failure_mode: Honeydew.FailureMode.Abandon)
          Honeydew.start_workers(:crawl_queue, Backend.Crawler, num: crawl_worker_count)
        end},
-      Backend.Crawler.StaleInstanceManager,
       Backend.Scheduler
     ]
 
+    children =
+      case Enum.member?(["true", 1], System.get_env("SKIP_CRAWL")) do
+        true -> children
+        false -> children ++ [Backend.Crawler.StaleInstanceManager]
+      end
+
     # See https://hexdocs.pm/elixir/Supervisor.html
     # for other strategies and supported options
     opts = [strategy: :one_for_one, name: Backend.Supervisor]

backend/lib/backend/crawler/crawlers/mastodon.ex

@@ -78,7 +78,7 @@ defmodule Backend.Crawler.Crawlers.Mastodon do
     # most recent status we have.
     min_timestamp =
       if statuses_seen == 0 do
-        get_last_crawl(domain)
+        get_last_successful_crawl_timestamp(domain)
       else
         min_timestamp
       end

backend/lib/backend/crawler/util.ex

@@ -1,6 +1,4 @@
 defmodule Backend.Crawler.Util do
-  import Ecto.Query
-  alias Backend.{Crawl, Repo}
   require Logger
 
   @spec get_config(atom) :: any
@@ -40,22 +38,6 @@ defmodule Backend.Crawler.Util do
     end
   end
 
-  @spec get_last_crawl(String.t()) :: NaiveDateTime.t() | nil
-  def get_last_crawl(domain) do
-    crawls =
-      Crawl
-      |> select([c], c.inserted_at)
-      |> where([c], is_nil(c.error) and c.instance_domain == ^domain)
-      |> order_by(desc: :id)
-      |> limit(1)
-      |> Repo.all()
-
-    case length(crawls) do
-      1 -> hd(crawls)
-      0 -> nil
-    end
-  end
-
   def get(url) do
     # TODO: add version number to user agent?
     HTTPoison.get(url, [{"User-Agent", "fediverse.space crawler"}],

backend/lib/backend/instance.ex

@@ -12,7 +12,7 @@ defmodule Backend.Instance do
 
     many_to_many :peers, Backend.Instance,
       join_through: Backend.InstancePeer,
-      join_keys: [source: :domain, target: :domain]
+      join_keys: [source_domain: :domain, target_domain: :domain]
 
     # This may look like it's duplicating :peers above, but it allows us to insert peer relationships quickly.
     # https://stackoverflow.com/a/56764241/3697202

backend/lib/backend/util.ex

@@ -1,5 +1,7 @@
 defmodule Backend.Util do
   import Backend.Crawler.Util
+  import Ecto.Query
+  alias Backend.{Crawl, Repo}
 
   @doc """
   Takes two lists and returns a list of the union thereof (without duplicates).
@@ -22,4 +24,93 @@ defmodule Backend.Util do
       String.ends_with?(domain, blacklisted_domain)
     end)
   end
+
+  @doc """
+  Returns the key to use for non-directed edges
+  (really, just the two domains sorted alphabetically)
+  """
+  @spec get_interaction_key(String.t(), String.t()) :: String.t()
+  def get_interaction_key(source, target) do
+    [source, target]
+    |> Enum.sort()
+    |> List.to_tuple()
+  end
+
+  @doc """
+  Gets the current UTC time as a NaiveDateTime in a format that can be inserted into the database.
+  """
+  def get_now() do
+    NaiveDateTime.truncate(NaiveDateTime.utc_now(), :second)
+  end
+
+  @doc """
+  Returns the later of two NaiveDateTimes.
+  """
+  @spec max_datetime(NaiveDateTime.t() | nil, NaiveDateTime.t() | nil) :: NaiveDateTime.t()
+  def max_datetime(datetime_one, nil) do
+    datetime_one
+  end
+
+  def max_datetime(nil, datetime_two) do
+    datetime_two
+  end
+
+  def max_datetime(datetime_one, datetime_two) do
+    case NaiveDateTime.compare(datetime_one, datetime_two) do
+      :gt -> datetime_one
+      _ -> datetime_two
+    end
+  end
+
+  @spec get_last_successful_crawl(String.t()) :: Crawl.t() | nil
+  def get_last_successful_crawl(domain) do
+    crawls =
+      Crawl
+      |> select([c], c)
+      |> where([c], is_nil(c.error) and c.instance_domain == ^domain)
+      |> order_by(desc: :id)
+      |> limit(1)
+      |> Repo.all()
+
+    case length(crawls) do
+      1 -> hd(crawls)
+      0 -> nil
+    end
+  end
+
+  @spec get_last_crawl(String.t()) :: Crawl.t() | nil
+  def get_last_crawl(domain) do
+    crawls =
+      Crawl
+      |> select([c], c)
+      |> where([c], c.instance_domain == ^domain)
+      |> order_by(desc: :id)
+      |> limit(1)
+      |> Repo.all()
+
+    case length(crawls) do
+      1 -> hd(crawls)
+      0 -> nil
+    end
+  end
+
+  @spec get_last_successful_crawl_timestamp(String.t()) :: NaiveDateTime.t() | nil
+  def get_last_successful_crawl_timestamp(domain) do
+    crawl = get_last_crawl(domain)
+
+    case crawl do
+      nil -> nil
+      _ -> crawl.inserted_at
+    end
+  end
+
+  @doc """
+  Takes two maps with numeric values and merges them, adding the values of duplicate keys.
+  """
+  def merge_count_maps(map1, map2) do
+    map1
+    |> Enum.reduce(map2, fn {key, val}, acc ->
+      Map.update(acc, key, val, &(&1 + val))
+    end)
+  end
 end

backend/lib/backend_web/controllers/fallback_controller.ex

@@ -0,0 +1,15 @@
+defmodule BackendWeb.FallbackController do
+  @moduledoc """
+  Translates controller action results into valid `Plug.Conn` responses.
+
+  See `Phoenix.Controller.action_fallback/1` for more details.
+  """
+  use BackendWeb, :controller
+
+  def call(conn, {:error, :not_found}) do
+    conn
+    |> put_status(:not_found)
+    |> put_view(BackendWeb.ErrorView)
+    |> render(:"404")
+  end
+end

backend/lib/backend_web/controllers/instance_controller.ex

@@ -0,0 +1,27 @@
+defmodule BackendWeb.InstanceController do
+  use BackendWeb, :controller
+
+  import Backend.Util
+  alias Backend.Api
+
+  action_fallback BackendWeb.FallbackController
+
+  def index(conn, _params) do
+    instances = Api.list_instances()
+    render(conn, "index.json", instances: instances)
+  end
+
+  def show(conn, %{"id" => domain}) do
+    instance = Api.get_instance!(domain)
+    last_crawl = get_last_crawl(domain)
+    render(conn, "show.json", instance: instance, crawl: last_crawl)
+  end
+
+  # def update(conn, %{"id" => id, "instance" => instance_params}) do
+  #   instance = Api.get_instance!(id)
+
+  #   with {:ok, %Instance{} = instance} <- Api.update_instance(instance, instance_params) do
+  #     render(conn, "show.json", instance: instance)
+  #   end
+  # end
+end

backend/lib/backend_web/router.ex

@@ -7,5 +7,7 @@ defmodule BackendWeb.Router do
 
   scope "/api", BackendWeb do
     pipe_through :api
+
+    resources "/instances", InstanceController, only: [:index, :show]
   end
 end

backend/lib/backend_web/views/changeset_view.ex

@@ -0,0 +1,19 @@
+defmodule BackendWeb.ChangesetView do
+  use BackendWeb, :view
+
+  @doc """
+  Traverses and translates changeset errors.
+
+  See `Ecto.Changeset.traverse_errors/2` and
+  `BackendWeb.ErrorHelpers.translate_error/1` for more details.
+  """
+  def translate_errors(changeset) do
+    Ecto.Changeset.traverse_errors(changeset, &translate_error/1)
+  end
+
+  def render("error.json", %{changeset: changeset}) do
+    # When encoded, the changeset returns its errors
+    # as a JSON object. So we just pass it forward.
+    %{errors: translate_errors(changeset)}
+  end
+end

backend/lib/backend_web/views/instance_view.ex

@@ -0,0 +1,45 @@
+defmodule BackendWeb.InstanceView do
+  use BackendWeb, :view
+  alias BackendWeb.InstanceView
+  require Logger
+
+  def render("index.json", %{instances: instances}) do
+    render_many(instances, InstanceView, "instance.json")
+  end
+
+  def render("show.json", %{instance: instance, crawl: crawl}) do
+    render_one(instance, InstanceView, "instance_detail.json", crawl: crawl)
+  end
+
+  def render("instance.json", %{instance: instance}) do
+    %{name: instance.domain}
+  end
+
+  def render("instance_detail.json", %{instance: instance, crawl: crawl}) do
+    Logger.info("keys: #{inspect(instance)}")
+
+    [status, last_updated] =
+      case crawl do
+        nil ->
+          ["not crawled", nil]
+
+        _ ->
+          case crawl.error do
+            nil -> ["success", crawl.inserted_at]
+            err -> [err, crawl.inserted_at]
+          end
+      end
+
+    %{
+      name: instance.domain,
+      description: instance.description,
+      version: instance.version,
+      userCount: instance.user_count,
+      statusCount: instance.status_count,
+      domainCount: length(instance.peers),
+      peers: render_many(instance.peers, InstanceView, "instance.json"),
+      lastUpdated: last_updated,
+      status: status
+    }
+  end
+end

backend/test/backend_web/controllers/instance_controller_test.exs

@@ -0,0 +1,88 @@
+defmodule BackendWeb.InstanceControllerTest do
+  use BackendWeb.ConnCase
+
+  alias Backend.Api
+  alias Backend.Api.Instance
+
+  @create_attrs %{
+    name: "some name"
+  }
+  @update_attrs %{
+    name: "some updated name"
+  }
+  @invalid_attrs %{name: nil}
+
+  def fixture(:instance) do
+    {:ok, instance} = Api.create_instance(@create_attrs)
+    instance
+  end
+
+  setup %{conn: conn} do
+    {:ok, conn: put_req_header(conn, "accept", "application/json")}
+  end
+
+  describe "index" do
+    test "lists all instances", %{conn: conn} do
+      conn = get(conn, Routes.instance_path(conn, :index))
+      assert json_response(conn, 200)["data"] == []
+    end
+  end
+
+  describe "create instance" do
+    test "renders instance when data is valid", %{conn: conn} do
+      conn = post(conn, Routes.instance_path(conn, :create), instance: @create_attrs)
+      assert %{"id" => id} = json_response(conn, 201)["data"]
+
+      conn = get(conn, Routes.instance_path(conn, :show, id))
+
+      assert %{
+               "id" => id,
+               "name" => "some name"
+             } = json_response(conn, 200)["data"]
+    end
+
+    test "renders errors when data is invalid", %{conn: conn} do
+      conn = post(conn, Routes.instance_path(conn, :create), instance: @invalid_attrs)
+      assert json_response(conn, 422)["errors"] != %{}
+    end
+  end
+
+  describe "update instance" do
+    setup [:create_instance]
+
+    test "renders instance when data is valid", %{conn: conn, instance: %Instance{id: id} = instance} do
+      conn = put(conn, Routes.instance_path(conn, :update, instance), instance: @update_attrs)
+      assert %{"id" => ^id} = json_response(conn, 200)["data"]
+
+      conn = get(conn, Routes.instance_path(conn, :show, id))
+
+      assert %{
+               "id" => id,
+               "name" => "some updated name"
+             } = json_response(conn, 200)["data"]
+    end
+
+    test "renders errors when data is invalid", %{conn: conn, instance: instance} do
+      conn = put(conn, Routes.instance_path(conn, :update, instance), instance: @invalid_attrs)
+      assert json_response(conn, 422)["errors"] != %{}
+    end
+  end
+
+  describe "delete instance" do
+    setup [:create_instance]
+
+    test "deletes chosen instance", %{conn: conn, instance: instance} do
+      conn = delete(conn, Routes.instance_path(conn, :delete, instance))
+      assert response(conn, 204)
+
+      assert_error_sent 404, fn ->
+        get(conn, Routes.instance_path(conn, :show, instance))
+      end
+    end
+  end
+
+  defp create_instance(_) do
+    instance = fixture(:instance)
+    {:ok, instance: instance}
+  end
+end