discover peers from mentions

backend/lib/backend/crawler/crawler.ex

@@ -5,7 +5,6 @@ defmodule Backend.Crawler.Crawler do
 
   alias __MODULE__
   alias Backend.Crawler.Crawlers.Mastodon
-  alias Backend.Crawler.ApiCrawler
   alias Backend.{Repo, Instance, Interaction, InstancePeer}
   import Ecto.Query
   require Logger
@@ -20,6 +19,7 @@ defmodule Backend.Crawler.Crawler do
     :result
   ]
 
+  @spec run(String.t()) :: any
   def run(domain) do
     Logger.info("Crawling #{domain}...")
     HTTPoison.start()
@@ -67,7 +67,12 @@ defmodule Backend.Crawler.Crawler do
       now = NaiveDateTime.truncate(NaiveDateTime.utc_now(), :second)
       peers_domains = result.peers
 
-      # Insert or update the instance we crawled
+      # We also get a list of the instances mentioned, in case the server doesn't publish a list of peers.
+      peers_from_mentions =
+        result.interactions
+        |> Enum.map(fn i -> Kernel.elem(i, 0) end)
+
+      ## Save the instance we crawled ##
       Repo.insert!(
         %Instance{
           domain: domain,
@@ -90,16 +95,18 @@ defmodule Backend.Crawler.Crawler do
         conflict_target: :domain
       )
 
-      # If we discovered new instances from the peers endpoint, add them
-      # TODO: also discover new instances from mentions (in case the peers endpoint isn't there)
+      # If we discovered new instances from the peers endpoint or from mentions, add them
       peers =
         peers_domains
+        |> MapSet.new()
+        |> (fn set -> MapSet.union(set, MapSet.new(peers_from_mentions)) end).()
+        |> MapSet.to_list()
         |> Enum.map(&%{domain: &1, inserted_at: now, updated_at: now})
 
       Instance
       |> Repo.insert_all(peers, on_conflict: :nothing, conflict_target: :domain)
 
-      ## Add the peer relationships ##
+      ## Save peer relationships ##
       Repo.transaction(fn ->
         # get current peers (a list of strings)
         current_peers =