normalize instance names

CHANGELOG.md

@@ -17,6 +17,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ### Fixed
 
+- Fixed some instances being duplicated (due to un-normalized data).
+
 ### Security
 
 ## [2.3.0 - 2019-08-02]

backend/config/config.exs

@@ -68,8 +68,7 @@ config :backend, :crawler,
   user_agent: "fediverse.space crawler",
   admin_phone: System.get_env("ADMIN_PHONE"),
   twilio_phone: System.get_env("TWILIO_PHONE"),
-  admin_email: System.get_env("ADMIN_EMAIL"),
-  frontend_domain: "https://www.fediverse.space"
+  admin_email: System.get_env("ADMIN_EMAIL")
 
 config :backend, Backend.Scheduler,
   jobs: [

backend/config/dev.exs

@@ -65,4 +65,5 @@ config :backend, :crawler,
   blacklist: [
     "gab.best",
     "4chan.icu"
-  ]
+  ],
+  frontend_domain: "localhost:3000"

backend/lib/backend/crawler/crawler.ex

@@ -148,6 +148,7 @@ defmodule Backend.Crawler do
       |> Map.keys()
       |> list_union(result.peers)
       |> Enum.filter(fn domain -> domain != nil and not is_blacklisted?(domain) end)
+      |> Enum.map(&clean_domain(&1))
 
     peers =
       peers_domains