From 287c7b56249a27801e27d13906518a62c280fb68 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tao=20Bror=20Bojl=C3=A9n?= Date: Fri, 2 Aug 2019 22:46:40 +0300 Subject: [PATCH] normalize instance names --- CHANGELOG.md | 2 ++ backend/config/config.exs | 3 +-- backend/config/dev.exs | 3 ++- backend/lib/backend/crawler/crawler.ex | 1 + 4 files changed, 6 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 470c943..30e0e7c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,6 +17,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Fixed +- Fixed some instances being duplicated (due to un-normalized data). + ### Security ## [2.3.0 - 2019-08-02] diff --git a/backend/config/config.exs b/backend/config/config.exs index 456386f..4005835 100644 --- a/backend/config/config.exs +++ b/backend/config/config.exs @@ -68,8 +68,7 @@ config :backend, :crawler, user_agent: "fediverse.space crawler", admin_phone: System.get_env("ADMIN_PHONE"), twilio_phone: System.get_env("TWILIO_PHONE"), - admin_email: System.get_env("ADMIN_EMAIL"), - frontend_domain: "https://www.fediverse.space" + admin_email: System.get_env("ADMIN_EMAIL") config :backend, Backend.Scheduler, jobs: [ diff --git a/backend/config/dev.exs b/backend/config/dev.exs index cc1c912..2dc65ca 100644 --- a/backend/config/dev.exs +++ b/backend/config/dev.exs @@ -65,4 +65,5 @@ config :backend, :crawler, blacklist: [ "gab.best", "4chan.icu" - ] + ], + frontend_domain: "localhost:3000" diff --git a/backend/lib/backend/crawler/crawler.ex b/backend/lib/backend/crawler/crawler.ex index d62b779..a9a1ab0 100644 --- a/backend/lib/backend/crawler/crawler.ex +++ b/backend/lib/backend/crawler/crawler.ex @@ -148,6 +148,7 @@ defmodule Backend.Crawler do |> Map.keys() |> list_union(result.peers) |> Enum.filter(fn domain -> domain != nil and not is_blacklisted?(domain) end) + |> Enum.map(&clean_domain(&1)) peers = peers_domains