From c2124468a74fda96338467e4ad81558baedeb4e4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tao=20Bror=20Bojl=C3=A9n?= Date: Wed, 21 Aug 2019 22:29:15 +0200 Subject: [PATCH] fail if some peers are invalid --- backend/lib/backend/crawler/crawler.ex | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/backend/lib/backend/crawler/crawler.ex b/backend/lib/backend/crawler/crawler.ex index d1a2e1b..8971470 100644 --- a/backend/lib/backend/crawler/crawler.ex +++ b/backend/lib/backend/crawler/crawler.ex @@ -197,6 +197,11 @@ defmodule Backend.Crawler do |> Enum.filter(fn domain -> domain != nil and not is_blacklisted?(domain) end) |> Enum.map(&clean_domain(&1)) + if not Enum.all?(peers_domains, &is_valid_domain?(&1)) do + invalid_peers = Enum.filter(peers_domains, &is_valid_domain?(&1)) + raise "#{domain} has invalid peers: #{inspect(invalid_peers)}" + end + peers = peers_domains |> Enum.map(&%{domain: &1, inserted_at: now, updated_at: now, next_crawl: now})