inex/index.community
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

fail if some peers are invalid

Browse source
This commit is contained in:
Tao Bror Bojlén 2019-08-21 22:29:15 +02:00
parent 67baa0ec04
commit c2124468a7
No known key found for this signature in database
GPG key ID: C6EC7AAB905F9E6F
1 changed files with 5 additions and 0 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

5
backend/lib/backend/crawler/crawler.ex
View file

@ -197,6 +197,11 @@ defmodule Backend.Crawler do
|> Enum.filter(fn domain -> domain != nil and not is_blacklisted?(domain) end) |> Enum.filter(fn domain -> domain != nil and not is_blacklisted?(domain) end)
|> Enum.map(&clean_domain(&1)) |> Enum.map(&clean_domain(&1))
if not Enum.all?(peers_domains, &is_valid_domain?(&1)) do
invalid_peers = Enum.filter(peers_domains, &is_valid_domain?(&1))
raise "#{domain} has invalid peers: #{inspect(invalid_peers)}"
end
peers = peers =
peers_domains peers_domains
|> Enum.map(&%{domain: &1, inserted_at: now, updated_at: now, next_crawl: now}) |> Enum.map(&%{domain: &1, inserted_at: now, updated_at: now, next_crawl: now})