inex/index.community
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

set infinite timeout on v expensive queries

Browse source
This commit is contained in:
Tao Bror Bojlén 2019-08-07 20:34:02 +03:00
parent 012fc1d85a
commit d9b9081ec3
No known key found for this signature in database
GPG key ID: C6EC7AAB905F9E6F
1 changed files with 51 additions and 45 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

28
backend/lib/backend/scheduler.ex
View file

@ -25,7 +25,7 @@ defmodule Backend.Scheduler do
i.inserted_at < i.inserted_at <
datetime_add(^NaiveDateTime.utc_now(), -1 * ^amount, ^unit) datetime_add(^NaiveDateTime.utc_now(), -1 * ^amount, ^unit)
) )
|> Repo.delete_all() |> Repo.delete_all(timeout: :infinity)
Logger.info("Pruned #{deleted_num} old crawls.") Logger.info("Pruned #{deleted_num} old crawls.")
end end
@ -59,7 +59,7 @@ defmodule Backend.Scheduler do
# we can take min() because every row is the same # we can take min() because every row is the same
interactions: min(c.interactions_seen) interactions: min(c.interactions_seen)
}) })
|> Repo.all() |> Repo.all(timeout: :infinity)
|> Enum.map(fn %{domain: domain, mentions: mentions, interactions: interactions} -> |> Enum.map(fn %{domain: domain, mentions: mentions, interactions: interactions} ->
%{ %{
domain: domain, domain: domain,
@ -72,7 +72,8 @@ defmodule Backend.Scheduler do
Instance Instance
|> Repo.insert_all(scores, |> Repo.insert_all(scores,
on_conflict: {:replace, [:insularity, :updated_at]}, on_conflict: {:replace, [:insularity, :updated_at]},
conflict_target: :domain conflict_target: :domain,
timeout: :infinity
) )
end end
@ -107,7 +108,7 @@ defmodule Backend.Scheduler do
second_earliest_crawl: min(c.inserted_at) second_earliest_crawl: min(c.inserted_at)
}) })
|> group_by([c], c.instance_domain) |> group_by([c], c.instance_domain)
|> Repo.all() |> Repo.all(timeout: :infinity)
|> Enum.map(fn %{ |> Enum.map(fn %{
instance_domain: domain, instance_domain: domain,
status_count: status_count, status_count: status_count,
@ -128,7 +129,8 @@ defmodule Backend.Scheduler do
Instance Instance
|> Repo.insert_all(instances, |> Repo.insert_all(instances,
on_conflict: {:replace, [:statuses_per_day, :updated_at]}, on_conflict: {:replace, [:statuses_per_day, :updated_at]},
conflict_target: :domain conflict_target: :domain,
timeout: :infinity
) )
end end
@ -167,12 +169,13 @@ defmodule Backend.Scheduler do
source_statuses_seen: min(c_source.statuses_seen), source_statuses_seen: min(c_source.statuses_seen),
target_statuses_seen: min(c_target.statuses_seen) target_statuses_seen: min(c_target.statuses_seen)
}) })
|> Repo.all() |> Repo.all(timeout: :infinity)
# Get edges and their weights # Get edges and their weights
Repo.transaction(fn -> Repo.transaction(
fn ->
Edge Edge
|> Repo.delete_all() |> Repo.delete_all(timeout: :infinity)
edges = edges =
interactions interactions
@ -197,7 +200,8 @@ defmodule Backend.Scheduler do
statuses_seen = source_statuses_seen + target_statuses_seen statuses_seen = source_statuses_seen + target_statuses_seen
Map.update(acc, key, {mentions, statuses_seen}, fn {curr_mentions, curr_statuses_seen} -> Map.update(acc, key, {mentions, statuses_seen}, fn {curr_mentions,
curr_statuses_seen} ->
{curr_mentions + mentions, curr_statuses_seen} {curr_mentions + mentions, curr_statuses_seen}
end) end)
end) end)
@ -212,8 +216,10 @@ defmodule Backend.Scheduler do
end) end)
Edge Edge
|> Repo.insert_all(edges) |> Repo.insert_all(edges, timeout: :infinity)
end) end,
timeout: :infinity
)
end end
@doc """ @doc """