fix process leak: only start httpoison once

backend/lib/backend/application.ex

@@ -16,11 +16,15 @@ defmodule Backend.Application do
       BackendWeb.Endpoint,
       # Crawler children
       :hackney_pool.child_spec(:crawler, timeout: 15000, max_connections: crawl_worker_count),
-      {Task,
-       fn ->
-         Honeydew.start_queue(:crawl_queue, failure_mode: Honeydew.FailureMode.Abandon)
-         Honeydew.start_workers(:crawl_queue, Backend.Crawler, num: crawl_worker_count)
-       end},
+      Supervisor.child_spec(
+        {Task,
+         fn ->
+           Honeydew.start_queue(:crawl_queue, failure_mode: Honeydew.FailureMode.Abandon)
+           Honeydew.start_workers(:crawl_queue, Backend.Crawler, num: crawl_worker_count)
+         end},
+        id: :start_honeydew
+      ),
+      Supervisor.child_spec({Task, fn -> HTTPoison.start() end}, id: :start_httpoison),
       Backend.Scheduler,
       Backend.Elasticsearch.Cluster
     ]

backend/lib/backend/crawler/crawler.ex

@@ -33,7 +33,6 @@ defmodule Backend.Crawler do
 
   def run(domain) do
     Logger.debug("Starting crawl of #{domain}")
-    HTTPoison.start()
 
     state = %Crawler{
       domain: domain,