never display opted-out instances anywhere

CHANGELOG.md

@@ -25,6 +25,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - Fixed some instances being duplicated (due to un-normalized data).
 - Fixed mobile instance view erroring for uncrawled instances.
 - Improved error handling in admin login page.
+- Instances that opt-out will no longer show up in search results ever, nor are they accessible through the API.
 
 ### Security
 

backend/lib/backend/api.ex

@@ -6,10 +6,16 @@ defmodule Backend.Api do
   @spec get_instance!(String.t()) :: Instance.t()
   def get_instance!(domain) do
     Instance
-    |> preload(:peers)
     |> Repo.get_by!(domain: domain)
   end
 
+  @spec get_instance_with_peers(String.t()) :: Instance.t() | nil
+  def get_instance_with_peers(domain) do
+    Instance
+    |> preload(:peers)
+    |> Repo.get_by(domain: domain)
+  end
+
   def update_instance(instance) do
     Repo.insert(
       instance,
@@ -111,6 +117,11 @@ defmodule Backend.Api do
         "size" => page_size,
         "query" => %{
           "bool" => %{
+            "filter" => %{
+              "term" => %{
+                "opt_out" => "false"
+              }
+            },
             "should" => [
               %{
                 "multi_match" => %{

backend/lib/backend/crawler/crawler.ex

@@ -129,7 +129,7 @@ defmodule Backend.Crawler do
       conflict_target: :domain
     )
 
-    Elasticsearch.put_document(Backend.Elasticsearch.Cluster, instance, "instances/_doc")
+    Elasticsearch.put_document!(Backend.Elasticsearch.Cluster, instance, "instances/_doc")
 
     # Save details of a new crawl
     curr_crawl =

backend/lib/backend/instance.ex

@@ -59,7 +59,8 @@ defmodule Backend.Instance do
         domain: instance.domain,
         description: instance.description,
         type: instance.type,
-        user_count: instance.user_count
+        user_count: instance.user_count,
+        opt_out: instance.opt_out
       }
     end
   end

backend/lib/backend_web/controllers/admin_controller.ex

@@ -20,13 +20,21 @@ defmodule BackendWeb.AdminController do
     with {:ok, domain} <- Auth.verify_token(token) do
       %{"optIn" => opt_in, "optOut" => opt_out} = params
 
-      instance = %Instance{
+      # Make sure to update ElasticSearch so that the instance is no longer returned in search results
+      es_instance =
+        Api.get_instance_with_peers(domain)
+        |> Map.put(:opt_in, opt_in)
+        |> Map.put(:opt_out, opt_out)
+
+      Elasticsearch.put_document!(Backend.Elasticsearch.Cluster, es_instance, "instances")
+
+      ecto_instance = %Instance{
         domain: domain,
         opt_in: opt_in,
         opt_out: opt_out
       }
 
-      with {:ok, updated_instance} <- Api.update_instance(instance) do
+      with {:ok, updated_instance} <- Api.update_instance(ecto_instance) do
         render(conn, "show.json", instance: updated_instance)
       end
     end

backend/lib/backend_web/controllers/instance_controller.ex

@@ -7,10 +7,15 @@ defmodule BackendWeb.InstanceController do
   action_fallback(BackendWeb.FallbackController)
 
   def show(conn, %{"id" => domain}) do
-    instance = Api.get_instance!(domain)
+    instance = Api.get_instance_with_peers(domain)
+
+    if instance == nil or instance.opt_out == true do
+      send_resp(conn, 404, "Not found")
+    else
       last_crawl = get_last_crawl(domain)
       render(conn, "show.json", instance: instance, crawl: last_crawl)
     end
+  end
 
   # def update(conn, %{"id" => id, "instance" => instance_params}) do
   #   instance = Api.get_instance!(id)

backend/priv/elasticsearch/instances.json

@@ -46,6 +46,9 @@
         },
         "user_count": {
           "type": "integer"
+        },
+        "opt_out": {
+          "type": "boolean"
         }
       }
     }