From 862f85b8fd47f0f372eadf32deb0db5626922e04 Mon Sep 17 00:00:00 2001
From: Houkime <>
Date: Mon, 1 Apr 2024 20:12:02 +0000
Subject: [PATCH 01/86] feature(redis): async connections

---
 selfprivacy_api/utils/redis_pool.py | 19 ++++++++++++-----
 tests/test_redis.py                 | 33 +++++++++++++++++++++++++++++
 2 files changed, 47 insertions(+), 5 deletions(-)
 create mode 100644 tests/test_redis.py

diff --git a/selfprivacy_api/utils/redis_pool.py b/selfprivacy_api/utils/redis_pool.py
index 3d35f01..04ccb51 100644
--- a/selfprivacy_api/utils/redis_pool.py
+++ b/selfprivacy_api/utils/redis_pool.py
@@ -2,6 +2,7 @@
 Redis pool module for selfprivacy_api
 """
 import redis
+import redis.asyncio as redis_async
 
 from selfprivacy_api.utils.singleton_metaclass import SingletonMetaclass
 
@@ -14,11 +15,18 @@ class RedisPool(metaclass=SingletonMetaclass):
     """
 
     def __init__(self):
+        url = RedisPool.connection_url(dbnumber=0)
+        # We need a normal sync pool because otherwise
+        # our whole API will need to be async
         self._pool = redis.ConnectionPool.from_url(
-            RedisPool.connection_url(dbnumber=0),
+            url,
+            decode_responses=True,
+        )
+        # We need an async pool for pubsub
+        self._async_pool = redis_async.ConnectionPool.from_url(
+            url,
             decode_responses=True,
         )
-        self._pubsub_connection = self.get_connection()
 
     @staticmethod
     def connection_url(dbnumber: int) -> str:
@@ -34,8 +42,9 @@ class RedisPool(metaclass=SingletonMetaclass):
         """
         return redis.Redis(connection_pool=self._pool)
 
-    def get_pubsub(self):
+    def get_connection_async(self) -> redis_async.Redis:
         """
-        Get a pubsub connection from the pool.
+        Get an async connection from the pool.
+        Async connections allow pubsub.
         """
-        return self._pubsub_connection.pubsub()
+        return redis_async.Redis(connection_pool=self._async_pool)
diff --git a/tests/test_redis.py b/tests/test_redis.py
new file mode 100644
index 0000000..48ec56e
--- /dev/null
+++ b/tests/test_redis.py
@@ -0,0 +1,33 @@
+import asyncio
+import pytest
+
+from selfprivacy_api.utils.redis_pool import RedisPool
+
+TEST_KEY = "test:test"
+
+
+@pytest.fixture()
+def empty_redis():
+    r = RedisPool().get_connection()
+    r.flushdb()
+    yield r
+    r.flushdb()
+
+
+async def write_to_test_key():
+    r = RedisPool().get_connection_async()
+    async with r.pipeline(transaction=True) as pipe:
+        ok1, ok2 = await pipe.set(TEST_KEY, "value1").set(TEST_KEY, "value2").execute()
+        assert ok1
+        assert ok2
+    assert await r.get(TEST_KEY) == "value2"
+    await r.close()
+
+
+def test_async_connection(empty_redis):
+    r = RedisPool().get_connection()
+    assert not r.exists(TEST_KEY)
+    # It _will_ report an error if it arises
+    asyncio.run(write_to_test_key())
+    # Confirming that we can read result from sync connection too
+    assert r.get(TEST_KEY) == "value2"

From 996cde15e14aa3a66e64e24d23aad86e95ed7742 Mon Sep 17 00:00:00 2001
From: Houkime <>
Date: Mon, 15 Apr 2024 13:35:44 +0000
Subject: [PATCH 02/86] chore(nixos): add pytest-asyncio

---
 flake.nix | 1 +
 1 file changed, 1 insertion(+)

diff --git a/flake.nix b/flake.nix
index f8b81aa..ab969a4 100644
--- a/flake.nix
+++ b/flake.nix
@@ -20,6 +20,7 @@
             pytest-datadir
             pytest-mock
             pytest-subprocess
+            pytest-asyncio
             black
             mypy
             pylsp-mypy

From 4d60b7264abdf3dd2af93ae7ad9128f020578e64 Mon Sep 17 00:00:00 2001
From: Houkime <>
Date: Mon, 15 Apr 2024 13:37:04 +0000
Subject: [PATCH 03/86] test(async): pubsub

---
 selfprivacy_api/utils/redis_pool.py | 12 +++++-
 tests/test_redis.py                 | 64 ++++++++++++++++++++++++++++-
 2 files changed, 73 insertions(+), 3 deletions(-)

diff --git a/selfprivacy_api/utils/redis_pool.py b/selfprivacy_api/utils/redis_pool.py
index 04ccb51..ea827d1 100644
--- a/selfprivacy_api/utils/redis_pool.py
+++ b/selfprivacy_api/utils/redis_pool.py
@@ -9,13 +9,15 @@ from selfprivacy_api.utils.singleton_metaclass import SingletonMetaclass
 REDIS_SOCKET = "/run/redis-sp-api/redis.sock"
 
 
-class RedisPool(metaclass=SingletonMetaclass):
+# class RedisPool(metaclass=SingletonMetaclass):
+class RedisPool:
     """
     Redis connection pool singleton.
     """
 
     def __init__(self):
-        url = RedisPool.connection_url(dbnumber=0)
+        self._dbnumber = 0
+        url = RedisPool.connection_url(dbnumber=self._dbnumber)
         # We need a normal sync pool because otherwise
         # our whole API will need to be async
         self._pool = redis.ConnectionPool.from_url(
@@ -48,3 +50,9 @@ class RedisPool(metaclass=SingletonMetaclass):
         Async connections allow pubsub.
         """
         return redis_async.Redis(connection_pool=self._async_pool)
+
+    async def subscribe_to_keys(self, pattern: str) -> redis_async.client.PubSub:
+        async_redis = self.get_connection_async()
+        pubsub = async_redis.pubsub()
+        await pubsub.psubscribe(f"__keyspace@{self._dbnumber}__:" + pattern)
+        return pubsub
diff --git a/tests/test_redis.py b/tests/test_redis.py
index 48ec56e..2def280 100644
--- a/tests/test_redis.py
+++ b/tests/test_redis.py
@@ -1,13 +1,18 @@
 import asyncio
 import pytest
+import pytest_asyncio
+from asyncio import streams
+import redis
+from typing import List
 
 from selfprivacy_api.utils.redis_pool import RedisPool
 
 TEST_KEY = "test:test"
+STOPWORD = "STOP"
 
 
 @pytest.fixture()
-def empty_redis():
+def empty_redis(event_loop):
     r = RedisPool().get_connection()
     r.flushdb()
     yield r
@@ -31,3 +36,60 @@ def test_async_connection(empty_redis):
     asyncio.run(write_to_test_key())
     # Confirming that we can read result from sync connection too
     assert r.get(TEST_KEY) == "value2"
+
+
+async def channel_reader(channel: redis.client.PubSub) -> List[dict]:
+    result: List[dict] = []
+    while True:
+        # Mypy cannot correctly detect that it is a coroutine
+        # But it is
+        message: dict = await channel.get_message(ignore_subscribe_messages=True, timeout=None)  # type: ignore
+        if message is not None:
+            result.append(message)
+            if message["data"] == STOPWORD:
+                break
+    return result
+
+
+@pytest.mark.asyncio
+async def test_pubsub(empty_redis, event_loop):
+    # Adapted from :
+    # https://redis.readthedocs.io/en/stable/examples/asyncio_examples.html
+    # Sanity checking because of previous event loop bugs
+    assert event_loop == asyncio.get_event_loop()
+    assert event_loop == asyncio.events.get_event_loop()
+    assert event_loop == asyncio.events._get_event_loop()
+    assert event_loop == asyncio.events.get_running_loop()
+
+    reader = streams.StreamReader(34)
+    assert event_loop == reader._loop
+    f = reader._loop.create_future()
+    f.set_result(3)
+    await f
+
+    r = RedisPool().get_connection_async()
+    async with r.pubsub() as pubsub:
+        await pubsub.subscribe("channel:1")
+        future = asyncio.create_task(channel_reader(pubsub))
+
+        await r.publish("channel:1", "Hello")
+        # message: dict = await pubsub.get_message(ignore_subscribe_messages=True, timeout=5.0) # type: ignore
+        # raise ValueError(message)
+        await r.publish("channel:1", "World")
+        await r.publish("channel:1", STOPWORD)
+
+        messages = await future
+
+        assert len(messages) == 3
+
+        message = messages[0]
+        assert "data" in message.keys()
+        assert message["data"] == "Hello"
+        message = messages[1]
+        assert "data" in message.keys()
+        assert message["data"] == "World"
+        message = messages[2]
+        assert "data" in message.keys()
+        assert message["data"] == STOPWORD
+
+    await r.close()

From 5bf5e7462f24a8882d04cdc5d723115415d5b5ff Mon Sep 17 00:00:00 2001
From: Houkime <>
Date: Mon, 22 Apr 2024 14:40:55 +0000
Subject: [PATCH 04/86] test(redis): test key event notifications

---
 tests/test_redis.py | 48 +++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 48 insertions(+)

diff --git a/tests/test_redis.py b/tests/test_redis.py
index 2def280..181d325 100644
--- a/tests/test_redis.py
+++ b/tests/test_redis.py
@@ -15,6 +15,8 @@ STOPWORD = "STOP"
 def empty_redis(event_loop):
     r = RedisPool().get_connection()
     r.flushdb()
+    r.config_set("notify-keyspace-events", "KEA")
+    assert r.config_get("notify-keyspace-events")["notify-keyspace-events"] == "AKE"
     yield r
     r.flushdb()
 
@@ -51,6 +53,15 @@ async def channel_reader(channel: redis.client.PubSub) -> List[dict]:
     return result
 
 
+async def channel_reader_onemessage(channel: redis.client.PubSub) -> dict:
+    while True:
+        # Mypy cannot correctly detect that it is a coroutine
+        # But it is
+        message: dict = await channel.get_message(ignore_subscribe_messages=True, timeout=None)  # type: ignore
+        if message is not None:
+            return message
+
+
 @pytest.mark.asyncio
 async def test_pubsub(empty_redis, event_loop):
     # Adapted from :
@@ -93,3 +104,40 @@ async def test_pubsub(empty_redis, event_loop):
         assert message["data"] == STOPWORD
 
     await r.close()
+
+
+@pytest.mark.asyncio
+async def test_keyspace_notifications_simple(empty_redis, event_loop):
+    r = RedisPool().get_connection_async()
+    await r.set(TEST_KEY, "I am not empty")
+    async with r.pubsub() as pubsub:
+        await pubsub.subscribe("__keyspace@0__:" + TEST_KEY)
+
+        future_message = asyncio.create_task(channel_reader_onemessage(pubsub))
+        empty_redis.set(TEST_KEY, "I am set!")
+        message = await future_message
+        assert message is not None
+        assert message["data"] is not None
+        assert message == {
+            "channel": f"__keyspace@0__:{TEST_KEY}",
+            "data": "set",
+            "pattern": None,
+            "type": "message",
+        }
+
+
+@pytest.mark.asyncio
+async def test_keyspace_notifications(empty_redis, event_loop):
+    pubsub = await RedisPool().subscribe_to_keys(TEST_KEY)
+    async with pubsub:
+        future_message = asyncio.create_task(channel_reader_onemessage(pubsub))
+        empty_redis.set(TEST_KEY, "I am set!")
+        message = await future_message
+        assert message is not None
+        assert message["data"] is not None
+        assert message == {
+            "channel": f"__keyspace@0__:{TEST_KEY}",
+            "data": "set",
+            "pattern": f"__keyspace@0__:{TEST_KEY}",
+            "type": "pmessage",
+        }

From 8d099c9a225ed6281df39aad32a97827072950d9 Mon Sep 17 00:00:00 2001
From: Houkime <>
Date: Mon, 22 Apr 2024 14:41:56 +0000
Subject: [PATCH 05/86] refactoring(jobs): break out a function returning all
 jobs

---
 selfprivacy_api/graphql/queries/jobs.py | 10 +++++++---
 1 file changed, 7 insertions(+), 3 deletions(-)

diff --git a/selfprivacy_api/graphql/queries/jobs.py b/selfprivacy_api/graphql/queries/jobs.py
index e7b99e6..337382a 100644
--- a/selfprivacy_api/graphql/queries/jobs.py
+++ b/selfprivacy_api/graphql/queries/jobs.py
@@ -11,13 +11,17 @@ from selfprivacy_api.graphql.common_types.jobs import (
 from selfprivacy_api.jobs import Jobs
 
 
+def get_all_jobs() -> typing.List[ApiJob]:
+    Jobs.get_jobs()
+
+    return [job_to_api_job(job) for job in Jobs.get_jobs()]
+
+
 @strawberry.type
 class Job:
     @strawberry.field
     def get_jobs(self) -> typing.List[ApiJob]:
-        Jobs.get_jobs()
-
-        return [job_to_api_job(job) for job in Jobs.get_jobs()]
+        return get_all_jobs()
 
     @strawberry.field
     def get_job(self, job_id: str) -> typing.Optional[ApiJob]:

From b204d4a9b3cc8b6a8dbf044f3d1dc9de665349d6 Mon Sep 17 00:00:00 2001
From: Houkime <>
Date: Mon, 22 Apr 2024 14:50:08 +0000
Subject: [PATCH 06/86] feature(redis): enable key space notifications by
 default

---
 selfprivacy_api/utils/redis_pool.py | 2 ++
 tests/test_redis.py                 | 1 -
 2 files changed, 2 insertions(+), 1 deletion(-)

diff --git a/selfprivacy_api/utils/redis_pool.py b/selfprivacy_api/utils/redis_pool.py
index ea827d1..39c536f 100644
--- a/selfprivacy_api/utils/redis_pool.py
+++ b/selfprivacy_api/utils/redis_pool.py
@@ -29,6 +29,8 @@ class RedisPool:
             url,
             decode_responses=True,
         )
+        # TODO: inefficient, this is probably done each time we connect
+        self.get_connection().config_set("notify-keyspace-events", "KEA")
 
     @staticmethod
     def connection_url(dbnumber: int) -> str:
diff --git a/tests/test_redis.py b/tests/test_redis.py
index 181d325..70ef43a 100644
--- a/tests/test_redis.py
+++ b/tests/test_redis.py
@@ -15,7 +15,6 @@ STOPWORD = "STOP"
 def empty_redis(event_loop):
     r = RedisPool().get_connection()
     r.flushdb()
-    r.config_set("notify-keyspace-events", "KEA")
     assert r.config_get("notify-keyspace-events")["notify-keyspace-events"] == "AKE"
     yield r
     r.flushdb()

From 43980f16ea43d302fd6628de1b841ac81c5523a1 Mon Sep 17 00:00:00 2001
From: Houkime <>
Date: Mon, 6 May 2024 14:54:13 +0000
Subject: [PATCH 07/86] feature(jobs): job update generator

---
 selfprivacy_api/jobs/__init__.py             | 29 ++++----
 selfprivacy_api/utils/redis_model_storage.py | 12 +++-
 tests/test_redis.py                          | 76 ++++++++++++++++++++
 3 files changed, 102 insertions(+), 15 deletions(-)

diff --git a/selfprivacy_api/jobs/__init__.py b/selfprivacy_api/jobs/__init__.py
index 4649bb0..3dd48c4 100644
--- a/selfprivacy_api/jobs/__init__.py
+++ b/selfprivacy_api/jobs/__init__.py
@@ -15,6 +15,7 @@ A job is a dictionary with the following keys:
     - result: result of the job
 """
 import typing
+import asyncio
 import datetime
 from uuid import UUID
 import uuid
@@ -23,6 +24,7 @@ from enum import Enum
 from pydantic import BaseModel
 
 from selfprivacy_api.utils.redis_pool import RedisPool
+from selfprivacy_api.utils.redis_model_storage import store_model_as_hash
 
 JOB_EXPIRATION_SECONDS = 10 * 24 * 60 * 60  # ten days
 
@@ -102,7 +104,7 @@ class Jobs:
             result=None,
         )
         redis = RedisPool().get_connection()
-        _store_job_as_hash(redis, _redis_key_from_uuid(job.uid), job)
+        store_model_as_hash(redis, _redis_key_from_uuid(job.uid), job)
         return job
 
     @staticmethod
@@ -218,7 +220,7 @@ class Jobs:
         redis = RedisPool().get_connection()
         key = _redis_key_from_uuid(job.uid)
         if redis.exists(key):
-            _store_job_as_hash(redis, key, job)
+            store_model_as_hash(redis, key, job)
             if status in (JobStatus.FINISHED, JobStatus.ERROR):
                 redis.expire(key, JOB_EXPIRATION_SECONDS)
 
@@ -294,17 +296,6 @@ def _progress_log_key_from_uuid(uuid_string) -> str:
     return PROGRESS_LOGS_PREFIX + str(uuid_string)
 
 
-def _store_job_as_hash(redis, redis_key, model) -> None:
-    for key, value in model.dict().items():
-        if isinstance(value, uuid.UUID):
-            value = str(value)
-        if isinstance(value, datetime.datetime):
-            value = value.isoformat()
-        if isinstance(value, JobStatus):
-            value = value.value
-        redis.hset(redis_key, key, str(value))
-
-
 def _job_from_hash(redis, redis_key) -> typing.Optional[Job]:
     if redis.exists(redis_key):
         job_dict = redis.hgetall(redis_key)
@@ -321,3 +312,15 @@ def _job_from_hash(redis, redis_key) -> typing.Optional[Job]:
 
         return Job(**job_dict)
     return None
+
+
+async def job_notifications() -> typing.AsyncGenerator[dict, None]:
+    channel = await RedisPool().subscribe_to_keys("jobs:*")
+    while True:
+        try:
+            # we cannot timeout here because we do not know when the next message is supposed to arrive
+            message: dict = await channel.get_message(ignore_subscribe_messages=True, timeout=None)  # type: ignore
+            if message is not None:
+                yield message
+        except GeneratorExit:
+            break
diff --git a/selfprivacy_api/utils/redis_model_storage.py b/selfprivacy_api/utils/redis_model_storage.py
index 06dfe8c..7d84210 100644
--- a/selfprivacy_api/utils/redis_model_storage.py
+++ b/selfprivacy_api/utils/redis_model_storage.py
@@ -1,15 +1,23 @@
+import uuid
+
 from datetime import datetime
 from typing import Optional
 from enum import Enum
 
 
 def store_model_as_hash(redis, redis_key, model):
-    for key, value in model.dict().items():
+    model_dict = model.dict()
+    for key, value in model_dict.items():
+        if isinstance(value, uuid.UUID):
+            value = str(value)
         if isinstance(value, datetime):
             value = value.isoformat()
         if isinstance(value, Enum):
             value = value.value
-        redis.hset(redis_key, key, str(value))
+        value = str(value)
+        model_dict[key] = value
+
+    redis.hset(redis_key, mapping=model_dict)
 
 
 def hash_as_model(redis, redis_key: str, model_class):
diff --git a/tests/test_redis.py b/tests/test_redis.py
index 70ef43a..02dfb21 100644
--- a/tests/test_redis.py
+++ b/tests/test_redis.py
@@ -7,6 +7,8 @@ from typing import List
 
 from selfprivacy_api.utils.redis_pool import RedisPool
 
+from selfprivacy_api.jobs import Jobs, job_notifications
+
 TEST_KEY = "test:test"
 STOPWORD = "STOP"
 
@@ -140,3 +142,77 @@ async def test_keyspace_notifications(empty_redis, event_loop):
             "pattern": f"__keyspace@0__:{TEST_KEY}",
             "type": "pmessage",
         }
+
+
+@pytest.mark.asyncio
+async def test_keyspace_notifications_patterns(empty_redis, event_loop):
+    pattern = "test*"
+    pubsub = await RedisPool().subscribe_to_keys(pattern)
+    async with pubsub:
+        future_message = asyncio.create_task(channel_reader_onemessage(pubsub))
+        empty_redis.set(TEST_KEY, "I am set!")
+        message = await future_message
+        assert message is not None
+        assert message["data"] is not None
+        assert message == {
+            "channel": f"__keyspace@0__:{TEST_KEY}",
+            "data": "set",
+            "pattern": f"__keyspace@0__:{pattern}",
+            "type": "pmessage",
+        }
+
+
+@pytest.mark.asyncio
+async def test_keyspace_notifications_jobs(empty_redis, event_loop):
+    pattern = "jobs:*"
+    pubsub = await RedisPool().subscribe_to_keys(pattern)
+    async with pubsub:
+        future_message = asyncio.create_task(channel_reader_onemessage(pubsub))
+        Jobs.add("testjob1", "test.test", "Testing aaaalll day")
+        message = await future_message
+        assert message is not None
+        assert message["data"] is not None
+        assert message["data"] == "hset"
+
+
+async def reader_of_jobs() -> List[dict]:
+    """
+    Reads 3 job updates and exits
+    """
+    result: List[dict] = []
+    async for message in job_notifications():
+        result.append(message)
+        if len(result) >= 3:
+            break
+    return result
+
+
+@pytest.mark.asyncio
+async def test_jobs_generator(empty_redis, event_loop):
+    # Will read exactly 3 job messages
+    future_messages = asyncio.create_task(reader_of_jobs())
+    await asyncio.sleep(1)
+
+    Jobs.add("testjob1", "test.test", "Testing aaaalll day")
+    Jobs.add("testjob2", "test.test", "Testing aaaalll day")
+    Jobs.add("testjob3", "test.test", "Testing aaaalll day")
+    Jobs.add("testjob4", "test.test", "Testing aaaalll day")
+
+    assert len(Jobs.get_jobs()) == 4
+    r = RedisPool().get_connection()
+    assert len(r.keys("jobs:*")) == 4
+
+    messages = await future_messages
+    assert len(messages) == 3
+    channels = [message["channel"] for message in messages]
+    operations = [message["data"] for message in messages]
+    assert set(operations) == set(["hset"])  # all of them are hsets
+
+    # Asserting that all of jobs emitted exactly one message
+    jobs = Jobs.get_jobs()
+    names = ["testjob1", "testjob2", "testjob3"]
+    ids = [str(job.uid) for job in jobs if job.name in names]
+    for id in ids:
+        assert id in " ".join(channels)
+    # Asserting that they came in order
+    assert "testjob4" not in " ".join(channels)

From 4b1becb4e22275ef61b9011698b4102038f270d1 Mon Sep 17 00:00:00 2001
From: Houkime <>
Date: Wed, 15 May 2024 11:29:20 +0000
Subject: [PATCH 08/86] feature(jobs): websocket connection

---
 selfprivacy_api/app.py               | 7 ++++++-
 tests/test_graphql/test_websocket.py | 6 ++++++
 2 files changed, 12 insertions(+), 1 deletion(-)
 create mode 100644 tests/test_graphql/test_websocket.py

diff --git a/selfprivacy_api/app.py b/selfprivacy_api/app.py
index 64ca85a..2f7e2f7 100644
--- a/selfprivacy_api/app.py
+++ b/selfprivacy_api/app.py
@@ -3,6 +3,7 @@
 from fastapi import FastAPI
 from fastapi.middleware.cors import CORSMiddleware
 from strawberry.fastapi import GraphQLRouter
+from strawberry.subscriptions import GRAPHQL_TRANSPORT_WS_PROTOCOL, GRAPHQL_WS_PROTOCOL
 
 import uvicorn
 
@@ -13,8 +14,12 @@ from selfprivacy_api.migrations import run_migrations
 
 app = FastAPI()
 
-graphql_app = GraphQLRouter(
+graphql_app: GraphQLRouter = GraphQLRouter(
     schema,
+    subscription_protocols=[
+        GRAPHQL_TRANSPORT_WS_PROTOCOL,
+        GRAPHQL_WS_PROTOCOL,
+    ],
 )
 
 app.add_middleware(
diff --git a/tests/test_graphql/test_websocket.py b/tests/test_graphql/test_websocket.py
new file mode 100644
index 0000000..fb2ac33
--- /dev/null
+++ b/tests/test_graphql/test_websocket.py
@@ -0,0 +1,6 @@
+
+def test_websocket_connection_bare(authorized_client):
+    client =authorized_client
+    with client.websocket_connect('/graphql', subprotocols=[ "graphql-transport-ws","graphql-ws"] ) as websocket:
+        assert websocket is not None
+        assert websocket.scope is not None

From 1fadf0214bf3edabedea83e1cb74f4c3d9f29bfb Mon Sep 17 00:00:00 2001
From: Houkime <>
Date: Wed, 15 May 2024 13:01:07 +0000
Subject: [PATCH 09/86] test(jobs): test Graphql job getting

---
 tests/common.py                 |  4 +++
 tests/test_graphql/test_jobs.py | 48 +++++++++++++++++++++++++++++++++
 2 files changed, 52 insertions(+)
 create mode 100644 tests/test_graphql/test_jobs.py

diff --git a/tests/common.py b/tests/common.py
index 5f69f3f..8c81f48 100644
--- a/tests/common.py
+++ b/tests/common.py
@@ -69,6 +69,10 @@ def generate_backup_query(query_array):
     return "query TestBackup {\n backup {" + "\n".join(query_array) + "}\n}"
 
 
+def generate_jobs_query(query_array):
+    return "query TestJobs {\n jobs {" + "\n".join(query_array) + "}\n}"
+
+
 def generate_service_query(query_array):
     return "query TestService {\n services {" + "\n".join(query_array) + "}\n}"
 
diff --git a/tests/test_graphql/test_jobs.py b/tests/test_graphql/test_jobs.py
new file mode 100644
index 0000000..8dfb102
--- /dev/null
+++ b/tests/test_graphql/test_jobs.py
@@ -0,0 +1,48 @@
+from tests.common import generate_jobs_query
+from tests.test_graphql.common import (
+    assert_ok,
+    assert_empty,
+    assert_errorcode,
+    get_data,
+)
+
+API_JOBS_QUERY = """
+getJobs {
+    uid
+    typeId
+    name
+    description
+    status
+    statusText
+    progress
+    createdAt
+    updatedAt
+    finishedAt
+    error
+    result
+}
+"""
+
+
+def graphql_send_query(client, query: str, variables: dict = {}):
+    return client.post("/graphql", json={"query": query, "variables": variables})
+
+
+def api_jobs(authorized_client):
+    response = graphql_send_query(
+        authorized_client, generate_jobs_query([API_JOBS_QUERY])
+    )
+    data = get_data(response)
+    result = data["jobs"]["getJobs"]
+    assert result is not None
+    return result
+
+
+def test_all_jobs_unauthorized(client):
+    response = graphql_send_query(client, generate_jobs_query([API_JOBS_QUERY]))
+    assert_empty(response)
+
+
+def test_all_jobs_when_none(authorized_client):
+    output = api_jobs(authorized_client)
+    assert output == []

From 4306c942311fb8ba1baba6197e09611298a55004 Mon Sep 17 00:00:00 2001
From: Houkime <>
Date: Wed, 15 May 2024 13:42:17 +0000
Subject: [PATCH 10/86] test(jobs) test API job format

---
 tests/test_graphql/test_jobs.py | 26 ++++++++++++++++++++++++++
 1 file changed, 26 insertions(+)

diff --git a/tests/test_graphql/test_jobs.py b/tests/test_graphql/test_jobs.py
index 8dfb102..68a6d20 100644
--- a/tests/test_graphql/test_jobs.py
+++ b/tests/test_graphql/test_jobs.py
@@ -1,4 +1,6 @@
 from tests.common import generate_jobs_query
+import tests.test_graphql.test_api_backup
+
 from tests.test_graphql.common import (
     assert_ok,
     assert_empty,
@@ -6,6 +8,8 @@ from tests.test_graphql.common import (
     get_data,
 )
 
+from selfprivacy_api.jobs import Jobs
+
 API_JOBS_QUERY = """
 getJobs {
     uid
@@ -46,3 +50,25 @@ def test_all_jobs_unauthorized(client):
 def test_all_jobs_when_none(authorized_client):
     output = api_jobs(authorized_client)
     assert output == []
+
+
+def test_all_jobs_when_some(authorized_client):
+    # We cannot make new jobs via API, at least directly
+    job = Jobs.add("bogus", "bogus.bogus", "fungus")
+    output = api_jobs(authorized_client)
+
+    len(output) == 1
+    api_job = output[0]
+
+    assert api_job["uid"] == str(job.uid)
+    assert api_job["typeId"] == job.type_id
+    assert api_job["name"] == job.name
+    assert api_job["description"] == job.description
+    assert api_job["status"] == job.status
+    assert api_job["statusText"] == job.status_text
+    assert api_job["progress"] == job.progress
+    assert api_job["createdAt"] == job.created_at.isoformat()
+    assert api_job["updatedAt"] == job.updated_at.isoformat()
+    assert api_job["finishedAt"] == None
+    assert api_job["error"] == None
+    assert api_job["result"] == None

From 098abd51499276bcb6aa7d2ff5be00bc6bc71c53 Mon Sep 17 00:00:00 2001
From: Houkime <>
Date: Wed, 15 May 2024 18:14:14 +0000
Subject: [PATCH 11/86] test(jobs): subscription query generating function

---
 tests/common.py | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/tests/common.py b/tests/common.py
index 8c81f48..3c05033 100644
--- a/tests/common.py
+++ b/tests/common.py
@@ -73,6 +73,10 @@ def generate_jobs_query(query_array):
     return "query TestJobs {\n jobs {" + "\n".join(query_array) + "}\n}"
 
 
+def generate_jobs_subscription(query_array):
+    return "subscription TestSubscription {\n jobs {" + "\n".join(query_array) + "}\n}"
+
+
 def generate_service_query(query_array):
     return "query TestService {\n services {" + "\n".join(query_array) + "}\n}"
 

From c19fa227c96f38b8d49ea4f671041f34e4dc2e49 Mon Sep 17 00:00:00 2001
From: Houkime <>
Date: Wed, 15 May 2024 18:15:16 +0000
Subject: [PATCH 12/86] test(websocket) test connection init

---
 tests/test_graphql/test_websocket.py | 48 ++++++++++++++++++++++++++--
 1 file changed, 46 insertions(+), 2 deletions(-)

diff --git a/tests/test_graphql/test_websocket.py b/tests/test_graphql/test_websocket.py
index fb2ac33..2431285 100644
--- a/tests/test_graphql/test_websocket.py
+++ b/tests/test_graphql/test_websocket.py
@@ -1,6 +1,50 @@
+from tests.common import generate_jobs_subscription
+from selfprivacy_api.graphql.queries.jobs import Job as _Job
+from selfprivacy_api.jobs import Jobs
+
+# JOBS_SUBSCRIPTION = """
+# jobUpdates {
+#     uid
+#     typeId
+#     name
+#     description
+#     status
+#     statusText
+#     progress
+#     createdAt
+#     updatedAt
+#     finishedAt
+#     error
+#     result
+# }
+# """
+
 
 def test_websocket_connection_bare(authorized_client):
-    client =authorized_client
-    with client.websocket_connect('/graphql', subprotocols=[ "graphql-transport-ws","graphql-ws"] ) as websocket:
+    client = authorized_client
+    with client.websocket_connect(
+        "/graphql", subprotocols=["graphql-transport-ws", "graphql-ws"]
+    ) as websocket:
         assert websocket is not None
         assert websocket.scope is not None
+
+
+def test_websocket_graphql_init(authorized_client):
+    client = authorized_client
+    with client.websocket_connect(
+        "/graphql", subprotocols=["graphql-transport-ws"]
+    ) as websocket:
+        websocket.send_json({"type": "connection_init", "payload": {}})
+        ack = websocket.receive_json()
+        assert ack == {"type": "connection_ack"}
+
+
+# def test_websocket_subscription(authorized_client):
+#     client = authorized_client
+#     with client.websocket_connect(
+#         "/graphql", subprotocols=["graphql-transport-ws", "graphql-ws"]
+#     ) as websocket:
+#         websocket.send(generate_jobs_subscription([JOBS_SUBSCRIPTION]))
+#         Jobs.add("bogus","bogus.bogus", "yyyaaaaayy")
+#         joblist = websocket.receive_json()
+#         raise NotImplementedError(joblist)

From 02d337c3f0a0fbe6def1ca97d2b744c64a4b1d9a Mon Sep 17 00:00:00 2001
From: Houkime <>
Date: Wed, 15 May 2024 18:31:16 +0000
Subject: [PATCH 13/86] test(websocket): ping pong test

---
 tests/test_graphql/test_websocket.py | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/tests/test_graphql/test_websocket.py b/tests/test_graphql/test_websocket.py
index 2431285..ef71312 100644
--- a/tests/test_graphql/test_websocket.py
+++ b/tests/test_graphql/test_websocket.py
@@ -29,7 +29,7 @@ def test_websocket_connection_bare(authorized_client):
         assert websocket.scope is not None
 
 
-def test_websocket_graphql_init(authorized_client):
+def test_websocket_graphql_ping(authorized_client):
     client = authorized_client
     with client.websocket_connect(
         "/graphql", subprotocols=["graphql-transport-ws"]
@@ -38,6 +38,11 @@ def test_websocket_graphql_init(authorized_client):
         ack = websocket.receive_json()
         assert ack == {"type": "connection_ack"}
 
+        # https://github.com/enisdenjo/graphql-ws/blob/master/PROTOCOL.md#ping
+        websocket.send_json({"type": "ping", "payload": {}})
+        pong = websocket.receive_json()
+        assert pong == {"type": "pong"}
+
 
 # def test_websocket_subscription(authorized_client):
 #     client = authorized_client

From 8348f11fafcc86280737666197acd1e26a9915dc Mon Sep 17 00:00:00 2001
From: Houkime <>
Date: Wed, 15 May 2024 18:36:17 +0000
Subject: [PATCH 14/86] test(websocket): separate ping and init

---
 tests/test_graphql/test_websocket.py | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/tests/test_graphql/test_websocket.py b/tests/test_graphql/test_websocket.py
index ef71312..d534269 100644
--- a/tests/test_graphql/test_websocket.py
+++ b/tests/test_graphql/test_websocket.py
@@ -29,7 +29,7 @@ def test_websocket_connection_bare(authorized_client):
         assert websocket.scope is not None
 
 
-def test_websocket_graphql_ping(authorized_client):
+def test_websocket_graphql_init(authorized_client):
     client = authorized_client
     with client.websocket_connect(
         "/graphql", subprotocols=["graphql-transport-ws"]
@@ -38,6 +38,12 @@ def test_websocket_graphql_ping(authorized_client):
         ack = websocket.receive_json()
         assert ack == {"type": "connection_ack"}
 
+
+def test_websocket_graphql_ping(authorized_client):
+    client = authorized_client
+    with client.websocket_connect(
+        "/graphql", subprotocols=["graphql-transport-ws"]
+    ) as websocket:
         # https://github.com/enisdenjo/graphql-ws/blob/master/PROTOCOL.md#ping
         websocket.send_json({"type": "ping", "payload": {}})
         pong = websocket.receive_json()

From 3b0600efb61bb0fcffce7d5059ef029a9a3926af Mon Sep 17 00:00:00 2001
From: Houkime <>
Date: Wed, 15 May 2024 20:41:36 +0000
Subject: [PATCH 15/86] feature(jobs): add subscription endpoint

---
 .../graphql/subscriptions/__init__.py         |  0
 selfprivacy_api/graphql/subscriptions/jobs.py | 20 +++++++++++++++++++
 2 files changed, 20 insertions(+)
 create mode 100644 selfprivacy_api/graphql/subscriptions/__init__.py
 create mode 100644 selfprivacy_api/graphql/subscriptions/jobs.py

diff --git a/selfprivacy_api/graphql/subscriptions/__init__.py b/selfprivacy_api/graphql/subscriptions/__init__.py
new file mode 100644
index 0000000..e69de29
diff --git a/selfprivacy_api/graphql/subscriptions/jobs.py b/selfprivacy_api/graphql/subscriptions/jobs.py
new file mode 100644
index 0000000..380badb
--- /dev/null
+++ b/selfprivacy_api/graphql/subscriptions/jobs.py
@@ -0,0 +1,20 @@
+# pylint: disable=too-few-public-methods
+import strawberry
+
+from typing import AsyncGenerator, List
+
+from selfprivacy_api.jobs import job_notifications
+
+from selfprivacy_api.graphql.common_types.jobs import ApiJob
+from selfprivacy_api.graphql.queries.jobs import get_all_jobs
+
+
+@strawberry.type
+class JobSubscriptions:
+    """Subscriptions related to jobs"""
+
+    @strawberry.subscription
+    async def job_updates(self) -> AsyncGenerator[List[ApiJob], None]:
+        # Send the complete list of jobs every time anything gets updated
+        async for notification in job_notifications():
+            yield get_all_jobs()

From 967e59271ff1f6242198e6e2add9468882164893 Mon Sep 17 00:00:00 2001
From: Houkime <>
Date: Wed, 15 May 2024 20:41:48 +0000
Subject: [PATCH 16/86] chore(jobs): shorter typehints and import sorting

---
 selfprivacy_api/graphql/queries/jobs.py | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/selfprivacy_api/graphql/queries/jobs.py b/selfprivacy_api/graphql/queries/jobs.py
index 337382a..3cc3bf7 100644
--- a/selfprivacy_api/graphql/queries/jobs.py
+++ b/selfprivacy_api/graphql/queries/jobs.py
@@ -1,17 +1,17 @@
 """Jobs status"""
 # pylint: disable=too-few-public-methods
-import typing
 import strawberry
+from typing import List, Optional
+
+from selfprivacy_api.jobs import Jobs
 from selfprivacy_api.graphql.common_types.jobs import (
     ApiJob,
     get_api_job_by_id,
     job_to_api_job,
 )
 
-from selfprivacy_api.jobs import Jobs
 
-
-def get_all_jobs() -> typing.List[ApiJob]:
+def get_all_jobs() -> List[ApiJob]:
     Jobs.get_jobs()
 
     return [job_to_api_job(job) for job in Jobs.get_jobs()]
@@ -20,9 +20,9 @@ def get_all_jobs() -> typing.List[ApiJob]:
 @strawberry.type
 class Job:
     @strawberry.field
-    def get_jobs(self) -> typing.List[ApiJob]:
+    def get_jobs(self) -> List[ApiJob]:
         return get_all_jobs()
 
     @strawberry.field
-    def get_job(self, job_id: str) -> typing.Optional[ApiJob]:
+    def get_job(self, job_id: str) -> Optional[ApiJob]:
         return get_api_job_by_id(job_id)

From 3910e416db564c248276527bd200cbb5f476cd79 Mon Sep 17 00:00:00 2001
From: Houkime <>
Date: Wed, 15 May 2024 20:43:17 +0000
Subject: [PATCH 17/86] test(jobs): test simple counting

---
 selfprivacy_api/graphql/schema.py    | 15 +++--
 tests/test_graphql/test_websocket.py | 92 +++++++++++++++++++++-------
 2 files changed, 81 insertions(+), 26 deletions(-)

diff --git a/selfprivacy_api/graphql/schema.py b/selfprivacy_api/graphql/schema.py
index e4e7264..078ee3d 100644
--- a/selfprivacy_api/graphql/schema.py
+++ b/selfprivacy_api/graphql/schema.py
@@ -28,6 +28,8 @@ from selfprivacy_api.graphql.queries.services import Services
 from selfprivacy_api.graphql.queries.storage import Storage
 from selfprivacy_api.graphql.queries.system import System
 
+from selfprivacy_api.graphql.subscriptions.jobs import JobSubscriptions
+
 from selfprivacy_api.graphql.mutations.users_mutations import UsersMutations
 from selfprivacy_api.graphql.queries.users import Users
 from selfprivacy_api.jobs.test import test_job
@@ -129,16 +131,19 @@ class Mutation(
             code=200,
         )
 
-    pass
-
 
 @strawberry.type
 class Subscription:
     """Root schema for subscriptions"""
 
-    @strawberry.subscription(permission_classes=[IsAuthenticated])
-    async def count(self, target: int = 100) -> AsyncGenerator[int, None]:
-        for i in range(target):
+    @strawberry.field(permission_classes=[IsAuthenticated])
+    def jobs(self) -> JobSubscriptions:
+        """Jobs subscriptions"""
+        return JobSubscriptions()
+
+    @strawberry.subscription
+    async def count(self) -> AsyncGenerator[int, None]:
+        for i in range(10):
             yield i
             await asyncio.sleep(0.5)
 
diff --git a/tests/test_graphql/test_websocket.py b/tests/test_graphql/test_websocket.py
index d534269..58681e0 100644
--- a/tests/test_graphql/test_websocket.py
+++ b/tests/test_graphql/test_websocket.py
@@ -2,22 +2,22 @@ from tests.common import generate_jobs_subscription
 from selfprivacy_api.graphql.queries.jobs import Job as _Job
 from selfprivacy_api.jobs import Jobs
 
-# JOBS_SUBSCRIPTION = """
-# jobUpdates {
-#     uid
-#     typeId
-#     name
-#     description
-#     status
-#     statusText
-#     progress
-#     createdAt
-#     updatedAt
-#     finishedAt
-#     error
-#     result
-# }
-# """
+JOBS_SUBSCRIPTION = """
+jobUpdates {
+    uid
+    typeId
+    name
+    description
+    status
+    statusText
+    progress
+    createdAt
+    updatedAt
+    finishedAt
+    error
+    result
+}
+"""
 
 
 def test_websocket_connection_bare(authorized_client):
@@ -50,12 +50,62 @@ def test_websocket_graphql_ping(authorized_client):
         assert pong == {"type": "pong"}
 
 
+def init_graphql(websocket):
+    websocket.send_json({"type": "connection_init", "payload": {}})
+    ack = websocket.receive_json()
+    assert ack == {"type": "connection_ack"}
+
+
+def test_websocket_subscription_minimal(authorized_client):
+    client = authorized_client
+    with client.websocket_connect(
+        "/graphql", subprotocols=["graphql-transport-ws"]
+    ) as websocket:
+        init_graphql(websocket)
+        websocket.send_json(
+            {
+                "id": "3aaa2445",
+                "type": "subscribe",
+                "payload": {
+                    "query": "subscription TestSubscription {count}",
+                },
+            }
+        )
+        response = websocket.receive_json()
+        assert response == {
+            "id": "3aaa2445",
+            "payload": {"data": {"count": 0}},
+            "type": "next",
+        }
+        response = websocket.receive_json()
+        assert response == {
+            "id": "3aaa2445",
+            "payload": {"data": {"count": 1}},
+            "type": "next",
+        }
+        response = websocket.receive_json()
+        assert response == {
+            "id": "3aaa2445",
+            "payload": {"data": {"count": 2}},
+            "type": "next",
+        }
+
+
 # def test_websocket_subscription(authorized_client):
 #     client = authorized_client
 #     with client.websocket_connect(
-#         "/graphql", subprotocols=["graphql-transport-ws", "graphql-ws"]
+#         "/graphql", subprotocols=["graphql-transport-ws"]
 #     ) as websocket:
-#         websocket.send(generate_jobs_subscription([JOBS_SUBSCRIPTION]))
-#         Jobs.add("bogus","bogus.bogus", "yyyaaaaayy")
-#         joblist = websocket.receive_json()
-#         raise NotImplementedError(joblist)
+#         init_graphql(websocket)
+#         websocket.send_json(
+#             {
+#                 "id": "3aaa2445",
+#                 "type": "subscribe",
+#                 "payload": {
+#                     "query": generate_jobs_subscription([JOBS_SUBSCRIPTION]),
+#                 },
+#             }
+#         )
+#         Jobs.add("bogus", "bogus.bogus", "yyyaaaaayy")
+#         response = websocket.receive_json()
+#         raise NotImplementedError(response)

From 6d2fdab07180f09b5e43ef04640a415b3c8e17a0 Mon Sep 17 00:00:00 2001
From: Houkime <>
Date: Wed, 22 May 2024 11:04:37 +0000
Subject: [PATCH 18/86] feature(jobs): UNSAFE endpoint to get job updates

---
 selfprivacy_api/graphql/queries/jobs.py       |  7 +-
 selfprivacy_api/graphql/schema.py             | 19 +++--
 selfprivacy_api/graphql/subscriptions/jobs.py | 14 +---
 tests/test_graphql/test_websocket.py          | 81 ++++++++++++++-----
 4 files changed, 83 insertions(+), 38 deletions(-)

diff --git a/selfprivacy_api/graphql/queries/jobs.py b/selfprivacy_api/graphql/queries/jobs.py
index 3cc3bf7..6a12838 100644
--- a/selfprivacy_api/graphql/queries/jobs.py
+++ b/selfprivacy_api/graphql/queries/jobs.py
@@ -12,9 +12,10 @@ from selfprivacy_api.graphql.common_types.jobs import (
 
 
 def get_all_jobs() -> List[ApiJob]:
-    Jobs.get_jobs()
-
-    return [job_to_api_job(job) for job in Jobs.get_jobs()]
+    jobs = Jobs.get_jobs()
+    api_jobs = [job_to_api_job(job) for job in jobs]
+    assert api_jobs is not None
+    return api_jobs
 
 
 @strawberry.type
diff --git a/selfprivacy_api/graphql/schema.py b/selfprivacy_api/graphql/schema.py
index 078ee3d..b8ed4e2 100644
--- a/selfprivacy_api/graphql/schema.py
+++ b/selfprivacy_api/graphql/schema.py
@@ -2,7 +2,7 @@
 # pylint: disable=too-few-public-methods
 
 import asyncio
-from typing import AsyncGenerator
+from typing import AsyncGenerator, List
 import strawberry
 from selfprivacy_api.graphql import IsAuthenticated
 from selfprivacy_api.graphql.mutations.deprecated_mutations import (
@@ -28,7 +28,9 @@ from selfprivacy_api.graphql.queries.services import Services
 from selfprivacy_api.graphql.queries.storage import Storage
 from selfprivacy_api.graphql.queries.system import System
 
-from selfprivacy_api.graphql.subscriptions.jobs import JobSubscriptions
+from selfprivacy_api.graphql.subscriptions.jobs import ApiJob
+from selfprivacy_api.jobs import job_notifications
+from selfprivacy_api.graphql.queries.jobs import get_all_jobs
 
 from selfprivacy_api.graphql.mutations.users_mutations import UsersMutations
 from selfprivacy_api.graphql.queries.users import Users
@@ -136,10 +138,15 @@ class Mutation(
 class Subscription:
     """Root schema for subscriptions"""
 
-    @strawberry.field(permission_classes=[IsAuthenticated])
-    def jobs(self) -> JobSubscriptions:
-        """Jobs subscriptions"""
-        return JobSubscriptions()
+    @strawberry.subscription
+    async def job_updates(self) -> AsyncGenerator[List[ApiJob], None]:
+        # Send the complete list of jobs every time anything gets updated
+        async for notification in job_notifications():
+            yield get_all_jobs()
+
+    # @strawberry.subscription
+    # async def job_updates(self) -> AsyncGenerator[List[ApiJob], None]:
+    #     return job_updates()
 
     @strawberry.subscription
     async def count(self) -> AsyncGenerator[int, None]:
diff --git a/selfprivacy_api/graphql/subscriptions/jobs.py b/selfprivacy_api/graphql/subscriptions/jobs.py
index 380badb..11d6263 100644
--- a/selfprivacy_api/graphql/subscriptions/jobs.py
+++ b/selfprivacy_api/graphql/subscriptions/jobs.py
@@ -1,5 +1,4 @@
 # pylint: disable=too-few-public-methods
-import strawberry
 
 from typing import AsyncGenerator, List
 
@@ -9,12 +8,7 @@ from selfprivacy_api.graphql.common_types.jobs import ApiJob
 from selfprivacy_api.graphql.queries.jobs import get_all_jobs
 
 
-@strawberry.type
-class JobSubscriptions:
-    """Subscriptions related to jobs"""
-
-    @strawberry.subscription
-    async def job_updates(self) -> AsyncGenerator[List[ApiJob], None]:
-        # Send the complete list of jobs every time anything gets updated
-        async for notification in job_notifications():
-            yield get_all_jobs()
+async def job_updates() -> AsyncGenerator[List[ApiJob], None]:
+    # Send the complete list of jobs every time anything gets updated
+    async for notification in job_notifications():
+        yield get_all_jobs()
diff --git a/tests/test_graphql/test_websocket.py b/tests/test_graphql/test_websocket.py
index 58681e0..ee33262 100644
--- a/tests/test_graphql/test_websocket.py
+++ b/tests/test_graphql/test_websocket.py
@@ -1,6 +1,13 @@
 from tests.common import generate_jobs_subscription
-from selfprivacy_api.graphql.queries.jobs import Job as _Job
+
+# from selfprivacy_api.graphql.subscriptions.jobs import JobSubscriptions
+import pytest
+import asyncio
+
 from selfprivacy_api.jobs import Jobs
+from time import sleep
+
+from tests.test_redis import empty_redis
 
 JOBS_SUBSCRIPTION = """
 jobUpdates {
@@ -91,21 +98,57 @@ def test_websocket_subscription_minimal(authorized_client):
         }
 
 
-# def test_websocket_subscription(authorized_client):
-#     client = authorized_client
-#     with client.websocket_connect(
-#         "/graphql", subprotocols=["graphql-transport-ws"]
-#     ) as websocket:
-#         init_graphql(websocket)
-#         websocket.send_json(
-#             {
-#                 "id": "3aaa2445",
-#                 "type": "subscribe",
-#                 "payload": {
-#                     "query": generate_jobs_subscription([JOBS_SUBSCRIPTION]),
-#                 },
-#             }
-#         )
-#         Jobs.add("bogus", "bogus.bogus", "yyyaaaaayy")
-#         response = websocket.receive_json()
-#         raise NotImplementedError(response)
+async def read_one_job(websocket):
+    # bug? We only get them starting from the second job update
+    # that's why we receive two jobs in the list them
+    # the first update gets lost somewhere
+    response = websocket.receive_json()
+    return response
+
+
+@pytest.mark.asyncio
+async def test_websocket_subscription(authorized_client, empty_redis, event_loop):
+    client = authorized_client
+    with client.websocket_connect(
+        "/graphql", subprotocols=["graphql-transport-ws"]
+    ) as websocket:
+        init_graphql(websocket)
+        websocket.send_json(
+            {
+                "id": "3aaa2445",
+                "type": "subscribe",
+                "payload": {
+                    "query": "subscription TestSubscription {"
+                    + JOBS_SUBSCRIPTION
+                    + "}",
+                },
+            }
+        )
+        future = asyncio.create_task(read_one_job(websocket))
+        jobs = []
+        jobs.append(Jobs.add("bogus", "bogus.bogus", "yyyaaaaayy it works"))
+        sleep(0.5)
+        jobs.append(Jobs.add("bogus2", "bogus.bogus", "yyyaaaaayy it works"))
+
+        response = await future
+        data = response["payload"]["data"]
+        jobs_received = data["jobUpdates"]
+        received_names = [job["name"] for job in jobs_received]
+        for job in jobs:
+            assert job.name in received_names
+
+        for job in jobs:
+            for api_job in jobs_received:
+                if (job.name) == api_job["name"]:
+                    assert api_job["uid"] == str(job.uid)
+                    assert api_job["typeId"] == job.type_id
+                    assert api_job["name"] == job.name
+                    assert api_job["description"] == job.description
+                    assert api_job["status"] == job.status
+                    assert api_job["statusText"] == job.status_text
+                    assert api_job["progress"] == job.progress
+                    assert api_job["createdAt"] == job.created_at.isoformat()
+                    assert api_job["updatedAt"] == job.updated_at.isoformat()
+                    assert api_job["finishedAt"] == None
+                    assert api_job["error"] == None
+                    assert api_job["result"] == None

From 39f584ad5c64f0423bef9a6fe1a0d6928dd34547 Mon Sep 17 00:00:00 2001
From: Houkime <>
Date: Mon, 27 May 2024 18:22:20 +0000
Subject: [PATCH 19/86] test(devices): provide devices for a service test to
 fix conditional test fail.

---
 tests/test_graphql/test_services.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tests/test_graphql/test_services.py b/tests/test_graphql/test_services.py
index 6e8dcf6..b7faf3d 100644
--- a/tests/test_graphql/test_services.py
+++ b/tests/test_graphql/test_services.py
@@ -543,8 +543,8 @@ def test_disable_enable(authorized_client, only_dummy_service):
     assert api_dummy_service["status"] == ServiceStatus.ACTIVE.value
 
 
-def test_move_immovable(authorized_client, only_dummy_service):
-    dummy_service = only_dummy_service
+def test_move_immovable(authorized_client, dummy_service_with_binds):
+    dummy_service = dummy_service_with_binds
     dummy_service.set_movable(False)
     root = BlockDevices().get_root_block_device()
     mutation_response = api_move(authorized_client, dummy_service, root.name)

From 8fd12a1775a969be56bd0f3c310c90e0df57e09b Mon Sep 17 00:00:00 2001
From: Houkime <>
Date: Mon, 27 May 2024 20:21:11 +0000
Subject: [PATCH 20/86] feature(websocket): add auth

---
 selfprivacy_api/graphql/schema.py    |  18 ++-
 tests/test_graphql/test_websocket.py | 169 ++++++++++++++++++---------
 2 files changed, 131 insertions(+), 56 deletions(-)

diff --git a/selfprivacy_api/graphql/schema.py b/selfprivacy_api/graphql/schema.py
index b8ed4e2..c6cf46b 100644
--- a/selfprivacy_api/graphql/schema.py
+++ b/selfprivacy_api/graphql/schema.py
@@ -4,6 +4,7 @@
 import asyncio
 from typing import AsyncGenerator, List
 import strawberry
+
 from selfprivacy_api.graphql import IsAuthenticated
 from selfprivacy_api.graphql.mutations.deprecated_mutations import (
     DeprecatedApiMutations,
@@ -134,12 +135,25 @@ class Mutation(
         )
 
 
+# A cruft for Websockets
+def authenticated(info) -> bool:
+    return IsAuthenticated().has_permission(source=None, info=info)
+
+
 @strawberry.type
 class Subscription:
-    """Root schema for subscriptions"""
+    """Root schema for subscriptions.
+    Every field here should be an AsyncIterator or AsyncGenerator
+    It is not a part of the spec but graphql-core (dep of strawberryql)
+    demands it while the spec is vague in this area."""
 
     @strawberry.subscription
-    async def job_updates(self) -> AsyncGenerator[List[ApiJob], None]:
+    async def job_updates(
+        self, info: strawberry.types.Info
+    ) -> AsyncGenerator[List[ApiJob], None]:
+        if not authenticated(info):
+            raise Exception(IsAuthenticated().message)
+
         # Send the complete list of jobs every time anything gets updated
         async for notification in job_notifications():
             yield get_all_jobs()
diff --git a/tests/test_graphql/test_websocket.py b/tests/test_graphql/test_websocket.py
index ee33262..5a92416 100644
--- a/tests/test_graphql/test_websocket.py
+++ b/tests/test_graphql/test_websocket.py
@@ -1,13 +1,20 @@
-from tests.common import generate_jobs_subscription
-
 # from selfprivacy_api.graphql.subscriptions.jobs import JobSubscriptions
 import pytest
 import asyncio
-
-from selfprivacy_api.jobs import Jobs
+from typing import Generator
 from time import sleep
 
-from tests.test_redis import empty_redis
+from starlette.testclient import WebSocketTestSession
+
+from selfprivacy_api.jobs import Jobs
+from selfprivacy_api.actions.api_tokens import TOKEN_REPO
+from selfprivacy_api.graphql import IsAuthenticated
+
+from tests.conftest import DEVICE_WE_AUTH_TESTS_WITH
+from tests.test_jobs import jobs as empty_jobs
+
+# We do not iterate through them yet
+TESTED_SUBPROTOCOLS = ["graphql-transport-ws"]
 
 JOBS_SUBSCRIPTION = """
 jobUpdates {
@@ -27,6 +34,48 @@ jobUpdates {
 """
 
 
+def connect_ws_authenticated(authorized_client) -> WebSocketTestSession:
+    token = "Bearer " + str(DEVICE_WE_AUTH_TESTS_WITH["token"])
+    return authorized_client.websocket_connect(
+        "/graphql",
+        subprotocols=TESTED_SUBPROTOCOLS,
+        params={"token": token},
+    )
+
+
+def connect_ws_not_authenticated(client) -> WebSocketTestSession:
+    return client.websocket_connect(
+        "/graphql",
+        subprotocols=TESTED_SUBPROTOCOLS,
+        params={"token": "I like vegan icecream but it is not a valid token"},
+    )
+
+
+def init_graphql(websocket):
+    websocket.send_json({"type": "connection_init", "payload": {}})
+    ack = websocket.receive_json()
+    assert ack == {"type": "connection_ack"}
+
+
+@pytest.fixture
+def authenticated_websocket(
+    authorized_client,
+) -> Generator[WebSocketTestSession, None, None]:
+    # We use authorized_client only tohave token in the repo, this client by itself is not enough to authorize websocket
+
+    ValueError(TOKEN_REPO.get_tokens())
+    with connect_ws_authenticated(authorized_client) as websocket:
+        yield websocket
+        sleep(1)
+
+
+@pytest.fixture
+def unauthenticated_websocket(client) -> Generator[WebSocketTestSession, None, None]:
+    with connect_ws_not_authenticated(client) as websocket:
+        yield websocket
+        sleep(1)
+
+
 def test_websocket_connection_bare(authorized_client):
     client = authorized_client
     with client.websocket_connect(
@@ -57,12 +106,6 @@ def test_websocket_graphql_ping(authorized_client):
         assert pong == {"type": "pong"}
 
 
-def init_graphql(websocket):
-    websocket.send_json({"type": "connection_init", "payload": {}})
-    ack = websocket.receive_json()
-    assert ack == {"type": "connection_ack"}
-
-
 def test_websocket_subscription_minimal(authorized_client):
     client = authorized_client
     with client.websocket_connect(
@@ -107,48 +150,66 @@ async def read_one_job(websocket):
 
 
 @pytest.mark.asyncio
-async def test_websocket_subscription(authorized_client, empty_redis, event_loop):
-    client = authorized_client
-    with client.websocket_connect(
-        "/graphql", subprotocols=["graphql-transport-ws"]
-    ) as websocket:
-        init_graphql(websocket)
-        websocket.send_json(
-            {
-                "id": "3aaa2445",
-                "type": "subscribe",
-                "payload": {
-                    "query": "subscription TestSubscription {"
-                    + JOBS_SUBSCRIPTION
-                    + "}",
-                },
-            }
-        )
-        future = asyncio.create_task(read_one_job(websocket))
-        jobs = []
-        jobs.append(Jobs.add("bogus", "bogus.bogus", "yyyaaaaayy it works"))
-        sleep(0.5)
-        jobs.append(Jobs.add("bogus2", "bogus.bogus", "yyyaaaaayy it works"))
+async def test_websocket_subscription(authenticated_websocket, event_loop, empty_jobs):
+    websocket = authenticated_websocket
+    init_graphql(websocket)
+    websocket.send_json(
+        {
+            "id": "3aaa2445",
+            "type": "subscribe",
+            "payload": {
+                "query": "subscription TestSubscription {" + JOBS_SUBSCRIPTION + "}",
+            },
+        }
+    )
+    future = asyncio.create_task(read_one_job(websocket))
+    jobs = []
+    jobs.append(Jobs.add("bogus", "bogus.bogus", "yyyaaaaayy it works"))
+    sleep(0.5)
+    jobs.append(Jobs.add("bogus2", "bogus.bogus", "yyyaaaaayy it works"))
 
-        response = await future
-        data = response["payload"]["data"]
-        jobs_received = data["jobUpdates"]
-        received_names = [job["name"] for job in jobs_received]
-        for job in jobs:
-            assert job.name in received_names
+    response = await future
+    data = response["payload"]["data"]
+    jobs_received = data["jobUpdates"]
+    received_names = [job["name"] for job in jobs_received]
+    for job in jobs:
+        assert job.name in received_names
 
-        for job in jobs:
-            for api_job in jobs_received:
-                if (job.name) == api_job["name"]:
-                    assert api_job["uid"] == str(job.uid)
-                    assert api_job["typeId"] == job.type_id
-                    assert api_job["name"] == job.name
-                    assert api_job["description"] == job.description
-                    assert api_job["status"] == job.status
-                    assert api_job["statusText"] == job.status_text
-                    assert api_job["progress"] == job.progress
-                    assert api_job["createdAt"] == job.created_at.isoformat()
-                    assert api_job["updatedAt"] == job.updated_at.isoformat()
-                    assert api_job["finishedAt"] == None
-                    assert api_job["error"] == None
-                    assert api_job["result"] == None
+    assert len(jobs_received) == 2
+
+    for job in jobs:
+        for api_job in jobs_received:
+            if (job.name) == api_job["name"]:
+                assert api_job["uid"] == str(job.uid)
+                assert api_job["typeId"] == job.type_id
+                assert api_job["name"] == job.name
+                assert api_job["description"] == job.description
+                assert api_job["status"] == job.status
+                assert api_job["statusText"] == job.status_text
+                assert api_job["progress"] == job.progress
+                assert api_job["createdAt"] == job.created_at.isoformat()
+                assert api_job["updatedAt"] == job.updated_at.isoformat()
+                assert api_job["finishedAt"] == None
+                assert api_job["error"] == None
+                assert api_job["result"] == None
+
+
+def test_websocket_subscription_unauthorized(unauthenticated_websocket):
+    websocket = unauthenticated_websocket
+    init_graphql(websocket)
+    websocket.send_json(
+        {
+            "id": "3aaa2445",
+            "type": "subscribe",
+            "payload": {
+                "query": "subscription TestSubscription {" + JOBS_SUBSCRIPTION + "}",
+            },
+        }
+    )
+
+    response = websocket.receive_json()
+    assert response == {
+        "id": "3aaa2445",
+        "payload": [{"message": IsAuthenticated.message}],
+        "type": "error",
+    }

From 950093a3b1bc46eddcc768031c51d052e1f8e3a0 Mon Sep 17 00:00:00 2001
From: Houkime <>
Date: Mon, 27 May 2024 20:38:51 +0000
Subject: [PATCH 21/86] feature(websocket): add auth to counter too

---
 selfprivacy_api/graphql/schema.py    | 17 +++----
 tests/test_graphql/test_websocket.py | 69 +++++++++++++++-------------
 2 files changed, 47 insertions(+), 39 deletions(-)

diff --git a/selfprivacy_api/graphql/schema.py b/selfprivacy_api/graphql/schema.py
index c6cf46b..3280396 100644
--- a/selfprivacy_api/graphql/schema.py
+++ b/selfprivacy_api/graphql/schema.py
@@ -136,10 +136,15 @@ class Mutation(
 
 
 # A cruft for Websockets
-def authenticated(info) -> bool:
+def authenticated(info: strawberry.types.Info) -> bool:
     return IsAuthenticated().has_permission(source=None, info=info)
 
 
+def reject_if_unauthenticated(info: strawberry.types.Info):
+    if not authenticated(info):
+        raise Exception(IsAuthenticated().message)
+
+
 @strawberry.type
 class Subscription:
     """Root schema for subscriptions.
@@ -151,19 +156,15 @@ class Subscription:
     async def job_updates(
         self, info: strawberry.types.Info
     ) -> AsyncGenerator[List[ApiJob], None]:
-        if not authenticated(info):
-            raise Exception(IsAuthenticated().message)
+        reject_if_unauthenticated(info)
 
         # Send the complete list of jobs every time anything gets updated
         async for notification in job_notifications():
             yield get_all_jobs()
 
-    # @strawberry.subscription
-    # async def job_updates(self) -> AsyncGenerator[List[ApiJob], None]:
-    #     return job_updates()
-
     @strawberry.subscription
-    async def count(self) -> AsyncGenerator[int, None]:
+    async def count(self, info: strawberry.types.Info) -> AsyncGenerator[int, None]:
+        reject_if_unauthenticated(info)
         for i in range(10):
             yield i
             await asyncio.sleep(0.5)
diff --git a/tests/test_graphql/test_websocket.py b/tests/test_graphql/test_websocket.py
index 5a92416..49cc944 100644
--- a/tests/test_graphql/test_websocket.py
+++ b/tests/test_graphql/test_websocket.py
@@ -106,41 +106,61 @@ def test_websocket_graphql_ping(authorized_client):
         assert pong == {"type": "pong"}
 
 
+def api_subscribe(websocket, id, subscription):
+    websocket.send_json(
+        {
+            "id": id,
+            "type": "subscribe",
+            "payload": {
+                "query": "subscription TestSubscription {" + subscription + "}",
+            },
+        }
+    )
+
+
 def test_websocket_subscription_minimal(authorized_client):
+    # Test a small endpoint that exists specifically for tests
     client = authorized_client
     with client.websocket_connect(
         "/graphql", subprotocols=["graphql-transport-ws"]
     ) as websocket:
         init_graphql(websocket)
-        websocket.send_json(
-            {
-                "id": "3aaa2445",
-                "type": "subscribe",
-                "payload": {
-                    "query": "subscription TestSubscription {count}",
-                },
-            }
-        )
+        arbitrary_id = "3aaa2445"
+        api_subscribe(websocket, arbitrary_id, "count")
         response = websocket.receive_json()
         assert response == {
-            "id": "3aaa2445",
+            "id": arbitrary_id,
             "payload": {"data": {"count": 0}},
             "type": "next",
         }
         response = websocket.receive_json()
         assert response == {
-            "id": "3aaa2445",
+            "id": arbitrary_id,
             "payload": {"data": {"count": 1}},
             "type": "next",
         }
         response = websocket.receive_json()
         assert response == {
-            "id": "3aaa2445",
+            "id": arbitrary_id,
             "payload": {"data": {"count": 2}},
             "type": "next",
         }
 
 
+def test_websocket_subscription_minimal_unauthorized(unauthenticated_websocket):
+    websocket = unauthenticated_websocket
+    init_graphql(websocket)
+    arbitrary_id = "3aaa2445"
+    api_subscribe(websocket, arbitrary_id, "count")
+
+    response = websocket.receive_json()
+    assert response == {
+        "id": arbitrary_id,
+        "payload": [{"message": IsAuthenticated.message}],
+        "type": "error",
+    }
+
+
 async def read_one_job(websocket):
     # bug? We only get them starting from the second job update
     # that's why we receive two jobs in the list them
@@ -153,15 +173,9 @@ async def read_one_job(websocket):
 async def test_websocket_subscription(authenticated_websocket, event_loop, empty_jobs):
     websocket = authenticated_websocket
     init_graphql(websocket)
-    websocket.send_json(
-        {
-            "id": "3aaa2445",
-            "type": "subscribe",
-            "payload": {
-                "query": "subscription TestSubscription {" + JOBS_SUBSCRIPTION + "}",
-            },
-        }
-    )
+    arbitrary_id = "3aaa2445"
+    api_subscribe(websocket, arbitrary_id, JOBS_SUBSCRIPTION)
+
     future = asyncio.create_task(read_one_job(websocket))
     jobs = []
     jobs.append(Jobs.add("bogus", "bogus.bogus", "yyyaaaaayy it works"))
@@ -197,19 +211,12 @@ async def test_websocket_subscription(authenticated_websocket, event_loop, empty
 def test_websocket_subscription_unauthorized(unauthenticated_websocket):
     websocket = unauthenticated_websocket
     init_graphql(websocket)
-    websocket.send_json(
-        {
-            "id": "3aaa2445",
-            "type": "subscribe",
-            "payload": {
-                "query": "subscription TestSubscription {" + JOBS_SUBSCRIPTION + "}",
-            },
-        }
-    )
+    id = "3aaa2445"
+    api_subscribe(websocket, id, JOBS_SUBSCRIPTION)
 
     response = websocket.receive_json()
     assert response == {
-        "id": "3aaa2445",
+        "id": id,
         "payload": [{"message": IsAuthenticated.message}],
         "type": "error",
     }

From f772005b1703c47c90f3b20dbdaecb541dc7cc07 Mon Sep 17 00:00:00 2001
From: Houkime <>
Date: Mon, 27 May 2024 21:13:57 +0000
Subject: [PATCH 22/86] refactor(jobs): offload job subscription logic to a
 separate file

---
 selfprivacy_api/graphql/schema.py    | 11 +++++------
 tests/test_graphql/test_websocket.py | 16 ++++++++++++----
 2 files changed, 17 insertions(+), 10 deletions(-)

diff --git a/selfprivacy_api/graphql/schema.py b/selfprivacy_api/graphql/schema.py
index 3280396..05e6bf9 100644
--- a/selfprivacy_api/graphql/schema.py
+++ b/selfprivacy_api/graphql/schema.py
@@ -30,8 +30,9 @@ from selfprivacy_api.graphql.queries.storage import Storage
 from selfprivacy_api.graphql.queries.system import System
 
 from selfprivacy_api.graphql.subscriptions.jobs import ApiJob
-from selfprivacy_api.jobs import job_notifications
-from selfprivacy_api.graphql.queries.jobs import get_all_jobs
+from selfprivacy_api.graphql.subscriptions.jobs import (
+    job_updates as job_update_generator,
+)
 
 from selfprivacy_api.graphql.mutations.users_mutations import UsersMutations
 from selfprivacy_api.graphql.queries.users import Users
@@ -157,12 +158,10 @@ class Subscription:
         self, info: strawberry.types.Info
     ) -> AsyncGenerator[List[ApiJob], None]:
         reject_if_unauthenticated(info)
-
-        # Send the complete list of jobs every time anything gets updated
-        async for notification in job_notifications():
-            yield get_all_jobs()
+        return job_update_generator()
 
     @strawberry.subscription
+    # Used for testing, consider deletion to shrink attack surface
     async def count(self, info: strawberry.types.Info) -> AsyncGenerator[int, None]:
         reject_if_unauthenticated(info)
         for i in range(10):
diff --git a/tests/test_graphql/test_websocket.py b/tests/test_graphql/test_websocket.py
index 49cc944..d538ca1 100644
--- a/tests/test_graphql/test_websocket.py
+++ b/tests/test_graphql/test_websocket.py
@@ -162,9 +162,9 @@ def test_websocket_subscription_minimal_unauthorized(unauthenticated_websocket):
 
 
 async def read_one_job(websocket):
-    # bug? We only get them starting from the second job update
-    # that's why we receive two jobs in the list them
-    # the first update gets lost somewhere
+    # Bug? We only get them starting from the second job update
+    # That's why we receive two jobs in the list them
+    # The first update gets lost somewhere
     response = websocket.receive_json()
     return response
 
@@ -215,8 +215,16 @@ def test_websocket_subscription_unauthorized(unauthenticated_websocket):
     api_subscribe(websocket, id, JOBS_SUBSCRIPTION)
 
     response = websocket.receive_json()
+    # I do not really know why strawberry gives more info on this
+    # One versus the counter
+    payload = response["payload"][0]
+    assert isinstance(payload, dict)
+    assert "locations" in payload.keys()
+    # It looks like this 'locations': [{'column': 32, 'line': 1}]
+    # We cannot test locations feasibly
+    del payload["locations"]
     assert response == {
         "id": id,
-        "payload": [{"message": IsAuthenticated.message}],
+        "payload": [{"message": IsAuthenticated.message, "path": ["jobUpdates"]}],
         "type": "error",
     }

From 17ae1621562f59733960b4a2a0c20cd1f82af47d Mon Sep 17 00:00:00 2001
From: Houkime <>
Date: Mon, 27 May 2024 21:15:47 +0000
Subject: [PATCH 23/86] test(websocket): remove excessive sleeping

---
 tests/test_graphql/test_websocket.py | 2 --
 1 file changed, 2 deletions(-)

diff --git a/tests/test_graphql/test_websocket.py b/tests/test_graphql/test_websocket.py
index d538ca1..27cfd55 100644
--- a/tests/test_graphql/test_websocket.py
+++ b/tests/test_graphql/test_websocket.py
@@ -66,14 +66,12 @@ def authenticated_websocket(
     ValueError(TOKEN_REPO.get_tokens())
     with connect_ws_authenticated(authorized_client) as websocket:
         yield websocket
-        sleep(1)
 
 
 @pytest.fixture
 def unauthenticated_websocket(client) -> Generator[WebSocketTestSession, None, None]:
     with connect_ws_not_authenticated(client) as websocket:
         yield websocket
-        sleep(1)
 
 
 def test_websocket_connection_bare(authorized_client):

From cb2a1421bf25f759e9fcea596efb2125a699d30e Mon Sep 17 00:00:00 2001
From: Houkime <>
Date: Mon, 27 May 2024 21:28:29 +0000
Subject: [PATCH 24/86] test(websocket): remove some duplication

---
 tests/test_graphql/test_websocket.py | 75 +++++++++++++---------------
 1 file changed, 36 insertions(+), 39 deletions(-)

diff --git a/tests/test_graphql/test_websocket.py b/tests/test_graphql/test_websocket.py
index 27cfd55..754fbbf 100644
--- a/tests/test_graphql/test_websocket.py
+++ b/tests/test_graphql/test_websocket.py
@@ -34,6 +34,18 @@ jobUpdates {
 """
 
 
+def api_subscribe(websocket, id, subscription):
+    websocket.send_json(
+        {
+            "id": id,
+            "type": "subscribe",
+            "payload": {
+                "query": "subscription TestSubscription {" + subscription + "}",
+            },
+        }
+    )
+
+
 def connect_ws_authenticated(authorized_client) -> WebSocketTestSession:
     token = "Bearer " + str(DEVICE_WE_AUTH_TESTS_WITH["token"])
     return authorized_client.websocket_connect(
@@ -61,7 +73,7 @@ def init_graphql(websocket):
 def authenticated_websocket(
     authorized_client,
 ) -> Generator[WebSocketTestSession, None, None]:
-    # We use authorized_client only tohave token in the repo, this client by itself is not enough to authorize websocket
+    # We use authorized_client only to have token in the repo, this client by itself is not enough to authorize websocket
 
     ValueError(TOKEN_REPO.get_tokens())
     with connect_ws_authenticated(authorized_client) as websocket:
@@ -104,45 +116,30 @@ def test_websocket_graphql_ping(authorized_client):
         assert pong == {"type": "pong"}
 
 
-def api_subscribe(websocket, id, subscription):
-    websocket.send_json(
-        {
-            "id": id,
-            "type": "subscribe",
-            "payload": {
-                "query": "subscription TestSubscription {" + subscription + "}",
-            },
-        }
-    )
-
-
-def test_websocket_subscription_minimal(authorized_client):
+def test_websocket_subscription_minimal(authorized_client, authenticated_websocket):
     # Test a small endpoint that exists specifically for tests
-    client = authorized_client
-    with client.websocket_connect(
-        "/graphql", subprotocols=["graphql-transport-ws"]
-    ) as websocket:
-        init_graphql(websocket)
-        arbitrary_id = "3aaa2445"
-        api_subscribe(websocket, arbitrary_id, "count")
-        response = websocket.receive_json()
-        assert response == {
-            "id": arbitrary_id,
-            "payload": {"data": {"count": 0}},
-            "type": "next",
-        }
-        response = websocket.receive_json()
-        assert response == {
-            "id": arbitrary_id,
-            "payload": {"data": {"count": 1}},
-            "type": "next",
-        }
-        response = websocket.receive_json()
-        assert response == {
-            "id": arbitrary_id,
-            "payload": {"data": {"count": 2}},
-            "type": "next",
-        }
+    websocket = authenticated_websocket
+    init_graphql(websocket)
+    arbitrary_id = "3aaa2445"
+    api_subscribe(websocket, arbitrary_id, "count")
+    response = websocket.receive_json()
+    assert response == {
+        "id": arbitrary_id,
+        "payload": {"data": {"count": 0}},
+        "type": "next",
+    }
+    response = websocket.receive_json()
+    assert response == {
+        "id": arbitrary_id,
+        "payload": {"data": {"count": 1}},
+        "type": "next",
+    }
+    response = websocket.receive_json()
+    assert response == {
+        "id": arbitrary_id,
+        "payload": {"data": {"count": 2}},
+        "type": "next",
+    }
 
 
 def test_websocket_subscription_minimal_unauthorized(unauthenticated_websocket):

From fc2ac0fe6d0e4a834922bdf818328b9c46882302 Mon Sep 17 00:00:00 2001
From: nhnn <nhnn@disroot.org>
Date: Mon, 27 May 2024 12:59:43 +0300
Subject: [PATCH 25/86] feat: graphql endpoint to fetch system logs from
 journald

---
 default.nix                             |   1 +
 selfprivacy_api/graphql/queries/logs.py | 123 ++++++++++++++++++++++
 selfprivacy_api/graphql/schema.py       |   6 ++
 tests/common.py                         |   4 +
 tests/test_graphql/test_api_logs.py     | 133 ++++++++++++++++++++++++
 5 files changed, 267 insertions(+)
 create mode 100644 selfprivacy_api/graphql/queries/logs.py
 create mode 100644 tests/test_graphql/test_api_logs.py

diff --git a/default.nix b/default.nix
index e7e6fcf..8d47c4d 100644
--- a/default.nix
+++ b/default.nix
@@ -14,6 +14,7 @@ pythonPackages.buildPythonPackage rec {
     pydantic
     pytz
     redis
+    systemd
     setuptools
     strawberry-graphql
     typing-extensions
diff --git a/selfprivacy_api/graphql/queries/logs.py b/selfprivacy_api/graphql/queries/logs.py
new file mode 100644
index 0000000..c16c950
--- /dev/null
+++ b/selfprivacy_api/graphql/queries/logs.py
@@ -0,0 +1,123 @@
+"""System logs"""
+from datetime import datetime
+import os
+import typing
+import strawberry
+from systemd import journal
+
+
+def get_events_from_journal(
+    j: journal.Reader, limit: int, next: typing.Callable[[journal.Reader], typing.Dict]
+):
+    events = []
+    i = 0
+    while i < limit:
+        entry = next(j)
+        if entry == None or entry == dict():
+            break
+        if entry["MESSAGE"] != "":
+            events.append(LogEntry(entry))
+            i += 1
+
+    return events
+
+
+@strawberry.type
+class LogEntry:
+    message: str = strawberry.field()
+    timestamp: datetime = strawberry.field()
+    priority: int = strawberry.field()
+    systemd_unit: typing.Optional[str] = strawberry.field()
+    systemd_slice: typing.Optional[str] = strawberry.field()
+
+    def __init__(self, journal_entry: typing.Dict):
+        self.entry = journal_entry
+        self.message = journal_entry["MESSAGE"]
+        self.timestamp = journal_entry["__REALTIME_TIMESTAMP"]
+        self.priority = journal_entry["PRIORITY"]
+        self.systemd_unit = journal_entry.get("_SYSTEMD_UNIT")
+        self.systemd_slice = journal_entry.get("_SYSTEMD_SLICE")
+
+    @strawberry.field()
+    def cursor(self) -> str:
+        return self.entry["__CURSOR"]
+
+
+@strawberry.type
+class PageMeta:
+    up_cursor: typing.Optional[str] = strawberry.field()
+    down_cursor: typing.Optional[str] = strawberry.field()
+
+    def __init__(
+        self, up_cursor: typing.Optional[str], down_cursor: typing.Optional[str]
+    ):
+        self.up_cursor = up_cursor
+        self.down_cursor = down_cursor
+
+
+@strawberry.type
+class PaginatedEntries:
+    page_meta: PageMeta = strawberry.field(description="Metadata to aid in pagination.")
+    entries: typing.List[LogEntry] = strawberry.field(
+        description="The list of log entries."
+    )
+
+    def __init__(self, meta: PageMeta, entries: typing.List[LogEntry]):
+        self.page_meta = meta
+        self.entries = entries
+
+    @staticmethod
+    def from_entries(entries: typing.List[LogEntry]):
+        if entries == []:
+            return PaginatedEntries(PageMeta(None, None), [])
+
+        return PaginatedEntries(
+            PageMeta(
+                entries[0].cursor(),
+                entries[-1].cursor(),
+            ),
+            entries,
+        )
+
+
+@strawberry.type
+class Logs:
+    @strawberry.field()
+    def paginated(
+        self,
+        limit: int = 20,
+        up_cursor: str
+        | None = None,  # All entries returned will be lesser than this cursor. Sets upper bound on results.
+        down_cursor: str
+        | None = None,  # All entries returned will be greater than this cursor. Sets lower bound on results.
+    ) -> PaginatedEntries:
+        if limit > 50:
+            raise Exception("You can't fetch more than 50 entries via single request.")
+        j = journal.Reader()
+
+        if up_cursor == None and down_cursor == None:
+            j.seek_tail()
+
+            events = get_events_from_journal(j, limit, lambda j: j.get_previous())
+            events.reverse()
+
+            return PaginatedEntries.from_entries(events)
+        elif up_cursor == None and down_cursor != None:
+            j.seek_cursor(down_cursor)
+            j.get_previous()  # pagination is exclusive
+
+            events = get_events_from_journal(j, limit, lambda j: j.get_previous())
+            events.reverse()
+
+            return PaginatedEntries.from_entries(events)
+        elif up_cursor != None and down_cursor == None:
+            j.seek_cursor(up_cursor)
+            j.get_next()  # pagination is exclusive
+
+            events = get_events_from_journal(j, limit, lambda j: j.get_next())
+
+            return PaginatedEntries.from_entries(events)
+        else:
+            raise NotImplemented(
+                "Pagination by both up_cursor and down_cursor is not implemented"
+            )
diff --git a/selfprivacy_api/graphql/schema.py b/selfprivacy_api/graphql/schema.py
index 05e6bf9..c65e233 100644
--- a/selfprivacy_api/graphql/schema.py
+++ b/selfprivacy_api/graphql/schema.py
@@ -25,6 +25,7 @@ from selfprivacy_api.graphql.mutations.backup_mutations import BackupMutations
 from selfprivacy_api.graphql.queries.api_queries import Api
 from selfprivacy_api.graphql.queries.backup import Backup
 from selfprivacy_api.graphql.queries.jobs import Job
+from selfprivacy_api.graphql.queries.logs import Logs
 from selfprivacy_api.graphql.queries.services import Services
 from selfprivacy_api.graphql.queries.storage import Storage
 from selfprivacy_api.graphql.queries.system import System
@@ -53,6 +54,11 @@ class Query:
         """System queries"""
         return System()
 
+    @strawberry.field(permission_classes=[IsAuthenticated])
+    def logs(self) -> Logs:
+        """Log queries"""
+        return Logs()
+
     @strawberry.field(permission_classes=[IsAuthenticated])
     def users(self) -> Users:
         """Users queries"""
diff --git a/tests/common.py b/tests/common.py
index 3c05033..1de3893 100644
--- a/tests/common.py
+++ b/tests/common.py
@@ -81,6 +81,10 @@ def generate_service_query(query_array):
     return "query TestService {\n services {" + "\n".join(query_array) + "}\n}"
 
 
+def generate_logs_query(query_array):
+    return "query TestService {\n logs {" + "\n".join(query_array) + "}\n}"
+
+
 def mnemonic_to_hex(mnemonic):
     return Mnemonic(language="english").to_entropy(mnemonic).hex()
 
diff --git a/tests/test_graphql/test_api_logs.py b/tests/test_graphql/test_api_logs.py
new file mode 100644
index 0000000..587b6c1
--- /dev/null
+++ b/tests/test_graphql/test_api_logs.py
@@ -0,0 +1,133 @@
+from datetime import datetime
+from systemd import journal
+
+
+def assert_log_entry_equals_to_journal_entry(api_entry, journal_entry):
+    assert api_entry["message"] == journal_entry["MESSAGE"]
+    assert (
+        datetime.fromisoformat(api_entry["timestamp"])
+        == journal_entry["__REALTIME_TIMESTAMP"]
+    )
+    assert api_entry["priority"] == journal_entry["PRIORITY"]
+    assert api_entry.get("systemdUnit") == journal_entry.get("_SYSTEMD_UNIT")
+    assert api_entry.get("systemdSlice") == journal_entry.get("_SYSTEMD_SLICE")
+
+
+def take_from_journal(j, limit, next):
+    entries = []
+    for _ in range(0, limit):
+        entry = next(j)
+        if entry["MESSAGE"] != "":
+            entries.append(entry)
+    return entries
+
+
+API_GET_LOGS_WITH_UP_BORDER = """
+query TestQuery($upCursor: String) {
+    logs {
+        paginated(limit: 4, upCursor: $upCursor) {
+            pageMeta {
+                upCursor
+                downCursor
+            }
+            entries {
+                message
+                timestamp
+                priority
+                systemdUnit
+                systemdSlice
+            }
+        }
+    }
+}
+"""
+
+API_GET_LOGS_WITH_DOWN_BORDER = """
+query TestQuery($downCursor: String) {
+    logs {
+        paginated(limit: 4, downCursor: $downCursor) {
+            pageMeta {
+                upCursor
+                downCursor
+            }
+            entries {
+                message
+                timestamp
+                priority
+                systemdUnit
+                systemdSlice
+            }
+        }
+    }
+}
+"""
+
+
+def test_graphql_get_logs_with_up_border(authorized_client):
+    j = journal.Reader()
+    j.seek_tail()
+
+    # < - cursor
+    # <- - log entry will be returned by API call.
+    # ...
+    # log <
+    # log <-
+    # log <-
+    # log <-
+    # log <-
+    # log
+
+    expected_entries = take_from_journal(j, 6, lambda j: j.get_previous())
+    expected_entries.reverse()
+
+    response = authorized_client.post(
+        "/graphql",
+        json={
+            "query": API_GET_LOGS_WITH_UP_BORDER,
+            "variables": {"upCursor": expected_entries[0]["__CURSOR"]},
+        },
+    )
+    assert response.status_code == 200
+
+    expected_entries = expected_entries[1:-1]
+    returned_entries = response.json()["data"]["logs"]["paginated"]["entries"]
+
+    assert len(returned_entries) == len(expected_entries)
+
+    for api_entry, journal_entry in zip(returned_entries, expected_entries):
+        assert_log_entry_equals_to_journal_entry(api_entry, journal_entry)
+
+
+def test_graphql_get_logs_with_down_border(authorized_client):
+    j = journal.Reader()
+    j.seek_head()
+    j.get_next()
+
+    # < - cursor
+    # <- - log entry will be returned by API call.
+    # log
+    # log <-
+    # log <-
+    # log <-
+    # log <-
+    # log <
+    # ...
+
+    expected_entries = take_from_journal(j, 5, lambda j: j.get_next())
+
+    response = authorized_client.post(
+        "/graphql",
+        json={
+            "query": API_GET_LOGS_WITH_DOWN_BORDER,
+            "variables": {"downCursor": expected_entries[-1]["__CURSOR"]},
+        },
+    )
+    assert response.status_code == 200
+
+    expected_entries = expected_entries[:-1]
+    returned_entries = response.json()["data"]["logs"]["paginated"]["entries"]
+
+    assert len(returned_entries) == len(expected_entries)
+
+    for api_entry, journal_entry in zip(returned_entries, expected_entries):
+        assert_log_entry_equals_to_journal_entry(api_entry, journal_entry)

From 3d2c79ecb1065d1db78fb1be0990175ca8b80743 Mon Sep 17 00:00:00 2001
From: nhnn <nhnn@disroot.org>
Date: Thu, 30 May 2024 10:05:36 +0300
Subject: [PATCH 26/86] feat: streaming of journald entries via graphql
 subscription

---
 default.nix                                   |  1 +
 selfprivacy_api/graphql/queries/logs.py       |  4 +--
 selfprivacy_api/graphql/schema.py             |  8 ++++-
 selfprivacy_api/graphql/subscriptions/logs.py | 31 ++++++++++++++++
 tests/test_graphql/test_api_logs.py           | 36 ++++++++++++++++++-
 5 files changed, 76 insertions(+), 4 deletions(-)
 create mode 100644 selfprivacy_api/graphql/subscriptions/logs.py

diff --git a/default.nix b/default.nix
index 8d47c4d..1af935e 100644
--- a/default.nix
+++ b/default.nix
@@ -19,6 +19,7 @@ pythonPackages.buildPythonPackage rec {
     strawberry-graphql
     typing-extensions
     uvicorn
+    websockets
   ];
   pythonImportsCheck = [ "selfprivacy_api" ];
   doCheck = false;
diff --git a/selfprivacy_api/graphql/queries/logs.py b/selfprivacy_api/graphql/queries/logs.py
index c16c950..b9e4af2 100644
--- a/selfprivacy_api/graphql/queries/logs.py
+++ b/selfprivacy_api/graphql/queries/logs.py
@@ -26,7 +26,7 @@ def get_events_from_journal(
 class LogEntry:
     message: str = strawberry.field()
     timestamp: datetime = strawberry.field()
-    priority: int = strawberry.field()
+    priority: typing.Optional[int] = strawberry.field()
     systemd_unit: typing.Optional[str] = strawberry.field()
     systemd_slice: typing.Optional[str] = strawberry.field()
 
@@ -34,7 +34,7 @@ class LogEntry:
         self.entry = journal_entry
         self.message = journal_entry["MESSAGE"]
         self.timestamp = journal_entry["__REALTIME_TIMESTAMP"]
-        self.priority = journal_entry["PRIORITY"]
+        self.priority = journal_entry.get("PRIORITY")
         self.systemd_unit = journal_entry.get("_SYSTEMD_UNIT")
         self.systemd_slice = journal_entry.get("_SYSTEMD_SLICE")
 
diff --git a/selfprivacy_api/graphql/schema.py b/selfprivacy_api/graphql/schema.py
index c65e233..f0d5a11 100644
--- a/selfprivacy_api/graphql/schema.py
+++ b/selfprivacy_api/graphql/schema.py
@@ -25,7 +25,7 @@ from selfprivacy_api.graphql.mutations.backup_mutations import BackupMutations
 from selfprivacy_api.graphql.queries.api_queries import Api
 from selfprivacy_api.graphql.queries.backup import Backup
 from selfprivacy_api.graphql.queries.jobs import Job
-from selfprivacy_api.graphql.queries.logs import Logs
+from selfprivacy_api.graphql.queries.logs import LogEntry, Logs
 from selfprivacy_api.graphql.queries.services import Services
 from selfprivacy_api.graphql.queries.storage import Storage
 from selfprivacy_api.graphql.queries.system import System
@@ -34,6 +34,7 @@ from selfprivacy_api.graphql.subscriptions.jobs import ApiJob
 from selfprivacy_api.graphql.subscriptions.jobs import (
     job_updates as job_update_generator,
 )
+from selfprivacy_api.graphql.subscriptions.logs import log_stream
 
 from selfprivacy_api.graphql.mutations.users_mutations import UsersMutations
 from selfprivacy_api.graphql.queries.users import Users
@@ -174,6 +175,11 @@ class Subscription:
             yield i
             await asyncio.sleep(0.5)
 
+    @strawberry.subscription
+    async def log_entries(self, info: strawberry.types.Info) -> AsyncGenerator[LogEntry, None]:
+        reject_if_unauthenticated(info)
+        return log_stream()
+
 
 schema = strawberry.Schema(
     query=Query,
diff --git a/selfprivacy_api/graphql/subscriptions/logs.py b/selfprivacy_api/graphql/subscriptions/logs.py
new file mode 100644
index 0000000..be8a004
--- /dev/null
+++ b/selfprivacy_api/graphql/subscriptions/logs.py
@@ -0,0 +1,31 @@
+from typing import AsyncGenerator, List
+from systemd import journal
+import asyncio
+
+from selfprivacy_api.graphql.queries.logs import LogEntry
+
+
+async def log_stream() -> AsyncGenerator[LogEntry, None]:
+    j = journal.Reader()
+
+    j.seek_tail()
+    j.get_previous()
+
+    queue = asyncio.Queue()
+
+    async def callback():
+        if j.process() != journal.APPEND:
+            return
+        for entry in j:
+            await queue.put(entry)
+
+    asyncio.get_event_loop().add_reader(j, lambda: asyncio.ensure_future(callback()))
+
+    while True:
+        entry = await queue.get()
+        try:
+            yield LogEntry(entry)
+        except:
+            asyncio.get_event_loop().remove_reader(j)
+            return
+        queue.task_done()
diff --git a/tests/test_graphql/test_api_logs.py b/tests/test_graphql/test_api_logs.py
index 587b6c1..18f4d32 100644
--- a/tests/test_graphql/test_api_logs.py
+++ b/tests/test_graphql/test_api_logs.py
@@ -1,6 +1,8 @@
 from datetime import datetime
 from systemd import journal
 
+from tests.test_graphql.test_websocket import init_graphql
+
 
 def assert_log_entry_equals_to_journal_entry(api_entry, journal_entry):
     assert api_entry["message"] == journal_entry["MESSAGE"]
@@ -8,7 +10,7 @@ def assert_log_entry_equals_to_journal_entry(api_entry, journal_entry):
         datetime.fromisoformat(api_entry["timestamp"])
         == journal_entry["__REALTIME_TIMESTAMP"]
     )
-    assert api_entry["priority"] == journal_entry["PRIORITY"]
+    assert api_entry.get("priority") == journal_entry.get("PRIORITY")
     assert api_entry.get("systemdUnit") == journal_entry.get("_SYSTEMD_UNIT")
     assert api_entry.get("systemdSlice") == journal_entry.get("_SYSTEMD_SLICE")
 
@@ -131,3 +133,35 @@ def test_graphql_get_logs_with_down_border(authorized_client):
 
     for api_entry, journal_entry in zip(returned_entries, expected_entries):
         assert_log_entry_equals_to_journal_entry(api_entry, journal_entry)
+
+
+def test_websocket_subscription_for_logs(authorized_client):
+    with authorized_client.websocket_connect(
+        "/graphql", subprotocols=["graphql-transport-ws"]
+    ) as websocket:
+        init_graphql(websocket)
+        websocket.send_json(
+            {
+                "id": "3aaa2445",
+                "type": "subscribe",
+                "payload": {
+                    "query": "subscription TestSubscription { logEntries { message } }",
+                },
+            }
+        )
+
+        def read_until(message, limit=5):
+            i = 0
+            while i < limit:
+                msg = websocket.receive_json()["payload"]["data"]["logEntries"][
+                    "message"
+                ]
+                if msg == message:
+                    return
+                else:
+                    continue
+            raise Exception("Failed to read websocket data, timeout")
+
+        for i in range(0, 10):
+            journal.send(f"Lorem ipsum number {i}")
+            read_until(f"Lorem ipsum number {i}")

From 8b2e4666dd28f53151a47467f5cccd564927d2b0 Mon Sep 17 00:00:00 2001
From: nhnn <nhnn@disroot.org>
Date: Tue, 11 Jun 2024 12:36:42 +0300
Subject: [PATCH 27/86] fix: rename PageMeta to LogsPageMeta

---
 selfprivacy_api/graphql/queries/logs.py | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/selfprivacy_api/graphql/queries/logs.py b/selfprivacy_api/graphql/queries/logs.py
index b9e4af2..52204cf 100644
--- a/selfprivacy_api/graphql/queries/logs.py
+++ b/selfprivacy_api/graphql/queries/logs.py
@@ -44,7 +44,7 @@ class LogEntry:
 
 
 @strawberry.type
-class PageMeta:
+class LogsPageMeta:
     up_cursor: typing.Optional[str] = strawberry.field()
     down_cursor: typing.Optional[str] = strawberry.field()
 
@@ -57,22 +57,22 @@ class PageMeta:
 
 @strawberry.type
 class PaginatedEntries:
-    page_meta: PageMeta = strawberry.field(description="Metadata to aid in pagination.")
+    page_meta: LogsPageMeta = strawberry.field(description="Metadata to aid in pagination.")
     entries: typing.List[LogEntry] = strawberry.field(
         description="The list of log entries."
     )
 
-    def __init__(self, meta: PageMeta, entries: typing.List[LogEntry]):
+    def __init__(self, meta: LogsPageMeta, entries: typing.List[LogEntry]):
         self.page_meta = meta
         self.entries = entries
 
     @staticmethod
     def from_entries(entries: typing.List[LogEntry]):
         if entries == []:
-            return PaginatedEntries(PageMeta(None, None), [])
+            return PaginatedEntries(LogsPageMeta(None, None), [])
 
         return PaginatedEntries(
-            PageMeta(
+            LogsPageMeta(
                 entries[0].cursor(),
                 entries[-1].cursor(),
             ),

From f90eb3fb4c917c1c3ac1faf9905b64b0b5ad4b8a Mon Sep 17 00:00:00 2001
From: dettlaff <dettlaff@riseup.net>
Date: Fri, 21 Jun 2024 23:35:04 +0300
Subject: [PATCH 28/86] feat: add flake services manager (#113)

Reviewed-on: https://git.selfprivacy.org/SelfPrivacy/selfprivacy-rest-api/pulls/113
Reviewed-by: Inex Code <inex.code@selfprivacy.org>
Reviewed-by: houkime <houkime@protonmail.com>
Co-authored-by: dettlaff <dettlaff@riseup.net>
Co-committed-by: dettlaff <dettlaff@riseup.net>
---
 .../services/flake_service_manager.py         |  53 ++++++++
 tests/test_flake_services_manager.py          | 127 ++++++++++++++++++
 .../no_services.nix                           |   4 +
 .../some_services.nix                         |  12 ++
 4 files changed, 196 insertions(+)
 create mode 100644 selfprivacy_api/services/flake_service_manager.py
 create mode 100644 tests/test_flake_services_manager.py
 create mode 100644 tests/test_flake_services_manager/no_services.nix
 create mode 100644 tests/test_flake_services_manager/some_services.nix

diff --git a/selfprivacy_api/services/flake_service_manager.py b/selfprivacy_api/services/flake_service_manager.py
new file mode 100644
index 0000000..63c2279
--- /dev/null
+++ b/selfprivacy_api/services/flake_service_manager.py
@@ -0,0 +1,53 @@
+import re
+from typing import Tuple, Optional
+
+FLAKE_CONFIG_PATH = "/etc/nixos/sp-modules/flake.nix"
+
+
+class FlakeServiceManager:
+    def __enter__(self) -> "FlakeServiceManager":
+        self.services = {}
+
+        with open(FLAKE_CONFIG_PATH, "r") as file:
+            for line in file:
+                service_name, url = self._extract_services(input_string=line)
+                if service_name and url:
+                    self.services[service_name] = url
+
+        return self
+
+    def _extract_services(
+        self, input_string: str
+    ) -> Tuple[Optional[str], Optional[str]]:
+        pattern = r"inputs\.([\w-]+)\.url\s*=\s*([\S]+);"
+        match = re.search(pattern, input_string)
+
+        if match:
+            variable_name = match.group(1)
+            url = match.group(2)
+            return variable_name, url
+        else:
+            return None, None
+
+    def __exit__(self, exc_type, exc_value, traceback) -> None:
+        with open(FLAKE_CONFIG_PATH, "w") as file:
+            file.write(
+                """
+{
+    description = "SelfPrivacy NixOS PoC modules/extensions/bundles/packages/etc";\n
+"""
+            )
+
+            for key, value in self.services.items():
+                file.write(
+                    f"""
+    inputs.{key}.url = {value};
+"""
+                )
+
+            file.write(
+                """
+    outputs = _: { };
+}
+"""
+            )
diff --git a/tests/test_flake_services_manager.py b/tests/test_flake_services_manager.py
new file mode 100644
index 0000000..4650b6d
--- /dev/null
+++ b/tests/test_flake_services_manager.py
@@ -0,0 +1,127 @@
+import pytest
+
+from selfprivacy_api.services.flake_service_manager import FlakeServiceManager
+
+all_services_file = """
+{
+    description = "SelfPrivacy NixOS PoC modules/extensions/bundles/packages/etc";
+
+
+    inputs.bitwarden.url = git+https://git.selfprivacy.org/SelfPrivacy/selfprivacy-nixos-config.git?ref=flakes&dir=sp-modules/bitwarden;
+
+    inputs.gitea.url = git+https://git.selfprivacy.org/SelfPrivacy/selfprivacy-nixos-config.git?ref=flakes&dir=sp-modules/gitea;
+
+    inputs.jitsi-meet.url = git+https://git.selfprivacy.org/SelfPrivacy/selfprivacy-nixos-config.git?ref=flakes&dir=sp-modules/jitsi-meet;
+
+    inputs.nextcloud.url = git+https://git.selfprivacy.org/SelfPrivacy/selfprivacy-nixos-config.git?ref=flakes&dir=sp-modules/nextcloud;
+
+    inputs.ocserv.url = git+https://git.selfprivacy.org/SelfPrivacy/selfprivacy-nixos-config.git?ref=flakes&dir=sp-modules/ocserv;
+
+    inputs.pleroma.url = git+https://git.selfprivacy.org/SelfPrivacy/selfprivacy-nixos-config.git?ref=flakes&dir=sp-modules/pleroma;
+
+    inputs.simple-nixos-mailserver.url = git+https://git.selfprivacy.org/SelfPrivacy/selfprivacy-nixos-config.git?ref=flakes&dir=sp-modules/simple-nixos-mailserver;
+
+    outputs = _: { };
+}
+"""
+
+
+some_services_file = """
+{
+    description = "SelfPrivacy NixOS PoC modules/extensions/bundles/packages/etc";
+
+
+    inputs.bitwarden.url = git+https://git.selfprivacy.org/SelfPrivacy/selfprivacy-nixos-config.git?ref=flakes&dir=sp-modules/bitwarden;
+
+    inputs.gitea.url = git+https://git.selfprivacy.org/SelfPrivacy/selfprivacy-nixos-config.git?ref=flakes&dir=sp-modules/gitea;
+
+    inputs.jitsi-meet.url = git+https://git.selfprivacy.org/SelfPrivacy/selfprivacy-nixos-config.git?ref=flakes&dir=sp-modules/jitsi-meet;
+
+    outputs = _: { };
+}
+"""
+
+
+@pytest.fixture
+def some_services_flake_mock(mocker, datadir):
+    flake_config_path = datadir / "some_services.nix"
+    mocker.patch(
+        "selfprivacy_api.services.flake_service_manager.FLAKE_CONFIG_PATH",
+        new=flake_config_path,
+    )
+    return flake_config_path
+
+
+@pytest.fixture
+def no_services_flake_mock(mocker, datadir):
+    flake_config_path = datadir / "no_services.nix"
+    mocker.patch(
+        "selfprivacy_api.services.flake_service_manager.FLAKE_CONFIG_PATH",
+        new=flake_config_path,
+    )
+    return flake_config_path
+
+
+# ---
+
+
+def test_read_services_list(some_services_flake_mock):
+    with FlakeServiceManager() as manager:
+        services = {
+            "bitwarden": "git+https://git.selfprivacy.org/SelfPrivacy/selfprivacy-nixos-config.git?ref=flakes&dir=sp-modules/bitwarden",
+            "gitea": "git+https://git.selfprivacy.org/SelfPrivacy/selfprivacy-nixos-config.git?ref=flakes&dir=sp-modules/gitea",
+            "jitsi-meet": "git+https://git.selfprivacy.org/SelfPrivacy/selfprivacy-nixos-config.git?ref=flakes&dir=sp-modules/jitsi-meet",
+        }
+        assert manager.services == services
+
+
+def test_change_services_list(some_services_flake_mock):
+    services = {
+        "bitwarden": "git+https://git.selfprivacy.org/SelfPrivacy/selfprivacy-nixos-config.git?ref=flakes&dir=sp-modules/bitwarden",
+        "gitea": "git+https://git.selfprivacy.org/SelfPrivacy/selfprivacy-nixos-config.git?ref=flakes&dir=sp-modules/gitea",
+        "jitsi-meet": "git+https://git.selfprivacy.org/SelfPrivacy/selfprivacy-nixos-config.git?ref=flakes&dir=sp-modules/jitsi-meet",
+        "nextcloud": "git+https://git.selfprivacy.org/SelfPrivacy/selfprivacy-nixos-config.git?ref=flakes&dir=sp-modules/nextcloud",
+        "ocserv": "git+https://git.selfprivacy.org/SelfPrivacy/selfprivacy-nixos-config.git?ref=flakes&dir=sp-modules/ocserv",
+        "pleroma": "git+https://git.selfprivacy.org/SelfPrivacy/selfprivacy-nixos-config.git?ref=flakes&dir=sp-modules/pleroma",
+        "simple-nixos-mailserver": "git+https://git.selfprivacy.org/SelfPrivacy/selfprivacy-nixos-config.git?ref=flakes&dir=sp-modules/simple-nixos-mailserver",
+    }
+
+    with FlakeServiceManager() as manager:
+        manager.services = services
+
+    with FlakeServiceManager() as manager:
+        assert manager.services == services
+
+    with open(some_services_flake_mock, "r", encoding="utf-8") as file:
+        file_content = file.read().strip()
+
+    assert all_services_file.strip() == file_content
+
+
+def test_read_empty_services_list(no_services_flake_mock):
+    with FlakeServiceManager() as manager:
+        services = {}
+        assert manager.services == services
+
+
+def test_change_empty_services_list(no_services_flake_mock):
+    services = {
+        "bitwarden": "git+https://git.selfprivacy.org/SelfPrivacy/selfprivacy-nixos-config.git?ref=flakes&dir=sp-modules/bitwarden",
+        "gitea": "git+https://git.selfprivacy.org/SelfPrivacy/selfprivacy-nixos-config.git?ref=flakes&dir=sp-modules/gitea",
+        "jitsi-meet": "git+https://git.selfprivacy.org/SelfPrivacy/selfprivacy-nixos-config.git?ref=flakes&dir=sp-modules/jitsi-meet",
+        "nextcloud": "git+https://git.selfprivacy.org/SelfPrivacy/selfprivacy-nixos-config.git?ref=flakes&dir=sp-modules/nextcloud",
+        "ocserv": "git+https://git.selfprivacy.org/SelfPrivacy/selfprivacy-nixos-config.git?ref=flakes&dir=sp-modules/ocserv",
+        "pleroma": "git+https://git.selfprivacy.org/SelfPrivacy/selfprivacy-nixos-config.git?ref=flakes&dir=sp-modules/pleroma",
+        "simple-nixos-mailserver": "git+https://git.selfprivacy.org/SelfPrivacy/selfprivacy-nixos-config.git?ref=flakes&dir=sp-modules/simple-nixos-mailserver",
+    }
+
+    with FlakeServiceManager() as manager:
+        manager.services = services
+
+    with FlakeServiceManager() as manager:
+        assert manager.services == services
+
+    with open(no_services_flake_mock, "r", encoding="utf-8") as file:
+        file_content = file.read().strip()
+
+    assert all_services_file.strip() == file_content
diff --git a/tests/test_flake_services_manager/no_services.nix b/tests/test_flake_services_manager/no_services.nix
new file mode 100644
index 0000000..5967016
--- /dev/null
+++ b/tests/test_flake_services_manager/no_services.nix
@@ -0,0 +1,4 @@
+{
+    description = "SelfPrivacy NixOS PoC modules/extensions/bundles/packages/etc";
+    outputs = _: { };
+}
diff --git a/tests/test_flake_services_manager/some_services.nix b/tests/test_flake_services_manager/some_services.nix
new file mode 100644
index 0000000..4bbb919
--- /dev/null
+++ b/tests/test_flake_services_manager/some_services.nix
@@ -0,0 +1,12 @@
+{
+    description = "SelfPrivacy NixOS PoC modules/extensions/bundles/packages/etc";
+
+
+    inputs.bitwarden.url = git+https://git.selfprivacy.org/SelfPrivacy/selfprivacy-nixos-config.git?ref=flakes&dir=sp-modules/bitwarden;
+
+    inputs.gitea.url = git+https://git.selfprivacy.org/SelfPrivacy/selfprivacy-nixos-config.git?ref=flakes&dir=sp-modules/gitea;
+
+    inputs.jitsi-meet.url = git+https://git.selfprivacy.org/SelfPrivacy/selfprivacy-nixos-config.git?ref=flakes&dir=sp-modules/jitsi-meet;
+
+    outputs = _: { };
+}

From 5602c960565ffe23a771f8f9fa221fb8ed04303d Mon Sep 17 00:00:00 2001
From: Maxim Leshchenko <cnmaks90@gmail.com>
Date: Thu, 27 Jun 2024 17:41:46 +0300
Subject: [PATCH 29/86] feat(services): rename "sda1" to "system disk" and etc
 (#122)

Closes #51

Reviewed-on: https://git.selfprivacy.org/SelfPrivacy/selfprivacy-rest-api/pulls/122
Reviewed-by: Inex Code <inex.code@selfprivacy.org>
Co-authored-by: Maxim Leshchenko <cnmaks90@gmail.com>
Co-committed-by: Maxim Leshchenko <cnmaks90@gmail.com>
---
 selfprivacy_api/actions/services.py    | 2 +-
 selfprivacy_api/utils/block_devices.py | 8 ++++++++
 2 files changed, 9 insertions(+), 1 deletion(-)

diff --git a/selfprivacy_api/actions/services.py b/selfprivacy_api/actions/services.py
index ebb0917..f9486d1 100644
--- a/selfprivacy_api/actions/services.py
+++ b/selfprivacy_api/actions/services.py
@@ -27,7 +27,7 @@ def move_service(service_id: str, volume_name: str) -> Job:
     job = Jobs.add(
         type_id=f"services.{service.get_id()}.move",
         name=f"Move {service.get_display_name()}",
-        description=f"Moving {service.get_display_name()} data to {volume.name}",
+        description=f"Moving {service.get_display_name()} data to {volume.get_display_name().lower()}",
     )
 
     move_service_task(service, volume, job)
diff --git a/selfprivacy_api/utils/block_devices.py b/selfprivacy_api/utils/block_devices.py
index 4de5b75..0db8fe0 100644
--- a/selfprivacy_api/utils/block_devices.py
+++ b/selfprivacy_api/utils/block_devices.py
@@ -90,6 +90,14 @@ class BlockDevice:
     def __hash__(self):
         return hash(self.name)
 
+    def get_display_name(self) -> str:
+        if self.is_root():
+            return "System disk"
+        elif self.model == "Volume":
+            return "Expandable volume"
+        else:
+            return self.name
+
     def is_root(self) -> bool:
         """
         Return True if the block device is the root device.

From 4823491e3eaf1ba77e38610aeeb467f9035dfe87 Mon Sep 17 00:00:00 2001
From: dettlaff <dettlaff@riseup.net>
Date: Thu, 6 Jun 2024 23:14:07 +0400
Subject: [PATCH 30/86] feat: add roundcube service

---
 selfprivacy_api/services/__init__.py          |   2 +
 .../services/roundcube/__init__.py            | 104 ++++++++++++++++++
 selfprivacy_api/services/roundcube/icon.py    |  16 +++
 3 files changed, 122 insertions(+)
 create mode 100644 selfprivacy_api/services/roundcube/__init__.py
 create mode 100644 selfprivacy_api/services/roundcube/icon.py

diff --git a/selfprivacy_api/services/__init__.py b/selfprivacy_api/services/__init__.py
index f9dfac2..267cc31 100644
--- a/selfprivacy_api/services/__init__.py
+++ b/selfprivacy_api/services/__init__.py
@@ -4,6 +4,7 @@ import typing
 from selfprivacy_api.services.bitwarden import Bitwarden
 from selfprivacy_api.services.gitea import Gitea
 from selfprivacy_api.services.jitsimeet import JitsiMeet
+from selfprivacy_api.services.roundcube import Roundcube
 from selfprivacy_api.services.mailserver import MailServer
 from selfprivacy_api.services.nextcloud import Nextcloud
 from selfprivacy_api.services.pleroma import Pleroma
@@ -19,6 +20,7 @@ services: list[Service] = [
     Pleroma(),
     Ocserv(),
     JitsiMeet(),
+    Roundcube(),
 ]
 
 
diff --git a/selfprivacy_api/services/roundcube/__init__.py b/selfprivacy_api/services/roundcube/__init__.py
new file mode 100644
index 0000000..4deed57
--- /dev/null
+++ b/selfprivacy_api/services/roundcube/__init__.py
@@ -0,0 +1,104 @@
+"""Class representing Roundcube service"""
+
+import base64
+import subprocess
+from typing import Optional, List
+
+from selfprivacy_api.jobs import Job
+from selfprivacy_api.utils.systemd import (
+    get_service_status_from_several_units,
+)
+from selfprivacy_api.services.service import Service, ServiceStatus
+from selfprivacy_api.utils import get_domain
+from selfprivacy_api.utils.block_devices import BlockDevice
+from selfprivacy_api.services.roundcube.icon import ROUNDCUBE_ICON
+
+
+class Roundcube(Service):
+    """Class representing roundcube service"""
+
+    @staticmethod
+    def get_id() -> str:
+        """Return service id."""
+        return "roundcube"
+
+    @staticmethod
+    def get_display_name() -> str:
+        """Return service display name."""
+        return "roundcube"
+
+    @staticmethod
+    def get_description() -> str:
+        """Return service description."""
+        return "Roundcube is a open source webmail software."
+
+    @staticmethod
+    def get_svg_icon() -> str:
+        """Read SVG icon from file and return it as base64 encoded string."""
+        return base64.b64encode(ROUNDCUBE_ICON.encode("utf-8")).decode("utf-8")
+
+    @staticmethod
+    def get_url() -> Optional[str]:
+        """Return service url."""
+        domain = get_domain()
+        return f"https://roundcube.{domain}"
+
+    @staticmethod
+    def get_subdomain() -> Optional[str]:
+        return "roundcube"
+
+    @staticmethod
+    def is_movable() -> bool:
+        return False
+
+    @staticmethod
+    def is_required() -> bool:
+        return False
+
+    @staticmethod
+    def get_backup_description() -> str:
+        return "Secrets that are used to encrypt the communication."
+
+    @staticmethod
+    def get_status() -> ServiceStatus:
+        return get_service_status_from_several_units(["phpfpm-roundcube.service"])
+
+    @staticmethod
+    def stop():
+        subprocess.run(
+            ["systemctl", "stop", "phpfpm-roundcube.service"],
+            check=False,
+        )
+
+    @staticmethod
+    def start():
+        subprocess.run(
+            ["systemctl", "start", "phpfpm-roundcube.service"],
+            check=False,
+        )
+
+    @staticmethod
+    def restart():
+        subprocess.run(
+            ["systemctl", "restart", "phpfpm-roundcube.service"],
+            check=False,
+        )
+
+    @staticmethod
+    def get_configuration():
+        return {}
+
+    @staticmethod
+    def set_configuration(config_items):
+        return super().set_configuration(config_items)
+
+    @staticmethod
+    def get_logs():
+        return ""
+
+    @staticmethod
+    def get_folders() -> List[str]:
+        return ["/var/lib/postgresql"]
+
+    def move_to_volume(self, volume: BlockDevice) -> Job:
+        raise NotImplementedError("roundcube service is not movable")
diff --git a/selfprivacy_api/services/roundcube/icon.py b/selfprivacy_api/services/roundcube/icon.py
new file mode 100644
index 0000000..5fb1288
--- /dev/null
+++ b/selfprivacy_api/services/roundcube/icon.py
@@ -0,0 +1,16 @@
+ROUNDCUBE_ICON = """
+<svg version="1.1" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="9.14 141.8 573.65 573.65">
+<style type="text/css">
+.st0{fill-rule:evenodd;clip-rule:evenodd;fill:#404F54;}
+.st1{fill-rule:evenodd;clip-rule:evenodd;fill:#E5E5E5;}
+.st2{fill-rule:evenodd;clip-rule:evenodd;fill:#CCCCCC;}
+.st3{fill-rule:evenodd;clip-rule:evenodd;fill:#37BEFF;}
+</style>
+<polygon class="st3" points="582.79,549.77 295.96,384.1 295.96,207.27 582.79,372.95 "/>
+<polygon class="st0" points="9.14,549.77 295.96,384.1 295.96,207.27 9.14,372.95 "/>
+<path class="st2" d="M295.96,141.8c109.56,0,198.41,88.85,198.41,198.41c0,109.56-88.85,198.41-198.41,198.41 c-109.56,0-198.41-88.85-198.41-198.41C97.55,230.65,186.4,141.8,295.96,141.8"/>
+<path class="st1" d="M295.96,141.8c109.6,0,198.48,88.85,198.48,198.41c0,109.56-88.88,198.41-198.48,198.41 c-62.91-42.34-88.94-127.64-88.94-198.3S233.05,184.22,295.96,141.8"/>
+<polygon class="st3" points="582.79,372.95 295.96,538.62 295.96,715.45 582.79,549.77 "/>
+<polygon class="st0" points="9.14,372.95 295.96,538.62 295.96,715.45 9.14,549.77 "/>
+</svg>
+"""

From 1b91168d060571efc9fc3b7ad22bcc9d60373bf7 Mon Sep 17 00:00:00 2001
From: dettlaff <dettlaff@riseup.net>
Date: Mon, 10 Jun 2024 05:52:26 +0400
Subject: [PATCH 31/86] style: fix imports

---
 selfprivacy_api/graphql/queries/api_queries.py | 2 ++
 selfprivacy_api/graphql/queries/backup.py      | 1 +
 selfprivacy_api/graphql/queries/common.py      | 1 +
 selfprivacy_api/graphql/queries/jobs.py        | 2 ++
 selfprivacy_api/graphql/queries/providers.py   | 1 +
 selfprivacy_api/graphql/queries/services.py    | 1 +
 selfprivacy_api/graphql/queries/storage.py     | 9 ++++++---
 selfprivacy_api/graphql/queries/system.py      | 2 ++
 selfprivacy_api/graphql/queries/users.py       | 1 +
 9 files changed, 17 insertions(+), 3 deletions(-)

diff --git a/selfprivacy_api/graphql/queries/api_queries.py b/selfprivacy_api/graphql/queries/api_queries.py
index 7052ded..77b0387 100644
--- a/selfprivacy_api/graphql/queries/api_queries.py
+++ b/selfprivacy_api/graphql/queries/api_queries.py
@@ -1,8 +1,10 @@
 """API access status"""
+
 # pylint: disable=too-few-public-methods
 import datetime
 import typing
 import strawberry
+
 from strawberry.types import Info
 from selfprivacy_api.actions.api_tokens import (
     get_api_tokens_with_caller_flag,
diff --git a/selfprivacy_api/graphql/queries/backup.py b/selfprivacy_api/graphql/queries/backup.py
index afb24ae..7695f0d 100644
--- a/selfprivacy_api/graphql/queries/backup.py
+++ b/selfprivacy_api/graphql/queries/backup.py
@@ -1,4 +1,5 @@
 """Backup"""
+
 # pylint: disable=too-few-public-methods
 import typing
 import strawberry
diff --git a/selfprivacy_api/graphql/queries/common.py b/selfprivacy_api/graphql/queries/common.py
index a1abbdc..09dbaf4 100644
--- a/selfprivacy_api/graphql/queries/common.py
+++ b/selfprivacy_api/graphql/queries/common.py
@@ -1,4 +1,5 @@
 """Common types and enums used by different types of queries."""
+
 from enum import Enum
 import datetime
 import typing
diff --git a/selfprivacy_api/graphql/queries/jobs.py b/selfprivacy_api/graphql/queries/jobs.py
index e7b99e6..eebba43 100644
--- a/selfprivacy_api/graphql/queries/jobs.py
+++ b/selfprivacy_api/graphql/queries/jobs.py
@@ -1,7 +1,9 @@
 """Jobs status"""
+
 # pylint: disable=too-few-public-methods
 import typing
 import strawberry
+
 from selfprivacy_api.graphql.common_types.jobs import (
     ApiJob,
     get_api_job_by_id,
diff --git a/selfprivacy_api/graphql/queries/providers.py b/selfprivacy_api/graphql/queries/providers.py
index 2995fe8..c08ea6c 100644
--- a/selfprivacy_api/graphql/queries/providers.py
+++ b/selfprivacy_api/graphql/queries/providers.py
@@ -1,4 +1,5 @@
 """Enums representing different service providers."""
+
 from enum import Enum
 import strawberry
 
diff --git a/selfprivacy_api/graphql/queries/services.py b/selfprivacy_api/graphql/queries/services.py
index 5398f81..3085d61 100644
--- a/selfprivacy_api/graphql/queries/services.py
+++ b/selfprivacy_api/graphql/queries/services.py
@@ -1,4 +1,5 @@
 """Services status"""
+
 # pylint: disable=too-few-public-methods
 import typing
 import strawberry
diff --git a/selfprivacy_api/graphql/queries/storage.py b/selfprivacy_api/graphql/queries/storage.py
index 4b9a291..c221d26 100644
--- a/selfprivacy_api/graphql/queries/storage.py
+++ b/selfprivacy_api/graphql/queries/storage.py
@@ -1,4 +1,5 @@
 """Storage queries."""
+
 # pylint: disable=too-few-public-methods
 import typing
 import strawberry
@@ -18,9 +19,11 @@ class Storage:
         """Get list of volumes"""
         return [
             StorageVolume(
-                total_space=str(volume.fssize)
-                if volume.fssize is not None
-                else str(volume.size),
+                total_space=(
+                    str(volume.fssize)
+                    if volume.fssize is not None
+                    else str(volume.size)
+                ),
                 free_space=str(volume.fsavail),
                 used_space=str(volume.fsused),
                 root=volume.is_root(),
diff --git a/selfprivacy_api/graphql/queries/system.py b/selfprivacy_api/graphql/queries/system.py
index 82c9260..55537d7 100644
--- a/selfprivacy_api/graphql/queries/system.py
+++ b/selfprivacy_api/graphql/queries/system.py
@@ -1,8 +1,10 @@
 """Common system information and settings"""
+
 # pylint: disable=too-few-public-methods
 import os
 import typing
 import strawberry
+
 from selfprivacy_api.graphql.common_types.dns import DnsRecord
 
 from selfprivacy_api.graphql.queries.common import Alert, Severity
diff --git a/selfprivacy_api/graphql/queries/users.py b/selfprivacy_api/graphql/queries/users.py
index d2c0555..992ce01 100644
--- a/selfprivacy_api/graphql/queries/users.py
+++ b/selfprivacy_api/graphql/queries/users.py
@@ -1,4 +1,5 @@
 """Users"""
+
 # pylint: disable=too-few-public-methods
 import typing
 import strawberry

From 9c50f8bba75a71abccf9c3064c22eeeb282b8962 Mon Sep 17 00:00:00 2001
From: dettlaff <dettlaff@riseup.net>
Date: Wed, 12 Jun 2024 17:56:43 +0400
Subject: [PATCH 32/86] fix from review

---
 selfprivacy_api/services/roundcube/__init__.py | 11 ++++++++---
 selfprivacy_api/services/roundcube/icon.py     | 17 ++++-------------
 2 files changed, 12 insertions(+), 16 deletions(-)

diff --git a/selfprivacy_api/services/roundcube/__init__.py b/selfprivacy_api/services/roundcube/__init__.py
index 4deed57..277292e 100644
--- a/selfprivacy_api/services/roundcube/__init__.py
+++ b/selfprivacy_api/services/roundcube/__init__.py
@@ -25,12 +25,12 @@ class Roundcube(Service):
     @staticmethod
     def get_display_name() -> str:
         """Return service display name."""
-        return "roundcube"
+        return "Roundcube"
 
     @staticmethod
     def get_description() -> str:
         """Return service description."""
-        return "Roundcube is a open source webmail software."
+        return "Roundcube is an open source webmail software."
 
     @staticmethod
     def get_svg_icon() -> str:
@@ -41,10 +41,15 @@ class Roundcube(Service):
     def get_url() -> Optional[str]:
         """Return service url."""
         domain = get_domain()
-        return f"https://roundcube.{domain}"
+        subdomain = get_subdomain()
+        return f"https://{subdomain}.{domain}"
 
     @staticmethod
     def get_subdomain() -> Optional[str]:
+        with ReadUserData() as data:
+            if "roundcube" in data["modules"]:
+                return data["modules"]["roundcube"]["subdomain"]
+
         return "roundcube"
 
     @staticmethod
diff --git a/selfprivacy_api/services/roundcube/icon.py b/selfprivacy_api/services/roundcube/icon.py
index 5fb1288..4a08207 100644
--- a/selfprivacy_api/services/roundcube/icon.py
+++ b/selfprivacy_api/services/roundcube/icon.py
@@ -1,16 +1,7 @@
 ROUNDCUBE_ICON = """
-<svg version="1.1" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="9.14 141.8 573.65 573.65">
-<style type="text/css">
-.st0{fill-rule:evenodd;clip-rule:evenodd;fill:#404F54;}
-.st1{fill-rule:evenodd;clip-rule:evenodd;fill:#E5E5E5;}
-.st2{fill-rule:evenodd;clip-rule:evenodd;fill:#CCCCCC;}
-.st3{fill-rule:evenodd;clip-rule:evenodd;fill:#37BEFF;}
-</style>
-<polygon class="st3" points="582.79,549.77 295.96,384.1 295.96,207.27 582.79,372.95 "/>
-<polygon class="st0" points="9.14,549.77 295.96,384.1 295.96,207.27 9.14,372.95 "/>
-<path class="st2" d="M295.96,141.8c109.56,0,198.41,88.85,198.41,198.41c0,109.56-88.85,198.41-198.41,198.41 c-109.56,0-198.41-88.85-198.41-198.41C97.55,230.65,186.4,141.8,295.96,141.8"/>
-<path class="st1" d="M295.96,141.8c109.6,0,198.48,88.85,198.48,198.41c0,109.56-88.88,198.41-198.48,198.41 c-62.91-42.34-88.94-127.64-88.94-198.3S233.05,184.22,295.96,141.8"/>
-<polygon class="st3" points="582.79,372.95 295.96,538.62 295.96,715.45 582.79,549.77 "/>
-<polygon class="st0" points="9.14,372.95 295.96,538.62 295.96,715.45 9.14,549.77 "/>
+<svg fill="none" version="1.1" viewBox="0 0 24 24" xmlns="http://www.w3.org/2000/svg">
+    <g transform="translate(29.07 -.3244)">
+        <path d="m-17.02 2.705c-4.01 2e-7 -7.283 3.273-7.283 7.283 0 0.00524-1.1e-5 0.01038 0 0.01562l-1.85 1.068v5.613l9.105 5.26 9.104-5.26v-5.613l-1.797-1.037c1.008e-4 -0.01573 0.00195-0.03112 0.00195-0.04688-1e-7 -4.01-3.271-7.283-7.281-7.283zm0 2.012c2.923 1e-7 5.27 2.349 5.27 5.271 0 2.923-2.347 5.27-5.27 5.27-2.923-1e-6 -5.271-2.347-5.271-5.27 0-2.923 2.349-5.271 5.271-5.271z" fill="#000" fill-rule="evenodd" stroke-linejoin="bevel"/>
+    </g>
 </svg>
 """

From a00c4d426858fa2b5836dbc04be0c8a4062e4f43 Mon Sep 17 00:00:00 2001
From: dettlaff <dettlaff@riseup.net>
Date: Wed, 12 Jun 2024 18:48:56 +0400
Subject: [PATCH 33/86] fix: change return get_folders

---
 selfprivacy_api/services/roundcube/__init__.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/selfprivacy_api/services/roundcube/__init__.py b/selfprivacy_api/services/roundcube/__init__.py
index 277292e..8884be5 100644
--- a/selfprivacy_api/services/roundcube/__init__.py
+++ b/selfprivacy_api/services/roundcube/__init__.py
@@ -103,7 +103,7 @@ class Roundcube(Service):
 
     @staticmethod
     def get_folders() -> List[str]:
-        return ["/var/lib/postgresql"]
+        return []
 
     def move_to_volume(self, volume: BlockDevice) -> Job:
         raise NotImplementedError("roundcube service is not movable")

From 31feeb211d39d4c2ce44aa88036dccbf4c133224 Mon Sep 17 00:00:00 2001
From: dettlaff <dettlaff@riseup.net>
Date: Thu, 13 Jun 2024 19:48:58 +0400
Subject: [PATCH 34/86] fix: change roundcube to webmail

---
 selfprivacy_api/services/roundcube/__init__.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/selfprivacy_api/services/roundcube/__init__.py b/selfprivacy_api/services/roundcube/__init__.py
index 8884be5..a2b2300 100644
--- a/selfprivacy_api/services/roundcube/__init__.py
+++ b/selfprivacy_api/services/roundcube/__init__.py
@@ -50,7 +50,7 @@ class Roundcube(Service):
             if "roundcube" in data["modules"]:
                 return data["modules"]["roundcube"]["subdomain"]
 
-        return "roundcube"
+        return "webmail"
 
     @staticmethod
     def is_movable() -> bool:

From 4d898f4ee803170ba66eb05e146fac7c814392c8 Mon Sep 17 00:00:00 2001
From: dettlaff <dettlaff@riseup.net>
Date: Sun, 16 Jun 2024 10:39:39 +0400
Subject: [PATCH 35/86] feat: add migration for services flake

---
 .../migrations/update_services_flake_list.py  | 34 +++++++++++++++++++
 1 file changed, 34 insertions(+)
 create mode 100644 selfprivacy_api/migrations/update_services_flake_list.py

diff --git a/selfprivacy_api/migrations/update_services_flake_list.py b/selfprivacy_api/migrations/update_services_flake_list.py
new file mode 100644
index 0000000..8d3b8e8
--- /dev/null
+++ b/selfprivacy_api/migrations/update_services_flake_list.py
@@ -0,0 +1,34 @@
+from selfprivacy_api.migrations.migration import Migration
+from selfprivacy_api.jobs import JobStatus, Jobs
+
+from selfprivacy_api.services.flake_service_manager import FlakeServiceManager
+
+CORRECT_SERVICES_LIST = {
+    "bitwarden": "git+https://git.selfprivacy.org/SelfPrivacy/selfprivacy-nixos-config.git?ref=flakes&dir=sp-modules/bitwarden",
+    "gitea": "git+https://git.selfprivacy.org/SelfPrivacy/selfprivacy-nixos-config.git?ref=flakes&dir=sp-modules/gitea",
+    "jitsi-meet": "git+https://git.selfprivacy.org/SelfPrivacy/selfprivacy-nixos-config.git?ref=flakes&dir=sp-modules/jitsi-meet",
+    "nextcloud": "git+https://git.selfprivacy.org/SelfPrivacy/selfprivacy-nixos-config.git?ref=flakes&dir=sp-modules/nextcloud",
+    "ocserv": "git+https://git.selfprivacy.org/SelfPrivacy/selfprivacy-nixos-config.git?ref=flakes&dir=sp-modules/ocserv",
+    "pleroma": "git+https://git.selfprivacy.org/SelfPrivacy/selfprivacy-nixos-config.git?ref=flakes&dir=sp-modules/pleroma",
+    "simple-nixos-mailserver": "git+https://git.selfprivacy.org/SelfPrivacy/selfprivacy-nixos-config.git?ref=flakes&dir=sp-modules/simple-nixos-mailserver",
+    "roundcube": "git+https://git.selfprivacy.org/SelfPrivacy/selfprivacy-nixos-config.git?ref=flakes&dir=sp-modules/roundcube",
+}
+
+
+class UpdateServicesFlakeList(Migration):
+    """Check if all required services are in the flake list"""
+
+    def get_migration_name(self):
+        return "update_services_flake_list"
+
+    def get_migration_description(self):
+        return "Check if all required services are in the flake list"
+
+    def is_migration_needed(self):
+        with FlakeServiceManager:
+            if not manager.services == CORRECT_SERVICES_LIST:
+                return True
+
+    def migrate(self):
+        with FlakeServiceManager:
+            manager.services = CORRECT_SERVICES_LIST

From 78dec5c3478d9fcfebb45018b3eec4f87e0e9719 Mon Sep 17 00:00:00 2001
From: dettlaff <dettlaff@riseup.net>
Date: Sun, 16 Jun 2024 11:59:01 +0400
Subject: [PATCH 36/86] feat: move get_subdomain to parent class

---
 selfprivacy_api/migrations/update_services_flake_list.py | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/selfprivacy_api/migrations/update_services_flake_list.py b/selfprivacy_api/migrations/update_services_flake_list.py
index 8d3b8e8..6f23903 100644
--- a/selfprivacy_api/migrations/update_services_flake_list.py
+++ b/selfprivacy_api/migrations/update_services_flake_list.py
@@ -25,10 +25,13 @@ class UpdateServicesFlakeList(Migration):
         return "Check if all required services are in the flake list"
 
     def is_migration_needed(self):
-        with FlakeServiceManager:
-            if not manager.services == CORRECT_SERVICES_LIST:
+        # this do not delete custom links for testing
+        for key, value in manager.services.items():
+            if key not in CORRECT_SERVICES_LIST:
                 return True
 
     def migrate(self):
         with FlakeServiceManager:
-            manager.services = CORRECT_SERVICES_LIST
+            for key, value in CORRECT_SERVICES_LIST.items():
+                if key not in manager.services:
+                    manager.services[key] = value

From 2b9b81890bb19e88d75d7e368a06a68f31c30630 Mon Sep 17 00:00:00 2001
From: dettlaff <dettlaff@riseup.net>
Date: Sun, 16 Jun 2024 12:00:09 +0400
Subject: [PATCH 37/86] feat: move get_subdomain to parent class really

---
 selfprivacy_api/services/roundcube/__init__.py | 8 --------
 selfprivacy_api/services/service.py            | 8 +++++---
 2 files changed, 5 insertions(+), 11 deletions(-)

diff --git a/selfprivacy_api/services/roundcube/__init__.py b/selfprivacy_api/services/roundcube/__init__.py
index a2b2300..61a23ea 100644
--- a/selfprivacy_api/services/roundcube/__init__.py
+++ b/selfprivacy_api/services/roundcube/__init__.py
@@ -44,14 +44,6 @@ class Roundcube(Service):
         subdomain = get_subdomain()
         return f"https://{subdomain}.{domain}"
 
-    @staticmethod
-    def get_subdomain() -> Optional[str]:
-        with ReadUserData() as data:
-            if "roundcube" in data["modules"]:
-                return data["modules"]["roundcube"]["subdomain"]
-
-        return "webmail"
-
     @staticmethod
     def is_movable() -> bool:
         return False
diff --git a/selfprivacy_api/services/service.py b/selfprivacy_api/services/service.py
index 64a1e80..262d690 100644
--- a/selfprivacy_api/services/service.py
+++ b/selfprivacy_api/services/service.py
@@ -1,4 +1,5 @@
 """Abstract class for a service running on a server"""
+
 from abc import ABC, abstractmethod
 from typing import List, Optional
 
@@ -73,13 +74,14 @@ class Service(ABC):
         """
         pass
 
-    @staticmethod
-    @abstractmethod
     def get_subdomain() -> Optional[str]:
         """
         The assigned primary subdomain for this service.
         """
-        pass
+        with ReadUserData() as data:
+            if self.get_display_name() in data["modules"]:
+                if "subdomain" in data["modules"][self.get_display_name()]:
+                    return data["modules"][self.get_display_name()]["subdomain"]
 
     @classmethod
     def get_user(cls) -> Optional[str]:

From 9125d03b352a730eda2d0c2c469a690eef5b158e Mon Sep 17 00:00:00 2001
From: dettlaff <dettlaff@riseup.net>
Date: Sun, 16 Jun 2024 12:05:58 +0400
Subject: [PATCH 38/86] fix: remove get sub domain from services

---
 selfprivacy_api/services/bitwarden/__init__.py    | 5 +----
 selfprivacy_api/services/gitea/__init__.py        | 5 +----
 selfprivacy_api/services/jitsimeet/__init__.py    | 5 +----
 selfprivacy_api/services/nextcloud/__init__.py    | 5 +----
 selfprivacy_api/services/ocserv/__init__.py       | 5 +----
 selfprivacy_api/services/pleroma/__init__.py      | 5 +----
 selfprivacy_api/services/test_service/__init__.py | 5 +----
 7 files changed, 7 insertions(+), 28 deletions(-)

diff --git a/selfprivacy_api/services/bitwarden/__init__.py b/selfprivacy_api/services/bitwarden/__init__.py
index 52f1466..6c44aeb 100644
--- a/selfprivacy_api/services/bitwarden/__init__.py
+++ b/selfprivacy_api/services/bitwarden/__init__.py
@@ -1,4 +1,5 @@
 """Class representing Bitwarden service"""
+
 import base64
 import subprocess
 from typing import Optional, List
@@ -43,10 +44,6 @@ class Bitwarden(Service):
         domain = get_domain()
         return f"https://password.{domain}"
 
-    @staticmethod
-    def get_subdomain() -> Optional[str]:
-        return "password"
-
     @staticmethod
     def is_movable() -> bool:
         return True
diff --git a/selfprivacy_api/services/gitea/__init__.py b/selfprivacy_api/services/gitea/__init__.py
index 311d59e..7029b48 100644
--- a/selfprivacy_api/services/gitea/__init__.py
+++ b/selfprivacy_api/services/gitea/__init__.py
@@ -1,4 +1,5 @@
 """Class representing Bitwarden service"""
+
 import base64
 import subprocess
 from typing import Optional, List
@@ -39,10 +40,6 @@ class Gitea(Service):
         domain = get_domain()
         return f"https://git.{domain}"
 
-    @staticmethod
-    def get_subdomain() -> Optional[str]:
-        return "git"
-
     @staticmethod
     def is_movable() -> bool:
         return True
diff --git a/selfprivacy_api/services/jitsimeet/__init__.py b/selfprivacy_api/services/jitsimeet/__init__.py
index 53d572c..05ef2f7 100644
--- a/selfprivacy_api/services/jitsimeet/__init__.py
+++ b/selfprivacy_api/services/jitsimeet/__init__.py
@@ -1,4 +1,5 @@
 """Class representing Jitsi Meet service"""
+
 import base64
 import subprocess
 from typing import Optional, List
@@ -42,10 +43,6 @@ class JitsiMeet(Service):
         domain = get_domain()
         return f"https://meet.{domain}"
 
-    @staticmethod
-    def get_subdomain() -> Optional[str]:
-        return "meet"
-
     @staticmethod
     def is_movable() -> bool:
         return False
diff --git a/selfprivacy_api/services/nextcloud/__init__.py b/selfprivacy_api/services/nextcloud/__init__.py
index 3e5b8d3..6aee771 100644
--- a/selfprivacy_api/services/nextcloud/__init__.py
+++ b/selfprivacy_api/services/nextcloud/__init__.py
@@ -1,4 +1,5 @@
 """Class representing Nextcloud service."""
+
 import base64
 import subprocess
 from typing import Optional, List
@@ -41,10 +42,6 @@ class Nextcloud(Service):
         domain = get_domain()
         return f"https://cloud.{domain}"
 
-    @staticmethod
-    def get_subdomain() -> Optional[str]:
-        return "cloud"
-
     @staticmethod
     def is_movable() -> bool:
         return True
diff --git a/selfprivacy_api/services/ocserv/__init__.py b/selfprivacy_api/services/ocserv/__init__.py
index 4dd802f..e21fe6e 100644
--- a/selfprivacy_api/services/ocserv/__init__.py
+++ b/selfprivacy_api/services/ocserv/__init__.py
@@ -1,4 +1,5 @@
 """Class representing ocserv service."""
+
 import base64
 import subprocess
 import typing
@@ -33,10 +34,6 @@ class Ocserv(Service):
         """Return service url."""
         return None
 
-    @staticmethod
-    def get_subdomain() -> typing.Optional[str]:
-        return "vpn"
-
     @staticmethod
     def is_movable() -> bool:
         return False
diff --git a/selfprivacy_api/services/pleroma/__init__.py b/selfprivacy_api/services/pleroma/__init__.py
index 44a9be8..eebd925 100644
--- a/selfprivacy_api/services/pleroma/__init__.py
+++ b/selfprivacy_api/services/pleroma/__init__.py
@@ -1,4 +1,5 @@
 """Class representing Nextcloud service."""
+
 import base64
 import subprocess
 from typing import Optional, List
@@ -37,10 +38,6 @@ class Pleroma(Service):
         domain = get_domain()
         return f"https://social.{domain}"
 
-    @staticmethod
-    def get_subdomain() -> Optional[str]:
-        return "social"
-
     @staticmethod
     def is_movable() -> bool:
         return True
diff --git a/selfprivacy_api/services/test_service/__init__.py b/selfprivacy_api/services/test_service/__init__.py
index caf4666..2058d57 100644
--- a/selfprivacy_api/services/test_service/__init__.py
+++ b/selfprivacy_api/services/test_service/__init__.py
@@ -1,4 +1,5 @@
 """Class representing Bitwarden service"""
+
 import base64
 import typing
 import subprocess
@@ -63,10 +64,6 @@ class DummyService(Service):
         domain = "test.com"
         return f"https://password.{domain}"
 
-    @staticmethod
-    def get_subdomain() -> typing.Optional[str]:
-        return "password"
-
     @classmethod
     def is_movable(cls) -> bool:
         return cls.movable

From 7b9420c244b5fd72c48500971a1e730066fe6d53 Mon Sep 17 00:00:00 2001
From: dettlaff <dettlaff@riseup.net>
Date: Sun, 16 Jun 2024 12:35:59 +0400
Subject: [PATCH 39/86] feat: rewrite get_url()

---
 selfprivacy_api/services/bitwarden/__init__.py | 8 --------
 selfprivacy_api/services/gitea/__init__.py     | 8 --------
 selfprivacy_api/services/jitsimeet/__init__.py | 7 -------
 selfprivacy_api/services/nextcloud/__init__.py | 7 -------
 selfprivacy_api/services/pleroma/__init__.py   | 8 --------
 selfprivacy_api/services/roundcube/__init__.py | 8 --------
 selfprivacy_api/services/service.py            | 7 ++++---
 7 files changed, 4 insertions(+), 49 deletions(-)

diff --git a/selfprivacy_api/services/bitwarden/__init__.py b/selfprivacy_api/services/bitwarden/__init__.py
index 6c44aeb..5d27569 100644
--- a/selfprivacy_api/services/bitwarden/__init__.py
+++ b/selfprivacy_api/services/bitwarden/__init__.py
@@ -4,8 +4,6 @@ import base64
 import subprocess
 from typing import Optional, List
 
-from selfprivacy_api.utils import get_domain
-
 from selfprivacy_api.utils.systemd import get_service_status
 from selfprivacy_api.services.service import Service, ServiceStatus
 from selfprivacy_api.services.bitwarden.icon import BITWARDEN_ICON
@@ -38,12 +36,6 @@ class Bitwarden(Service):
     def get_user() -> str:
         return "vaultwarden"
 
-    @staticmethod
-    def get_url() -> Optional[str]:
-        """Return service url."""
-        domain = get_domain()
-        return f"https://password.{domain}"
-
     @staticmethod
     def is_movable() -> bool:
         return True
diff --git a/selfprivacy_api/services/gitea/__init__.py b/selfprivacy_api/services/gitea/__init__.py
index 7029b48..3247040 100644
--- a/selfprivacy_api/services/gitea/__init__.py
+++ b/selfprivacy_api/services/gitea/__init__.py
@@ -4,8 +4,6 @@ import base64
 import subprocess
 from typing import Optional, List
 
-from selfprivacy_api.utils import get_domain
-
 from selfprivacy_api.utils.systemd import get_service_status
 from selfprivacy_api.services.service import Service, ServiceStatus
 from selfprivacy_api.services.gitea.icon import GITEA_ICON
@@ -34,12 +32,6 @@ class Gitea(Service):
         """Read SVG icon from file and return it as base64 encoded string."""
         return base64.b64encode(GITEA_ICON.encode("utf-8")).decode("utf-8")
 
-    @staticmethod
-    def get_url() -> Optional[str]:
-        """Return service url."""
-        domain = get_domain()
-        return f"https://git.{domain}"
-
     @staticmethod
     def is_movable() -> bool:
         return True
diff --git a/selfprivacy_api/services/jitsimeet/__init__.py b/selfprivacy_api/services/jitsimeet/__init__.py
index 05ef2f7..074c546 100644
--- a/selfprivacy_api/services/jitsimeet/__init__.py
+++ b/selfprivacy_api/services/jitsimeet/__init__.py
@@ -9,7 +9,6 @@ from selfprivacy_api.utils.systemd import (
     get_service_status_from_several_units,
 )
 from selfprivacy_api.services.service import Service, ServiceStatus
-from selfprivacy_api.utils import get_domain
 from selfprivacy_api.utils.block_devices import BlockDevice
 from selfprivacy_api.services.jitsimeet.icon import JITSI_ICON
 
@@ -37,12 +36,6 @@ class JitsiMeet(Service):
         """Read SVG icon from file and return it as base64 encoded string."""
         return base64.b64encode(JITSI_ICON.encode("utf-8")).decode("utf-8")
 
-    @staticmethod
-    def get_url() -> Optional[str]:
-        """Return service url."""
-        domain = get_domain()
-        return f"https://meet.{domain}"
-
     @staticmethod
     def is_movable() -> bool:
         return False
diff --git a/selfprivacy_api/services/nextcloud/__init__.py b/selfprivacy_api/services/nextcloud/__init__.py
index 6aee771..eee9fa7 100644
--- a/selfprivacy_api/services/nextcloud/__init__.py
+++ b/selfprivacy_api/services/nextcloud/__init__.py
@@ -4,7 +4,6 @@ import base64
 import subprocess
 from typing import Optional, List
 
-from selfprivacy_api.utils import get_domain
 from selfprivacy_api.jobs import Job, Jobs
 
 from selfprivacy_api.utils.systemd import get_service_status
@@ -36,12 +35,6 @@ class Nextcloud(Service):
         """Read SVG icon from file and return it as base64 encoded string."""
         return base64.b64encode(NEXTCLOUD_ICON.encode("utf-8")).decode("utf-8")
 
-    @staticmethod
-    def get_url() -> Optional[str]:
-        """Return service url."""
-        domain = get_domain()
-        return f"https://cloud.{domain}"
-
     @staticmethod
     def is_movable() -> bool:
         return True
diff --git a/selfprivacy_api/services/pleroma/__init__.py b/selfprivacy_api/services/pleroma/__init__.py
index eebd925..16ee70c 100644
--- a/selfprivacy_api/services/pleroma/__init__.py
+++ b/selfprivacy_api/services/pleroma/__init__.py
@@ -4,8 +4,6 @@ import base64
 import subprocess
 from typing import Optional, List
 
-from selfprivacy_api.utils import get_domain
-
 from selfprivacy_api.services.owned_path import OwnedPath
 from selfprivacy_api.utils.systemd import get_service_status
 from selfprivacy_api.services.service import Service, ServiceStatus
@@ -32,12 +30,6 @@ class Pleroma(Service):
     def get_svg_icon() -> str:
         return base64.b64encode(PLEROMA_ICON.encode("utf-8")).decode("utf-8")
 
-    @staticmethod
-    def get_url() -> Optional[str]:
-        """Return service url."""
-        domain = get_domain()
-        return f"https://social.{domain}"
-
     @staticmethod
     def is_movable() -> bool:
         return True
diff --git a/selfprivacy_api/services/roundcube/__init__.py b/selfprivacy_api/services/roundcube/__init__.py
index 61a23ea..41bef32 100644
--- a/selfprivacy_api/services/roundcube/__init__.py
+++ b/selfprivacy_api/services/roundcube/__init__.py
@@ -9,7 +9,6 @@ from selfprivacy_api.utils.systemd import (
     get_service_status_from_several_units,
 )
 from selfprivacy_api.services.service import Service, ServiceStatus
-from selfprivacy_api.utils import get_domain
 from selfprivacy_api.utils.block_devices import BlockDevice
 from selfprivacy_api.services.roundcube.icon import ROUNDCUBE_ICON
 
@@ -37,13 +36,6 @@ class Roundcube(Service):
         """Read SVG icon from file and return it as base64 encoded string."""
         return base64.b64encode(ROUNDCUBE_ICON.encode("utf-8")).decode("utf-8")
 
-    @staticmethod
-    def get_url() -> Optional[str]:
-        """Return service url."""
-        domain = get_domain()
-        subdomain = get_subdomain()
-        return f"https://{subdomain}.{domain}"
-
     @staticmethod
     def is_movable() -> bool:
         return False
diff --git a/selfprivacy_api/services/service.py b/selfprivacy_api/services/service.py
index 262d690..ed55a0e 100644
--- a/selfprivacy_api/services/service.py
+++ b/selfprivacy_api/services/service.py
@@ -4,6 +4,7 @@ from abc import ABC, abstractmethod
 from typing import List, Optional
 
 from selfprivacy_api import utils
+from selfprivacy_api.utils import get_domain
 from selfprivacy_api.utils import ReadUserData, WriteUserData
 from selfprivacy_api.utils.waitloop import wait_until_true
 from selfprivacy_api.utils.block_devices import BlockDevice, BlockDevices
@@ -66,13 +67,13 @@ class Service(ABC):
         """
         pass
 
-    @staticmethod
-    @abstractmethod
     def get_url() -> Optional[str]:
         """
         The url of the service if it is accessible from the internet browser.
         """
-        pass
+        domain = get_domain()
+        subdomain = self.get_subdomain()
+        return f"https://{subdomain}.{domain}"
 
     def get_subdomain() -> Optional[str]:
         """

From 82a0b557e154e6d95ba02c4a44819df9f0ed83ed Mon Sep 17 00:00:00 2001
From: dettlaff <dettlaff@riseup.net>
Date: Sun, 16 Jun 2024 23:48:25 +0400
Subject: [PATCH 40/86] feat: add migration for userdata

---
 selfprivacy_api/migrations/update_userdata.py | 25 +++++++++++++++++++
 1 file changed, 25 insertions(+)
 create mode 100644 selfprivacy_api/migrations/update_userdata.py

diff --git a/selfprivacy_api/migrations/update_userdata.py b/selfprivacy_api/migrations/update_userdata.py
new file mode 100644
index 0000000..9011731
--- /dev/null
+++ b/selfprivacy_api/migrations/update_userdata.py
@@ -0,0 +1,25 @@
+from selfprivacy_api.migrations.migration import Migration
+
+from selfprivacy_api.utils import ReadUserData, WriteUserData
+
+
+class UpdateServicesFlakeList(Migration):
+    """Check if all required services are in the flake list"""
+
+    def get_migration_name(self):
+        return "update_services_flake_list"
+
+    def get_migration_description(self):
+        return "Check if all required services are in the flake list"
+
+    def is_migration_needed(self):
+        with ReadUserData() as data:
+            if "roundcube" not in data["modules"]:
+                return True
+
+    def migrate(self):
+        with WriteUserData() as data:
+            data["modules"]["roundcube"] = {
+                "enable": True,
+                "subdomain": "roundcube",
+            }

From 416a0a8725209da40aa2c4a9b5cbb667532a6396 Mon Sep 17 00:00:00 2001
From: dettlaff <dettlaff@riseup.net>
Date: Wed, 19 Jun 2024 15:20:31 +0400
Subject: [PATCH 41/86] fix: from review

---
 .../{update_userdata.py => add_roundcube_to_userdata.py} | 9 ++++-----
 selfprivacy_api/migrations/update_services_flake_list.py | 1 -
 selfprivacy_api/services/roundcube/__init__.py           | 6 +++++-
 3 files changed, 9 insertions(+), 7 deletions(-)
 rename selfprivacy_api/migrations/{update_userdata.py => add_roundcube_to_userdata.py} (67%)

diff --git a/selfprivacy_api/migrations/update_userdata.py b/selfprivacy_api/migrations/add_roundcube_to_userdata.py
similarity index 67%
rename from selfprivacy_api/migrations/update_userdata.py
rename to selfprivacy_api/migrations/add_roundcube_to_userdata.py
index 9011731..2c6bafb 100644
--- a/selfprivacy_api/migrations/update_userdata.py
+++ b/selfprivacy_api/migrations/add_roundcube_to_userdata.py
@@ -3,14 +3,14 @@ from selfprivacy_api.migrations.migration import Migration
 from selfprivacy_api.utils import ReadUserData, WriteUserData
 
 
-class UpdateServicesFlakeList(Migration):
-    """Check if all required services are in the flake list"""
+class AddRoundcubeToUserdata(Migration):
+    """Add Roundcube to userdata.json if it does not exist"""
 
     def get_migration_name(self):
-        return "update_services_flake_list"
+        return "add_roundcube_to_userdata"
 
     def get_migration_description(self):
-        return "Check if all required services are in the flake list"
+        return "Add Roundcube to userdata.json if it does not exist"
 
     def is_migration_needed(self):
         with ReadUserData() as data:
@@ -20,6 +20,5 @@ class UpdateServicesFlakeList(Migration):
     def migrate(self):
         with WriteUserData() as data:
             data["modules"]["roundcube"] = {
-                "enable": True,
                 "subdomain": "roundcube",
             }
diff --git a/selfprivacy_api/migrations/update_services_flake_list.py b/selfprivacy_api/migrations/update_services_flake_list.py
index 6f23903..02671e8 100644
--- a/selfprivacy_api/migrations/update_services_flake_list.py
+++ b/selfprivacy_api/migrations/update_services_flake_list.py
@@ -25,7 +25,6 @@ class UpdateServicesFlakeList(Migration):
         return "Check if all required services are in the flake list"
 
     def is_migration_needed(self):
-        # this do not delete custom links for testing
         for key, value in manager.services.items():
             if key not in CORRECT_SERVICES_LIST:
                 return True
diff --git a/selfprivacy_api/services/roundcube/__init__.py b/selfprivacy_api/services/roundcube/__init__.py
index 41bef32..7572770 100644
--- a/selfprivacy_api/services/roundcube/__init__.py
+++ b/selfprivacy_api/services/roundcube/__init__.py
@@ -44,9 +44,13 @@ class Roundcube(Service):
     def is_required() -> bool:
         return False
 
+    @staticmethod
+    def can_be_backed_up() -> bool:
+        return False
+
     @staticmethod
     def get_backup_description() -> str:
-        return "Secrets that are used to encrypt the communication."
+        return "Nothing to backup."
 
     @staticmethod
     def get_status() -> ServiceStatus:

From 02bc74f4c4a6cd3227a0cd31d1d08ca1af3b16c2 Mon Sep 17 00:00:00 2001
From: dettlaff <dettlaff@riseup.net>
Date: Wed, 19 Jun 2024 16:18:21 +0400
Subject: [PATCH 42/86] fix: only roundcube migration, other services removed

---
 .../migrations/update_services_flake_list.py  | 24 ++++++-------------
 1 file changed, 7 insertions(+), 17 deletions(-)

diff --git a/selfprivacy_api/migrations/update_services_flake_list.py b/selfprivacy_api/migrations/update_services_flake_list.py
index 02671e8..88f7307 100644
--- a/selfprivacy_api/migrations/update_services_flake_list.py
+++ b/selfprivacy_api/migrations/update_services_flake_list.py
@@ -3,17 +3,6 @@ from selfprivacy_api.jobs import JobStatus, Jobs
 
 from selfprivacy_api.services.flake_service_manager import FlakeServiceManager
 
-CORRECT_SERVICES_LIST = {
-    "bitwarden": "git+https://git.selfprivacy.org/SelfPrivacy/selfprivacy-nixos-config.git?ref=flakes&dir=sp-modules/bitwarden",
-    "gitea": "git+https://git.selfprivacy.org/SelfPrivacy/selfprivacy-nixos-config.git?ref=flakes&dir=sp-modules/gitea",
-    "jitsi-meet": "git+https://git.selfprivacy.org/SelfPrivacy/selfprivacy-nixos-config.git?ref=flakes&dir=sp-modules/jitsi-meet",
-    "nextcloud": "git+https://git.selfprivacy.org/SelfPrivacy/selfprivacy-nixos-config.git?ref=flakes&dir=sp-modules/nextcloud",
-    "ocserv": "git+https://git.selfprivacy.org/SelfPrivacy/selfprivacy-nixos-config.git?ref=flakes&dir=sp-modules/ocserv",
-    "pleroma": "git+https://git.selfprivacy.org/SelfPrivacy/selfprivacy-nixos-config.git?ref=flakes&dir=sp-modules/pleroma",
-    "simple-nixos-mailserver": "git+https://git.selfprivacy.org/SelfPrivacy/selfprivacy-nixos-config.git?ref=flakes&dir=sp-modules/simple-nixos-mailserver",
-    "roundcube": "git+https://git.selfprivacy.org/SelfPrivacy/selfprivacy-nixos-config.git?ref=flakes&dir=sp-modules/roundcube",
-}
-
 
 class UpdateServicesFlakeList(Migration):
     """Check if all required services are in the flake list"""
@@ -25,12 +14,13 @@ class UpdateServicesFlakeList(Migration):
         return "Check if all required services are in the flake list"
 
     def is_migration_needed(self):
-        for key, value in manager.services.items():
-            if key not in CORRECT_SERVICES_LIST:
+        with FlakeServiceManager() as manager:
+            if "roundcube" not in manager.services:
                 return True
 
     def migrate(self):
-        with FlakeServiceManager:
-            for key, value in CORRECT_SERVICES_LIST.items():
-                if key not in manager.services:
-                    manager.services[key] = value
+        with FlakeServiceManager() as manager:
+            if "roundcube" not in manager.services:
+                manager.services[
+                    "roundcube"
+                ] = "git+https://git.selfprivacy.org/SelfPrivacy/selfprivacy-nixos-config.git?ref=flakes&dir=sp-modules/roundcube"

From f1cc84b8c83306b07eeafb463a0611c4ac2b2481 Mon Sep 17 00:00:00 2001
From: nhnn <nhnn@disroot.org>
Date: Fri, 21 Jun 2024 15:14:43 +0300
Subject: [PATCH 43/86] fix: add migrations to migration list in
 migrations/__init__.py

---
 selfprivacy_api/migrations/__init__.py | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/selfprivacy_api/migrations/__init__.py b/selfprivacy_api/migrations/__init__.py
index 2a2cbaa..badf40a 100644
--- a/selfprivacy_api/migrations/__init__.py
+++ b/selfprivacy_api/migrations/__init__.py
@@ -14,10 +14,14 @@ from selfprivacy_api.migrations.write_token_to_redis import WriteTokenToRedis
 from selfprivacy_api.migrations.check_for_system_rebuild_jobs import (
     CheckForSystemRebuildJobs,
 )
+from selfprivacy_api.migrations.update_services_flake_list import UpdateServicesFlakeList
+from selfprivacy_api.migrations.add_roundcube_to_userdata import AddRoundcubeToUserdata
 
 migrations = [
     WriteTokenToRedis(),
     CheckForSystemRebuildJobs(),
+    UpdateServicesFlakeList(),
+    AddRoundcubeToUserdata(),
 ]
 
 

From 306b7f898d7e88ae8fb8c1b429a6a15fb2d7f427 Mon Sep 17 00:00:00 2001
From: Inex Code <inex.code@selfprivacy.org>
Date: Wed, 26 Jun 2024 14:21:26 +0300
Subject: [PATCH 44/86] Revert "feat: rewrite get_url()"

This reverts commit f834c85401073a19b18bcafbda837d82f6e5f383.
---
 selfprivacy_api/services/bitwarden/__init__.py | 8 ++++++++
 selfprivacy_api/services/gitea/__init__.py     | 8 ++++++++
 selfprivacy_api/services/jitsimeet/__init__.py | 7 +++++++
 selfprivacy_api/services/nextcloud/__init__.py | 7 +++++++
 selfprivacy_api/services/pleroma/__init__.py   | 8 ++++++++
 selfprivacy_api/services/roundcube/__init__.py | 8 ++++++++
 selfprivacy_api/services/service.py            | 7 +++----
 7 files changed, 49 insertions(+), 4 deletions(-)

diff --git a/selfprivacy_api/services/bitwarden/__init__.py b/selfprivacy_api/services/bitwarden/__init__.py
index 5d27569..6c44aeb 100644
--- a/selfprivacy_api/services/bitwarden/__init__.py
+++ b/selfprivacy_api/services/bitwarden/__init__.py
@@ -4,6 +4,8 @@ import base64
 import subprocess
 from typing import Optional, List
 
+from selfprivacy_api.utils import get_domain
+
 from selfprivacy_api.utils.systemd import get_service_status
 from selfprivacy_api.services.service import Service, ServiceStatus
 from selfprivacy_api.services.bitwarden.icon import BITWARDEN_ICON
@@ -36,6 +38,12 @@ class Bitwarden(Service):
     def get_user() -> str:
         return "vaultwarden"
 
+    @staticmethod
+    def get_url() -> Optional[str]:
+        """Return service url."""
+        domain = get_domain()
+        return f"https://password.{domain}"
+
     @staticmethod
     def is_movable() -> bool:
         return True
diff --git a/selfprivacy_api/services/gitea/__init__.py b/selfprivacy_api/services/gitea/__init__.py
index 3247040..7029b48 100644
--- a/selfprivacy_api/services/gitea/__init__.py
+++ b/selfprivacy_api/services/gitea/__init__.py
@@ -4,6 +4,8 @@ import base64
 import subprocess
 from typing import Optional, List
 
+from selfprivacy_api.utils import get_domain
+
 from selfprivacy_api.utils.systemd import get_service_status
 from selfprivacy_api.services.service import Service, ServiceStatus
 from selfprivacy_api.services.gitea.icon import GITEA_ICON
@@ -32,6 +34,12 @@ class Gitea(Service):
         """Read SVG icon from file and return it as base64 encoded string."""
         return base64.b64encode(GITEA_ICON.encode("utf-8")).decode("utf-8")
 
+    @staticmethod
+    def get_url() -> Optional[str]:
+        """Return service url."""
+        domain = get_domain()
+        return f"https://git.{domain}"
+
     @staticmethod
     def is_movable() -> bool:
         return True
diff --git a/selfprivacy_api/services/jitsimeet/__init__.py b/selfprivacy_api/services/jitsimeet/__init__.py
index 074c546..05ef2f7 100644
--- a/selfprivacy_api/services/jitsimeet/__init__.py
+++ b/selfprivacy_api/services/jitsimeet/__init__.py
@@ -9,6 +9,7 @@ from selfprivacy_api.utils.systemd import (
     get_service_status_from_several_units,
 )
 from selfprivacy_api.services.service import Service, ServiceStatus
+from selfprivacy_api.utils import get_domain
 from selfprivacy_api.utils.block_devices import BlockDevice
 from selfprivacy_api.services.jitsimeet.icon import JITSI_ICON
 
@@ -36,6 +37,12 @@ class JitsiMeet(Service):
         """Read SVG icon from file and return it as base64 encoded string."""
         return base64.b64encode(JITSI_ICON.encode("utf-8")).decode("utf-8")
 
+    @staticmethod
+    def get_url() -> Optional[str]:
+        """Return service url."""
+        domain = get_domain()
+        return f"https://meet.{domain}"
+
     @staticmethod
     def is_movable() -> bool:
         return False
diff --git a/selfprivacy_api/services/nextcloud/__init__.py b/selfprivacy_api/services/nextcloud/__init__.py
index eee9fa7..6aee771 100644
--- a/selfprivacy_api/services/nextcloud/__init__.py
+++ b/selfprivacy_api/services/nextcloud/__init__.py
@@ -4,6 +4,7 @@ import base64
 import subprocess
 from typing import Optional, List
 
+from selfprivacy_api.utils import get_domain
 from selfprivacy_api.jobs import Job, Jobs
 
 from selfprivacy_api.utils.systemd import get_service_status
@@ -35,6 +36,12 @@ class Nextcloud(Service):
         """Read SVG icon from file and return it as base64 encoded string."""
         return base64.b64encode(NEXTCLOUD_ICON.encode("utf-8")).decode("utf-8")
 
+    @staticmethod
+    def get_url() -> Optional[str]:
+        """Return service url."""
+        domain = get_domain()
+        return f"https://cloud.{domain}"
+
     @staticmethod
     def is_movable() -> bool:
         return True
diff --git a/selfprivacy_api/services/pleroma/__init__.py b/selfprivacy_api/services/pleroma/__init__.py
index 16ee70c..eebd925 100644
--- a/selfprivacy_api/services/pleroma/__init__.py
+++ b/selfprivacy_api/services/pleroma/__init__.py
@@ -4,6 +4,8 @@ import base64
 import subprocess
 from typing import Optional, List
 
+from selfprivacy_api.utils import get_domain
+
 from selfprivacy_api.services.owned_path import OwnedPath
 from selfprivacy_api.utils.systemd import get_service_status
 from selfprivacy_api.services.service import Service, ServiceStatus
@@ -30,6 +32,12 @@ class Pleroma(Service):
     def get_svg_icon() -> str:
         return base64.b64encode(PLEROMA_ICON.encode("utf-8")).decode("utf-8")
 
+    @staticmethod
+    def get_url() -> Optional[str]:
+        """Return service url."""
+        domain = get_domain()
+        return f"https://social.{domain}"
+
     @staticmethod
     def is_movable() -> bool:
         return True
diff --git a/selfprivacy_api/services/roundcube/__init__.py b/selfprivacy_api/services/roundcube/__init__.py
index 7572770..9a52c3a 100644
--- a/selfprivacy_api/services/roundcube/__init__.py
+++ b/selfprivacy_api/services/roundcube/__init__.py
@@ -9,6 +9,7 @@ from selfprivacy_api.utils.systemd import (
     get_service_status_from_several_units,
 )
 from selfprivacy_api.services.service import Service, ServiceStatus
+from selfprivacy_api.utils import get_domain
 from selfprivacy_api.utils.block_devices import BlockDevice
 from selfprivacy_api.services.roundcube.icon import ROUNDCUBE_ICON
 
@@ -36,6 +37,13 @@ class Roundcube(Service):
         """Read SVG icon from file and return it as base64 encoded string."""
         return base64.b64encode(ROUNDCUBE_ICON.encode("utf-8")).decode("utf-8")
 
+    @staticmethod
+    def get_url() -> Optional[str]:
+        """Return service url."""
+        domain = get_domain()
+        subdomain = get_subdomain()
+        return f"https://{subdomain}.{domain}"
+
     @staticmethod
     def is_movable() -> bool:
         return False
diff --git a/selfprivacy_api/services/service.py b/selfprivacy_api/services/service.py
index ed55a0e..262d690 100644
--- a/selfprivacy_api/services/service.py
+++ b/selfprivacy_api/services/service.py
@@ -4,7 +4,6 @@ from abc import ABC, abstractmethod
 from typing import List, Optional
 
 from selfprivacy_api import utils
-from selfprivacy_api.utils import get_domain
 from selfprivacy_api.utils import ReadUserData, WriteUserData
 from selfprivacy_api.utils.waitloop import wait_until_true
 from selfprivacy_api.utils.block_devices import BlockDevice, BlockDevices
@@ -67,13 +66,13 @@ class Service(ABC):
         """
         pass
 
+    @staticmethod
+    @abstractmethod
     def get_url() -> Optional[str]:
         """
         The url of the service if it is accessible from the internet browser.
         """
-        domain = get_domain()
-        subdomain = self.get_subdomain()
-        return f"https://{subdomain}.{domain}"
+        pass
 
     def get_subdomain() -> Optional[str]:
         """

From 8bb916628719c1ace839898fb31241609ee05eea Mon Sep 17 00:00:00 2001
From: Inex Code <inex.code@selfprivacy.org>
Date: Wed, 26 Jun 2024 14:21:49 +0300
Subject: [PATCH 45/86] Revert "fix: remove get sub domain from services"

This reverts commit 46fd7a237c6c1f8be0946313f45deac61a7d2a67.
---
 selfprivacy_api/services/bitwarden/__init__.py    | 5 ++++-
 selfprivacy_api/services/gitea/__init__.py        | 5 ++++-
 selfprivacy_api/services/jitsimeet/__init__.py    | 5 ++++-
 selfprivacy_api/services/nextcloud/__init__.py    | 5 ++++-
 selfprivacy_api/services/ocserv/__init__.py       | 5 ++++-
 selfprivacy_api/services/pleroma/__init__.py      | 5 ++++-
 selfprivacy_api/services/test_service/__init__.py | 5 ++++-
 7 files changed, 28 insertions(+), 7 deletions(-)

diff --git a/selfprivacy_api/services/bitwarden/__init__.py b/selfprivacy_api/services/bitwarden/__init__.py
index 6c44aeb..52f1466 100644
--- a/selfprivacy_api/services/bitwarden/__init__.py
+++ b/selfprivacy_api/services/bitwarden/__init__.py
@@ -1,5 +1,4 @@
 """Class representing Bitwarden service"""
-
 import base64
 import subprocess
 from typing import Optional, List
@@ -44,6 +43,10 @@ class Bitwarden(Service):
         domain = get_domain()
         return f"https://password.{domain}"
 
+    @staticmethod
+    def get_subdomain() -> Optional[str]:
+        return "password"
+
     @staticmethod
     def is_movable() -> bool:
         return True
diff --git a/selfprivacy_api/services/gitea/__init__.py b/selfprivacy_api/services/gitea/__init__.py
index 7029b48..311d59e 100644
--- a/selfprivacy_api/services/gitea/__init__.py
+++ b/selfprivacy_api/services/gitea/__init__.py
@@ -1,5 +1,4 @@
 """Class representing Bitwarden service"""
-
 import base64
 import subprocess
 from typing import Optional, List
@@ -40,6 +39,10 @@ class Gitea(Service):
         domain = get_domain()
         return f"https://git.{domain}"
 
+    @staticmethod
+    def get_subdomain() -> Optional[str]:
+        return "git"
+
     @staticmethod
     def is_movable() -> bool:
         return True
diff --git a/selfprivacy_api/services/jitsimeet/__init__.py b/selfprivacy_api/services/jitsimeet/__init__.py
index 05ef2f7..53d572c 100644
--- a/selfprivacy_api/services/jitsimeet/__init__.py
+++ b/selfprivacy_api/services/jitsimeet/__init__.py
@@ -1,5 +1,4 @@
 """Class representing Jitsi Meet service"""
-
 import base64
 import subprocess
 from typing import Optional, List
@@ -43,6 +42,10 @@ class JitsiMeet(Service):
         domain = get_domain()
         return f"https://meet.{domain}"
 
+    @staticmethod
+    def get_subdomain() -> Optional[str]:
+        return "meet"
+
     @staticmethod
     def is_movable() -> bool:
         return False
diff --git a/selfprivacy_api/services/nextcloud/__init__.py b/selfprivacy_api/services/nextcloud/__init__.py
index 6aee771..3e5b8d3 100644
--- a/selfprivacy_api/services/nextcloud/__init__.py
+++ b/selfprivacy_api/services/nextcloud/__init__.py
@@ -1,5 +1,4 @@
 """Class representing Nextcloud service."""
-
 import base64
 import subprocess
 from typing import Optional, List
@@ -42,6 +41,10 @@ class Nextcloud(Service):
         domain = get_domain()
         return f"https://cloud.{domain}"
 
+    @staticmethod
+    def get_subdomain() -> Optional[str]:
+        return "cloud"
+
     @staticmethod
     def is_movable() -> bool:
         return True
diff --git a/selfprivacy_api/services/ocserv/__init__.py b/selfprivacy_api/services/ocserv/__init__.py
index e21fe6e..4dd802f 100644
--- a/selfprivacy_api/services/ocserv/__init__.py
+++ b/selfprivacy_api/services/ocserv/__init__.py
@@ -1,5 +1,4 @@
 """Class representing ocserv service."""
-
 import base64
 import subprocess
 import typing
@@ -34,6 +33,10 @@ class Ocserv(Service):
         """Return service url."""
         return None
 
+    @staticmethod
+    def get_subdomain() -> typing.Optional[str]:
+        return "vpn"
+
     @staticmethod
     def is_movable() -> bool:
         return False
diff --git a/selfprivacy_api/services/pleroma/__init__.py b/selfprivacy_api/services/pleroma/__init__.py
index eebd925..44a9be8 100644
--- a/selfprivacy_api/services/pleroma/__init__.py
+++ b/selfprivacy_api/services/pleroma/__init__.py
@@ -1,5 +1,4 @@
 """Class representing Nextcloud service."""
-
 import base64
 import subprocess
 from typing import Optional, List
@@ -38,6 +37,10 @@ class Pleroma(Service):
         domain = get_domain()
         return f"https://social.{domain}"
 
+    @staticmethod
+    def get_subdomain() -> Optional[str]:
+        return "social"
+
     @staticmethod
     def is_movable() -> bool:
         return True
diff --git a/selfprivacy_api/services/test_service/__init__.py b/selfprivacy_api/services/test_service/__init__.py
index 2058d57..caf4666 100644
--- a/selfprivacy_api/services/test_service/__init__.py
+++ b/selfprivacy_api/services/test_service/__init__.py
@@ -1,5 +1,4 @@
 """Class representing Bitwarden service"""
-
 import base64
 import typing
 import subprocess
@@ -64,6 +63,10 @@ class DummyService(Service):
         domain = "test.com"
         return f"https://password.{domain}"
 
+    @staticmethod
+    def get_subdomain() -> typing.Optional[str]:
+        return "password"
+
     @classmethod
     def is_movable(cls) -> bool:
         return cls.movable

From c42e2ef3ac068e597378e0788b4a2e3d1b496ecf Mon Sep 17 00:00:00 2001
From: Inex Code <inex.code@selfprivacy.org>
Date: Wed, 26 Jun 2024 14:21:58 +0300
Subject: [PATCH 46/86] Revert "feat: move get_subdomain to parent class
 really"

This reverts commit 4eaefc832113563441528f8fc8011d6959926e30.
---
 selfprivacy_api/services/roundcube/__init__.py | 8 ++++++++
 selfprivacy_api/services/service.py            | 8 +++-----
 2 files changed, 11 insertions(+), 5 deletions(-)

diff --git a/selfprivacy_api/services/roundcube/__init__.py b/selfprivacy_api/services/roundcube/__init__.py
index 9a52c3a..b61282b 100644
--- a/selfprivacy_api/services/roundcube/__init__.py
+++ b/selfprivacy_api/services/roundcube/__init__.py
@@ -44,6 +44,14 @@ class Roundcube(Service):
         subdomain = get_subdomain()
         return f"https://{subdomain}.{domain}"
 
+    @staticmethod
+    def get_subdomain() -> Optional[str]:
+        with ReadUserData() as data:
+            if "roundcube" in data["modules"]:
+                return data["modules"]["roundcube"]["subdomain"]
+
+        return "webmail"
+
     @staticmethod
     def is_movable() -> bool:
         return False
diff --git a/selfprivacy_api/services/service.py b/selfprivacy_api/services/service.py
index 262d690..64a1e80 100644
--- a/selfprivacy_api/services/service.py
+++ b/selfprivacy_api/services/service.py
@@ -1,5 +1,4 @@
 """Abstract class for a service running on a server"""
-
 from abc import ABC, abstractmethod
 from typing import List, Optional
 
@@ -74,14 +73,13 @@ class Service(ABC):
         """
         pass
 
+    @staticmethod
+    @abstractmethod
     def get_subdomain() -> Optional[str]:
         """
         The assigned primary subdomain for this service.
         """
-        with ReadUserData() as data:
-            if self.get_display_name() in data["modules"]:
-                if "subdomain" in data["modules"][self.get_display_name()]:
-                    return data["modules"][self.get_display_name()]["subdomain"]
+        pass
 
     @classmethod
     def get_user(cls) -> Optional[str]:

From 6e0bf4f2a34b9ce55692404d374d4ea1d333bb14 Mon Sep 17 00:00:00 2001
From: Inex Code <inex.code@selfprivacy.org>
Date: Wed, 26 Jun 2024 15:00:51 +0300
Subject: [PATCH 47/86] chore: PR cleanup

---
 selfprivacy_api/migrations/__init__.py        |  6 ++--
 selfprivacy_api/migrations/add_roundcube.py   | 36 +++++++++++++++++++
 .../migrations/add_roundcube_to_userdata.py   | 24 -------------
 .../check_for_system_rebuild_jobs.py          |  9 ++---
 selfprivacy_api/migrations/migration.py       |  8 ++---
 .../migrations/update_services_flake_list.py  | 26 --------------
 .../migrations/write_token_to_redis.py        |  9 ++---
 .../services/bitwarden/__init__.py            |  8 ++---
 .../services/flake_service_manager.py         |  6 ++--
 selfprivacy_api/services/gitea/__init__.py    |  8 ++---
 .../services/jitsimeet/__init__.py            |  8 ++---
 .../services/mailserver/__init__.py           |  8 ++---
 .../services/nextcloud/__init__.py            |  9 +++--
 selfprivacy_api/services/ocserv/__init__.py   |  8 ++---
 selfprivacy_api/services/pleroma/__init__.py  |  8 ++---
 .../services/roundcube/__init__.py            | 16 ++++-----
 selfprivacy_api/services/service.py           |  8 ++---
 .../services/test_service/__init__.py         |  8 ++---
 tests/data/turned_on.json                     |  3 ++
 tests/test_flake_services_manager.py          | 28 +++++++--------
 .../no_services.nix                           |  4 +--
 .../some_services.nix                         | 10 +++---
 22 files changed, 123 insertions(+), 135 deletions(-)
 create mode 100644 selfprivacy_api/migrations/add_roundcube.py
 delete mode 100644 selfprivacy_api/migrations/add_roundcube_to_userdata.py
 delete mode 100644 selfprivacy_api/migrations/update_services_flake_list.py

diff --git a/selfprivacy_api/migrations/__init__.py b/selfprivacy_api/migrations/__init__.py
index badf40a..c7f660d 100644
--- a/selfprivacy_api/migrations/__init__.py
+++ b/selfprivacy_api/migrations/__init__.py
@@ -14,14 +14,12 @@ from selfprivacy_api.migrations.write_token_to_redis import WriteTokenToRedis
 from selfprivacy_api.migrations.check_for_system_rebuild_jobs import (
     CheckForSystemRebuildJobs,
 )
-from selfprivacy_api.migrations.update_services_flake_list import UpdateServicesFlakeList
-from selfprivacy_api.migrations.add_roundcube_to_userdata import AddRoundcubeToUserdata
+from selfprivacy_api.migrations.add_roundcube import AddRoundcube
 
 migrations = [
     WriteTokenToRedis(),
     CheckForSystemRebuildJobs(),
-    UpdateServicesFlakeList(),
-    AddRoundcubeToUserdata(),
+    AddRoundcube(),
 ]
 
 
diff --git a/selfprivacy_api/migrations/add_roundcube.py b/selfprivacy_api/migrations/add_roundcube.py
new file mode 100644
index 0000000..3c422c2
--- /dev/null
+++ b/selfprivacy_api/migrations/add_roundcube.py
@@ -0,0 +1,36 @@
+from selfprivacy_api.migrations.migration import Migration
+
+from selfprivacy_api.services.flake_service_manager import FlakeServiceManager
+from selfprivacy_api.utils import ReadUserData, WriteUserData
+
+
+class AddRoundcube(Migration):
+    """Adds the Roundcube if it is not present."""
+
+    def get_migration_name(self) -> str:
+        return "add_roundcube"
+
+    def get_migration_description(self) -> str:
+        return "Adds the Roundcube if it is not present."
+
+    def is_migration_needed(self) -> bool:
+        with FlakeServiceManager() as manager:
+            if "roundcube" not in manager.services:
+                return True
+        with ReadUserData() as data:
+            if "roundcube" not in data["modules"]:
+                return True
+        return False
+
+    def migrate(self) -> None:
+        with FlakeServiceManager() as manager:
+            if "roundcube" not in manager.services:
+                manager.services[
+                    "roundcube"
+                ] = "git+https://git.selfprivacy.org/SelfPrivacy/selfprivacy-nixos-config.git?ref=flakes&dir=sp-modules/roundcube"
+        with WriteUserData() as data:
+            if "roundcube" not in data["modules"]:
+                data["modules"]["roundcube"] = {
+                    "enable": False,
+                    "subdomain": "roundcube",
+                }
diff --git a/selfprivacy_api/migrations/add_roundcube_to_userdata.py b/selfprivacy_api/migrations/add_roundcube_to_userdata.py
deleted file mode 100644
index 2c6bafb..0000000
--- a/selfprivacy_api/migrations/add_roundcube_to_userdata.py
+++ /dev/null
@@ -1,24 +0,0 @@
-from selfprivacy_api.migrations.migration import Migration
-
-from selfprivacy_api.utils import ReadUserData, WriteUserData
-
-
-class AddRoundcubeToUserdata(Migration):
-    """Add Roundcube to userdata.json if it does not exist"""
-
-    def get_migration_name(self):
-        return "add_roundcube_to_userdata"
-
-    def get_migration_description(self):
-        return "Add Roundcube to userdata.json if it does not exist"
-
-    def is_migration_needed(self):
-        with ReadUserData() as data:
-            if "roundcube" not in data["modules"]:
-                return True
-
-    def migrate(self):
-        with WriteUserData() as data:
-            data["modules"]["roundcube"] = {
-                "subdomain": "roundcube",
-            }
diff --git a/selfprivacy_api/migrations/check_for_system_rebuild_jobs.py b/selfprivacy_api/migrations/check_for_system_rebuild_jobs.py
index 9bbac8a..bb8eb74 100644
--- a/selfprivacy_api/migrations/check_for_system_rebuild_jobs.py
+++ b/selfprivacy_api/migrations/check_for_system_rebuild_jobs.py
@@ -5,13 +5,13 @@ from selfprivacy_api.jobs import JobStatus, Jobs
 class CheckForSystemRebuildJobs(Migration):
     """Check if there are unfinished system rebuild jobs and finish them"""
 
-    def get_migration_name(self):
+    def get_migration_name(self) -> str:
         return "check_for_system_rebuild_jobs"
 
-    def get_migration_description(self):
+    def get_migration_description(self) -> str:
         return "Check if there are unfinished system rebuild jobs and finish them"
 
-    def is_migration_needed(self):
+    def is_migration_needed(self) -> bool:
         # Check if there are any unfinished system rebuild jobs
         for job in Jobs.get_jobs():
             if (
@@ -25,8 +25,9 @@ class CheckForSystemRebuildJobs(Migration):
                 JobStatus.RUNNING,
             ]:
                 return True
+        return False
 
-    def migrate(self):
+    def migrate(self) -> None:
         # As the API is restarted, we assume that the jobs are finished
         for job in Jobs.get_jobs():
             if (
diff --git a/selfprivacy_api/migrations/migration.py b/selfprivacy_api/migrations/migration.py
index 1116672..8eb047d 100644
--- a/selfprivacy_api/migrations/migration.py
+++ b/selfprivacy_api/migrations/migration.py
@@ -12,17 +12,17 @@ class Migration(ABC):
     """
 
     @abstractmethod
-    def get_migration_name(self):
+    def get_migration_name(self) -> str:
         pass
 
     @abstractmethod
-    def get_migration_description(self):
+    def get_migration_description(self) -> str:
         pass
 
     @abstractmethod
-    def is_migration_needed(self):
+    def is_migration_needed(self) -> bool:
         pass
 
     @abstractmethod
-    def migrate(self):
+    def migrate(self) -> None:
         pass
diff --git a/selfprivacy_api/migrations/update_services_flake_list.py b/selfprivacy_api/migrations/update_services_flake_list.py
deleted file mode 100644
index 88f7307..0000000
--- a/selfprivacy_api/migrations/update_services_flake_list.py
+++ /dev/null
@@ -1,26 +0,0 @@
-from selfprivacy_api.migrations.migration import Migration
-from selfprivacy_api.jobs import JobStatus, Jobs
-
-from selfprivacy_api.services.flake_service_manager import FlakeServiceManager
-
-
-class UpdateServicesFlakeList(Migration):
-    """Check if all required services are in the flake list"""
-
-    def get_migration_name(self):
-        return "update_services_flake_list"
-
-    def get_migration_description(self):
-        return "Check if all required services are in the flake list"
-
-    def is_migration_needed(self):
-        with FlakeServiceManager() as manager:
-            if "roundcube" not in manager.services:
-                return True
-
-    def migrate(self):
-        with FlakeServiceManager() as manager:
-            if "roundcube" not in manager.services:
-                manager.services[
-                    "roundcube"
-                ] = "git+https://git.selfprivacy.org/SelfPrivacy/selfprivacy-nixos-config.git?ref=flakes&dir=sp-modules/roundcube"
diff --git a/selfprivacy_api/migrations/write_token_to_redis.py b/selfprivacy_api/migrations/write_token_to_redis.py
index aab4f72..ccf1c04 100644
--- a/selfprivacy_api/migrations/write_token_to_redis.py
+++ b/selfprivacy_api/migrations/write_token_to_redis.py
@@ -15,10 +15,10 @@ from selfprivacy_api.utils import ReadUserData, UserDataFiles
 class WriteTokenToRedis(Migration):
     """Load Json tokens into Redis"""
 
-    def get_migration_name(self):
+    def get_migration_name(self) -> str:
         return "write_token_to_redis"
 
-    def get_migration_description(self):
+    def get_migration_description(self) -> str:
         return "Loads the initial token into redis token storage"
 
     def is_repo_empty(self, repo: AbstractTokensRepository) -> bool:
@@ -38,7 +38,7 @@ class WriteTokenToRedis(Migration):
             print(e)
             return None
 
-    def is_migration_needed(self):
+    def is_migration_needed(self) -> bool:
         try:
             if self.get_token_from_json() is not None and self.is_repo_empty(
                 RedisTokensRepository()
@@ -47,8 +47,9 @@ class WriteTokenToRedis(Migration):
         except Exception as e:
             print(e)
             return False
+        return False
 
-    def migrate(self):
+    def migrate(self) -> None:
         # Write info about providers to userdata.json
         try:
             token = self.get_token_from_json()
diff --git a/selfprivacy_api/services/bitwarden/__init__.py b/selfprivacy_api/services/bitwarden/__init__.py
index 52f1466..56ee6e5 100644
--- a/selfprivacy_api/services/bitwarden/__init__.py
+++ b/selfprivacy_api/services/bitwarden/__init__.py
@@ -37,14 +37,14 @@ class Bitwarden(Service):
     def get_user() -> str:
         return "vaultwarden"
 
-    @staticmethod
-    def get_url() -> Optional[str]:
+    @classmethod
+    def get_url(cls) -> Optional[str]:
         """Return service url."""
         domain = get_domain()
         return f"https://password.{domain}"
 
-    @staticmethod
-    def get_subdomain() -> Optional[str]:
+    @classmethod
+    def get_subdomain(cls) -> Optional[str]:
         return "password"
 
     @staticmethod
diff --git a/selfprivacy_api/services/flake_service_manager.py b/selfprivacy_api/services/flake_service_manager.py
index 63c2279..8b76e5d 100644
--- a/selfprivacy_api/services/flake_service_manager.py
+++ b/selfprivacy_api/services/flake_service_manager.py
@@ -34,20 +34,20 @@ class FlakeServiceManager:
             file.write(
                 """
 {
-    description = "SelfPrivacy NixOS PoC modules/extensions/bundles/packages/etc";\n
+  description = "SelfPrivacy NixOS PoC modules/extensions/bundles/packages/etc";\n
 """
             )
 
             for key, value in self.services.items():
                 file.write(
                     f"""
-    inputs.{key}.url = {value};
+  inputs.{key}.url = {value};
 """
                 )
 
             file.write(
                 """
-    outputs = _: { };
+  outputs = _: { };
 }
 """
             )
diff --git a/selfprivacy_api/services/gitea/__init__.py b/selfprivacy_api/services/gitea/__init__.py
index 311d59e..88df4ed 100644
--- a/selfprivacy_api/services/gitea/__init__.py
+++ b/selfprivacy_api/services/gitea/__init__.py
@@ -33,14 +33,14 @@ class Gitea(Service):
         """Read SVG icon from file and return it as base64 encoded string."""
         return base64.b64encode(GITEA_ICON.encode("utf-8")).decode("utf-8")
 
-    @staticmethod
-    def get_url() -> Optional[str]:
+    @classmethod
+    def get_url(cls) -> Optional[str]:
         """Return service url."""
         domain = get_domain()
         return f"https://git.{domain}"
 
-    @staticmethod
-    def get_subdomain() -> Optional[str]:
+    @classmethod
+    def get_subdomain(cls) -> Optional[str]:
         return "git"
 
     @staticmethod
diff --git a/selfprivacy_api/services/jitsimeet/__init__.py b/selfprivacy_api/services/jitsimeet/__init__.py
index 53d572c..27a497a 100644
--- a/selfprivacy_api/services/jitsimeet/__init__.py
+++ b/selfprivacy_api/services/jitsimeet/__init__.py
@@ -36,14 +36,14 @@ class JitsiMeet(Service):
         """Read SVG icon from file and return it as base64 encoded string."""
         return base64.b64encode(JITSI_ICON.encode("utf-8")).decode("utf-8")
 
-    @staticmethod
-    def get_url() -> Optional[str]:
+    @classmethod
+    def get_url(cls) -> Optional[str]:
         """Return service url."""
         domain = get_domain()
         return f"https://meet.{domain}"
 
-    @staticmethod
-    def get_subdomain() -> Optional[str]:
+    @classmethod
+    def get_subdomain(cls) -> Optional[str]:
         return "meet"
 
     @staticmethod
diff --git a/selfprivacy_api/services/mailserver/__init__.py b/selfprivacy_api/services/mailserver/__init__.py
index d2e9b5d..aba302d 100644
--- a/selfprivacy_api/services/mailserver/__init__.py
+++ b/selfprivacy_api/services/mailserver/__init__.py
@@ -35,13 +35,13 @@ class MailServer(Service):
     def get_user() -> str:
         return "virtualMail"
 
-    @staticmethod
-    def get_url() -> Optional[str]:
+    @classmethod
+    def get_url(cls) -> Optional[str]:
         """Return service url."""
         return None
 
-    @staticmethod
-    def get_subdomain() -> Optional[str]:
+    @classmethod
+    def get_subdomain(cls) -> Optional[str]:
         return None
 
     @staticmethod
diff --git a/selfprivacy_api/services/nextcloud/__init__.py b/selfprivacy_api/services/nextcloud/__init__.py
index 3e5b8d3..275b11d 100644
--- a/selfprivacy_api/services/nextcloud/__init__.py
+++ b/selfprivacy_api/services/nextcloud/__init__.py
@@ -4,7 +4,6 @@ import subprocess
 from typing import Optional, List
 
 from selfprivacy_api.utils import get_domain
-from selfprivacy_api.jobs import Job, Jobs
 
 from selfprivacy_api.utils.systemd import get_service_status
 from selfprivacy_api.services.service import Service, ServiceStatus
@@ -35,14 +34,14 @@ class Nextcloud(Service):
         """Read SVG icon from file and return it as base64 encoded string."""
         return base64.b64encode(NEXTCLOUD_ICON.encode("utf-8")).decode("utf-8")
 
-    @staticmethod
-    def get_url() -> Optional[str]:
+    @classmethod
+    def get_url(cls) -> Optional[str]:
         """Return service url."""
         domain = get_domain()
         return f"https://cloud.{domain}"
 
-    @staticmethod
-    def get_subdomain() -> Optional[str]:
+    @classmethod
+    def get_subdomain(cls) -> Optional[str]:
         return "cloud"
 
     @staticmethod
diff --git a/selfprivacy_api/services/ocserv/__init__.py b/selfprivacy_api/services/ocserv/__init__.py
index 4dd802f..f600772 100644
--- a/selfprivacy_api/services/ocserv/__init__.py
+++ b/selfprivacy_api/services/ocserv/__init__.py
@@ -28,13 +28,13 @@ class Ocserv(Service):
     def get_svg_icon() -> str:
         return base64.b64encode(OCSERV_ICON.encode("utf-8")).decode("utf-8")
 
-    @staticmethod
-    def get_url() -> typing.Optional[str]:
+    @classmethod
+    def get_url(cls) -> typing.Optional[str]:
         """Return service url."""
         return None
 
-    @staticmethod
-    def get_subdomain() -> typing.Optional[str]:
+    @classmethod
+    def get_subdomain(cls) -> typing.Optional[str]:
         return "vpn"
 
     @staticmethod
diff --git a/selfprivacy_api/services/pleroma/__init__.py b/selfprivacy_api/services/pleroma/__init__.py
index 44a9be8..64edd96 100644
--- a/selfprivacy_api/services/pleroma/__init__.py
+++ b/selfprivacy_api/services/pleroma/__init__.py
@@ -31,14 +31,14 @@ class Pleroma(Service):
     def get_svg_icon() -> str:
         return base64.b64encode(PLEROMA_ICON.encode("utf-8")).decode("utf-8")
 
-    @staticmethod
-    def get_url() -> Optional[str]:
+    @classmethod
+    def get_url(cls) -> Optional[str]:
         """Return service url."""
         domain = get_domain()
         return f"https://social.{domain}"
 
-    @staticmethod
-    def get_subdomain() -> Optional[str]:
+    @classmethod
+    def get_subdomain(cls) -> Optional[str]:
         return "social"
 
     @staticmethod
diff --git a/selfprivacy_api/services/roundcube/__init__.py b/selfprivacy_api/services/roundcube/__init__.py
index b61282b..22604f5 100644
--- a/selfprivacy_api/services/roundcube/__init__.py
+++ b/selfprivacy_api/services/roundcube/__init__.py
@@ -2,14 +2,14 @@
 
 import base64
 import subprocess
-from typing import Optional, List
+from typing import List, Optional
 
 from selfprivacy_api.jobs import Job
 from selfprivacy_api.utils.systemd import (
     get_service_status_from_several_units,
 )
 from selfprivacy_api.services.service import Service, ServiceStatus
-from selfprivacy_api.utils import get_domain
+from selfprivacy_api.utils import ReadUserData, get_domain
 from selfprivacy_api.utils.block_devices import BlockDevice
 from selfprivacy_api.services.roundcube.icon import ROUNDCUBE_ICON
 
@@ -37,20 +37,20 @@ class Roundcube(Service):
         """Read SVG icon from file and return it as base64 encoded string."""
         return base64.b64encode(ROUNDCUBE_ICON.encode("utf-8")).decode("utf-8")
 
-    @staticmethod
-    def get_url() -> Optional[str]:
+    @classmethod
+    def get_url(cls) -> Optional[str]:
         """Return service url."""
         domain = get_domain()
-        subdomain = get_subdomain()
+        subdomain = cls.get_subdomain()
         return f"https://{subdomain}.{domain}"
 
-    @staticmethod
-    def get_subdomain() -> Optional[str]:
+    @classmethod
+    def get_subdomain(cls) -> Optional[str]:
         with ReadUserData() as data:
             if "roundcube" in data["modules"]:
                 return data["modules"]["roundcube"]["subdomain"]
 
-        return "webmail"
+        return "roundcube"
 
     @staticmethod
     def is_movable() -> bool:
diff --git a/selfprivacy_api/services/service.py b/selfprivacy_api/services/service.py
index 64a1e80..6e3decf 100644
--- a/selfprivacy_api/services/service.py
+++ b/selfprivacy_api/services/service.py
@@ -65,17 +65,17 @@ class Service(ABC):
         """
         pass
 
-    @staticmethod
+    @classmethod
     @abstractmethod
-    def get_url() -> Optional[str]:
+    def get_url(cls) -> Optional[str]:
         """
         The url of the service if it is accessible from the internet browser.
         """
         pass
 
-    @staticmethod
+    @classmethod
     @abstractmethod
-    def get_subdomain() -> Optional[str]:
+    def get_subdomain(cls) -> Optional[str]:
         """
         The assigned primary subdomain for this service.
         """
diff --git a/selfprivacy_api/services/test_service/__init__.py b/selfprivacy_api/services/test_service/__init__.py
index caf4666..de3c493 100644
--- a/selfprivacy_api/services/test_service/__init__.py
+++ b/selfprivacy_api/services/test_service/__init__.py
@@ -57,14 +57,14 @@ class DummyService(Service):
         # return ""
         return base64.b64encode(BITWARDEN_ICON.encode("utf-8")).decode("utf-8")
 
-    @staticmethod
-    def get_url() -> typing.Optional[str]:
+    @classmethod
+    def get_url(cls) -> typing.Optional[str]:
         """Return service url."""
         domain = "test.com"
         return f"https://password.{domain}"
 
-    @staticmethod
-    def get_subdomain() -> typing.Optional[str]:
+    @classmethod
+    def get_subdomain(cls) -> typing.Optional[str]:
         return "password"
 
     @classmethod
diff --git a/tests/data/turned_on.json b/tests/data/turned_on.json
index 0bcc2f0..9c285b1 100644
--- a/tests/data/turned_on.json
+++ b/tests/data/turned_on.json
@@ -62,6 +62,9 @@
         "simple-nixos-mailserver": {
             "enable": true,
             "location": "sdb"
+        },
+        "roundcube": {
+            "enable": true
         }
     },
     "volumes": [
diff --git a/tests/test_flake_services_manager.py b/tests/test_flake_services_manager.py
index 4650b6d..93c6e1d 100644
--- a/tests/test_flake_services_manager.py
+++ b/tests/test_flake_services_manager.py
@@ -4,40 +4,40 @@ from selfprivacy_api.services.flake_service_manager import FlakeServiceManager
 
 all_services_file = """
 {
-    description = "SelfPrivacy NixOS PoC modules/extensions/bundles/packages/etc";
+  description = "SelfPrivacy NixOS PoC modules/extensions/bundles/packages/etc";
 
 
-    inputs.bitwarden.url = git+https://git.selfprivacy.org/SelfPrivacy/selfprivacy-nixos-config.git?ref=flakes&dir=sp-modules/bitwarden;
+  inputs.bitwarden.url = git+https://git.selfprivacy.org/SelfPrivacy/selfprivacy-nixos-config.git?ref=flakes&dir=sp-modules/bitwarden;
 
-    inputs.gitea.url = git+https://git.selfprivacy.org/SelfPrivacy/selfprivacy-nixos-config.git?ref=flakes&dir=sp-modules/gitea;
+  inputs.gitea.url = git+https://git.selfprivacy.org/SelfPrivacy/selfprivacy-nixos-config.git?ref=flakes&dir=sp-modules/gitea;
 
-    inputs.jitsi-meet.url = git+https://git.selfprivacy.org/SelfPrivacy/selfprivacy-nixos-config.git?ref=flakes&dir=sp-modules/jitsi-meet;
+  inputs.jitsi-meet.url = git+https://git.selfprivacy.org/SelfPrivacy/selfprivacy-nixos-config.git?ref=flakes&dir=sp-modules/jitsi-meet;
 
-    inputs.nextcloud.url = git+https://git.selfprivacy.org/SelfPrivacy/selfprivacy-nixos-config.git?ref=flakes&dir=sp-modules/nextcloud;
+  inputs.nextcloud.url = git+https://git.selfprivacy.org/SelfPrivacy/selfprivacy-nixos-config.git?ref=flakes&dir=sp-modules/nextcloud;
 
-    inputs.ocserv.url = git+https://git.selfprivacy.org/SelfPrivacy/selfprivacy-nixos-config.git?ref=flakes&dir=sp-modules/ocserv;
+  inputs.ocserv.url = git+https://git.selfprivacy.org/SelfPrivacy/selfprivacy-nixos-config.git?ref=flakes&dir=sp-modules/ocserv;
 
-    inputs.pleroma.url = git+https://git.selfprivacy.org/SelfPrivacy/selfprivacy-nixos-config.git?ref=flakes&dir=sp-modules/pleroma;
+  inputs.pleroma.url = git+https://git.selfprivacy.org/SelfPrivacy/selfprivacy-nixos-config.git?ref=flakes&dir=sp-modules/pleroma;
 
-    inputs.simple-nixos-mailserver.url = git+https://git.selfprivacy.org/SelfPrivacy/selfprivacy-nixos-config.git?ref=flakes&dir=sp-modules/simple-nixos-mailserver;
+  inputs.simple-nixos-mailserver.url = git+https://git.selfprivacy.org/SelfPrivacy/selfprivacy-nixos-config.git?ref=flakes&dir=sp-modules/simple-nixos-mailserver;
 
-    outputs = _: { };
+  outputs = _: { };
 }
 """
 
 
 some_services_file = """
 {
-    description = "SelfPrivacy NixOS PoC modules/extensions/bundles/packages/etc";
+  description = "SelfPrivacy NixOS PoC modules/extensions/bundles/packages/etc";
 
 
-    inputs.bitwarden.url = git+https://git.selfprivacy.org/SelfPrivacy/selfprivacy-nixos-config.git?ref=flakes&dir=sp-modules/bitwarden;
+  inputs.bitwarden.url = git+https://git.selfprivacy.org/SelfPrivacy/selfprivacy-nixos-config.git?ref=flakes&dir=sp-modules/bitwarden;
 
-    inputs.gitea.url = git+https://git.selfprivacy.org/SelfPrivacy/selfprivacy-nixos-config.git?ref=flakes&dir=sp-modules/gitea;
+  inputs.gitea.url = git+https://git.selfprivacy.org/SelfPrivacy/selfprivacy-nixos-config.git?ref=flakes&dir=sp-modules/gitea;
 
-    inputs.jitsi-meet.url = git+https://git.selfprivacy.org/SelfPrivacy/selfprivacy-nixos-config.git?ref=flakes&dir=sp-modules/jitsi-meet;
+  inputs.jitsi-meet.url = git+https://git.selfprivacy.org/SelfPrivacy/selfprivacy-nixos-config.git?ref=flakes&dir=sp-modules/jitsi-meet;
 
-    outputs = _: { };
+  outputs = _: { };
 }
 """
 
diff --git a/tests/test_flake_services_manager/no_services.nix b/tests/test_flake_services_manager/no_services.nix
index 5967016..8588bc7 100644
--- a/tests/test_flake_services_manager/no_services.nix
+++ b/tests/test_flake_services_manager/no_services.nix
@@ -1,4 +1,4 @@
 {
-    description = "SelfPrivacy NixOS PoC modules/extensions/bundles/packages/etc";
-    outputs = _: { };
+  description = "SelfPrivacy NixOS PoC modules/extensions/bundles/packages/etc";
+  outputs = _: { };
 }
diff --git a/tests/test_flake_services_manager/some_services.nix b/tests/test_flake_services_manager/some_services.nix
index 4bbb919..8c2e6af 100644
--- a/tests/test_flake_services_manager/some_services.nix
+++ b/tests/test_flake_services_manager/some_services.nix
@@ -1,12 +1,12 @@
 {
-    description = "SelfPrivacy NixOS PoC modules/extensions/bundles/packages/etc";
+  description = "SelfPrivacy NixOS PoC modules/extensions/bundles/packages/etc";
 
 
-    inputs.bitwarden.url = git+https://git.selfprivacy.org/SelfPrivacy/selfprivacy-nixos-config.git?ref=flakes&dir=sp-modules/bitwarden;
+  inputs.bitwarden.url = git+https://git.selfprivacy.org/SelfPrivacy/selfprivacy-nixos-config.git?ref=flakes&dir=sp-modules/bitwarden;
 
-    inputs.gitea.url = git+https://git.selfprivacy.org/SelfPrivacy/selfprivacy-nixos-config.git?ref=flakes&dir=sp-modules/gitea;
+  inputs.gitea.url = git+https://git.selfprivacy.org/SelfPrivacy/selfprivacy-nixos-config.git?ref=flakes&dir=sp-modules/gitea;
 
-    inputs.jitsi-meet.url = git+https://git.selfprivacy.org/SelfPrivacy/selfprivacy-nixos-config.git?ref=flakes&dir=sp-modules/jitsi-meet;
+  inputs.jitsi-meet.url = git+https://git.selfprivacy.org/SelfPrivacy/selfprivacy-nixos-config.git?ref=flakes&dir=sp-modules/jitsi-meet;
 
-    outputs = _: { };
+  outputs = _: { };
 }

From 7522c2d796ac8060843e2824851dc67ca19d7a8c Mon Sep 17 00:00:00 2001
From: Inex Code <inex.code@selfprivacy.org>
Date: Sun, 30 Jun 2024 23:02:07 +0400
Subject: [PATCH 48/86] refactor: Change gitea to Forgejo

---
 selfprivacy_api/jobs/migrate_to_binds.py      |  6 ++---
 selfprivacy_api/services/__init__.py          |  4 +--
 .../services/{gitea => forgejo}/__init__.py   | 26 +++++++++++--------
 .../services/{gitea => forgejo}/gitea.svg     |  0
 .../services/{gitea => forgejo}/icon.py       |  2 +-
 tests/test_services_systemctl.py              |  8 +++---
 6 files changed, 25 insertions(+), 21 deletions(-)
 rename selfprivacy_api/services/{gitea => forgejo}/__init__.py (72%)
 rename selfprivacy_api/services/{gitea => forgejo}/gitea.svg (100%)
 rename selfprivacy_api/services/{gitea => forgejo}/icon.py (98%)

diff --git a/selfprivacy_api/jobs/migrate_to_binds.py b/selfprivacy_api/jobs/migrate_to_binds.py
index 3250c9a..782b361 100644
--- a/selfprivacy_api/jobs/migrate_to_binds.py
+++ b/selfprivacy_api/jobs/migrate_to_binds.py
@@ -6,7 +6,7 @@ import shutil
 from pydantic import BaseModel
 from selfprivacy_api.jobs import Job, JobStatus, Jobs
 from selfprivacy_api.services.bitwarden import Bitwarden
-from selfprivacy_api.services.gitea import Gitea
+from selfprivacy_api.services.forgejo import Forgejo
 from selfprivacy_api.services.mailserver import MailServer
 from selfprivacy_api.services.nextcloud import Nextcloud
 from selfprivacy_api.services.pleroma import Pleroma
@@ -230,7 +230,7 @@ def migrate_to_binds(config: BindMigrationConfig, job: Job):
         status_text="Migrating Gitea.",
     )
 
-    Gitea().stop()
+    Forgejo().stop()
 
     if not pathlib.Path("/volumes/sda1/gitea").exists():
         if not pathlib.Path("/volumes/sdb/gitea").exists():
@@ -241,7 +241,7 @@ def migrate_to_binds(config: BindMigrationConfig, job: Job):
                 group="gitea",
             )
 
-    Gitea().start()
+    Forgejo().start()
 
     # Perform migration of Mail server
 
diff --git a/selfprivacy_api/services/__init__.py b/selfprivacy_api/services/__init__.py
index f9dfac2..da02eba 100644
--- a/selfprivacy_api/services/__init__.py
+++ b/selfprivacy_api/services/__init__.py
@@ -2,7 +2,7 @@
 
 import typing
 from selfprivacy_api.services.bitwarden import Bitwarden
-from selfprivacy_api.services.gitea import Gitea
+from selfprivacy_api.services.forgejo import Forgejo
 from selfprivacy_api.services.jitsimeet import JitsiMeet
 from selfprivacy_api.services.mailserver import MailServer
 from selfprivacy_api.services.nextcloud import Nextcloud
@@ -13,7 +13,7 @@ import selfprivacy_api.utils.network as network_utils
 
 services: list[Service] = [
     Bitwarden(),
-    Gitea(),
+    Forgejo(),
     MailServer(),
     Nextcloud(),
     Pleroma(),
diff --git a/selfprivacy_api/services/gitea/__init__.py b/selfprivacy_api/services/forgejo/__init__.py
similarity index 72%
rename from selfprivacy_api/services/gitea/__init__.py
rename to selfprivacy_api/services/forgejo/__init__.py
index 311d59e..d035736 100644
--- a/selfprivacy_api/services/gitea/__init__.py
+++ b/selfprivacy_api/services/forgejo/__init__.py
@@ -7,31 +7,34 @@ from selfprivacy_api.utils import get_domain
 
 from selfprivacy_api.utils.systemd import get_service_status
 from selfprivacy_api.services.service import Service, ServiceStatus
-from selfprivacy_api.services.gitea.icon import GITEA_ICON
+from selfprivacy_api.services.forgejo.icon import FORGEJO_ICON
 
 
-class Gitea(Service):
-    """Class representing Gitea service"""
+class Forgejo(Service):
+    """Class representing Forgejo service.
+
+    Previously was Gitea, so some IDs are still called gitea for compatibility.
+    """
 
     @staticmethod
     def get_id() -> str:
-        """Return service id."""
+        """Return service id. For compatibility keep in gitea."""
         return "gitea"
 
     @staticmethod
     def get_display_name() -> str:
         """Return service display name."""
-        return "Gitea"
+        return "Forgejo"
 
     @staticmethod
     def get_description() -> str:
         """Return service description."""
-        return "Gitea is a Git forge."
+        return "Forgejo is a Git forge."
 
     @staticmethod
     def get_svg_icon() -> str:
         """Read SVG icon from file and return it as base64 encoded string."""
-        return base64.b64encode(GITEA_ICON.encode("utf-8")).decode("utf-8")
+        return base64.b64encode(FORGEJO_ICON.encode("utf-8")).decode("utf-8")
 
     @staticmethod
     def get_url() -> Optional[str]:
@@ -65,19 +68,19 @@ class Gitea(Service):
         Return code 3 means service is stopped.
         Return code 4 means service is off.
         """
-        return get_service_status("gitea.service")
+        return get_service_status("forgejo.service")
 
     @staticmethod
     def stop():
-        subprocess.run(["systemctl", "stop", "gitea.service"])
+        subprocess.run(["systemctl", "stop", "forgejo.service"])
 
     @staticmethod
     def start():
-        subprocess.run(["systemctl", "start", "gitea.service"])
+        subprocess.run(["systemctl", "start", "forgejo.service"])
 
     @staticmethod
     def restart():
-        subprocess.run(["systemctl", "restart", "gitea.service"])
+        subprocess.run(["systemctl", "restart", "forgejo.service"])
 
     @staticmethod
     def get_configuration():
@@ -93,4 +96,5 @@ class Gitea(Service):
 
     @staticmethod
     def get_folders() -> List[str]:
+        """The data folder is still called gitea for compatibility."""
         return ["/var/lib/gitea"]
diff --git a/selfprivacy_api/services/gitea/gitea.svg b/selfprivacy_api/services/forgejo/gitea.svg
similarity index 100%
rename from selfprivacy_api/services/gitea/gitea.svg
rename to selfprivacy_api/services/forgejo/gitea.svg
diff --git a/selfprivacy_api/services/gitea/icon.py b/selfprivacy_api/services/forgejo/icon.py
similarity index 98%
rename from selfprivacy_api/services/gitea/icon.py
rename to selfprivacy_api/services/forgejo/icon.py
index 569f96a..5e600cf 100644
--- a/selfprivacy_api/services/gitea/icon.py
+++ b/selfprivacy_api/services/forgejo/icon.py
@@ -1,4 +1,4 @@
-GITEA_ICON = """
+FORGEJO_ICON = """
 <svg width="24" height="24" viewBox="0 0 24 24" fill="none" xmlns="http://www.w3.org/2000/svg">
 <path d="M2.60007 10.5899L8.38007 4.79995L10.0701 6.49995C9.83007 7.34995 10.2201 8.27995 11.0001 8.72995V14.2699C10.4001 14.6099 10.0001 15.2599 10.0001 15.9999C10.0001 16.5304 10.2108 17.0391 10.5859 17.4142C10.9609 17.7892 11.4696 17.9999 12.0001 17.9999C12.5305 17.9999 13.0392 17.7892 13.4143 17.4142C13.7894 17.0391 14.0001 16.5304 14.0001 15.9999C14.0001 15.2599 13.6001 14.6099 13.0001 14.2699V9.40995L15.0701 11.4999C15.0001 11.6499 15.0001 11.8199 15.0001 11.9999C15.0001 12.5304 15.2108 13.0391 15.5859 13.4142C15.9609 13.7892 16.4696 13.9999 17.0001 13.9999C17.5305 13.9999 18.0392 13.7892 18.4143 13.4142C18.7894 13.0391 19.0001 12.5304 19.0001 11.9999C19.0001 11.4695 18.7894 10.9608 18.4143 10.5857C18.0392 10.2107 17.5305 9.99995 17.0001 9.99995C16.8201 9.99995 16.6501 9.99995 16.5001 10.0699L13.9301 7.49995C14.1901 6.56995 13.7101 5.54995 12.7801 5.15995C12.3501 4.99995 11.9001 4.95995 11.5001 5.06995L9.80007 3.37995L10.5901 2.59995C11.3701 1.80995 12.6301 1.80995 13.4101 2.59995L21.4001 10.5899C22.1901 11.3699 22.1901 12.6299 21.4001 13.4099L13.4101 21.3999C12.6301 22.1899 11.3701 22.1899 10.5901 21.3999L2.60007 13.4099C1.81007 12.6299 1.81007 11.3699 2.60007 10.5899Z" fill="black"/>
 </svg>
diff --git a/tests/test_services_systemctl.py b/tests/test_services_systemctl.py
index 8b247e0..43805e8 100644
--- a/tests/test_services_systemctl.py
+++ b/tests/test_services_systemctl.py
@@ -2,7 +2,7 @@ import pytest
 
 from selfprivacy_api.services.service import ServiceStatus
 from selfprivacy_api.services.bitwarden import Bitwarden
-from selfprivacy_api.services.gitea import Gitea
+from selfprivacy_api.services.forgejo import Forgejo
 from selfprivacy_api.services.mailserver import MailServer
 from selfprivacy_api.services.nextcloud import Nextcloud
 from selfprivacy_api.services.ocserv import Ocserv
@@ -22,7 +22,7 @@ def call_args_asserts(mocked_object):
             "dovecot2.service",
             "postfix.service",
             "vaultwarden.service",
-            "gitea.service",
+            "forgejo.service",
             "phpfpm-nextcloud.service",
             "ocserv.service",
             "pleroma.service",
@@ -77,7 +77,7 @@ def mock_popen_systemctl_service_not_ok(mocker):
 def test_systemctl_ok(mock_popen_systemctl_service_ok):
     assert MailServer.get_status() == ServiceStatus.ACTIVE
     assert Bitwarden.get_status() == ServiceStatus.ACTIVE
-    assert Gitea.get_status() == ServiceStatus.ACTIVE
+    assert Forgejo.get_status() == ServiceStatus.ACTIVE
     assert Nextcloud.get_status() == ServiceStatus.ACTIVE
     assert Ocserv.get_status() == ServiceStatus.ACTIVE
     assert Pleroma.get_status() == ServiceStatus.ACTIVE
@@ -87,7 +87,7 @@ def test_systemctl_ok(mock_popen_systemctl_service_ok):
 def test_systemctl_failed_service(mock_popen_systemctl_service_not_ok):
     assert MailServer.get_status() == ServiceStatus.FAILED
     assert Bitwarden.get_status() == ServiceStatus.FAILED
-    assert Gitea.get_status() == ServiceStatus.FAILED
+    assert Forgejo.get_status() == ServiceStatus.FAILED
     assert Nextcloud.get_status() == ServiceStatus.FAILED
     assert Ocserv.get_status() == ServiceStatus.FAILED
     assert Pleroma.get_status() == ServiceStatus.FAILED

From 4066be38ec11aabf47b03afd35778a53c6d28942 Mon Sep 17 00:00:00 2001
From: Inex Code <inex.code@selfprivacy.org>
Date: Mon, 1 Jul 2024 19:25:54 +0400
Subject: [PATCH 49/86] chore: Bump version to 3.2.2

---
 selfprivacy_api/dependencies.py | 2 +-
 setup.py                        | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/selfprivacy_api/dependencies.py b/selfprivacy_api/dependencies.py
index b9d0904..69ce319 100644
--- a/selfprivacy_api/dependencies.py
+++ b/selfprivacy_api/dependencies.py
@@ -27,4 +27,4 @@ async def get_token_header(
 
 def get_api_version() -> str:
     """Get API version"""
-    return "3.2.1"
+    return "3.2.2"
diff --git a/setup.py b/setup.py
index 473ece8..23c544e 100644
--- a/setup.py
+++ b/setup.py
@@ -2,7 +2,7 @@ from setuptools import setup, find_packages
 
 setup(
     name="selfprivacy_api",
-    version="3.2.1",
+    version="3.2.2",
     packages=find_packages(),
     scripts=[
         "selfprivacy_api/app.py",

From b6118465a071a88577a1e6b2bfa59524b4094ecb Mon Sep 17 00:00:00 2001
From: Houkime <>
Date: Mon, 1 Apr 2024 20:12:02 +0000
Subject: [PATCH 50/86] feature(redis): async connections

---
 selfprivacy_api/utils/redis_pool.py | 19 ++++++++++++-----
 tests/test_redis.py                 | 33 +++++++++++++++++++++++++++++
 2 files changed, 47 insertions(+), 5 deletions(-)
 create mode 100644 tests/test_redis.py

diff --git a/selfprivacy_api/utils/redis_pool.py b/selfprivacy_api/utils/redis_pool.py
index 3d35f01..04ccb51 100644
--- a/selfprivacy_api/utils/redis_pool.py
+++ b/selfprivacy_api/utils/redis_pool.py
@@ -2,6 +2,7 @@
 Redis pool module for selfprivacy_api
 """
 import redis
+import redis.asyncio as redis_async
 
 from selfprivacy_api.utils.singleton_metaclass import SingletonMetaclass
 
@@ -14,11 +15,18 @@ class RedisPool(metaclass=SingletonMetaclass):
     """
 
     def __init__(self):
+        url = RedisPool.connection_url(dbnumber=0)
+        # We need a normal sync pool because otherwise
+        # our whole API will need to be async
         self._pool = redis.ConnectionPool.from_url(
-            RedisPool.connection_url(dbnumber=0),
+            url,
+            decode_responses=True,
+        )
+        # We need an async pool for pubsub
+        self._async_pool = redis_async.ConnectionPool.from_url(
+            url,
             decode_responses=True,
         )
-        self._pubsub_connection = self.get_connection()
 
     @staticmethod
     def connection_url(dbnumber: int) -> str:
@@ -34,8 +42,9 @@ class RedisPool(metaclass=SingletonMetaclass):
         """
         return redis.Redis(connection_pool=self._pool)
 
-    def get_pubsub(self):
+    def get_connection_async(self) -> redis_async.Redis:
         """
-        Get a pubsub connection from the pool.
+        Get an async connection from the pool.
+        Async connections allow pubsub.
         """
-        return self._pubsub_connection.pubsub()
+        return redis_async.Redis(connection_pool=self._async_pool)
diff --git a/tests/test_redis.py b/tests/test_redis.py
new file mode 100644
index 0000000..48ec56e
--- /dev/null
+++ b/tests/test_redis.py
@@ -0,0 +1,33 @@
+import asyncio
+import pytest
+
+from selfprivacy_api.utils.redis_pool import RedisPool
+
+TEST_KEY = "test:test"
+
+
+@pytest.fixture()
+def empty_redis():
+    r = RedisPool().get_connection()
+    r.flushdb()
+    yield r
+    r.flushdb()
+
+
+async def write_to_test_key():
+    r = RedisPool().get_connection_async()
+    async with r.pipeline(transaction=True) as pipe:
+        ok1, ok2 = await pipe.set(TEST_KEY, "value1").set(TEST_KEY, "value2").execute()
+        assert ok1
+        assert ok2
+    assert await r.get(TEST_KEY) == "value2"
+    await r.close()
+
+
+def test_async_connection(empty_redis):
+    r = RedisPool().get_connection()
+    assert not r.exists(TEST_KEY)
+    # It _will_ report an error if it arises
+    asyncio.run(write_to_test_key())
+    # Confirming that we can read result from sync connection too
+    assert r.get(TEST_KEY) == "value2"

From 94386fc53d33656d40101a9977ea06fa51c07b8f Mon Sep 17 00:00:00 2001
From: Houkime <>
Date: Mon, 15 Apr 2024 13:35:44 +0000
Subject: [PATCH 51/86] chore(nixos): add pytest-asyncio

---
 flake.nix | 1 +
 1 file changed, 1 insertion(+)

diff --git a/flake.nix b/flake.nix
index f8b81aa..ab969a4 100644
--- a/flake.nix
+++ b/flake.nix
@@ -20,6 +20,7 @@
             pytest-datadir
             pytest-mock
             pytest-subprocess
+            pytest-asyncio
             black
             mypy
             pylsp-mypy

From f08dc3ad232a16cc70f8b22fd1db08dcb58e37a6 Mon Sep 17 00:00:00 2001
From: Houkime <>
Date: Mon, 15 Apr 2024 13:37:04 +0000
Subject: [PATCH 52/86] test(async): pubsub

---
 selfprivacy_api/utils/redis_pool.py | 12 +++++-
 tests/test_redis.py                 | 64 ++++++++++++++++++++++++++++-
 2 files changed, 73 insertions(+), 3 deletions(-)

diff --git a/selfprivacy_api/utils/redis_pool.py b/selfprivacy_api/utils/redis_pool.py
index 04ccb51..ea827d1 100644
--- a/selfprivacy_api/utils/redis_pool.py
+++ b/selfprivacy_api/utils/redis_pool.py
@@ -9,13 +9,15 @@ from selfprivacy_api.utils.singleton_metaclass import SingletonMetaclass
 REDIS_SOCKET = "/run/redis-sp-api/redis.sock"
 
 
-class RedisPool(metaclass=SingletonMetaclass):
+# class RedisPool(metaclass=SingletonMetaclass):
+class RedisPool:
     """
     Redis connection pool singleton.
     """
 
     def __init__(self):
-        url = RedisPool.connection_url(dbnumber=0)
+        self._dbnumber = 0
+        url = RedisPool.connection_url(dbnumber=self._dbnumber)
         # We need a normal sync pool because otherwise
         # our whole API will need to be async
         self._pool = redis.ConnectionPool.from_url(
@@ -48,3 +50,9 @@ class RedisPool(metaclass=SingletonMetaclass):
         Async connections allow pubsub.
         """
         return redis_async.Redis(connection_pool=self._async_pool)
+
+    async def subscribe_to_keys(self, pattern: str) -> redis_async.client.PubSub:
+        async_redis = self.get_connection_async()
+        pubsub = async_redis.pubsub()
+        await pubsub.psubscribe(f"__keyspace@{self._dbnumber}__:" + pattern)
+        return pubsub
diff --git a/tests/test_redis.py b/tests/test_redis.py
index 48ec56e..2def280 100644
--- a/tests/test_redis.py
+++ b/tests/test_redis.py
@@ -1,13 +1,18 @@
 import asyncio
 import pytest
+import pytest_asyncio
+from asyncio import streams
+import redis
+from typing import List
 
 from selfprivacy_api.utils.redis_pool import RedisPool
 
 TEST_KEY = "test:test"
+STOPWORD = "STOP"
 
 
 @pytest.fixture()
-def empty_redis():
+def empty_redis(event_loop):
     r = RedisPool().get_connection()
     r.flushdb()
     yield r
@@ -31,3 +36,60 @@ def test_async_connection(empty_redis):
     asyncio.run(write_to_test_key())
     # Confirming that we can read result from sync connection too
     assert r.get(TEST_KEY) == "value2"
+
+
+async def channel_reader(channel: redis.client.PubSub) -> List[dict]:
+    result: List[dict] = []
+    while True:
+        # Mypy cannot correctly detect that it is a coroutine
+        # But it is
+        message: dict = await channel.get_message(ignore_subscribe_messages=True, timeout=None)  # type: ignore
+        if message is not None:
+            result.append(message)
+            if message["data"] == STOPWORD:
+                break
+    return result
+
+
+@pytest.mark.asyncio
+async def test_pubsub(empty_redis, event_loop):
+    # Adapted from :
+    # https://redis.readthedocs.io/en/stable/examples/asyncio_examples.html
+    # Sanity checking because of previous event loop bugs
+    assert event_loop == asyncio.get_event_loop()
+    assert event_loop == asyncio.events.get_event_loop()
+    assert event_loop == asyncio.events._get_event_loop()
+    assert event_loop == asyncio.events.get_running_loop()
+
+    reader = streams.StreamReader(34)
+    assert event_loop == reader._loop
+    f = reader._loop.create_future()
+    f.set_result(3)
+    await f
+
+    r = RedisPool().get_connection_async()
+    async with r.pubsub() as pubsub:
+        await pubsub.subscribe("channel:1")
+        future = asyncio.create_task(channel_reader(pubsub))
+
+        await r.publish("channel:1", "Hello")
+        # message: dict = await pubsub.get_message(ignore_subscribe_messages=True, timeout=5.0) # type: ignore
+        # raise ValueError(message)
+        await r.publish("channel:1", "World")
+        await r.publish("channel:1", STOPWORD)
+
+        messages = await future
+
+        assert len(messages) == 3
+
+        message = messages[0]
+        assert "data" in message.keys()
+        assert message["data"] == "Hello"
+        message = messages[1]
+        assert "data" in message.keys()
+        assert message["data"] == "World"
+        message = messages[2]
+        assert "data" in message.keys()
+        assert message["data"] == STOPWORD
+
+    await r.close()

From 5558577927a4711b06890349117d86f694bd2704 Mon Sep 17 00:00:00 2001
From: Houkime <>
Date: Mon, 22 Apr 2024 14:40:55 +0000
Subject: [PATCH 53/86] test(redis): test key event notifications

---
 tests/test_redis.py | 48 +++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 48 insertions(+)

diff --git a/tests/test_redis.py b/tests/test_redis.py
index 2def280..181d325 100644
--- a/tests/test_redis.py
+++ b/tests/test_redis.py
@@ -15,6 +15,8 @@ STOPWORD = "STOP"
 def empty_redis(event_loop):
     r = RedisPool().get_connection()
     r.flushdb()
+    r.config_set("notify-keyspace-events", "KEA")
+    assert r.config_get("notify-keyspace-events")["notify-keyspace-events"] == "AKE"
     yield r
     r.flushdb()
 
@@ -51,6 +53,15 @@ async def channel_reader(channel: redis.client.PubSub) -> List[dict]:
     return result
 
 
+async def channel_reader_onemessage(channel: redis.client.PubSub) -> dict:
+    while True:
+        # Mypy cannot correctly detect that it is a coroutine
+        # But it is
+        message: dict = await channel.get_message(ignore_subscribe_messages=True, timeout=None)  # type: ignore
+        if message is not None:
+            return message
+
+
 @pytest.mark.asyncio
 async def test_pubsub(empty_redis, event_loop):
     # Adapted from :
@@ -93,3 +104,40 @@ async def test_pubsub(empty_redis, event_loop):
         assert message["data"] == STOPWORD
 
     await r.close()
+
+
+@pytest.mark.asyncio
+async def test_keyspace_notifications_simple(empty_redis, event_loop):
+    r = RedisPool().get_connection_async()
+    await r.set(TEST_KEY, "I am not empty")
+    async with r.pubsub() as pubsub:
+        await pubsub.subscribe("__keyspace@0__:" + TEST_KEY)
+
+        future_message = asyncio.create_task(channel_reader_onemessage(pubsub))
+        empty_redis.set(TEST_KEY, "I am set!")
+        message = await future_message
+        assert message is not None
+        assert message["data"] is not None
+        assert message == {
+            "channel": f"__keyspace@0__:{TEST_KEY}",
+            "data": "set",
+            "pattern": None,
+            "type": "message",
+        }
+
+
+@pytest.mark.asyncio
+async def test_keyspace_notifications(empty_redis, event_loop):
+    pubsub = await RedisPool().subscribe_to_keys(TEST_KEY)
+    async with pubsub:
+        future_message = asyncio.create_task(channel_reader_onemessage(pubsub))
+        empty_redis.set(TEST_KEY, "I am set!")
+        message = await future_message
+        assert message is not None
+        assert message["data"] is not None
+        assert message == {
+            "channel": f"__keyspace@0__:{TEST_KEY}",
+            "data": "set",
+            "pattern": f"__keyspace@0__:{TEST_KEY}",
+            "type": "pmessage",
+        }

From fff8a49992c3af9ba76931e72aa0502c3450c903 Mon Sep 17 00:00:00 2001
From: Houkime <>
Date: Mon, 22 Apr 2024 14:41:56 +0000
Subject: [PATCH 54/86] refactoring(jobs): break out a function returning all
 jobs

---
 selfprivacy_api/graphql/queries/jobs.py | 10 +++++++---
 1 file changed, 7 insertions(+), 3 deletions(-)

diff --git a/selfprivacy_api/graphql/queries/jobs.py b/selfprivacy_api/graphql/queries/jobs.py
index e7b99e6..337382a 100644
--- a/selfprivacy_api/graphql/queries/jobs.py
+++ b/selfprivacy_api/graphql/queries/jobs.py
@@ -11,13 +11,17 @@ from selfprivacy_api.graphql.common_types.jobs import (
 from selfprivacy_api.jobs import Jobs
 
 
+def get_all_jobs() -> typing.List[ApiJob]:
+    Jobs.get_jobs()
+
+    return [job_to_api_job(job) for job in Jobs.get_jobs()]
+
+
 @strawberry.type
 class Job:
     @strawberry.field
     def get_jobs(self) -> typing.List[ApiJob]:
-        Jobs.get_jobs()
-
-        return [job_to_api_job(job) for job in Jobs.get_jobs()]
+        return get_all_jobs()
 
     @strawberry.field
     def get_job(self, job_id: str) -> typing.Optional[ApiJob]:

From 6510d4cac6d336139f319d879843151a8fe92335 Mon Sep 17 00:00:00 2001
From: Houkime <>
Date: Mon, 22 Apr 2024 14:50:08 +0000
Subject: [PATCH 55/86] feature(redis): enable key space notifications by
 default

---
 selfprivacy_api/utils/redis_pool.py | 2 ++
 tests/test_redis.py                 | 1 -
 2 files changed, 2 insertions(+), 1 deletion(-)

diff --git a/selfprivacy_api/utils/redis_pool.py b/selfprivacy_api/utils/redis_pool.py
index ea827d1..39c536f 100644
--- a/selfprivacy_api/utils/redis_pool.py
+++ b/selfprivacy_api/utils/redis_pool.py
@@ -29,6 +29,8 @@ class RedisPool:
             url,
             decode_responses=True,
         )
+        # TODO: inefficient, this is probably done each time we connect
+        self.get_connection().config_set("notify-keyspace-events", "KEA")
 
     @staticmethod
     def connection_url(dbnumber: int) -> str:
diff --git a/tests/test_redis.py b/tests/test_redis.py
index 181d325..70ef43a 100644
--- a/tests/test_redis.py
+++ b/tests/test_redis.py
@@ -15,7 +15,6 @@ STOPWORD = "STOP"
 def empty_redis(event_loop):
     r = RedisPool().get_connection()
     r.flushdb()
-    r.config_set("notify-keyspace-events", "KEA")
     assert r.config_get("notify-keyspace-events")["notify-keyspace-events"] == "AKE"
     yield r
     r.flushdb()

From 9bfffcd820d0e43d68258a98b0e7c920cdd478f7 Mon Sep 17 00:00:00 2001
From: Houkime <>
Date: Mon, 6 May 2024 14:54:13 +0000
Subject: [PATCH 56/86] feature(jobs): job update generator

---
 selfprivacy_api/jobs/__init__.py             | 29 ++++----
 selfprivacy_api/utils/redis_model_storage.py | 12 +++-
 tests/test_redis.py                          | 76 ++++++++++++++++++++
 3 files changed, 102 insertions(+), 15 deletions(-)

diff --git a/selfprivacy_api/jobs/__init__.py b/selfprivacy_api/jobs/__init__.py
index 4649bb0..3dd48c4 100644
--- a/selfprivacy_api/jobs/__init__.py
+++ b/selfprivacy_api/jobs/__init__.py
@@ -15,6 +15,7 @@ A job is a dictionary with the following keys:
     - result: result of the job
 """
 import typing
+import asyncio
 import datetime
 from uuid import UUID
 import uuid
@@ -23,6 +24,7 @@ from enum import Enum
 from pydantic import BaseModel
 
 from selfprivacy_api.utils.redis_pool import RedisPool
+from selfprivacy_api.utils.redis_model_storage import store_model_as_hash
 
 JOB_EXPIRATION_SECONDS = 10 * 24 * 60 * 60  # ten days
 
@@ -102,7 +104,7 @@ class Jobs:
             result=None,
         )
         redis = RedisPool().get_connection()
-        _store_job_as_hash(redis, _redis_key_from_uuid(job.uid), job)
+        store_model_as_hash(redis, _redis_key_from_uuid(job.uid), job)
         return job
 
     @staticmethod
@@ -218,7 +220,7 @@ class Jobs:
         redis = RedisPool().get_connection()
         key = _redis_key_from_uuid(job.uid)
         if redis.exists(key):
-            _store_job_as_hash(redis, key, job)
+            store_model_as_hash(redis, key, job)
             if status in (JobStatus.FINISHED, JobStatus.ERROR):
                 redis.expire(key, JOB_EXPIRATION_SECONDS)
 
@@ -294,17 +296,6 @@ def _progress_log_key_from_uuid(uuid_string) -> str:
     return PROGRESS_LOGS_PREFIX + str(uuid_string)
 
 
-def _store_job_as_hash(redis, redis_key, model) -> None:
-    for key, value in model.dict().items():
-        if isinstance(value, uuid.UUID):
-            value = str(value)
-        if isinstance(value, datetime.datetime):
-            value = value.isoformat()
-        if isinstance(value, JobStatus):
-            value = value.value
-        redis.hset(redis_key, key, str(value))
-
-
 def _job_from_hash(redis, redis_key) -> typing.Optional[Job]:
     if redis.exists(redis_key):
         job_dict = redis.hgetall(redis_key)
@@ -321,3 +312,15 @@ def _job_from_hash(redis, redis_key) -> typing.Optional[Job]:
 
         return Job(**job_dict)
     return None
+
+
+async def job_notifications() -> typing.AsyncGenerator[dict, None]:
+    channel = await RedisPool().subscribe_to_keys("jobs:*")
+    while True:
+        try:
+            # we cannot timeout here because we do not know when the next message is supposed to arrive
+            message: dict = await channel.get_message(ignore_subscribe_messages=True, timeout=None)  # type: ignore
+            if message is not None:
+                yield message
+        except GeneratorExit:
+            break
diff --git a/selfprivacy_api/utils/redis_model_storage.py b/selfprivacy_api/utils/redis_model_storage.py
index 06dfe8c..7d84210 100644
--- a/selfprivacy_api/utils/redis_model_storage.py
+++ b/selfprivacy_api/utils/redis_model_storage.py
@@ -1,15 +1,23 @@
+import uuid
+
 from datetime import datetime
 from typing import Optional
 from enum import Enum
 
 
 def store_model_as_hash(redis, redis_key, model):
-    for key, value in model.dict().items():
+    model_dict = model.dict()
+    for key, value in model_dict.items():
+        if isinstance(value, uuid.UUID):
+            value = str(value)
         if isinstance(value, datetime):
             value = value.isoformat()
         if isinstance(value, Enum):
             value = value.value
-        redis.hset(redis_key, key, str(value))
+        value = str(value)
+        model_dict[key] = value
+
+    redis.hset(redis_key, mapping=model_dict)
 
 
 def hash_as_model(redis, redis_key: str, model_class):
diff --git a/tests/test_redis.py b/tests/test_redis.py
index 70ef43a..02dfb21 100644
--- a/tests/test_redis.py
+++ b/tests/test_redis.py
@@ -7,6 +7,8 @@ from typing import List
 
 from selfprivacy_api.utils.redis_pool import RedisPool
 
+from selfprivacy_api.jobs import Jobs, job_notifications
+
 TEST_KEY = "test:test"
 STOPWORD = "STOP"
 
@@ -140,3 +142,77 @@ async def test_keyspace_notifications(empty_redis, event_loop):
             "pattern": f"__keyspace@0__:{TEST_KEY}",
             "type": "pmessage",
         }
+
+
+@pytest.mark.asyncio
+async def test_keyspace_notifications_patterns(empty_redis, event_loop):
+    pattern = "test*"
+    pubsub = await RedisPool().subscribe_to_keys(pattern)
+    async with pubsub:
+        future_message = asyncio.create_task(channel_reader_onemessage(pubsub))
+        empty_redis.set(TEST_KEY, "I am set!")
+        message = await future_message
+        assert message is not None
+        assert message["data"] is not None
+        assert message == {
+            "channel": f"__keyspace@0__:{TEST_KEY}",
+            "data": "set",
+            "pattern": f"__keyspace@0__:{pattern}",
+            "type": "pmessage",
+        }
+
+
+@pytest.mark.asyncio
+async def test_keyspace_notifications_jobs(empty_redis, event_loop):
+    pattern = "jobs:*"
+    pubsub = await RedisPool().subscribe_to_keys(pattern)
+    async with pubsub:
+        future_message = asyncio.create_task(channel_reader_onemessage(pubsub))
+        Jobs.add("testjob1", "test.test", "Testing aaaalll day")
+        message = await future_message
+        assert message is not None
+        assert message["data"] is not None
+        assert message["data"] == "hset"
+
+
+async def reader_of_jobs() -> List[dict]:
+    """
+    Reads 3 job updates and exits
+    """
+    result: List[dict] = []
+    async for message in job_notifications():
+        result.append(message)
+        if len(result) >= 3:
+            break
+    return result
+
+
+@pytest.mark.asyncio
+async def test_jobs_generator(empty_redis, event_loop):
+    # Will read exactly 3 job messages
+    future_messages = asyncio.create_task(reader_of_jobs())
+    await asyncio.sleep(1)
+
+    Jobs.add("testjob1", "test.test", "Testing aaaalll day")
+    Jobs.add("testjob2", "test.test", "Testing aaaalll day")
+    Jobs.add("testjob3", "test.test", "Testing aaaalll day")
+    Jobs.add("testjob4", "test.test", "Testing aaaalll day")
+
+    assert len(Jobs.get_jobs()) == 4
+    r = RedisPool().get_connection()
+    assert len(r.keys("jobs:*")) == 4
+
+    messages = await future_messages
+    assert len(messages) == 3
+    channels = [message["channel"] for message in messages]
+    operations = [message["data"] for message in messages]
+    assert set(operations) == set(["hset"])  # all of them are hsets
+
+    # Asserting that all of jobs emitted exactly one message
+    jobs = Jobs.get_jobs()
+    names = ["testjob1", "testjob2", "testjob3"]
+    ids = [str(job.uid) for job in jobs if job.name in names]
+    for id in ids:
+        assert id in " ".join(channels)
+    # Asserting that they came in order
+    assert "testjob4" not in " ".join(channels)

From 63d2e48a98c65d16ee2f3b2c1b38b71a29888689 Mon Sep 17 00:00:00 2001
From: Houkime <>
Date: Wed, 15 May 2024 11:29:20 +0000
Subject: [PATCH 57/86] feature(jobs): websocket connection

---
 selfprivacy_api/app.py               | 7 ++++++-
 tests/test_graphql/test_websocket.py | 6 ++++++
 2 files changed, 12 insertions(+), 1 deletion(-)
 create mode 100644 tests/test_graphql/test_websocket.py

diff --git a/selfprivacy_api/app.py b/selfprivacy_api/app.py
index 64ca85a..2f7e2f7 100644
--- a/selfprivacy_api/app.py
+++ b/selfprivacy_api/app.py
@@ -3,6 +3,7 @@
 from fastapi import FastAPI
 from fastapi.middleware.cors import CORSMiddleware
 from strawberry.fastapi import GraphQLRouter
+from strawberry.subscriptions import GRAPHQL_TRANSPORT_WS_PROTOCOL, GRAPHQL_WS_PROTOCOL
 
 import uvicorn
 
@@ -13,8 +14,12 @@ from selfprivacy_api.migrations import run_migrations
 
 app = FastAPI()
 
-graphql_app = GraphQLRouter(
+graphql_app: GraphQLRouter = GraphQLRouter(
     schema,
+    subscription_protocols=[
+        GRAPHQL_TRANSPORT_WS_PROTOCOL,
+        GRAPHQL_WS_PROTOCOL,
+    ],
 )
 
 app.add_middleware(
diff --git a/tests/test_graphql/test_websocket.py b/tests/test_graphql/test_websocket.py
new file mode 100644
index 0000000..fb2ac33
--- /dev/null
+++ b/tests/test_graphql/test_websocket.py
@@ -0,0 +1,6 @@
+
+def test_websocket_connection_bare(authorized_client):
+    client =authorized_client
+    with client.websocket_connect('/graphql', subprotocols=[ "graphql-transport-ws","graphql-ws"] ) as websocket:
+        assert websocket is not None
+        assert websocket.scope is not None

From c4aa757ca4f23b1fd1d7a47825a7f5bcd31e8efa Mon Sep 17 00:00:00 2001
From: Houkime <>
Date: Wed, 15 May 2024 13:01:07 +0000
Subject: [PATCH 58/86] test(jobs): test Graphql job getting

---
 tests/common.py                 |  4 +++
 tests/test_graphql/test_jobs.py | 48 +++++++++++++++++++++++++++++++++
 2 files changed, 52 insertions(+)
 create mode 100644 tests/test_graphql/test_jobs.py

diff --git a/tests/common.py b/tests/common.py
index 5f69f3f..8c81f48 100644
--- a/tests/common.py
+++ b/tests/common.py
@@ -69,6 +69,10 @@ def generate_backup_query(query_array):
     return "query TestBackup {\n backup {" + "\n".join(query_array) + "}\n}"
 
 
+def generate_jobs_query(query_array):
+    return "query TestJobs {\n jobs {" + "\n".join(query_array) + "}\n}"
+
+
 def generate_service_query(query_array):
     return "query TestService {\n services {" + "\n".join(query_array) + "}\n}"
 
diff --git a/tests/test_graphql/test_jobs.py b/tests/test_graphql/test_jobs.py
new file mode 100644
index 0000000..8dfb102
--- /dev/null
+++ b/tests/test_graphql/test_jobs.py
@@ -0,0 +1,48 @@
+from tests.common import generate_jobs_query
+from tests.test_graphql.common import (
+    assert_ok,
+    assert_empty,
+    assert_errorcode,
+    get_data,
+)
+
+API_JOBS_QUERY = """
+getJobs {
+    uid
+    typeId
+    name
+    description
+    status
+    statusText
+    progress
+    createdAt
+    updatedAt
+    finishedAt
+    error
+    result
+}
+"""
+
+
+def graphql_send_query(client, query: str, variables: dict = {}):
+    return client.post("/graphql", json={"query": query, "variables": variables})
+
+
+def api_jobs(authorized_client):
+    response = graphql_send_query(
+        authorized_client, generate_jobs_query([API_JOBS_QUERY])
+    )
+    data = get_data(response)
+    result = data["jobs"]["getJobs"]
+    assert result is not None
+    return result
+
+
+def test_all_jobs_unauthorized(client):
+    response = graphql_send_query(client, generate_jobs_query([API_JOBS_QUERY]))
+    assert_empty(response)
+
+
+def test_all_jobs_when_none(authorized_client):
+    output = api_jobs(authorized_client)
+    assert output == []

From 2d9f48650e398e72c6b22d13b0327da28e13fb26 Mon Sep 17 00:00:00 2001
From: Houkime <>
Date: Wed, 15 May 2024 13:42:17 +0000
Subject: [PATCH 59/86] test(jobs) test API job format

---
 tests/test_graphql/test_jobs.py | 26 ++++++++++++++++++++++++++
 1 file changed, 26 insertions(+)

diff --git a/tests/test_graphql/test_jobs.py b/tests/test_graphql/test_jobs.py
index 8dfb102..68a6d20 100644
--- a/tests/test_graphql/test_jobs.py
+++ b/tests/test_graphql/test_jobs.py
@@ -1,4 +1,6 @@
 from tests.common import generate_jobs_query
+import tests.test_graphql.test_api_backup
+
 from tests.test_graphql.common import (
     assert_ok,
     assert_empty,
@@ -6,6 +8,8 @@ from tests.test_graphql.common import (
     get_data,
 )
 
+from selfprivacy_api.jobs import Jobs
+
 API_JOBS_QUERY = """
 getJobs {
     uid
@@ -46,3 +50,25 @@ def test_all_jobs_unauthorized(client):
 def test_all_jobs_when_none(authorized_client):
     output = api_jobs(authorized_client)
     assert output == []
+
+
+def test_all_jobs_when_some(authorized_client):
+    # We cannot make new jobs via API, at least directly
+    job = Jobs.add("bogus", "bogus.bogus", "fungus")
+    output = api_jobs(authorized_client)
+
+    len(output) == 1
+    api_job = output[0]
+
+    assert api_job["uid"] == str(job.uid)
+    assert api_job["typeId"] == job.type_id
+    assert api_job["name"] == job.name
+    assert api_job["description"] == job.description
+    assert api_job["status"] == job.status
+    assert api_job["statusText"] == job.status_text
+    assert api_job["progress"] == job.progress
+    assert api_job["createdAt"] == job.created_at.isoformat()
+    assert api_job["updatedAt"] == job.updated_at.isoformat()
+    assert api_job["finishedAt"] == None
+    assert api_job["error"] == None
+    assert api_job["result"] == None

From 00c42d966099580d841ea32b886fec8cfe5ff10e Mon Sep 17 00:00:00 2001
From: Houkime <>
Date: Wed, 15 May 2024 18:14:14 +0000
Subject: [PATCH 60/86] test(jobs): subscription query generating function

---
 tests/common.py | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/tests/common.py b/tests/common.py
index 8c81f48..3c05033 100644
--- a/tests/common.py
+++ b/tests/common.py
@@ -73,6 +73,10 @@ def generate_jobs_query(query_array):
     return "query TestJobs {\n jobs {" + "\n".join(query_array) + "}\n}"
 
 
+def generate_jobs_subscription(query_array):
+    return "subscription TestSubscription {\n jobs {" + "\n".join(query_array) + "}\n}"
+
+
 def generate_service_query(query_array):
     return "query TestService {\n services {" + "\n".join(query_array) + "}\n}"
 

From 9add0b1dc1db9725de183d8ae8840994558ce6da Mon Sep 17 00:00:00 2001
From: Houkime <>
Date: Wed, 15 May 2024 18:15:16 +0000
Subject: [PATCH 61/86] test(websocket) test connection init

---
 tests/test_graphql/test_websocket.py | 48 ++++++++++++++++++++++++++--
 1 file changed, 46 insertions(+), 2 deletions(-)

diff --git a/tests/test_graphql/test_websocket.py b/tests/test_graphql/test_websocket.py
index fb2ac33..2431285 100644
--- a/tests/test_graphql/test_websocket.py
+++ b/tests/test_graphql/test_websocket.py
@@ -1,6 +1,50 @@
+from tests.common import generate_jobs_subscription
+from selfprivacy_api.graphql.queries.jobs import Job as _Job
+from selfprivacy_api.jobs import Jobs
+
+# JOBS_SUBSCRIPTION = """
+# jobUpdates {
+#     uid
+#     typeId
+#     name
+#     description
+#     status
+#     statusText
+#     progress
+#     createdAt
+#     updatedAt
+#     finishedAt
+#     error
+#     result
+# }
+# """
+
 
 def test_websocket_connection_bare(authorized_client):
-    client =authorized_client
-    with client.websocket_connect('/graphql', subprotocols=[ "graphql-transport-ws","graphql-ws"] ) as websocket:
+    client = authorized_client
+    with client.websocket_connect(
+        "/graphql", subprotocols=["graphql-transport-ws", "graphql-ws"]
+    ) as websocket:
         assert websocket is not None
         assert websocket.scope is not None
+
+
+def test_websocket_graphql_init(authorized_client):
+    client = authorized_client
+    with client.websocket_connect(
+        "/graphql", subprotocols=["graphql-transport-ws"]
+    ) as websocket:
+        websocket.send_json({"type": "connection_init", "payload": {}})
+        ack = websocket.receive_json()
+        assert ack == {"type": "connection_ack"}
+
+
+# def test_websocket_subscription(authorized_client):
+#     client = authorized_client
+#     with client.websocket_connect(
+#         "/graphql", subprotocols=["graphql-transport-ws", "graphql-ws"]
+#     ) as websocket:
+#         websocket.send(generate_jobs_subscription([JOBS_SUBSCRIPTION]))
+#         Jobs.add("bogus","bogus.bogus", "yyyaaaaayy")
+#         joblist = websocket.receive_json()
+#         raise NotImplementedError(joblist)

From a2a4b461e7054f712ef19b15caf95e2b56b83d52 Mon Sep 17 00:00:00 2001
From: Houkime <>
Date: Wed, 15 May 2024 18:31:16 +0000
Subject: [PATCH 62/86] test(websocket): ping pong test

---
 tests/test_graphql/test_websocket.py | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/tests/test_graphql/test_websocket.py b/tests/test_graphql/test_websocket.py
index 2431285..ef71312 100644
--- a/tests/test_graphql/test_websocket.py
+++ b/tests/test_graphql/test_websocket.py
@@ -29,7 +29,7 @@ def test_websocket_connection_bare(authorized_client):
         assert websocket.scope is not None
 
 
-def test_websocket_graphql_init(authorized_client):
+def test_websocket_graphql_ping(authorized_client):
     client = authorized_client
     with client.websocket_connect(
         "/graphql", subprotocols=["graphql-transport-ws"]
@@ -38,6 +38,11 @@ def test_websocket_graphql_init(authorized_client):
         ack = websocket.receive_json()
         assert ack == {"type": "connection_ack"}
 
+        # https://github.com/enisdenjo/graphql-ws/blob/master/PROTOCOL.md#ping
+        websocket.send_json({"type": "ping", "payload": {}})
+        pong = websocket.receive_json()
+        assert pong == {"type": "pong"}
+
 
 # def test_websocket_subscription(authorized_client):
 #     client = authorized_client

From f14866bdbc2ba3d54dbfbc59605c7b30452f44f9 Mon Sep 17 00:00:00 2001
From: Houkime <>
Date: Wed, 15 May 2024 18:36:17 +0000
Subject: [PATCH 63/86] test(websocket): separate ping and init

---
 tests/test_graphql/test_websocket.py | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/tests/test_graphql/test_websocket.py b/tests/test_graphql/test_websocket.py
index ef71312..d534269 100644
--- a/tests/test_graphql/test_websocket.py
+++ b/tests/test_graphql/test_websocket.py
@@ -29,7 +29,7 @@ def test_websocket_connection_bare(authorized_client):
         assert websocket.scope is not None
 
 
-def test_websocket_graphql_ping(authorized_client):
+def test_websocket_graphql_init(authorized_client):
     client = authorized_client
     with client.websocket_connect(
         "/graphql", subprotocols=["graphql-transport-ws"]
@@ -38,6 +38,12 @@ def test_websocket_graphql_ping(authorized_client):
         ack = websocket.receive_json()
         assert ack == {"type": "connection_ack"}
 
+
+def test_websocket_graphql_ping(authorized_client):
+    client = authorized_client
+    with client.websocket_connect(
+        "/graphql", subprotocols=["graphql-transport-ws"]
+    ) as websocket:
         # https://github.com/enisdenjo/graphql-ws/blob/master/PROTOCOL.md#ping
         websocket.send_json({"type": "ping", "payload": {}})
         pong = websocket.receive_json()

From ed777e3ebf5da70ff91c2fd5e99a8312ae7250df Mon Sep 17 00:00:00 2001
From: Houkime <>
Date: Wed, 15 May 2024 20:41:36 +0000
Subject: [PATCH 64/86] feature(jobs): add subscription endpoint

---
 .../graphql/subscriptions/__init__.py         |  0
 selfprivacy_api/graphql/subscriptions/jobs.py | 20 +++++++++++++++++++
 2 files changed, 20 insertions(+)
 create mode 100644 selfprivacy_api/graphql/subscriptions/__init__.py
 create mode 100644 selfprivacy_api/graphql/subscriptions/jobs.py

diff --git a/selfprivacy_api/graphql/subscriptions/__init__.py b/selfprivacy_api/graphql/subscriptions/__init__.py
new file mode 100644
index 0000000..e69de29
diff --git a/selfprivacy_api/graphql/subscriptions/jobs.py b/selfprivacy_api/graphql/subscriptions/jobs.py
new file mode 100644
index 0000000..380badb
--- /dev/null
+++ b/selfprivacy_api/graphql/subscriptions/jobs.py
@@ -0,0 +1,20 @@
+# pylint: disable=too-few-public-methods
+import strawberry
+
+from typing import AsyncGenerator, List
+
+from selfprivacy_api.jobs import job_notifications
+
+from selfprivacy_api.graphql.common_types.jobs import ApiJob
+from selfprivacy_api.graphql.queries.jobs import get_all_jobs
+
+
+@strawberry.type
+class JobSubscriptions:
+    """Subscriptions related to jobs"""
+
+    @strawberry.subscription
+    async def job_updates(self) -> AsyncGenerator[List[ApiJob], None]:
+        # Send the complete list of jobs every time anything gets updated
+        async for notification in job_notifications():
+            yield get_all_jobs()

From cbe5c56270609cb4fb7c42c19da520a1cdfbf9d3 Mon Sep 17 00:00:00 2001
From: Houkime <>
Date: Wed, 15 May 2024 20:41:48 +0000
Subject: [PATCH 65/86] chore(jobs): shorter typehints and import sorting

---
 selfprivacy_api/graphql/queries/jobs.py | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/selfprivacy_api/graphql/queries/jobs.py b/selfprivacy_api/graphql/queries/jobs.py
index 337382a..3cc3bf7 100644
--- a/selfprivacy_api/graphql/queries/jobs.py
+++ b/selfprivacy_api/graphql/queries/jobs.py
@@ -1,17 +1,17 @@
 """Jobs status"""
 # pylint: disable=too-few-public-methods
-import typing
 import strawberry
+from typing import List, Optional
+
+from selfprivacy_api.jobs import Jobs
 from selfprivacy_api.graphql.common_types.jobs import (
     ApiJob,
     get_api_job_by_id,
     job_to_api_job,
 )
 
-from selfprivacy_api.jobs import Jobs
 
-
-def get_all_jobs() -> typing.List[ApiJob]:
+def get_all_jobs() -> List[ApiJob]:
     Jobs.get_jobs()
 
     return [job_to_api_job(job) for job in Jobs.get_jobs()]
@@ -20,9 +20,9 @@ def get_all_jobs() -> typing.List[ApiJob]:
 @strawberry.type
 class Job:
     @strawberry.field
-    def get_jobs(self) -> typing.List[ApiJob]:
+    def get_jobs(self) -> List[ApiJob]:
         return get_all_jobs()
 
     @strawberry.field
-    def get_job(self, job_id: str) -> typing.Optional[ApiJob]:
+    def get_job(self, job_id: str) -> Optional[ApiJob]:
         return get_api_job_by_id(job_id)

From 51ccde8b0750f88dd5cb3c19b213d30c274cb5f2 Mon Sep 17 00:00:00 2001
From: Houkime <>
Date: Wed, 15 May 2024 20:43:17 +0000
Subject: [PATCH 66/86] test(jobs): test simple counting

---
 selfprivacy_api/graphql/schema.py    | 15 +++--
 tests/test_graphql/test_websocket.py | 92 +++++++++++++++++++++-------
 2 files changed, 81 insertions(+), 26 deletions(-)

diff --git a/selfprivacy_api/graphql/schema.py b/selfprivacy_api/graphql/schema.py
index e4e7264..078ee3d 100644
--- a/selfprivacy_api/graphql/schema.py
+++ b/selfprivacy_api/graphql/schema.py
@@ -28,6 +28,8 @@ from selfprivacy_api.graphql.queries.services import Services
 from selfprivacy_api.graphql.queries.storage import Storage
 from selfprivacy_api.graphql.queries.system import System
 
+from selfprivacy_api.graphql.subscriptions.jobs import JobSubscriptions
+
 from selfprivacy_api.graphql.mutations.users_mutations import UsersMutations
 from selfprivacy_api.graphql.queries.users import Users
 from selfprivacy_api.jobs.test import test_job
@@ -129,16 +131,19 @@ class Mutation(
             code=200,
         )
 
-    pass
-
 
 @strawberry.type
 class Subscription:
     """Root schema for subscriptions"""
 
-    @strawberry.subscription(permission_classes=[IsAuthenticated])
-    async def count(self, target: int = 100) -> AsyncGenerator[int, None]:
-        for i in range(target):
+    @strawberry.field(permission_classes=[IsAuthenticated])
+    def jobs(self) -> JobSubscriptions:
+        """Jobs subscriptions"""
+        return JobSubscriptions()
+
+    @strawberry.subscription
+    async def count(self) -> AsyncGenerator[int, None]:
+        for i in range(10):
             yield i
             await asyncio.sleep(0.5)
 
diff --git a/tests/test_graphql/test_websocket.py b/tests/test_graphql/test_websocket.py
index d534269..58681e0 100644
--- a/tests/test_graphql/test_websocket.py
+++ b/tests/test_graphql/test_websocket.py
@@ -2,22 +2,22 @@ from tests.common import generate_jobs_subscription
 from selfprivacy_api.graphql.queries.jobs import Job as _Job
 from selfprivacy_api.jobs import Jobs
 
-# JOBS_SUBSCRIPTION = """
-# jobUpdates {
-#     uid
-#     typeId
-#     name
-#     description
-#     status
-#     statusText
-#     progress
-#     createdAt
-#     updatedAt
-#     finishedAt
-#     error
-#     result
-# }
-# """
+JOBS_SUBSCRIPTION = """
+jobUpdates {
+    uid
+    typeId
+    name
+    description
+    status
+    statusText
+    progress
+    createdAt
+    updatedAt
+    finishedAt
+    error
+    result
+}
+"""
 
 
 def test_websocket_connection_bare(authorized_client):
@@ -50,12 +50,62 @@ def test_websocket_graphql_ping(authorized_client):
         assert pong == {"type": "pong"}
 
 
+def init_graphql(websocket):
+    websocket.send_json({"type": "connection_init", "payload": {}})
+    ack = websocket.receive_json()
+    assert ack == {"type": "connection_ack"}
+
+
+def test_websocket_subscription_minimal(authorized_client):
+    client = authorized_client
+    with client.websocket_connect(
+        "/graphql", subprotocols=["graphql-transport-ws"]
+    ) as websocket:
+        init_graphql(websocket)
+        websocket.send_json(
+            {
+                "id": "3aaa2445",
+                "type": "subscribe",
+                "payload": {
+                    "query": "subscription TestSubscription {count}",
+                },
+            }
+        )
+        response = websocket.receive_json()
+        assert response == {
+            "id": "3aaa2445",
+            "payload": {"data": {"count": 0}},
+            "type": "next",
+        }
+        response = websocket.receive_json()
+        assert response == {
+            "id": "3aaa2445",
+            "payload": {"data": {"count": 1}},
+            "type": "next",
+        }
+        response = websocket.receive_json()
+        assert response == {
+            "id": "3aaa2445",
+            "payload": {"data": {"count": 2}},
+            "type": "next",
+        }
+
+
 # def test_websocket_subscription(authorized_client):
 #     client = authorized_client
 #     with client.websocket_connect(
-#         "/graphql", subprotocols=["graphql-transport-ws", "graphql-ws"]
+#         "/graphql", subprotocols=["graphql-transport-ws"]
 #     ) as websocket:
-#         websocket.send(generate_jobs_subscription([JOBS_SUBSCRIPTION]))
-#         Jobs.add("bogus","bogus.bogus", "yyyaaaaayy")
-#         joblist = websocket.receive_json()
-#         raise NotImplementedError(joblist)
+#         init_graphql(websocket)
+#         websocket.send_json(
+#             {
+#                 "id": "3aaa2445",
+#                 "type": "subscribe",
+#                 "payload": {
+#                     "query": generate_jobs_subscription([JOBS_SUBSCRIPTION]),
+#                 },
+#             }
+#         )
+#         Jobs.add("bogus", "bogus.bogus", "yyyaaaaayy")
+#         response = websocket.receive_json()
+#         raise NotImplementedError(response)

From 442538ee4361b4cf045e0b0b9673f05d1985f35d Mon Sep 17 00:00:00 2001
From: Houkime <>
Date: Wed, 22 May 2024 11:04:37 +0000
Subject: [PATCH 67/86] feature(jobs): UNSAFE endpoint to get job updates

---
 selfprivacy_api/graphql/queries/jobs.py       |  7 +-
 selfprivacy_api/graphql/schema.py             | 19 +++--
 selfprivacy_api/graphql/subscriptions/jobs.py | 14 +---
 tests/test_graphql/test_websocket.py          | 81 ++++++++++++++-----
 4 files changed, 83 insertions(+), 38 deletions(-)

diff --git a/selfprivacy_api/graphql/queries/jobs.py b/selfprivacy_api/graphql/queries/jobs.py
index 3cc3bf7..6a12838 100644
--- a/selfprivacy_api/graphql/queries/jobs.py
+++ b/selfprivacy_api/graphql/queries/jobs.py
@@ -12,9 +12,10 @@ from selfprivacy_api.graphql.common_types.jobs import (
 
 
 def get_all_jobs() -> List[ApiJob]:
-    Jobs.get_jobs()
-
-    return [job_to_api_job(job) for job in Jobs.get_jobs()]
+    jobs = Jobs.get_jobs()
+    api_jobs = [job_to_api_job(job) for job in jobs]
+    assert api_jobs is not None
+    return api_jobs
 
 
 @strawberry.type
diff --git a/selfprivacy_api/graphql/schema.py b/selfprivacy_api/graphql/schema.py
index 078ee3d..b8ed4e2 100644
--- a/selfprivacy_api/graphql/schema.py
+++ b/selfprivacy_api/graphql/schema.py
@@ -2,7 +2,7 @@
 # pylint: disable=too-few-public-methods
 
 import asyncio
-from typing import AsyncGenerator
+from typing import AsyncGenerator, List
 import strawberry
 from selfprivacy_api.graphql import IsAuthenticated
 from selfprivacy_api.graphql.mutations.deprecated_mutations import (
@@ -28,7 +28,9 @@ from selfprivacy_api.graphql.queries.services import Services
 from selfprivacy_api.graphql.queries.storage import Storage
 from selfprivacy_api.graphql.queries.system import System
 
-from selfprivacy_api.graphql.subscriptions.jobs import JobSubscriptions
+from selfprivacy_api.graphql.subscriptions.jobs import ApiJob
+from selfprivacy_api.jobs import job_notifications
+from selfprivacy_api.graphql.queries.jobs import get_all_jobs
 
 from selfprivacy_api.graphql.mutations.users_mutations import UsersMutations
 from selfprivacy_api.graphql.queries.users import Users
@@ -136,10 +138,15 @@ class Mutation(
 class Subscription:
     """Root schema for subscriptions"""
 
-    @strawberry.field(permission_classes=[IsAuthenticated])
-    def jobs(self) -> JobSubscriptions:
-        """Jobs subscriptions"""
-        return JobSubscriptions()
+    @strawberry.subscription
+    async def job_updates(self) -> AsyncGenerator[List[ApiJob], None]:
+        # Send the complete list of jobs every time anything gets updated
+        async for notification in job_notifications():
+            yield get_all_jobs()
+
+    # @strawberry.subscription
+    # async def job_updates(self) -> AsyncGenerator[List[ApiJob], None]:
+    #     return job_updates()
 
     @strawberry.subscription
     async def count(self) -> AsyncGenerator[int, None]:
diff --git a/selfprivacy_api/graphql/subscriptions/jobs.py b/selfprivacy_api/graphql/subscriptions/jobs.py
index 380badb..11d6263 100644
--- a/selfprivacy_api/graphql/subscriptions/jobs.py
+++ b/selfprivacy_api/graphql/subscriptions/jobs.py
@@ -1,5 +1,4 @@
 # pylint: disable=too-few-public-methods
-import strawberry
 
 from typing import AsyncGenerator, List
 
@@ -9,12 +8,7 @@ from selfprivacy_api.graphql.common_types.jobs import ApiJob
 from selfprivacy_api.graphql.queries.jobs import get_all_jobs
 
 
-@strawberry.type
-class JobSubscriptions:
-    """Subscriptions related to jobs"""
-
-    @strawberry.subscription
-    async def job_updates(self) -> AsyncGenerator[List[ApiJob], None]:
-        # Send the complete list of jobs every time anything gets updated
-        async for notification in job_notifications():
-            yield get_all_jobs()
+async def job_updates() -> AsyncGenerator[List[ApiJob], None]:
+    # Send the complete list of jobs every time anything gets updated
+    async for notification in job_notifications():
+        yield get_all_jobs()
diff --git a/tests/test_graphql/test_websocket.py b/tests/test_graphql/test_websocket.py
index 58681e0..ee33262 100644
--- a/tests/test_graphql/test_websocket.py
+++ b/tests/test_graphql/test_websocket.py
@@ -1,6 +1,13 @@
 from tests.common import generate_jobs_subscription
-from selfprivacy_api.graphql.queries.jobs import Job as _Job
+
+# from selfprivacy_api.graphql.subscriptions.jobs import JobSubscriptions
+import pytest
+import asyncio
+
 from selfprivacy_api.jobs import Jobs
+from time import sleep
+
+from tests.test_redis import empty_redis
 
 JOBS_SUBSCRIPTION = """
 jobUpdates {
@@ -91,21 +98,57 @@ def test_websocket_subscription_minimal(authorized_client):
         }
 
 
-# def test_websocket_subscription(authorized_client):
-#     client = authorized_client
-#     with client.websocket_connect(
-#         "/graphql", subprotocols=["graphql-transport-ws"]
-#     ) as websocket:
-#         init_graphql(websocket)
-#         websocket.send_json(
-#             {
-#                 "id": "3aaa2445",
-#                 "type": "subscribe",
-#                 "payload": {
-#                     "query": generate_jobs_subscription([JOBS_SUBSCRIPTION]),
-#                 },
-#             }
-#         )
-#         Jobs.add("bogus", "bogus.bogus", "yyyaaaaayy")
-#         response = websocket.receive_json()
-#         raise NotImplementedError(response)
+async def read_one_job(websocket):
+    # bug? We only get them starting from the second job update
+    # that's why we receive two jobs in the list them
+    # the first update gets lost somewhere
+    response = websocket.receive_json()
+    return response
+
+
+@pytest.mark.asyncio
+async def test_websocket_subscription(authorized_client, empty_redis, event_loop):
+    client = authorized_client
+    with client.websocket_connect(
+        "/graphql", subprotocols=["graphql-transport-ws"]
+    ) as websocket:
+        init_graphql(websocket)
+        websocket.send_json(
+            {
+                "id": "3aaa2445",
+                "type": "subscribe",
+                "payload": {
+                    "query": "subscription TestSubscription {"
+                    + JOBS_SUBSCRIPTION
+                    + "}",
+                },
+            }
+        )
+        future = asyncio.create_task(read_one_job(websocket))
+        jobs = []
+        jobs.append(Jobs.add("bogus", "bogus.bogus", "yyyaaaaayy it works"))
+        sleep(0.5)
+        jobs.append(Jobs.add("bogus2", "bogus.bogus", "yyyaaaaayy it works"))
+
+        response = await future
+        data = response["payload"]["data"]
+        jobs_received = data["jobUpdates"]
+        received_names = [job["name"] for job in jobs_received]
+        for job in jobs:
+            assert job.name in received_names
+
+        for job in jobs:
+            for api_job in jobs_received:
+                if (job.name) == api_job["name"]:
+                    assert api_job["uid"] == str(job.uid)
+                    assert api_job["typeId"] == job.type_id
+                    assert api_job["name"] == job.name
+                    assert api_job["description"] == job.description
+                    assert api_job["status"] == job.status
+                    assert api_job["statusText"] == job.status_text
+                    assert api_job["progress"] == job.progress
+                    assert api_job["createdAt"] == job.created_at.isoformat()
+                    assert api_job["updatedAt"] == job.updated_at.isoformat()
+                    assert api_job["finishedAt"] == None
+                    assert api_job["error"] == None
+                    assert api_job["result"] == None

From 0fda29cdd7489608563d1250bf0a54fb7e41599c Mon Sep 17 00:00:00 2001
From: Houkime <>
Date: Mon, 27 May 2024 18:22:20 +0000
Subject: [PATCH 68/86] test(devices): provide devices for a service test to
 fix conditional test fail.

---
 tests/test_graphql/test_services.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tests/test_graphql/test_services.py b/tests/test_graphql/test_services.py
index 6e8dcf6..b7faf3d 100644
--- a/tests/test_graphql/test_services.py
+++ b/tests/test_graphql/test_services.py
@@ -543,8 +543,8 @@ def test_disable_enable(authorized_client, only_dummy_service):
     assert api_dummy_service["status"] == ServiceStatus.ACTIVE.value
 
 
-def test_move_immovable(authorized_client, only_dummy_service):
-    dummy_service = only_dummy_service
+def test_move_immovable(authorized_client, dummy_service_with_binds):
+    dummy_service = dummy_service_with_binds
     dummy_service.set_movable(False)
     root = BlockDevices().get_root_block_device()
     mutation_response = api_move(authorized_client, dummy_service, root.name)

From cb641e4f37d1ec73595b058d49b93721adc2555d Mon Sep 17 00:00:00 2001
From: Houkime <>
Date: Mon, 27 May 2024 20:21:11 +0000
Subject: [PATCH 69/86] feature(websocket): add auth

---
 selfprivacy_api/graphql/schema.py    |  18 ++-
 tests/test_graphql/test_websocket.py | 169 ++++++++++++++++++---------
 2 files changed, 131 insertions(+), 56 deletions(-)

diff --git a/selfprivacy_api/graphql/schema.py b/selfprivacy_api/graphql/schema.py
index b8ed4e2..c6cf46b 100644
--- a/selfprivacy_api/graphql/schema.py
+++ b/selfprivacy_api/graphql/schema.py
@@ -4,6 +4,7 @@
 import asyncio
 from typing import AsyncGenerator, List
 import strawberry
+
 from selfprivacy_api.graphql import IsAuthenticated
 from selfprivacy_api.graphql.mutations.deprecated_mutations import (
     DeprecatedApiMutations,
@@ -134,12 +135,25 @@ class Mutation(
         )
 
 
+# A cruft for Websockets
+def authenticated(info) -> bool:
+    return IsAuthenticated().has_permission(source=None, info=info)
+
+
 @strawberry.type
 class Subscription:
-    """Root schema for subscriptions"""
+    """Root schema for subscriptions.
+    Every field here should be an AsyncIterator or AsyncGenerator
+    It is not a part of the spec but graphql-core (dep of strawberryql)
+    demands it while the spec is vague in this area."""
 
     @strawberry.subscription
-    async def job_updates(self) -> AsyncGenerator[List[ApiJob], None]:
+    async def job_updates(
+        self, info: strawberry.types.Info
+    ) -> AsyncGenerator[List[ApiJob], None]:
+        if not authenticated(info):
+            raise Exception(IsAuthenticated().message)
+
         # Send the complete list of jobs every time anything gets updated
         async for notification in job_notifications():
             yield get_all_jobs()
diff --git a/tests/test_graphql/test_websocket.py b/tests/test_graphql/test_websocket.py
index ee33262..5a92416 100644
--- a/tests/test_graphql/test_websocket.py
+++ b/tests/test_graphql/test_websocket.py
@@ -1,13 +1,20 @@
-from tests.common import generate_jobs_subscription
-
 # from selfprivacy_api.graphql.subscriptions.jobs import JobSubscriptions
 import pytest
 import asyncio
-
-from selfprivacy_api.jobs import Jobs
+from typing import Generator
 from time import sleep
 
-from tests.test_redis import empty_redis
+from starlette.testclient import WebSocketTestSession
+
+from selfprivacy_api.jobs import Jobs
+from selfprivacy_api.actions.api_tokens import TOKEN_REPO
+from selfprivacy_api.graphql import IsAuthenticated
+
+from tests.conftest import DEVICE_WE_AUTH_TESTS_WITH
+from tests.test_jobs import jobs as empty_jobs
+
+# We do not iterate through them yet
+TESTED_SUBPROTOCOLS = ["graphql-transport-ws"]
 
 JOBS_SUBSCRIPTION = """
 jobUpdates {
@@ -27,6 +34,48 @@ jobUpdates {
 """
 
 
+def connect_ws_authenticated(authorized_client) -> WebSocketTestSession:
+    token = "Bearer " + str(DEVICE_WE_AUTH_TESTS_WITH["token"])
+    return authorized_client.websocket_connect(
+        "/graphql",
+        subprotocols=TESTED_SUBPROTOCOLS,
+        params={"token": token},
+    )
+
+
+def connect_ws_not_authenticated(client) -> WebSocketTestSession:
+    return client.websocket_connect(
+        "/graphql",
+        subprotocols=TESTED_SUBPROTOCOLS,
+        params={"token": "I like vegan icecream but it is not a valid token"},
+    )
+
+
+def init_graphql(websocket):
+    websocket.send_json({"type": "connection_init", "payload": {}})
+    ack = websocket.receive_json()
+    assert ack == {"type": "connection_ack"}
+
+
+@pytest.fixture
+def authenticated_websocket(
+    authorized_client,
+) -> Generator[WebSocketTestSession, None, None]:
+    # We use authorized_client only tohave token in the repo, this client by itself is not enough to authorize websocket
+
+    ValueError(TOKEN_REPO.get_tokens())
+    with connect_ws_authenticated(authorized_client) as websocket:
+        yield websocket
+        sleep(1)
+
+
+@pytest.fixture
+def unauthenticated_websocket(client) -> Generator[WebSocketTestSession, None, None]:
+    with connect_ws_not_authenticated(client) as websocket:
+        yield websocket
+        sleep(1)
+
+
 def test_websocket_connection_bare(authorized_client):
     client = authorized_client
     with client.websocket_connect(
@@ -57,12 +106,6 @@ def test_websocket_graphql_ping(authorized_client):
         assert pong == {"type": "pong"}
 
 
-def init_graphql(websocket):
-    websocket.send_json({"type": "connection_init", "payload": {}})
-    ack = websocket.receive_json()
-    assert ack == {"type": "connection_ack"}
-
-
 def test_websocket_subscription_minimal(authorized_client):
     client = authorized_client
     with client.websocket_connect(
@@ -107,48 +150,66 @@ async def read_one_job(websocket):
 
 
 @pytest.mark.asyncio
-async def test_websocket_subscription(authorized_client, empty_redis, event_loop):
-    client = authorized_client
-    with client.websocket_connect(
-        "/graphql", subprotocols=["graphql-transport-ws"]
-    ) as websocket:
-        init_graphql(websocket)
-        websocket.send_json(
-            {
-                "id": "3aaa2445",
-                "type": "subscribe",
-                "payload": {
-                    "query": "subscription TestSubscription {"
-                    + JOBS_SUBSCRIPTION
-                    + "}",
-                },
-            }
-        )
-        future = asyncio.create_task(read_one_job(websocket))
-        jobs = []
-        jobs.append(Jobs.add("bogus", "bogus.bogus", "yyyaaaaayy it works"))
-        sleep(0.5)
-        jobs.append(Jobs.add("bogus2", "bogus.bogus", "yyyaaaaayy it works"))
+async def test_websocket_subscription(authenticated_websocket, event_loop, empty_jobs):
+    websocket = authenticated_websocket
+    init_graphql(websocket)
+    websocket.send_json(
+        {
+            "id": "3aaa2445",
+            "type": "subscribe",
+            "payload": {
+                "query": "subscription TestSubscription {" + JOBS_SUBSCRIPTION + "}",
+            },
+        }
+    )
+    future = asyncio.create_task(read_one_job(websocket))
+    jobs = []
+    jobs.append(Jobs.add("bogus", "bogus.bogus", "yyyaaaaayy it works"))
+    sleep(0.5)
+    jobs.append(Jobs.add("bogus2", "bogus.bogus", "yyyaaaaayy it works"))
 
-        response = await future
-        data = response["payload"]["data"]
-        jobs_received = data["jobUpdates"]
-        received_names = [job["name"] for job in jobs_received]
-        for job in jobs:
-            assert job.name in received_names
+    response = await future
+    data = response["payload"]["data"]
+    jobs_received = data["jobUpdates"]
+    received_names = [job["name"] for job in jobs_received]
+    for job in jobs:
+        assert job.name in received_names
 
-        for job in jobs:
-            for api_job in jobs_received:
-                if (job.name) == api_job["name"]:
-                    assert api_job["uid"] == str(job.uid)
-                    assert api_job["typeId"] == job.type_id
-                    assert api_job["name"] == job.name
-                    assert api_job["description"] == job.description
-                    assert api_job["status"] == job.status
-                    assert api_job["statusText"] == job.status_text
-                    assert api_job["progress"] == job.progress
-                    assert api_job["createdAt"] == job.created_at.isoformat()
-                    assert api_job["updatedAt"] == job.updated_at.isoformat()
-                    assert api_job["finishedAt"] == None
-                    assert api_job["error"] == None
-                    assert api_job["result"] == None
+    assert len(jobs_received) == 2
+
+    for job in jobs:
+        for api_job in jobs_received:
+            if (job.name) == api_job["name"]:
+                assert api_job["uid"] == str(job.uid)
+                assert api_job["typeId"] == job.type_id
+                assert api_job["name"] == job.name
+                assert api_job["description"] == job.description
+                assert api_job["status"] == job.status
+                assert api_job["statusText"] == job.status_text
+                assert api_job["progress"] == job.progress
+                assert api_job["createdAt"] == job.created_at.isoformat()
+                assert api_job["updatedAt"] == job.updated_at.isoformat()
+                assert api_job["finishedAt"] == None
+                assert api_job["error"] == None
+                assert api_job["result"] == None
+
+
+def test_websocket_subscription_unauthorized(unauthenticated_websocket):
+    websocket = unauthenticated_websocket
+    init_graphql(websocket)
+    websocket.send_json(
+        {
+            "id": "3aaa2445",
+            "type": "subscribe",
+            "payload": {
+                "query": "subscription TestSubscription {" + JOBS_SUBSCRIPTION + "}",
+            },
+        }
+    )
+
+    response = websocket.receive_json()
+    assert response == {
+        "id": "3aaa2445",
+        "payload": [{"message": IsAuthenticated.message}],
+        "type": "error",
+    }

From ccf71078b85c0e036d51da5c816ea00665edc2e8 Mon Sep 17 00:00:00 2001
From: Houkime <>
Date: Mon, 27 May 2024 20:38:51 +0000
Subject: [PATCH 70/86] feature(websocket): add auth to counter too

---
 selfprivacy_api/graphql/schema.py    | 17 +++----
 tests/test_graphql/test_websocket.py | 69 +++++++++++++++-------------
 2 files changed, 47 insertions(+), 39 deletions(-)

diff --git a/selfprivacy_api/graphql/schema.py b/selfprivacy_api/graphql/schema.py
index c6cf46b..3280396 100644
--- a/selfprivacy_api/graphql/schema.py
+++ b/selfprivacy_api/graphql/schema.py
@@ -136,10 +136,15 @@ class Mutation(
 
 
 # A cruft for Websockets
-def authenticated(info) -> bool:
+def authenticated(info: strawberry.types.Info) -> bool:
     return IsAuthenticated().has_permission(source=None, info=info)
 
 
+def reject_if_unauthenticated(info: strawberry.types.Info):
+    if not authenticated(info):
+        raise Exception(IsAuthenticated().message)
+
+
 @strawberry.type
 class Subscription:
     """Root schema for subscriptions.
@@ -151,19 +156,15 @@ class Subscription:
     async def job_updates(
         self, info: strawberry.types.Info
     ) -> AsyncGenerator[List[ApiJob], None]:
-        if not authenticated(info):
-            raise Exception(IsAuthenticated().message)
+        reject_if_unauthenticated(info)
 
         # Send the complete list of jobs every time anything gets updated
         async for notification in job_notifications():
             yield get_all_jobs()
 
-    # @strawberry.subscription
-    # async def job_updates(self) -> AsyncGenerator[List[ApiJob], None]:
-    #     return job_updates()
-
     @strawberry.subscription
-    async def count(self) -> AsyncGenerator[int, None]:
+    async def count(self, info: strawberry.types.Info) -> AsyncGenerator[int, None]:
+        reject_if_unauthenticated(info)
         for i in range(10):
             yield i
             await asyncio.sleep(0.5)
diff --git a/tests/test_graphql/test_websocket.py b/tests/test_graphql/test_websocket.py
index 5a92416..49cc944 100644
--- a/tests/test_graphql/test_websocket.py
+++ b/tests/test_graphql/test_websocket.py
@@ -106,41 +106,61 @@ def test_websocket_graphql_ping(authorized_client):
         assert pong == {"type": "pong"}
 
 
+def api_subscribe(websocket, id, subscription):
+    websocket.send_json(
+        {
+            "id": id,
+            "type": "subscribe",
+            "payload": {
+                "query": "subscription TestSubscription {" + subscription + "}",
+            },
+        }
+    )
+
+
 def test_websocket_subscription_minimal(authorized_client):
+    # Test a small endpoint that exists specifically for tests
     client = authorized_client
     with client.websocket_connect(
         "/graphql", subprotocols=["graphql-transport-ws"]
     ) as websocket:
         init_graphql(websocket)
-        websocket.send_json(
-            {
-                "id": "3aaa2445",
-                "type": "subscribe",
-                "payload": {
-                    "query": "subscription TestSubscription {count}",
-                },
-            }
-        )
+        arbitrary_id = "3aaa2445"
+        api_subscribe(websocket, arbitrary_id, "count")
         response = websocket.receive_json()
         assert response == {
-            "id": "3aaa2445",
+            "id": arbitrary_id,
             "payload": {"data": {"count": 0}},
             "type": "next",
         }
         response = websocket.receive_json()
         assert response == {
-            "id": "3aaa2445",
+            "id": arbitrary_id,
             "payload": {"data": {"count": 1}},
             "type": "next",
         }
         response = websocket.receive_json()
         assert response == {
-            "id": "3aaa2445",
+            "id": arbitrary_id,
             "payload": {"data": {"count": 2}},
             "type": "next",
         }
 
 
+def test_websocket_subscription_minimal_unauthorized(unauthenticated_websocket):
+    websocket = unauthenticated_websocket
+    init_graphql(websocket)
+    arbitrary_id = "3aaa2445"
+    api_subscribe(websocket, arbitrary_id, "count")
+
+    response = websocket.receive_json()
+    assert response == {
+        "id": arbitrary_id,
+        "payload": [{"message": IsAuthenticated.message}],
+        "type": "error",
+    }
+
+
 async def read_one_job(websocket):
     # bug? We only get them starting from the second job update
     # that's why we receive two jobs in the list them
@@ -153,15 +173,9 @@ async def read_one_job(websocket):
 async def test_websocket_subscription(authenticated_websocket, event_loop, empty_jobs):
     websocket = authenticated_websocket
     init_graphql(websocket)
-    websocket.send_json(
-        {
-            "id": "3aaa2445",
-            "type": "subscribe",
-            "payload": {
-                "query": "subscription TestSubscription {" + JOBS_SUBSCRIPTION + "}",
-            },
-        }
-    )
+    arbitrary_id = "3aaa2445"
+    api_subscribe(websocket, arbitrary_id, JOBS_SUBSCRIPTION)
+
     future = asyncio.create_task(read_one_job(websocket))
     jobs = []
     jobs.append(Jobs.add("bogus", "bogus.bogus", "yyyaaaaayy it works"))
@@ -197,19 +211,12 @@ async def test_websocket_subscription(authenticated_websocket, event_loop, empty
 def test_websocket_subscription_unauthorized(unauthenticated_websocket):
     websocket = unauthenticated_websocket
     init_graphql(websocket)
-    websocket.send_json(
-        {
-            "id": "3aaa2445",
-            "type": "subscribe",
-            "payload": {
-                "query": "subscription TestSubscription {" + JOBS_SUBSCRIPTION + "}",
-            },
-        }
-    )
+    id = "3aaa2445"
+    api_subscribe(websocket, id, JOBS_SUBSCRIPTION)
 
     response = websocket.receive_json()
     assert response == {
-        "id": "3aaa2445",
+        "id": id,
         "payload": [{"message": IsAuthenticated.message}],
         "type": "error",
     }

From 05ffa036b3d916720ed3276603f5db4bd890309e Mon Sep 17 00:00:00 2001
From: Houkime <>
Date: Mon, 27 May 2024 21:13:57 +0000
Subject: [PATCH 71/86] refactor(jobs): offload job subscription logic to a
 separate file

---
 selfprivacy_api/graphql/schema.py    | 11 +++++------
 tests/test_graphql/test_websocket.py | 16 ++++++++++++----
 2 files changed, 17 insertions(+), 10 deletions(-)

diff --git a/selfprivacy_api/graphql/schema.py b/selfprivacy_api/graphql/schema.py
index 3280396..05e6bf9 100644
--- a/selfprivacy_api/graphql/schema.py
+++ b/selfprivacy_api/graphql/schema.py
@@ -30,8 +30,9 @@ from selfprivacy_api.graphql.queries.storage import Storage
 from selfprivacy_api.graphql.queries.system import System
 
 from selfprivacy_api.graphql.subscriptions.jobs import ApiJob
-from selfprivacy_api.jobs import job_notifications
-from selfprivacy_api.graphql.queries.jobs import get_all_jobs
+from selfprivacy_api.graphql.subscriptions.jobs import (
+    job_updates as job_update_generator,
+)
 
 from selfprivacy_api.graphql.mutations.users_mutations import UsersMutations
 from selfprivacy_api.graphql.queries.users import Users
@@ -157,12 +158,10 @@ class Subscription:
         self, info: strawberry.types.Info
     ) -> AsyncGenerator[List[ApiJob], None]:
         reject_if_unauthenticated(info)
-
-        # Send the complete list of jobs every time anything gets updated
-        async for notification in job_notifications():
-            yield get_all_jobs()
+        return job_update_generator()
 
     @strawberry.subscription
+    # Used for testing, consider deletion to shrink attack surface
     async def count(self, info: strawberry.types.Info) -> AsyncGenerator[int, None]:
         reject_if_unauthenticated(info)
         for i in range(10):
diff --git a/tests/test_graphql/test_websocket.py b/tests/test_graphql/test_websocket.py
index 49cc944..d538ca1 100644
--- a/tests/test_graphql/test_websocket.py
+++ b/tests/test_graphql/test_websocket.py
@@ -162,9 +162,9 @@ def test_websocket_subscription_minimal_unauthorized(unauthenticated_websocket):
 
 
 async def read_one_job(websocket):
-    # bug? We only get them starting from the second job update
-    # that's why we receive two jobs in the list them
-    # the first update gets lost somewhere
+    # Bug? We only get them starting from the second job update
+    # That's why we receive two jobs in the list them
+    # The first update gets lost somewhere
     response = websocket.receive_json()
     return response
 
@@ -215,8 +215,16 @@ def test_websocket_subscription_unauthorized(unauthenticated_websocket):
     api_subscribe(websocket, id, JOBS_SUBSCRIPTION)
 
     response = websocket.receive_json()
+    # I do not really know why strawberry gives more info on this
+    # One versus the counter
+    payload = response["payload"][0]
+    assert isinstance(payload, dict)
+    assert "locations" in payload.keys()
+    # It looks like this 'locations': [{'column': 32, 'line': 1}]
+    # We cannot test locations feasibly
+    del payload["locations"]
     assert response == {
         "id": id,
-        "payload": [{"message": IsAuthenticated.message}],
+        "payload": [{"message": IsAuthenticated.message, "path": ["jobUpdates"]}],
         "type": "error",
     }

From 57378a794089eb39bea3ec4da0ce92066859825e Mon Sep 17 00:00:00 2001
From: Houkime <>
Date: Mon, 27 May 2024 21:15:47 +0000
Subject: [PATCH 72/86] test(websocket): remove excessive sleeping

---
 tests/test_graphql/test_websocket.py | 2 --
 1 file changed, 2 deletions(-)

diff --git a/tests/test_graphql/test_websocket.py b/tests/test_graphql/test_websocket.py
index d538ca1..27cfd55 100644
--- a/tests/test_graphql/test_websocket.py
+++ b/tests/test_graphql/test_websocket.py
@@ -66,14 +66,12 @@ def authenticated_websocket(
     ValueError(TOKEN_REPO.get_tokens())
     with connect_ws_authenticated(authorized_client) as websocket:
         yield websocket
-        sleep(1)
 
 
 @pytest.fixture
 def unauthenticated_websocket(client) -> Generator[WebSocketTestSession, None, None]:
     with connect_ws_not_authenticated(client) as websocket:
         yield websocket
-        sleep(1)
 
 
 def test_websocket_connection_bare(authorized_client):

From 41f6d8b6d2078a0c62f4567ffa7279a5d9c9a198 Mon Sep 17 00:00:00 2001
From: Houkime <>
Date: Mon, 27 May 2024 21:28:29 +0000
Subject: [PATCH 73/86] test(websocket): remove some duplication

---
 tests/test_graphql/test_websocket.py | 75 +++++++++++++---------------
 1 file changed, 36 insertions(+), 39 deletions(-)

diff --git a/tests/test_graphql/test_websocket.py b/tests/test_graphql/test_websocket.py
index 27cfd55..754fbbf 100644
--- a/tests/test_graphql/test_websocket.py
+++ b/tests/test_graphql/test_websocket.py
@@ -34,6 +34,18 @@ jobUpdates {
 """
 
 
+def api_subscribe(websocket, id, subscription):
+    websocket.send_json(
+        {
+            "id": id,
+            "type": "subscribe",
+            "payload": {
+                "query": "subscription TestSubscription {" + subscription + "}",
+            },
+        }
+    )
+
+
 def connect_ws_authenticated(authorized_client) -> WebSocketTestSession:
     token = "Bearer " + str(DEVICE_WE_AUTH_TESTS_WITH["token"])
     return authorized_client.websocket_connect(
@@ -61,7 +73,7 @@ def init_graphql(websocket):
 def authenticated_websocket(
     authorized_client,
 ) -> Generator[WebSocketTestSession, None, None]:
-    # We use authorized_client only tohave token in the repo, this client by itself is not enough to authorize websocket
+    # We use authorized_client only to have token in the repo, this client by itself is not enough to authorize websocket
 
     ValueError(TOKEN_REPO.get_tokens())
     with connect_ws_authenticated(authorized_client) as websocket:
@@ -104,45 +116,30 @@ def test_websocket_graphql_ping(authorized_client):
         assert pong == {"type": "pong"}
 
 
-def api_subscribe(websocket, id, subscription):
-    websocket.send_json(
-        {
-            "id": id,
-            "type": "subscribe",
-            "payload": {
-                "query": "subscription TestSubscription {" + subscription + "}",
-            },
-        }
-    )
-
-
-def test_websocket_subscription_minimal(authorized_client):
+def test_websocket_subscription_minimal(authorized_client, authenticated_websocket):
     # Test a small endpoint that exists specifically for tests
-    client = authorized_client
-    with client.websocket_connect(
-        "/graphql", subprotocols=["graphql-transport-ws"]
-    ) as websocket:
-        init_graphql(websocket)
-        arbitrary_id = "3aaa2445"
-        api_subscribe(websocket, arbitrary_id, "count")
-        response = websocket.receive_json()
-        assert response == {
-            "id": arbitrary_id,
-            "payload": {"data": {"count": 0}},
-            "type": "next",
-        }
-        response = websocket.receive_json()
-        assert response == {
-            "id": arbitrary_id,
-            "payload": {"data": {"count": 1}},
-            "type": "next",
-        }
-        response = websocket.receive_json()
-        assert response == {
-            "id": arbitrary_id,
-            "payload": {"data": {"count": 2}},
-            "type": "next",
-        }
+    websocket = authenticated_websocket
+    init_graphql(websocket)
+    arbitrary_id = "3aaa2445"
+    api_subscribe(websocket, arbitrary_id, "count")
+    response = websocket.receive_json()
+    assert response == {
+        "id": arbitrary_id,
+        "payload": {"data": {"count": 0}},
+        "type": "next",
+    }
+    response = websocket.receive_json()
+    assert response == {
+        "id": arbitrary_id,
+        "payload": {"data": {"count": 1}},
+        "type": "next",
+    }
+    response = websocket.receive_json()
+    assert response == {
+        "id": arbitrary_id,
+        "payload": {"data": {"count": 2}},
+        "type": "next",
+    }
 
 
 def test_websocket_subscription_minimal_unauthorized(unauthenticated_websocket):

From 9accf861c51a4f96f00eeb06a4da839d4ba92cfa Mon Sep 17 00:00:00 2001
From: Houkime <>
Date: Mon, 17 Jun 2024 11:34:23 +0000
Subject: [PATCH 74/86] fix(websockets): add websockets dep so that uvicorn
 works

---
 default.nix                                |  1 +
 tests/test_websocket_uvicorn_standalone.py | 39 ++++++++++++++++++++++
 2 files changed, 40 insertions(+)
 create mode 100644 tests/test_websocket_uvicorn_standalone.py

diff --git a/default.nix b/default.nix
index e7e6fcf..f6d85d6 100644
--- a/default.nix
+++ b/default.nix
@@ -18,6 +18,7 @@ pythonPackages.buildPythonPackage rec {
     strawberry-graphql
     typing-extensions
     uvicorn
+    websockets
   ];
   pythonImportsCheck = [ "selfprivacy_api" ];
   doCheck = false;
diff --git a/tests/test_websocket_uvicorn_standalone.py b/tests/test_websocket_uvicorn_standalone.py
new file mode 100644
index 0000000..43a53ef
--- /dev/null
+++ b/tests/test_websocket_uvicorn_standalone.py
@@ -0,0 +1,39 @@
+import pytest
+from fastapi import FastAPI, WebSocket
+import uvicorn
+
+# import subprocess
+from multiprocessing import Process
+import asyncio
+from time import sleep
+from websockets import client
+
+app = FastAPI()
+
+
+@app.websocket("/")
+async def websocket_endpoint(websocket: WebSocket):
+    await websocket.accept()
+    while True:
+        data = await websocket.receive_text()
+        await websocket.send_text(f"You sent: {data}")
+
+
+def run_uvicorn():
+    uvicorn.run(app, port=5000)
+    return True
+
+
+@pytest.mark.asyncio
+async def test_uvcorn_ws_works_in_prod():
+    proc = Process(target=run_uvicorn)
+    proc.start()
+    sleep(2)
+
+    ws = await client.connect("ws://127.0.0.1:5000")
+
+    await ws.send("hohoho")
+    message = await ws.read_message()
+    assert message == "You sent: hohoho"
+    await ws.close()
+    proc.kill()

From a7be03a6d31d3017bf9ffe87b02680c62e6aeb5a Mon Sep 17 00:00:00 2001
From: Inex Code <inex.code@selfprivacy.org>
Date: Thu, 4 Jul 2024 18:49:17 +0400
Subject: [PATCH 75/86] refactor: Remove setting KEA

This is already done via NixOS config
---
 selfprivacy_api/utils/redis_pool.py | 2 --
 1 file changed, 2 deletions(-)

diff --git a/selfprivacy_api/utils/redis_pool.py b/selfprivacy_api/utils/redis_pool.py
index 39c536f..ea827d1 100644
--- a/selfprivacy_api/utils/redis_pool.py
+++ b/selfprivacy_api/utils/redis_pool.py
@@ -29,8 +29,6 @@ class RedisPool:
             url,
             decode_responses=True,
         )
-        # TODO: inefficient, this is probably done each time we connect
-        self.get_connection().config_set("notify-keyspace-events", "KEA")
 
     @staticmethod
     def connection_url(dbnumber: int) -> str:

From ceee6e4db9a7def34d8e2193a6088b2076e39fb8 Mon Sep 17 00:00:00 2001
From: Inex Code <inex.code@selfprivacy.org>
Date: Thu, 4 Jul 2024 21:08:40 +0400
Subject: [PATCH 76/86] fix: Read auth token from the connection initialization
 payload

Websockets do not provide headers, and sending a token as a query param is also not good (it gets into server's logs),
As an alternative, we can provide a token in the first ws payload.

Read more: https://strawberry.rocks/docs/general/subscriptions#authenticating-subscriptions
---
 selfprivacy_api/graphql/__init__.py | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/selfprivacy_api/graphql/__init__.py b/selfprivacy_api/graphql/__init__.py
index 6124a1a..edd8a78 100644
--- a/selfprivacy_api/graphql/__init__.py
+++ b/selfprivacy_api/graphql/__init__.py
@@ -16,6 +16,10 @@ class IsAuthenticated(BasePermission):
         token = info.context["request"].headers.get("Authorization")
         if token is None:
             token = info.context["request"].query_params.get("token")
+        if token is None:
+            connection_params = info.context.get("connection_params")
+            if connection_params is not None:
+                token = connection_params.get("Authorization")
         if token is None:
             return False
         return is_token_valid(token.replace("Bearer ", ""))

From 5f3fc0d96e37aed82aebc59424f461c5e5d312f1 Mon Sep 17 00:00:00 2001
From: Inex Code <inex.code@selfprivacy.org>
Date: Wed, 10 Jul 2024 19:18:22 +0400
Subject: [PATCH 77/86] chore: formatting

---
 selfprivacy_api/graphql/queries/logs.py | 4 +++-
 selfprivacy_api/graphql/schema.py       | 5 ++++-
 2 files changed, 7 insertions(+), 2 deletions(-)

diff --git a/selfprivacy_api/graphql/queries/logs.py b/selfprivacy_api/graphql/queries/logs.py
index 52204cf..6aa864f 100644
--- a/selfprivacy_api/graphql/queries/logs.py
+++ b/selfprivacy_api/graphql/queries/logs.py
@@ -57,7 +57,9 @@ class LogsPageMeta:
 
 @strawberry.type
 class PaginatedEntries:
-    page_meta: LogsPageMeta = strawberry.field(description="Metadata to aid in pagination.")
+    page_meta: LogsPageMeta = strawberry.field(
+        description="Metadata to aid in pagination."
+    )
     entries: typing.List[LogEntry] = strawberry.field(
         description="The list of log entries."
     )
diff --git a/selfprivacy_api/graphql/schema.py b/selfprivacy_api/graphql/schema.py
index f0d5a11..a515d0c 100644
--- a/selfprivacy_api/graphql/schema.py
+++ b/selfprivacy_api/graphql/schema.py
@@ -176,7 +176,10 @@ class Subscription:
             await asyncio.sleep(0.5)
 
     @strawberry.subscription
-    async def log_entries(self, info: strawberry.types.Info) -> AsyncGenerator[LogEntry, None]:
+    async def log_entries(
+        self,
+        info: strawberry.types.Info,
+    ) -> AsyncGenerator[LogEntry, None]:
         reject_if_unauthenticated(info)
         return log_stream()
 

From faa8952e9c79b8e01fa03a291bab3d18b636fb1f Mon Sep 17 00:00:00 2001
From: Inex Code <inex.code@selfprivacy.org>
Date: Wed, 10 Jul 2024 19:51:10 +0400
Subject: [PATCH 78/86] chore: Bump version to 3.3.0

---
 selfprivacy_api/dependencies.py | 2 +-
 setup.py                        | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/selfprivacy_api/dependencies.py b/selfprivacy_api/dependencies.py
index 69ce319..b2e2b19 100644
--- a/selfprivacy_api/dependencies.py
+++ b/selfprivacy_api/dependencies.py
@@ -27,4 +27,4 @@ async def get_token_header(
 
 def get_api_version() -> str:
     """Get API version"""
-    return "3.2.2"
+    return "3.3.0"
diff --git a/setup.py b/setup.py
index 23c544e..aaf333e 100644
--- a/setup.py
+++ b/setup.py
@@ -2,7 +2,7 @@ from setuptools import setup, find_packages
 
 setup(
     name="selfprivacy_api",
-    version="3.2.2",
+    version="3.3.0",
     packages=find_packages(),
     scripts=[
         "selfprivacy_api/app.py",

From 4ca9b9f54e0c3991eade23276ce924881ea1d976 Mon Sep 17 00:00:00 2001
From: Inex Code <inex.code@selfprivacy.org>
Date: Wed, 10 Jul 2024 21:46:14 +0400
Subject: [PATCH 79/86] fix: Wait for ws logs test to init

---
 tests/test_graphql/test_api_logs.py | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/tests/test_graphql/test_api_logs.py b/tests/test_graphql/test_api_logs.py
index 18f4d32..6875531 100644
--- a/tests/test_graphql/test_api_logs.py
+++ b/tests/test_graphql/test_api_logs.py
@@ -1,3 +1,5 @@
+import asyncio
+import pytest
 from datetime import datetime
 from systemd import journal
 
@@ -135,7 +137,8 @@ def test_graphql_get_logs_with_down_border(authorized_client):
         assert_log_entry_equals_to_journal_entry(api_entry, journal_entry)
 
 
-def test_websocket_subscription_for_logs(authorized_client):
+@pytest.mark.asyncio
+async def test_websocket_subscription_for_logs(authorized_client):
     with authorized_client.websocket_connect(
         "/graphql", subprotocols=["graphql-transport-ws"]
     ) as websocket:
@@ -149,6 +152,7 @@ def test_websocket_subscription_for_logs(authorized_client):
                 },
             }
         )
+        await asyncio.sleep(1)
 
         def read_until(message, limit=5):
             i = 0
@@ -159,6 +163,7 @@ def test_websocket_subscription_for_logs(authorized_client):
                 if msg == message:
                     return
                 else:
+                    i += 1
                     continue
             raise Exception("Failed to read websocket data, timeout")
 

From 859ac4dbc6ba3664ac2e46cd12bd3fdbac9f6311 Mon Sep 17 00:00:00 2001
From: Inex Code <inex.code@selfprivacy.org>
Date: Thu, 11 Jul 2024 19:08:04 +0400
Subject: [PATCH 80/86] chore: Update nixpkgs

---
 flake.lock | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/flake.lock b/flake.lock
index 1f52d36..ba47e51 100644
--- a/flake.lock
+++ b/flake.lock
@@ -2,11 +2,11 @@
   "nodes": {
     "nixpkgs": {
       "locked": {
-        "lastModified": 1709677081,
-        "narHash": "sha256-tix36Y7u0rkn6mTm0lA45b45oab2cFLqAzDbJxeXS+c=",
+        "lastModified": 1719957072,
+        "narHash": "sha256-gvFhEf5nszouwLAkT9nWsDzocUTqLWHuL++dvNjMp9I=",
         "owner": "nixos",
         "repo": "nixpkgs",
-        "rev": "880992dcc006a5e00dd0591446fdf723e6a51a64",
+        "rev": "7144d6241f02d171d25fba3edeaf15e0f2592105",
         "type": "github"
       },
       "original": {

From c857678c9a9651abaee13f0621382ae36f1bf85a Mon Sep 17 00:00:00 2001
From: Inex Code <inex.code@selfprivacy.org>
Date: Thu, 11 Jul 2024 20:20:08 +0400
Subject: [PATCH 81/86] docs: Update Contributing file

---
 CONTRIBUTING.md | 15 ++++++++-------
 1 file changed, 8 insertions(+), 7 deletions(-)

diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
index 45ebd2a..a188a00 100644
--- a/CONTRIBUTING.md
+++ b/CONTRIBUTING.md
@@ -13,9 +13,9 @@ the [repository](https://git.selfprivacy.org/SelfPrivacy/selfprivacy-rest-api),
 
     For detailed installation information, please review and follow: [link](https://nixos.org/manual/nix/stable/installation/installing-binary.html#installing-a-binary-distribution).
 
-3. **Change directory to the cloned repository and start a nix shell:**
+3. **Change directory to the cloned repository and start a nix development shell:**
 
-    ```cd selfprivacy-rest-api && nix-shell```
+    ```cd selfprivacy-rest-api && nix develop```
 
     Nix will install all of the necessary packages for development work, all further actions will take place only within nix-shell.
 
@@ -31,7 +31,7 @@ the [repository](https://git.selfprivacy.org/SelfPrivacy/selfprivacy-rest-api),
 
     Copy the path that starts with ```/nix/store/``` and ends with ```env/bin/python```
 
-    ```/nix/store/???-python3-3.9.??-env/bin/python```
+    ```/nix/store/???-python3-3.10.??-env/bin/python```
 
     Click on the python version selection in the lower right corner, and replace the path to the interpreter in the project with the one you copied from the terminal.
 
@@ -43,12 +43,13 @@ the [repository](https://git.selfprivacy.org/SelfPrivacy/selfprivacy-rest-api),
 
 ## What to do after making changes to the repository?
 
-**Run unit tests** using ```pytest .```
-Make sure that all tests pass successfully and the API works correctly. For convenience, you can use the built-in VScode interface.
+**Run unit tests** using ```pytest-vm``` inside of the development shell. This will run all the test inside a virtual machine, which is necessary for the tests to pass successfully.
+Make sure that all tests pass successfully and the API works correctly.
 
-How to review the percentage of code coverage? Execute the command:
+The ```pytest-vm``` command will also print out the coverage of the tests. To export the report to an XML file, use the following command:
+
+```coverage xml```
 
-```coverage run -m pytest && coverage xml && coverage report```
 
 Next, use the recommended extension ```ryanluker.vscode-coverage-gutters```, navigate to one of the test files, and click the "watch" button on the bottom panel of VScode.
 

From 94b0276f743622873a59fe46e131f64b7286e196 Mon Sep 17 00:00:00 2001
From: nhnn <nhnn@disroot.org>
Date: Fri, 12 Jul 2024 20:50:43 +0300
Subject: [PATCH 82/86] fix: extract business logic to utils/systemd_journal.py

---
 selfprivacy_api/graphql/queries/logs.py  | 53 ++++-------------------
 selfprivacy_api/graphql/schema.py        |  3 +-
 selfprivacy_api/utils/systemd_journal.py | 55 ++++++++++++++++++++++++
 3 files changed, 64 insertions(+), 47 deletions(-)
 create mode 100644 selfprivacy_api/utils/systemd_journal.py

diff --git a/selfprivacy_api/graphql/queries/logs.py b/selfprivacy_api/graphql/queries/logs.py
index 6aa864f..6841f30 100644
--- a/selfprivacy_api/graphql/queries/logs.py
+++ b/selfprivacy_api/graphql/queries/logs.py
@@ -1,25 +1,8 @@
 """System logs"""
 from datetime import datetime
-import os
 import typing
 import strawberry
-from systemd import journal
-
-
-def get_events_from_journal(
-    j: journal.Reader, limit: int, next: typing.Callable[[journal.Reader], typing.Dict]
-):
-    events = []
-    i = 0
-    while i < limit:
-        entry = next(j)
-        if entry == None or entry == dict():
-            break
-        if entry["MESSAGE"] != "":
-            events.append(LogEntry(entry))
-            i += 1
-
-    return events
+from selfprivacy_api.utils.systemd_journal import get_paginated_logs
 
 
 @strawberry.type
@@ -95,31 +78,11 @@ class Logs:
     ) -> PaginatedEntries:
         if limit > 50:
             raise Exception("You can't fetch more than 50 entries via single request.")
-        j = journal.Reader()
-
-        if up_cursor == None and down_cursor == None:
-            j.seek_tail()
-
-            events = get_events_from_journal(j, limit, lambda j: j.get_previous())
-            events.reverse()
-
-            return PaginatedEntries.from_entries(events)
-        elif up_cursor == None and down_cursor != None:
-            j.seek_cursor(down_cursor)
-            j.get_previous()  # pagination is exclusive
-
-            events = get_events_from_journal(j, limit, lambda j: j.get_previous())
-            events.reverse()
-
-            return PaginatedEntries.from_entries(events)
-        elif up_cursor != None and down_cursor == None:
-            j.seek_cursor(up_cursor)
-            j.get_next()  # pagination is exclusive
-
-            events = get_events_from_journal(j, limit, lambda j: j.get_next())
-
-            return PaginatedEntries.from_entries(events)
-        else:
-            raise NotImplemented(
-                "Pagination by both up_cursor and down_cursor is not implemented"
+        return PaginatedEntries.from_entries(
+            list(
+                map(
+                    lambda x: LogEntry(x),
+                    get_paginated_logs(limit, up_cursor, down_cursor),
+                )
             )
+        )
diff --git a/selfprivacy_api/graphql/schema.py b/selfprivacy_api/graphql/schema.py
index a515d0c..534bacf 100644
--- a/selfprivacy_api/graphql/schema.py
+++ b/selfprivacy_api/graphql/schema.py
@@ -177,8 +177,7 @@ class Subscription:
 
     @strawberry.subscription
     async def log_entries(
-        self,
-        info: strawberry.types.Info,
+        self, info: strawberry.types.Info
     ) -> AsyncGenerator[LogEntry, None]:
         reject_if_unauthenticated(info)
         return log_stream()
diff --git a/selfprivacy_api/utils/systemd_journal.py b/selfprivacy_api/utils/systemd_journal.py
new file mode 100644
index 0000000..6c03c93
--- /dev/null
+++ b/selfprivacy_api/utils/systemd_journal.py
@@ -0,0 +1,55 @@
+import typing
+from systemd import journal
+
+
+def get_events_from_journal(
+    j: journal.Reader, limit: int, next: typing.Callable[[journal.Reader], typing.Dict]
+):
+    events = []
+    i = 0
+    while i < limit:
+        entry = next(j)
+        if entry is None or entry == dict():
+            break
+        if entry["MESSAGE"] != "":
+            events.append(entry)
+            i += 1
+
+    return events
+
+
+def get_paginated_logs(
+    limit: int = 20,
+    up_cursor: str
+    | None = None,  # All entries returned will be lesser than this cursor. Sets upper bound on results.
+    down_cursor: str
+    | None = None,  # All entries returned will be greater than this cursor. Sets lower bound on results.
+):
+    j = journal.Reader()
+
+    if up_cursor is None and down_cursor is None:
+        j.seek_tail()
+
+        events = get_events_from_journal(j, limit, lambda j: j.get_previous())
+        events.reverse()
+
+        return events
+    elif up_cursor is None and down_cursor is not None:
+        j.seek_cursor(down_cursor)
+        j.get_previous()  # pagination is exclusive
+
+        events = get_events_from_journal(j, limit, lambda j: j.get_previous())
+        events.reverse()
+
+        return events
+    elif up_cursor is not None and down_cursor is None:
+        j.seek_cursor(up_cursor)
+        j.get_next()  # pagination is exclusive
+
+        events = get_events_from_journal(j, limit, lambda j: j.get_next())
+
+        return events
+    else:
+        raise NotImplementedError(
+            "Pagination by both up_cursor and down_cursor is not implemented"
+        )

From cc4b41165766a1f1a0b8981887ff127a65f3605e Mon Sep 17 00:00:00 2001
From: Inex Code <inex.code@selfprivacy.org>
Date: Mon, 15 Jul 2024 16:59:15 +0400
Subject: [PATCH 83/86] refactor: Replace strawberry.types.Info with just Info

---
 selfprivacy_api/graphql/schema.py | 15 ++++++---------
 1 file changed, 6 insertions(+), 9 deletions(-)

diff --git a/selfprivacy_api/graphql/schema.py b/selfprivacy_api/graphql/schema.py
index 534bacf..b49a629 100644
--- a/selfprivacy_api/graphql/schema.py
+++ b/selfprivacy_api/graphql/schema.py
@@ -4,6 +4,7 @@
 import asyncio
 from typing import AsyncGenerator, List
 import strawberry
+from strawberry.types import Info
 
 from selfprivacy_api.graphql import IsAuthenticated
 from selfprivacy_api.graphql.mutations.deprecated_mutations import (
@@ -144,11 +145,11 @@ class Mutation(
 
 
 # A cruft for Websockets
-def authenticated(info: strawberry.types.Info) -> bool:
+def authenticated(info: Info) -> bool:
     return IsAuthenticated().has_permission(source=None, info=info)
 
 
-def reject_if_unauthenticated(info: strawberry.types.Info):
+def reject_if_unauthenticated(info: Info):
     if not authenticated(info):
         raise Exception(IsAuthenticated().message)
 
@@ -161,24 +162,20 @@ class Subscription:
     demands it while the spec is vague in this area."""
 
     @strawberry.subscription
-    async def job_updates(
-        self, info: strawberry.types.Info
-    ) -> AsyncGenerator[List[ApiJob], None]:
+    async def job_updates(self, info: Info) -> AsyncGenerator[List[ApiJob], None]:
         reject_if_unauthenticated(info)
         return job_update_generator()
 
     @strawberry.subscription
     # Used for testing, consider deletion to shrink attack surface
-    async def count(self, info: strawberry.types.Info) -> AsyncGenerator[int, None]:
+    async def count(self, info: Info) -> AsyncGenerator[int, None]:
         reject_if_unauthenticated(info)
         for i in range(10):
             yield i
             await asyncio.sleep(0.5)
 
     @strawberry.subscription
-    async def log_entries(
-        self, info: strawberry.types.Info
-    ) -> AsyncGenerator[LogEntry, None]:
+    async def log_entries(self, info: Info) -> AsyncGenerator[LogEntry, None]:
         reject_if_unauthenticated(info)
         return log_stream()
 

From 5c5e098bab3a3d8ea7acf3a42893010464db247c Mon Sep 17 00:00:00 2001
From: Inex Code <inex.code@selfprivacy.org>
Date: Mon, 15 Jul 2024 17:01:33 +0400
Subject: [PATCH 84/86] style: do not break line before logic operator

---
 selfprivacy_api/graphql/queries/logs.py  | 8 ++++----
 selfprivacy_api/utils/systemd_journal.py | 8 ++++----
 2 files changed, 8 insertions(+), 8 deletions(-)

diff --git a/selfprivacy_api/graphql/queries/logs.py b/selfprivacy_api/graphql/queries/logs.py
index 6841f30..cf8fe21 100644
--- a/selfprivacy_api/graphql/queries/logs.py
+++ b/selfprivacy_api/graphql/queries/logs.py
@@ -71,10 +71,10 @@ class Logs:
     def paginated(
         self,
         limit: int = 20,
-        up_cursor: str
-        | None = None,  # All entries returned will be lesser than this cursor. Sets upper bound on results.
-        down_cursor: str
-        | None = None,  # All entries returned will be greater than this cursor. Sets lower bound on results.
+        # All entries returned will be lesser than this cursor. Sets upper bound on results.
+        up_cursor: str | None = None,
+        # All entries returned will be greater than this cursor. Sets lower bound on results.
+        down_cursor: str | None = None,
     ) -> PaginatedEntries:
         if limit > 50:
             raise Exception("You can't fetch more than 50 entries via single request.")
diff --git a/selfprivacy_api/utils/systemd_journal.py b/selfprivacy_api/utils/systemd_journal.py
index 6c03c93..48e97b8 100644
--- a/selfprivacy_api/utils/systemd_journal.py
+++ b/selfprivacy_api/utils/systemd_journal.py
@@ -20,10 +20,10 @@ def get_events_from_journal(
 
 def get_paginated_logs(
     limit: int = 20,
-    up_cursor: str
-    | None = None,  # All entries returned will be lesser than this cursor. Sets upper bound on results.
-    down_cursor: str
-    | None = None,  # All entries returned will be greater than this cursor. Sets lower bound on results.
+    # All entries returned will be lesser than this cursor. Sets upper bound on results.
+    up_cursor: str | None = None,
+    # All entries returned will be greater than this cursor. Sets lower bound on results.
+    down_cursor: str | None = None,
 ):
     j = journal.Reader()
 

From d8fe54e0e941bed8564ee0dab76cc4b5740c570d Mon Sep 17 00:00:00 2001
From: Inex Code <inex.code@selfprivacy.org>
Date: Mon, 15 Jul 2024 17:05:38 +0400
Subject: [PATCH 85/86] fix: do not use bare 'except'

---
 selfprivacy_api/graphql/subscriptions/logs.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/selfprivacy_api/graphql/subscriptions/logs.py b/selfprivacy_api/graphql/subscriptions/logs.py
index be8a004..1e47dba 100644
--- a/selfprivacy_api/graphql/subscriptions/logs.py
+++ b/selfprivacy_api/graphql/subscriptions/logs.py
@@ -1,4 +1,4 @@
-from typing import AsyncGenerator, List
+from typing import AsyncGenerator
 from systemd import journal
 import asyncio
 
@@ -25,7 +25,7 @@ async def log_stream() -> AsyncGenerator[LogEntry, None]:
         entry = await queue.get()
         try:
             yield LogEntry(entry)
-        except:
+        except Exception:
             asyncio.get_event_loop().remove_reader(j)
             return
         queue.task_done()

From a00aae1bee3edf71332290629ea2f711ef91f8ba Mon Sep 17 00:00:00 2001
From: Alexander Tomokhov <alexoundos@selfprivacy.org>
Date: Wed, 17 Apr 2024 15:37:28 +0400
Subject: [PATCH 86/86] fix: remove '-v' in pytest-vm

---
 flake.nix | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/flake.nix b/flake.nix
index ab969a4..4c8880e 100644
--- a/flake.nix
+++ b/flake.nix
@@ -66,7 +66,7 @@
           SCRIPT=$(cat <<EOF
           start_all()
           machine.succeed("ln -sf $NIXOS_VM_SHARED_DIR_GUEST -T ${vmtest-src-dir} >&2")
-          machine.succeed("cd ${vmtest-src-dir} && coverage run -m pytest -v $@ >&2")
+          machine.succeed("cd ${vmtest-src-dir} && coverage run -m pytest $@ >&2")
           machine.succeed("cd ${vmtest-src-dir} && coverage report >&2")
           EOF
           )