From 13d3261d36d878af138f5e411a28409ef183eb82 Mon Sep 17 00:00:00 2001 From: def Date: Thu, 24 Nov 2022 06:08:58 +0400 Subject: [PATCH 01/50] feat: add nix-collect-garbage job --- selfprivacy_api/jobs/nix_collect_garbage.py | 32 +++++++++++++++++++++ 1 file changed, 32 insertions(+) create mode 100644 selfprivacy_api/jobs/nix_collect_garbage.py diff --git a/selfprivacy_api/jobs/nix_collect_garbage.py b/selfprivacy_api/jobs/nix_collect_garbage.py new file mode 100644 index 0000000..18e2a63 --- /dev/null +++ b/selfprivacy_api/jobs/nix_collect_garbage.py @@ -0,0 +1,32 @@ +import re +import subprocess + +from selfprivacy_api.jobs import Job, JobStatus, Jobs +from selfprivacy_api.utils.huey import huey + + +@huey.task() +def nix_collect_garbage(job: Job): + + Jobs.update( + job=job, + status=JobStatus.RUNNING, + progress=0, + status_text="Start cleaning.", + ) + + output = subprocess.check_output("nix-collect-garbage -d") + + pat = re.compile(r"linking saves ([+-]?\d+\.\d+ \w+).+?([+-]?\d+\.\d+ \w+) freed") + match = re.search( + pat, + output, + ) + + Jobs.update( + job=job, + status=JobStatus.FINISHED, + progress=100, + status_text="Сleaning completed.", + result=f"Currently hard linking saves {match.group(1)}, {match.group(2)} freed", + ) From 6b93df9630fc9b3036d801802dd4785e152917de Mon Sep 17 00:00:00 2001 From: def Date: Thu, 24 Nov 2022 06:32:37 +0400 Subject: [PATCH 02/50] fix: subprocess.check_output --- selfprivacy_api/jobs/nix_collect_garbage.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/selfprivacy_api/jobs/nix_collect_garbage.py b/selfprivacy_api/jobs/nix_collect_garbage.py index 18e2a63..9bb6268 100644 --- a/selfprivacy_api/jobs/nix_collect_garbage.py +++ b/selfprivacy_api/jobs/nix_collect_garbage.py @@ -15,7 +15,7 @@ def nix_collect_garbage(job: Job): status_text="Start cleaning.", ) - output = subprocess.check_output("nix-collect-garbage -d") + output = subprocess.check_output(["nix-collect-garbage", "-d"]) pat = re.compile(r"linking saves ([+-]?\d+\.\d+ \w+).+?([+-]?\d+\.\d+ \w+) freed") match = re.search( From 4ce96c303d778bf865be2ac2094b73902bf7c1a2 Mon Sep 17 00:00:00 2001 From: def Date: Sat, 3 Dec 2022 22:27:10 +0400 Subject: [PATCH 03/50] feat: loading percentage --- selfprivacy_api/jobs/nix_collect_garbage.py | 71 +++++++++++++++++---- 1 file changed, 60 insertions(+), 11 deletions(-) diff --git a/selfprivacy_api/jobs/nix_collect_garbage.py b/selfprivacy_api/jobs/nix_collect_garbage.py index 9bb6268..2c81324 100644 --- a/selfprivacy_api/jobs/nix_collect_garbage.py +++ b/selfprivacy_api/jobs/nix_collect_garbage.py @@ -1,3 +1,4 @@ +from time import sleep import re import subprocess @@ -12,21 +13,69 @@ def nix_collect_garbage(job: Job): job=job, status=JobStatus.RUNNING, progress=0, - status_text="Start cleaning.", + status_text="Сalculate the number of dead packages...", ) - output = subprocess.check_output(["nix-collect-garbage", "-d"]) - - pat = re.compile(r"linking saves ([+-]?\d+\.\d+ \w+).+?([+-]?\d+\.\d+ \w+) freed") - match = re.search( - pat, - output, + output = subprocess.check_output( + ["nix-store --gc --print-dead", "--gc", "--print-dead"] ) + dead_packages = len(re.findall("/nix/store/", output.decode("utf-8"))) + package_equal_to_percent = 100 / dead_packages + Jobs.update( job=job, - status=JobStatus.FINISHED, - progress=100, - status_text="Сleaning completed.", - result=f"Currently hard linking saves {match.group(1)}, {match.group(2)} freed", + status=JobStatus.RUNNING, + progress=0, + status_text=f"Found {dead_packages} packages to remove!", ) + + def _parse_line(line): + pattern = re.compile(r"[+-]?\d+\.\d+ \w+ freed") + match = re.search( + pattern, + line, + ) + + if match is None: + Jobs.update( + job=job, + status=JobStatus.FINISHED, + progress=100, + status_text="Completed with an error", + result="We are sorry, result was not found :(", + ) + + else: + Jobs.update( + job=job, + status=JobStatus.FINISHED, + progress=100, + status_text="Сleaning completed.", + result=f"{match.group(0)} have been cleared", + ) + + def _stream_process(process): + go = process.poll() is None + percent = 0 + + for line in process.stdout: + if "deleting '/nix/store/" in line: + percent += package_equal_to_percent + + Jobs.update( + job=job, + status=JobStatus.RUNNING, + progress=int(percent), + status_text="Сleaning...", + ) + + elif "store paths deleted," in line: + _parse_line(line) + + return go + + process = subprocess.Popen( + ["nix-collect-garbage", "-d"], stdout=subprocess.PIPE, stderr=subprocess.STDOUT + ) + _stream_process(process) From 3ecd522bd2dd092ca4063ae2daeed37f32788f37 Mon Sep 17 00:00:00 2001 From: def Date: Sat, 3 Dec 2022 22:30:41 +0400 Subject: [PATCH 04/50] refactor: delete unnecessary import --- selfprivacy_api/jobs/nix_collect_garbage.py | 1 - 1 file changed, 1 deletion(-) diff --git a/selfprivacy_api/jobs/nix_collect_garbage.py b/selfprivacy_api/jobs/nix_collect_garbage.py index 2c81324..78d306e 100644 --- a/selfprivacy_api/jobs/nix_collect_garbage.py +++ b/selfprivacy_api/jobs/nix_collect_garbage.py @@ -1,4 +1,3 @@ -from time import sleep import re import subprocess From 510b94039e68372fce9e72329a3385616467bf60 Mon Sep 17 00:00:00 2001 From: def Date: Tue, 13 Dec 2022 05:44:52 +0400 Subject: [PATCH 05/50] refactor: nix-collect-garbage is now pure --- selfprivacy_api/jobs/nix_collect_garbage.py | 131 +++++++++++------- .../test_graphql/test_nix_collect_garbage.py | 32 +++++ 2 files changed, 110 insertions(+), 53 deletions(-) create mode 100644 tests/test_graphql/test_nix_collect_garbage.py diff --git a/selfprivacy_api/jobs/nix_collect_garbage.py b/selfprivacy_api/jobs/nix_collect_garbage.py index 78d306e..3d84ca3 100644 --- a/selfprivacy_api/jobs/nix_collect_garbage.py +++ b/selfprivacy_api/jobs/nix_collect_garbage.py @@ -5,8 +5,73 @@ from selfprivacy_api.jobs import Job, JobStatus, Jobs from selfprivacy_api.utils.huey import huey +def run_nix_store_print_dead(): + return subprocess.check_output(["nix-store", "--gc", "--print-dead"]) + + +def run_nix_collect_garbage(): + return subprocess.Popen( + ["nix-collect-garbage", "-d"], stdout=subprocess.PIPE, stderr=subprocess.STDOUT + ) + + +def parse_line(line, job: Job): + pattern = re.compile(r"[+-]?\d+\.\d+ \w+ freed") + match = re.search( + pattern, + line, + ) + + if match is None: + Jobs.update( + job=job, + status=JobStatus.FINISHED, + progress=100, + status_text="Completed with an error", + result="We are sorry, result was not found :(", + ) + + else: + Jobs.update( + job=job, + status=JobStatus.FINISHED, + progress=100, + status_text="Сleaning completed.", + result=f"{match.group(0)} have been cleared", + ) + + +def stream_process( + process, + package_equal_to_percent, + job: Job, +): + go = process.poll() is None + percent = 0 + + for line in process.stdout: + if "deleting '/nix/store/" in line: + percent += package_equal_to_percent + + Jobs.update( + job=job, + status=JobStatus.RUNNING, + progress=int(percent), + status_text="Сleaning...", + ) + + elif "store paths deleted," in line: + parse_line(line, job) + + return go + + @huey.task() -def nix_collect_garbage(job: Job): +def nix_collect_garbage( + job: Job, + run_nix_store=run_nix_store_print_dead, + run_nix_collect=run_nix_collect_garbage, +): # innocent as a pure function Jobs.update( job=job, @@ -15,11 +80,19 @@ def nix_collect_garbage(job: Job): status_text="Сalculate the number of dead packages...", ) - output = subprocess.check_output( - ["nix-store --gc --print-dead", "--gc", "--print-dead"] - ) + output = run_nix_store() dead_packages = len(re.findall("/nix/store/", output.decode("utf-8"))) + + if dead_packages == 0: + Jobs.update( + job=job, + status=JobStatus.FINISHED, + progress=100, + status_text="Nothing to clear", + result="System is clear", + ) + package_equal_to_percent = 100 / dead_packages Jobs.update( @@ -29,52 +102,4 @@ def nix_collect_garbage(job: Job): status_text=f"Found {dead_packages} packages to remove!", ) - def _parse_line(line): - pattern = re.compile(r"[+-]?\d+\.\d+ \w+ freed") - match = re.search( - pattern, - line, - ) - - if match is None: - Jobs.update( - job=job, - status=JobStatus.FINISHED, - progress=100, - status_text="Completed with an error", - result="We are sorry, result was not found :(", - ) - - else: - Jobs.update( - job=job, - status=JobStatus.FINISHED, - progress=100, - status_text="Сleaning completed.", - result=f"{match.group(0)} have been cleared", - ) - - def _stream_process(process): - go = process.poll() is None - percent = 0 - - for line in process.stdout: - if "deleting '/nix/store/" in line: - percent += package_equal_to_percent - - Jobs.update( - job=job, - status=JobStatus.RUNNING, - progress=int(percent), - status_text="Сleaning...", - ) - - elif "store paths deleted," in line: - _parse_line(line) - - return go - - process = subprocess.Popen( - ["nix-collect-garbage", "-d"], stdout=subprocess.PIPE, stderr=subprocess.STDOUT - ) - _stream_process(process) + stream_process(run_nix_collect, package_equal_to_percent, job) diff --git a/tests/test_graphql/test_nix_collect_garbage.py b/tests/test_graphql/test_nix_collect_garbage.py new file mode 100644 index 0000000..a14dcb3 --- /dev/null +++ b/tests/test_graphql/test_nix_collect_garbage.py @@ -0,0 +1,32 @@ +# pylint: disable=redefined-outer-name +# pylint: disable=unused-argument +# pylint: disable=missing-function-docstring + +import pytest + +from selfprivacy_api.jobs.nix_collect_garbage import nix_collect_garbage + + + + + + created_at: datetime.datetime + updated_at: datetime.datetime + uid: UUID + type_id: str + name: str + description: str + status: JobStatus + + +def test_nix_collect_garbage(job( + created_at = "2019-12-04", + updated_at = "2019-12-04", + uid = UUID, + type_id = "typeid", + name = "name", + description: "desc", + status = status(CREATED = "CREATED"), +)): + + assert nix_collect_garbage() is not None From 2340a0f8e90e5e135d6ec1c0b4d52de876ac15f7 Mon Sep 17 00:00:00 2001 From: def Date: Mon, 19 Dec 2022 06:27:44 +0400 Subject: [PATCH 06/50] test: fix nix collect garbage, add tests --- selfprivacy_api/jobs/nix_collect_garbage.py | 105 +++++++----- .../test_graphql/test_nix_collect_garbage.py | 154 +++++++++++++++--- 2 files changed, 199 insertions(+), 60 deletions(-) diff --git a/selfprivacy_api/jobs/nix_collect_garbage.py b/selfprivacy_api/jobs/nix_collect_garbage.py index 3d84ca3..32de683 100644 --- a/selfprivacy_api/jobs/nix_collect_garbage.py +++ b/selfprivacy_api/jobs/nix_collect_garbage.py @@ -1,105 +1,128 @@ import re import subprocess -from selfprivacy_api.jobs import Job, JobStatus, Jobs -from selfprivacy_api.utils.huey import huey +from selfprivacy_api.jobs import JobStatus, Jobs + + +COMPLETED_WITH_ERROR = "Completed with an error" +RESULT_WAAS_NOT_FOUND_ERROR = "We are sorry, result was not found :(" +CLEAR_COMPLETED = "Сleaning completed." def run_nix_store_print_dead(): - return subprocess.check_output(["nix-store", "--gc", "--print-dead"]) + return subprocess.check_output(["nix-store", "--gc", "--print-dead"]).decode( + "utf-8" + ) def run_nix_collect_garbage(): return subprocess.Popen( ["nix-collect-garbage", "-d"], stdout=subprocess.PIPE, stderr=subprocess.STDOUT - ) + ).stdout -def parse_line(line, job: Job): - pattern = re.compile(r"[+-]?\d+\.\d+ \w+ freed") +def set_job_status_wrapper(Jobs, job): + def set_job_status(status, progress, status_text, result="Default result"): + Jobs.update( + job=job, + status=status, + progress=progress, + status_text=status_text, + result=result, + ) + + return set_job_status + + +def parse_line(line): + pattern = re.compile(r"[+-]?\d+\.\d+ \w+(?= freed)") match = re.search( pattern, line, ) if match is None: - Jobs.update( - job=job, - status=JobStatus.FINISHED, - progress=100, - status_text="Completed with an error", - result="We are sorry, result was not found :(", + return ( + JobStatus.FINISHED, + 100, + COMPLETED_WITH_ERROR, + RESULT_WAAS_NOT_FOUND_ERROR, ) else: - Jobs.update( - job=job, - status=JobStatus.FINISHED, - progress=100, - status_text="Сleaning completed.", - result=f"{match.group(0)} have been cleared", + return ( + JobStatus.FINISHED, + 100, + CLEAR_COMPLETED, + f"{match.group(0)} have been cleared", ) def stream_process( - process, + stream, package_equal_to_percent, - job: Job, + set_job_status, ): - go = process.poll() is None percent = 0 - for line in process.stdout: + for line in stream: if "deleting '/nix/store/" in line: percent += package_equal_to_percent - Jobs.update( - job=job, + set_job_status( status=JobStatus.RUNNING, progress=int(percent), status_text="Сleaning...", ) elif "store paths deleted," in line: - parse_line(line, job) - - return go + status = parse_line(line) + set_job_status( + status=status[0], + progress=status[1], + status_text=status[2], + result=status[3], + ) + + +def get_dead_packages(output): + dead = len(re.findall("/nix/store/", output)) + percent = None + if dead != 0: + percent = 100 / dead + return dead, percent -@huey.task() def nix_collect_garbage( - job: Job, + job, + jobs=Jobs, run_nix_store=run_nix_store_print_dead, run_nix_collect=run_nix_collect_garbage, + set_job_status=None, ): # innocent as a pure function + set_job_status = set_job_status or set_job_status_wrapper(jobs, job) - Jobs.update( - job=job, + set_job_status( status=JobStatus.RUNNING, progress=0, status_text="Сalculate the number of dead packages...", ) - output = run_nix_store() - - dead_packages = len(re.findall("/nix/store/", output.decode("utf-8"))) + dead_packages, package_equal_to_percent = get_dead_packages(run_nix_store()) if dead_packages == 0: - Jobs.update( - job=job, + set_job_status( status=JobStatus.FINISHED, progress=100, status_text="Nothing to clear", result="System is clear", ) + return - package_equal_to_percent = 100 / dead_packages - - Jobs.update( - job=job, + set_job_status( status=JobStatus.RUNNING, progress=0, status_text=f"Found {dead_packages} packages to remove!", ) - stream_process(run_nix_collect, package_equal_to_percent, job) + stream_process(run_nix_collect(), package_equal_to_percent, set_job_status) diff --git a/tests/test_graphql/test_nix_collect_garbage.py b/tests/test_graphql/test_nix_collect_garbage.py index a14dcb3..b903027 100644 --- a/tests/test_graphql/test_nix_collect_garbage.py +++ b/tests/test_graphql/test_nix_collect_garbage.py @@ -3,30 +3,146 @@ # pylint: disable=missing-function-docstring import pytest +from selfprivacy_api.jobs import JobStatus -from selfprivacy_api.jobs.nix_collect_garbage import nix_collect_garbage +from selfprivacy_api.jobs.nix_collect_garbage import ( + get_dead_packages, + nix_collect_garbage, + parse_line, + CLEAR_COMPLETED, + COMPLETED_WITH_ERROR, + stream_process, + RESULT_WAAS_NOT_FOUND_ERROR, +) +output_print_dead = """ +finding garbage collector roots... +determining live/dead paths... +/nix/store/02k8pmw00p7p7mf2dg3n057771w7liia-python3.10-cchardet-2.1.7 +/nix/store/03vc6dznx8njbvyd3gfhfa4n5j4lvhbl-python3.10-async-timeout-4.0.2 +/nix/store/03ybv2dvfk7c3cpb527y5kzf6i35ch41-python3.10-pycparser-2.21 +/nix/store/04dn9slfqwhqisn1j3jv531lms9w5wlj-python3.10-hypothesis-6.50.1.drv +/nix/store/04hhx2z1iyi3b48hxykiw1g03lp46jk7-python-remove-bin-bytecode-hook +""" - - created_at: datetime.datetime - updated_at: datetime.datetime - uid: UUID - type_id: str - name: str - description: str - status: JobStatus +output_collect_garbage = """ +removing old generations of profile /nix/var/nix/profiles/per-user/def/channels +finding garbage collector roots... +deleting garbage... +deleting '/nix/store/02k8pmw00p7p7mf2dg3n057771w7liia-python3.10-cchardet-2.1.7' +deleting '/nix/store/03vc6dznx8njbvyd3gfhfa4n5j4lvhbl-python3.10-async-timeout-4.0.2' +deleting '/nix/store/03ybv2dvfk7c3cpb527y5kzf6i35ch41-python3.10-pycparser-2.21' +deleting '/nix/store/04dn9slfqwhqisn1j3jv531lms9w5wlj-python3.10-hypothesis-6.50.1.drv' +deleting '/nix/store/04hhx2z1iyi3b48hxykiw1g03lp46jk7-python-remove-bin-bytecode-hook' +deleting unused links... +note: currently hard linking saves -0.00 MiB +190 store paths deleted, 425.51 MiB freed +""" -def test_nix_collect_garbage(job( - created_at = "2019-12-04", - updated_at = "2019-12-04", - uid = UUID, - type_id = "typeid", - name = "name", - description: "desc", - status = status(CREATED = "CREATED"), -)): +def test_parse_line(): + txt = "190 store paths deleted, 425.51 MiB freed" + output = ( + JobStatus.FINISHED, + 100, + CLEAR_COMPLETED, + "425.51 MiB have been cleared", + ) + assert parse_line(txt) == output - assert nix_collect_garbage() is not None + +def test_parse_line_with_blank_line(): + txt = "" + output = ( + JobStatus.FINISHED, + 100, + COMPLETED_WITH_ERROR, + RESULT_WAAS_NOT_FOUND_ERROR, + ) + assert parse_line(txt) == output + + +def test_get_dead_packages(): + assert get_dead_packages(output_print_dead) == (5, 20.0) + + +def test_get_dead_packages_zero(): + assert get_dead_packages("") == (0, None) + + +def test_stream_process(): + log_event = [] + reference = [ + (JobStatus.RUNNING, 20, "Сleaning...", ""), + (JobStatus.RUNNING, 40, "Сleaning...", ""), + (JobStatus.RUNNING, 60, "Сleaning...", ""), + (JobStatus.RUNNING, 80, "Сleaning...", ""), + (JobStatus.RUNNING, 100, "Сleaning...", ""), + ( + JobStatus.FINISHED, + 100, + "Сleaning completed.", + "425.51 MiB have been cleared", + ), + ] + + def set_job_status(status, progress, status_text, result=""): + log_event.append((status, progress, status_text, result)) + + stream_process(output_collect_garbage.split("\n"), 20.0, set_job_status) + assert log_event == reference + + +def test_nix_collect_garbage(): + log_event = [] + reference = [ + (JobStatus.RUNNING, 0, "Сalculate the number of dead packages...", ""), + (JobStatus.RUNNING, 0, "Found 5 packages to remove!", ""), + (JobStatus.RUNNING, 20, "Сleaning...", ""), + (JobStatus.RUNNING, 40, "Сleaning...", ""), + (JobStatus.RUNNING, 60, "Сleaning...", ""), + (JobStatus.RUNNING, 80, "Сleaning...", ""), + (JobStatus.RUNNING, 100, "Сleaning...", ""), + ( + JobStatus.FINISHED, + 100, + "Сleaning completed.", + "425.51 MiB have been cleared", + ), + ] + + def set_job_status(status="", progress="", status_text="", result=""): + log_event.append((status, progress, status_text, result)) + + nix_collect_garbage( + None, + None, + lambda: output_print_dead, + lambda: output_collect_garbage.split("\n"), + set_job_status, + ) + + assert log_event == reference + + +def test_nix_collect_garbage_zero_trash(): + log_event = [] + reference = [ + (JobStatus.RUNNING, 0, "Сalculate the number of dead packages...", ""), + (JobStatus.FINISHED, 100, "Nothing to clear", "System is clear"), + ] + + def set_job_status(status="", progress="", status_text="", result=""): + log_event.append((status, progress, status_text, result)) + + nix_collect_garbage( + None, + None, + lambda: "", + lambda: output_collect_garbage.split("\n"), + set_job_status, + ) + + assert log_event == reference From fcdd61006b8ea509bd6363c58947ae605a784ec9 Mon Sep 17 00:00:00 2001 From: def Date: Tue, 27 Dec 2022 04:15:02 +0400 Subject: [PATCH 07/50] fix: types, add tests --- selfprivacy_api/jobs/nix_collect_garbage.py | 7 +++-- selfprivacy_api/task_registry.py | 1 + .../test_graphql/test_nix_collect_garbage.py | 27 +++++++++++++++++-- 3 files changed, 31 insertions(+), 4 deletions(-) diff --git a/selfprivacy_api/jobs/nix_collect_garbage.py b/selfprivacy_api/jobs/nix_collect_garbage.py index 32de683..587da65 100644 --- a/selfprivacy_api/jobs/nix_collect_garbage.py +++ b/selfprivacy_api/jobs/nix_collect_garbage.py @@ -2,10 +2,11 @@ import re import subprocess from selfprivacy_api.jobs import JobStatus, Jobs +from selfprivacy_api.utils.huey import huey COMPLETED_WITH_ERROR = "Completed with an error" -RESULT_WAAS_NOT_FOUND_ERROR = "We are sorry, result was not found :(" +RESULT_WAS_NOT_FOUND_ERROR = "We are sorry, result was not found :(" CLEAR_COMPLETED = "Сleaning completed." @@ -46,7 +47,7 @@ def parse_line(line): JobStatus.FINISHED, 100, COMPLETED_WITH_ERROR, - RESULT_WAAS_NOT_FOUND_ERROR, + RESULT_WAS_NOT_FOUND_ERROR, ) else: @@ -72,6 +73,7 @@ def stream_process( set_job_status( status=JobStatus.RUNNING, progress=int(percent), + progress=int(percent), status_text="Сleaning...", ) @@ -93,6 +95,7 @@ def get_dead_packages(output): return dead, percent +@huey.task() def nix_collect_garbage( job, jobs=Jobs, diff --git a/selfprivacy_api/task_registry.py b/selfprivacy_api/task_registry.py index 82eaf06..f9f92e1 100644 --- a/selfprivacy_api/task_registry.py +++ b/selfprivacy_api/task_registry.py @@ -2,3 +2,4 @@ from selfprivacy_api.utils.huey import huey from selfprivacy_api.jobs.test import test_job from selfprivacy_api.restic_controller.tasks import * from selfprivacy_api.services.generic_service_mover import move_service +from selfprivacy_api.jobs.nix_collect_garbage import nix_collect_garbage diff --git a/tests/test_graphql/test_nix_collect_garbage.py b/tests/test_graphql/test_nix_collect_garbage.py index b903027..4305f5c 100644 --- a/tests/test_graphql/test_nix_collect_garbage.py +++ b/tests/test_graphql/test_nix_collect_garbage.py @@ -4,6 +4,11 @@ import pytest from selfprivacy_api.jobs import JobStatus +from selfprivacy_api.graphql import schema +import asyncio +import strawberry + +# from selfprivacy_api.graphql.schema import Subscription from selfprivacy_api.jobs.nix_collect_garbage import ( get_dead_packages, @@ -12,7 +17,7 @@ from selfprivacy_api.jobs.nix_collect_garbage import ( CLEAR_COMPLETED, COMPLETED_WITH_ERROR, stream_process, - RESULT_WAAS_NOT_FOUND_ERROR, + RESULT_WAS_NOT_FOUND_ERROR, ) @@ -59,7 +64,7 @@ def test_parse_line_with_blank_line(): JobStatus.FINISHED, 100, COMPLETED_WITH_ERROR, - RESULT_WAAS_NOT_FOUND_ERROR, + RESULT_WAS_NOT_FOUND_ERROR, ) assert parse_line(txt) == output @@ -146,3 +151,21 @@ def test_nix_collect_garbage_zero_trash(): ) assert log_event == reference + + +@pytest.mark.asyncio +async def test_graphql_nix_collect_garbage(): + query = """ + subscription { + nixCollectGarbage() + } + """ + + schema_for_garbage = strawberry.Schema( + query=schema.Query, mutation=schema.Mutation, subscription=schema.Subscription + ) + + sub = await schema_for_garbage.subscribe(query) + for result in sub: + assert not result.errors + assert result.data == {} From 88d66fea8cbc302984f31386c84acea50bf55a80 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=92=D0=B0=D1=88=C3=90dettlaff?= Date: Thu, 23 Mar 2023 20:49:30 +0300 Subject: [PATCH 08/50] fix: percentage --- dump.rdb | Bin 0 -> 88 bytes selfprivacy_api/jobs/nix_collect_garbage.py | 12 ++-- setup.py | 0 shell.nix | 52 ++++-------------- .../test_graphql/test_nix_collect_garbage.py | 29 +++++----- 5 files changed, 30 insertions(+), 63 deletions(-) create mode 100644 dump.rdb mode change 100755 => 100644 setup.py diff --git a/dump.rdb b/dump.rdb new file mode 100644 index 0000000000000000000000000000000000000000..2f3b490816640dcfd897e08c8eb137f27ec5a6e2 GIT binary patch literal 88 zcmWG?b@2=~FfcUu#aWb^l3A=uO-d|IJ;3n)^rLMb4lXTO2>_mAByj)$ literal 0 HcmV?d00001 diff --git a/selfprivacy_api/jobs/nix_collect_garbage.py b/selfprivacy_api/jobs/nix_collect_garbage.py index 587da65..3d13382 100644 --- a/selfprivacy_api/jobs/nix_collect_garbage.py +++ b/selfprivacy_api/jobs/nix_collect_garbage.py @@ -61,19 +61,19 @@ def parse_line(line): def stream_process( stream, - package_equal_to_percent, + total_dead_packages, set_job_status, ): - percent = 0 + completed_packages = 0 for line in stream: if "deleting '/nix/store/" in line: - percent += package_equal_to_percent + completed_packages += 1 + percent = int((completed_packages / total_dead_packages) * 100) set_job_status( status=JobStatus.RUNNING, - progress=int(percent), - progress=int(percent), + progress=percent, status_text="Сleaning...", ) @@ -95,7 +95,7 @@ def get_dead_packages(output): return dead, percent -@huey.task() +# @huey.task() # ломает все к фигам def nix_collect_garbage( job, jobs=Jobs, diff --git a/setup.py b/setup.py old mode 100755 new mode 100644 diff --git a/shell.nix b/shell.nix index 0ccb99d..b620ee2 100644 --- a/shell.nix +++ b/shell.nix @@ -1,6 +1,6 @@ -{ pkgs ? import { } }: +{ pkgs ? import { } }: let - sp-python = pkgs.python39.withPackages (p: with p; [ + sp-python = pkgs.python310.withPackages (p: with p; [ setuptools portalocker pytz @@ -18,54 +18,24 @@ let black fastapi uvicorn - (buildPythonPackage rec { - pname = "strawberry-graphql"; - version = "0.123.0"; - format = "pyproject"; - patches = [ - ./strawberry-graphql.patch - ]; - propagatedBuildInputs = [ - typing-extensions - python-multipart - python-dateutil - # flask - pydantic - pygments - poetry - # flask-cors - (buildPythonPackage rec { - pname = "graphql-core"; - version = "3.2.0"; - format = "setuptools"; - src = fetchPypi { - inherit pname version; - sha256 = "sha256-huKgvgCL/eGe94OI3opyWh2UKpGQykMcJKYIN5c4A84="; - }; - checkInputs = [ - pytest-asyncio - pytest-benchmark - pytestCheckHook - ]; - pythonImportsCheck = [ - "graphql" - ]; - }) - ]; - src = fetchPypi { - inherit pname version; - sha256 = "KsmZ5Xv8tUg6yBxieAEtvoKoRG60VS+iVGV0X6oCExo="; - }; - }) + redis + strawberry-graphql ]); in pkgs.mkShell { buildInputs = [ sp-python pkgs.black + pkgs.redis + pkgs.restic ]; shellHook = '' PYTHONPATH=${sp-python}/${sp-python.sitePackages} + # envs set with export and as attributes are treated differently. + # for example. printenv will not fetch the value of an attribute. + export USE_REDIS_PORT=6379 + pkill redis-server + redis-server --bind 127.0.0.1 --port $USE_REDIS_PORT >/dev/null & # maybe set more env-vars ''; } diff --git a/tests/test_graphql/test_nix_collect_garbage.py b/tests/test_graphql/test_nix_collect_garbage.py index 4305f5c..0db620a 100644 --- a/tests/test_graphql/test_nix_collect_garbage.py +++ b/tests/test_graphql/test_nix_collect_garbage.py @@ -96,26 +96,21 @@ def test_stream_process(): def set_job_status(status, progress, status_text, result=""): log_event.append((status, progress, status_text, result)) - stream_process(output_collect_garbage.split("\n"), 20.0, set_job_status) + stream_process(output_collect_garbage.split("\n"), 5, set_job_status) assert log_event == reference def test_nix_collect_garbage(): log_event = [] reference = [ - (JobStatus.RUNNING, 0, "Сalculate the number of dead packages...", ""), - (JobStatus.RUNNING, 0, "Found 5 packages to remove!", ""), - (JobStatus.RUNNING, 20, "Сleaning...", ""), - (JobStatus.RUNNING, 40, "Сleaning...", ""), - (JobStatus.RUNNING, 60, "Сleaning...", ""), - (JobStatus.RUNNING, 80, "Сleaning...", ""), - (JobStatus.RUNNING, 100, "Сleaning...", ""), - ( - JobStatus.FINISHED, - 100, - "Сleaning completed.", - "425.51 MiB have been cleared", - ), + (JobStatus.RUNNING, 0, 'Сalculate the number of dead packages...', ''), + (JobStatus.RUNNING, 0, 'Found 5 packages to remove!', ''), + (JobStatus.RUNNING, 5, 'Сleaning...', ''), + (JobStatus.RUNNING, 10, 'Сleaning...', ''), + (JobStatus.RUNNING, 15, 'Сleaning...', ''), + (JobStatus.RUNNING, 20, 'Сleaning...', ''), + (JobStatus.RUNNING, 25, 'Сleaning...', ''), + (JobStatus.FINISHED, 100, 'Сleaning completed.', '425.51 MiB have been cleared'), ] def set_job_status(status="", progress="", status_text="", result=""): @@ -128,6 +123,8 @@ def test_nix_collect_garbage(): lambda: output_collect_garbage.split("\n"), set_job_status, ) + print("log_event:", log_event) + print("reference:", reference) assert log_event == reference @@ -152,7 +149,7 @@ def test_nix_collect_garbage_zero_trash(): assert log_event == reference - +# андр констракнш @pytest.mark.asyncio async def test_graphql_nix_collect_garbage(): query = """ @@ -166,6 +163,6 @@ async def test_graphql_nix_collect_garbage(): ) sub = await schema_for_garbage.subscribe(query) - for result in sub: + async for result in sub: assert not result.errors assert result.data == {} From cf06d4c8d545badcfffaf392f7d466d1359c64a5 Mon Sep 17 00:00:00 2001 From: dettlaff Date: Mon, 3 Apr 2023 17:41:41 +0300 Subject: [PATCH 09/50] refactor: delete purest of nix_collect_garbage() --- .gitignore | 3 +++ selfprivacy_api/jobs/nix_collect_garbage.py | 13 +++++-------- 2 files changed, 8 insertions(+), 8 deletions(-) mode change 100755 => 100644 .gitignore diff --git a/.gitignore b/.gitignore old mode 100755 new mode 100644 index 7941396..48a9439 --- a/.gitignore +++ b/.gitignore @@ -147,3 +147,6 @@ cython_debug/ # End of https://www.toptal.com/developers/gitignore/api/flask *.db + +# Redis db +*.rdb \ No newline at end of file diff --git a/selfprivacy_api/jobs/nix_collect_garbage.py b/selfprivacy_api/jobs/nix_collect_garbage.py index 3d13382..ec894fd 100644 --- a/selfprivacy_api/jobs/nix_collect_garbage.py +++ b/selfprivacy_api/jobs/nix_collect_garbage.py @@ -95,15 +95,12 @@ def get_dead_packages(output): return dead, percent -# @huey.task() # ломает все к фигам +# @huey.task() def nix_collect_garbage( job, jobs=Jobs, - run_nix_store=run_nix_store_print_dead, - run_nix_collect=run_nix_collect_garbage, - set_job_status=None, -): # innocent as a pure function - set_job_status = set_job_status or set_job_status_wrapper(jobs, job) +): + set_job_status = set_job_status_wrapper(jobs, job) set_job_status( status=JobStatus.RUNNING, @@ -111,7 +108,7 @@ def nix_collect_garbage( status_text="Сalculate the number of dead packages...", ) - dead_packages, package_equal_to_percent = get_dead_packages(run_nix_store()) + dead_packages, package_equal_to_percent = get_dead_packages(run_nix_store_print_dead()) if dead_packages == 0: set_job_status( @@ -128,4 +125,4 @@ def nix_collect_garbage( status_text=f"Found {dead_packages} packages to remove!", ) - stream_process(run_nix_collect(), package_equal_to_percent, set_job_status) + stream_process(run_nix_collect_garbage(), package_equal_to_percent, set_job_status) From a67f3e822adf6a0a03d8ca6a8a206b158e7525f3 Mon Sep 17 00:00:00 2001 From: dettlaff Date: Wed, 5 Apr 2023 13:49:56 +0300 Subject: [PATCH 10/50] fix: do a redis reset() --- dump.rdb | Bin 88 -> 88 bytes selfprivacy_api/jobs/nix_collect_garbage.py | 7 +- .../test_graphql/test_nix_collect_garbage.py | 126 ++++++++++++------ 3 files changed, 86 insertions(+), 47 deletions(-) diff --git a/dump.rdb b/dump.rdb index 2f3b490816640dcfd897e08c8eb137f27ec5a6e2..7a274dbe6fa11a2889c1664f1b0cbee5dde9742c 100644 GIT binary patch delta 45 zcma!um|&n$&#ImBi=(tSHAOc!HTRH$CNIM;j>P;l-K50g)B_Cvryq~qzCu^z4FGn_ B5$ON` delta 45 zcma!um|&n$mn)OCL B6mI|k diff --git a/selfprivacy_api/jobs/nix_collect_garbage.py b/selfprivacy_api/jobs/nix_collect_garbage.py index ec894fd..82e00e2 100644 --- a/selfprivacy_api/jobs/nix_collect_garbage.py +++ b/selfprivacy_api/jobs/nix_collect_garbage.py @@ -98,9 +98,8 @@ def get_dead_packages(output): # @huey.task() def nix_collect_garbage( job, - jobs=Jobs, ): - set_job_status = set_job_status_wrapper(jobs, job) + set_job_status = set_job_status_wrapper(Jobs, job) set_job_status( status=JobStatus.RUNNING, @@ -108,7 +107,9 @@ def nix_collect_garbage( status_text="Сalculate the number of dead packages...", ) - dead_packages, package_equal_to_percent = get_dead_packages(run_nix_store_print_dead()) + dead_packages, package_equal_to_percent = get_dead_packages( + run_nix_store_print_dead() + ) if dead_packages == 0: set_job_status( diff --git a/tests/test_graphql/test_nix_collect_garbage.py b/tests/test_graphql/test_nix_collect_garbage.py index 0db620a..98848b4 100644 --- a/tests/test_graphql/test_nix_collect_garbage.py +++ b/tests/test_graphql/test_nix_collect_garbage.py @@ -3,11 +3,11 @@ # pylint: disable=missing-function-docstring import pytest -from selfprivacy_api.jobs import JobStatus -from selfprivacy_api.graphql import schema -import asyncio import strawberry +from selfprivacy_api.jobs import JobStatus, Jobs +from selfprivacy_api.graphql import schema + # from selfprivacy_api.graphql.schema import Subscription from selfprivacy_api.jobs.nix_collect_garbage import ( @@ -21,7 +21,7 @@ from selfprivacy_api.jobs.nix_collect_garbage import ( ) -output_print_dead = """ +OUTPUT_PRINT_DEAD = """ finding garbage collector roots... determining live/dead paths... /nix/store/02k8pmw00p7p7mf2dg3n057771w7liia-python3.10-cchardet-2.1.7 @@ -32,7 +32,7 @@ determining live/dead paths... """ -output_collect_garbage = """ +OUTPUT_COLLECT_GARBAGE = """ removing old generations of profile /nix/var/nix/profiles/per-user/def/channels finding garbage collector roots... deleting garbage... @@ -46,8 +46,53 @@ note: currently hard linking saves -0.00 MiB 190 store paths deleted, 425.51 MiB freed """ +log_event = [] + + +def set_job_status(status="", progress="", status_text="", result=""): + log_event.append((status, progress, status_text, result)) + + +@pytest.fixture +def mock_set_job_status(mocker): + mock = mocker.patch( + "selfprivacy_api.jobs.nix_collect_garbage.set_job_status_wrapper", + autospec=True, + return_value=set_job_status, + ) + return mock + + +@pytest.fixture +def mock_run_nix_collect_garbage(mocker): + mock = mocker.patch( + "selfprivacy_api.jobs.nix_collect_garbage.run_nix_collect_garbage", + autospec=True, + return_value=OUTPUT_COLLECT_GARBAGE.split("\n"), + ) + return mock + + +@pytest.fixture +def mock_run_nix_store_print_dead(mocker): + mock = mocker.patch( + "selfprivacy_api.jobs.nix_collect_garbage.run_nix_store_print_dead", + autospec=True, + return_value="", + ) + return mock + + +@pytest.fixture +def job_reset(): + Jobs.reset() + + +# --- + + +def test_parse_line(job_reset): -def test_parse_line(): txt = "190 store paths deleted, 425.51 MiB freed" output = ( JobStatus.FINISHED, @@ -58,7 +103,7 @@ def test_parse_line(): assert parse_line(txt) == output -def test_parse_line_with_blank_line(): +def test_parse_line_with_blank_line(job_reset): txt = "" output = ( JobStatus.FINISHED, @@ -69,11 +114,11 @@ def test_parse_line_with_blank_line(): assert parse_line(txt) == output -def test_get_dead_packages(): - assert get_dead_packages(output_print_dead) == (5, 20.0) +def test_get_dead_packages(job_resetм): + assert get_dead_packages(OUTPUT_PRINT_DEAD) == (5, 20.0) -def test_get_dead_packages_zero(): +def test_get_dead_packages_zero(job_reset): assert get_dead_packages("") == (0, None) @@ -96,59 +141,52 @@ def test_stream_process(): def set_job_status(status, progress, status_text, result=""): log_event.append((status, progress, status_text, result)) - stream_process(output_collect_garbage.split("\n"), 5, set_job_status) + stream_process(OUTPUT_COLLECT_GARBAGE.split("\n"), 5, set_job_status) assert log_event == reference -def test_nix_collect_garbage(): +def test_nix_collect_garbage( + mock_set_job_status, mock_run_nix_collect_garbage, job_reset +): log_event = [] reference = [ - (JobStatus.RUNNING, 0, 'Сalculate the number of dead packages...', ''), - (JobStatus.RUNNING, 0, 'Found 5 packages to remove!', ''), - (JobStatus.RUNNING, 5, 'Сleaning...', ''), - (JobStatus.RUNNING, 10, 'Сleaning...', ''), - (JobStatus.RUNNING, 15, 'Сleaning...', ''), - (JobStatus.RUNNING, 20, 'Сleaning...', ''), - (JobStatus.RUNNING, 25, 'Сleaning...', ''), - (JobStatus.FINISHED, 100, 'Сleaning completed.', '425.51 MiB have been cleared'), + (JobStatus.RUNNING, 0, "Сalculate the number of dead packages...", ""), + (JobStatus.RUNNING, 0, "Found 5 packages to remove!", ""), + (JobStatus.RUNNING, 5, "Сleaning...", ""), + (JobStatus.RUNNING, 10, "Сleaning...", ""), + (JobStatus.RUNNING, 15, "Сleaning...", ""), + (JobStatus.RUNNING, 20, "Сleaning...", ""), + (JobStatus.RUNNING, 25, "Сleaning...", ""), + ( + JobStatus.FINISHED, + 100, + "Сleaning completed.", + "425.51 MiB have been cleared", + ), ] - def set_job_status(status="", progress="", status_text="", result=""): - log_event.append((status, progress, status_text, result)) - - nix_collect_garbage( - None, - None, - lambda: output_print_dead, - lambda: output_collect_garbage.split("\n"), - set_job_status, - ) - print("log_event:", log_event) - print("reference:", reference) + nix_collect_garbage(None) assert log_event == reference -def test_nix_collect_garbage_zero_trash(): - log_event = [] +def test_nix_collect_garbage_zero_trash( + mock_set_job_status, + mock_run_nix_collect_garbage, + mock_run_nix_store_print_dead, + job_reset, +): + reference = [ (JobStatus.RUNNING, 0, "Сalculate the number of dead packages...", ""), (JobStatus.FINISHED, 100, "Nothing to clear", "System is clear"), ] - def set_job_status(status="", progress="", status_text="", result=""): - log_event.append((status, progress, status_text, result)) - - nix_collect_garbage( - None, - None, - lambda: "", - lambda: output_collect_garbage.split("\n"), - set_job_status, - ) + nix_collect_garbage(None) assert log_event == reference + # андр констракнш @pytest.mark.asyncio async def test_graphql_nix_collect_garbage(): From 57ec87bf43ecfc2bdb59b8ba68b186157e7370b0 Mon Sep 17 00:00:00 2001 From: def Date: Thu, 24 Nov 2022 06:08:58 +0400 Subject: [PATCH 11/50] feat: add nix-collect-garbage job --- selfprivacy_api/jobs/nix_collect_garbage.py | 32 +++++++++++++++++++++ 1 file changed, 32 insertions(+) create mode 100644 selfprivacy_api/jobs/nix_collect_garbage.py diff --git a/selfprivacy_api/jobs/nix_collect_garbage.py b/selfprivacy_api/jobs/nix_collect_garbage.py new file mode 100644 index 0000000..18e2a63 --- /dev/null +++ b/selfprivacy_api/jobs/nix_collect_garbage.py @@ -0,0 +1,32 @@ +import re +import subprocess + +from selfprivacy_api.jobs import Job, JobStatus, Jobs +from selfprivacy_api.utils.huey import huey + + +@huey.task() +def nix_collect_garbage(job: Job): + + Jobs.update( + job=job, + status=JobStatus.RUNNING, + progress=0, + status_text="Start cleaning.", + ) + + output = subprocess.check_output("nix-collect-garbage -d") + + pat = re.compile(r"linking saves ([+-]?\d+\.\d+ \w+).+?([+-]?\d+\.\d+ \w+) freed") + match = re.search( + pat, + output, + ) + + Jobs.update( + job=job, + status=JobStatus.FINISHED, + progress=100, + status_text="Сleaning completed.", + result=f"Currently hard linking saves {match.group(1)}, {match.group(2)} freed", + ) From ba168fc92b482a8dc131e17a98b2d056c55fa3f2 Mon Sep 17 00:00:00 2001 From: def Date: Thu, 24 Nov 2022 06:32:37 +0400 Subject: [PATCH 12/50] fix: subprocess.check_output --- selfprivacy_api/jobs/nix_collect_garbage.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/selfprivacy_api/jobs/nix_collect_garbage.py b/selfprivacy_api/jobs/nix_collect_garbage.py index 18e2a63..9bb6268 100644 --- a/selfprivacy_api/jobs/nix_collect_garbage.py +++ b/selfprivacy_api/jobs/nix_collect_garbage.py @@ -15,7 +15,7 @@ def nix_collect_garbage(job: Job): status_text="Start cleaning.", ) - output = subprocess.check_output("nix-collect-garbage -d") + output = subprocess.check_output(["nix-collect-garbage", "-d"]) pat = re.compile(r"linking saves ([+-]?\d+\.\d+ \w+).+?([+-]?\d+\.\d+ \w+) freed") match = re.search( From 6ef9222e329d8ffbf6d6f19dd3017e114845ff89 Mon Sep 17 00:00:00 2001 From: def Date: Sat, 3 Dec 2022 22:27:10 +0400 Subject: [PATCH 13/50] feat: loading percentage --- selfprivacy_api/jobs/nix_collect_garbage.py | 71 +++++++++++++++++---- 1 file changed, 60 insertions(+), 11 deletions(-) diff --git a/selfprivacy_api/jobs/nix_collect_garbage.py b/selfprivacy_api/jobs/nix_collect_garbage.py index 9bb6268..2c81324 100644 --- a/selfprivacy_api/jobs/nix_collect_garbage.py +++ b/selfprivacy_api/jobs/nix_collect_garbage.py @@ -1,3 +1,4 @@ +from time import sleep import re import subprocess @@ -12,21 +13,69 @@ def nix_collect_garbage(job: Job): job=job, status=JobStatus.RUNNING, progress=0, - status_text="Start cleaning.", + status_text="Сalculate the number of dead packages...", ) - output = subprocess.check_output(["nix-collect-garbage", "-d"]) - - pat = re.compile(r"linking saves ([+-]?\d+\.\d+ \w+).+?([+-]?\d+\.\d+ \w+) freed") - match = re.search( - pat, - output, + output = subprocess.check_output( + ["nix-store --gc --print-dead", "--gc", "--print-dead"] ) + dead_packages = len(re.findall("/nix/store/", output.decode("utf-8"))) + package_equal_to_percent = 100 / dead_packages + Jobs.update( job=job, - status=JobStatus.FINISHED, - progress=100, - status_text="Сleaning completed.", - result=f"Currently hard linking saves {match.group(1)}, {match.group(2)} freed", + status=JobStatus.RUNNING, + progress=0, + status_text=f"Found {dead_packages} packages to remove!", ) + + def _parse_line(line): + pattern = re.compile(r"[+-]?\d+\.\d+ \w+ freed") + match = re.search( + pattern, + line, + ) + + if match is None: + Jobs.update( + job=job, + status=JobStatus.FINISHED, + progress=100, + status_text="Completed with an error", + result="We are sorry, result was not found :(", + ) + + else: + Jobs.update( + job=job, + status=JobStatus.FINISHED, + progress=100, + status_text="Сleaning completed.", + result=f"{match.group(0)} have been cleared", + ) + + def _stream_process(process): + go = process.poll() is None + percent = 0 + + for line in process.stdout: + if "deleting '/nix/store/" in line: + percent += package_equal_to_percent + + Jobs.update( + job=job, + status=JobStatus.RUNNING, + progress=int(percent), + status_text="Сleaning...", + ) + + elif "store paths deleted," in line: + _parse_line(line) + + return go + + process = subprocess.Popen( + ["nix-collect-garbage", "-d"], stdout=subprocess.PIPE, stderr=subprocess.STDOUT + ) + _stream_process(process) From f42594f1fb22f18a36acdbf2746a6b4170b64a83 Mon Sep 17 00:00:00 2001 From: def Date: Sat, 3 Dec 2022 22:30:41 +0400 Subject: [PATCH 14/50] refactor: delete unnecessary import --- selfprivacy_api/jobs/nix_collect_garbage.py | 1 - 1 file changed, 1 deletion(-) diff --git a/selfprivacy_api/jobs/nix_collect_garbage.py b/selfprivacy_api/jobs/nix_collect_garbage.py index 2c81324..78d306e 100644 --- a/selfprivacy_api/jobs/nix_collect_garbage.py +++ b/selfprivacy_api/jobs/nix_collect_garbage.py @@ -1,4 +1,3 @@ -from time import sleep import re import subprocess From 7e3adffb68facc59fe56fc2d4d788b78c7a15553 Mon Sep 17 00:00:00 2001 From: def Date: Tue, 13 Dec 2022 05:44:52 +0400 Subject: [PATCH 15/50] refactor: nix-collect-garbage is now pure --- selfprivacy_api/jobs/nix_collect_garbage.py | 131 +++++++++++------- .../test_graphql/test_nix_collect_garbage.py | 32 +++++ 2 files changed, 110 insertions(+), 53 deletions(-) create mode 100644 tests/test_graphql/test_nix_collect_garbage.py diff --git a/selfprivacy_api/jobs/nix_collect_garbage.py b/selfprivacy_api/jobs/nix_collect_garbage.py index 78d306e..3d84ca3 100644 --- a/selfprivacy_api/jobs/nix_collect_garbage.py +++ b/selfprivacy_api/jobs/nix_collect_garbage.py @@ -5,8 +5,73 @@ from selfprivacy_api.jobs import Job, JobStatus, Jobs from selfprivacy_api.utils.huey import huey +def run_nix_store_print_dead(): + return subprocess.check_output(["nix-store", "--gc", "--print-dead"]) + + +def run_nix_collect_garbage(): + return subprocess.Popen( + ["nix-collect-garbage", "-d"], stdout=subprocess.PIPE, stderr=subprocess.STDOUT + ) + + +def parse_line(line, job: Job): + pattern = re.compile(r"[+-]?\d+\.\d+ \w+ freed") + match = re.search( + pattern, + line, + ) + + if match is None: + Jobs.update( + job=job, + status=JobStatus.FINISHED, + progress=100, + status_text="Completed with an error", + result="We are sorry, result was not found :(", + ) + + else: + Jobs.update( + job=job, + status=JobStatus.FINISHED, + progress=100, + status_text="Сleaning completed.", + result=f"{match.group(0)} have been cleared", + ) + + +def stream_process( + process, + package_equal_to_percent, + job: Job, +): + go = process.poll() is None + percent = 0 + + for line in process.stdout: + if "deleting '/nix/store/" in line: + percent += package_equal_to_percent + + Jobs.update( + job=job, + status=JobStatus.RUNNING, + progress=int(percent), + status_text="Сleaning...", + ) + + elif "store paths deleted," in line: + parse_line(line, job) + + return go + + @huey.task() -def nix_collect_garbage(job: Job): +def nix_collect_garbage( + job: Job, + run_nix_store=run_nix_store_print_dead, + run_nix_collect=run_nix_collect_garbage, +): # innocent as a pure function Jobs.update( job=job, @@ -15,11 +80,19 @@ def nix_collect_garbage(job: Job): status_text="Сalculate the number of dead packages...", ) - output = subprocess.check_output( - ["nix-store --gc --print-dead", "--gc", "--print-dead"] - ) + output = run_nix_store() dead_packages = len(re.findall("/nix/store/", output.decode("utf-8"))) + + if dead_packages == 0: + Jobs.update( + job=job, + status=JobStatus.FINISHED, + progress=100, + status_text="Nothing to clear", + result="System is clear", + ) + package_equal_to_percent = 100 / dead_packages Jobs.update( @@ -29,52 +102,4 @@ def nix_collect_garbage(job: Job): status_text=f"Found {dead_packages} packages to remove!", ) - def _parse_line(line): - pattern = re.compile(r"[+-]?\d+\.\d+ \w+ freed") - match = re.search( - pattern, - line, - ) - - if match is None: - Jobs.update( - job=job, - status=JobStatus.FINISHED, - progress=100, - status_text="Completed with an error", - result="We are sorry, result was not found :(", - ) - - else: - Jobs.update( - job=job, - status=JobStatus.FINISHED, - progress=100, - status_text="Сleaning completed.", - result=f"{match.group(0)} have been cleared", - ) - - def _stream_process(process): - go = process.poll() is None - percent = 0 - - for line in process.stdout: - if "deleting '/nix/store/" in line: - percent += package_equal_to_percent - - Jobs.update( - job=job, - status=JobStatus.RUNNING, - progress=int(percent), - status_text="Сleaning...", - ) - - elif "store paths deleted," in line: - _parse_line(line) - - return go - - process = subprocess.Popen( - ["nix-collect-garbage", "-d"], stdout=subprocess.PIPE, stderr=subprocess.STDOUT - ) - _stream_process(process) + stream_process(run_nix_collect, package_equal_to_percent, job) diff --git a/tests/test_graphql/test_nix_collect_garbage.py b/tests/test_graphql/test_nix_collect_garbage.py new file mode 100644 index 0000000..a14dcb3 --- /dev/null +++ b/tests/test_graphql/test_nix_collect_garbage.py @@ -0,0 +1,32 @@ +# pylint: disable=redefined-outer-name +# pylint: disable=unused-argument +# pylint: disable=missing-function-docstring + +import pytest + +from selfprivacy_api.jobs.nix_collect_garbage import nix_collect_garbage + + + + + + created_at: datetime.datetime + updated_at: datetime.datetime + uid: UUID + type_id: str + name: str + description: str + status: JobStatus + + +def test_nix_collect_garbage(job( + created_at = "2019-12-04", + updated_at = "2019-12-04", + uid = UUID, + type_id = "typeid", + name = "name", + description: "desc", + status = status(CREATED = "CREATED"), +)): + + assert nix_collect_garbage() is not None From 7553202a696571d442dd80b61d7ed0d26c5a5b8a Mon Sep 17 00:00:00 2001 From: def Date: Mon, 19 Dec 2022 06:27:44 +0400 Subject: [PATCH 16/50] test: fix nix collect garbage, add tests --- selfprivacy_api/jobs/nix_collect_garbage.py | 105 +++++++----- .../test_graphql/test_nix_collect_garbage.py | 154 +++++++++++++++--- 2 files changed, 199 insertions(+), 60 deletions(-) diff --git a/selfprivacy_api/jobs/nix_collect_garbage.py b/selfprivacy_api/jobs/nix_collect_garbage.py index 3d84ca3..32de683 100644 --- a/selfprivacy_api/jobs/nix_collect_garbage.py +++ b/selfprivacy_api/jobs/nix_collect_garbage.py @@ -1,105 +1,128 @@ import re import subprocess -from selfprivacy_api.jobs import Job, JobStatus, Jobs -from selfprivacy_api.utils.huey import huey +from selfprivacy_api.jobs import JobStatus, Jobs + + +COMPLETED_WITH_ERROR = "Completed with an error" +RESULT_WAAS_NOT_FOUND_ERROR = "We are sorry, result was not found :(" +CLEAR_COMPLETED = "Сleaning completed." def run_nix_store_print_dead(): - return subprocess.check_output(["nix-store", "--gc", "--print-dead"]) + return subprocess.check_output(["nix-store", "--gc", "--print-dead"]).decode( + "utf-8" + ) def run_nix_collect_garbage(): return subprocess.Popen( ["nix-collect-garbage", "-d"], stdout=subprocess.PIPE, stderr=subprocess.STDOUT - ) + ).stdout -def parse_line(line, job: Job): - pattern = re.compile(r"[+-]?\d+\.\d+ \w+ freed") +def set_job_status_wrapper(Jobs, job): + def set_job_status(status, progress, status_text, result="Default result"): + Jobs.update( + job=job, + status=status, + progress=progress, + status_text=status_text, + result=result, + ) + + return set_job_status + + +def parse_line(line): + pattern = re.compile(r"[+-]?\d+\.\d+ \w+(?= freed)") match = re.search( pattern, line, ) if match is None: - Jobs.update( - job=job, - status=JobStatus.FINISHED, - progress=100, - status_text="Completed with an error", - result="We are sorry, result was not found :(", + return ( + JobStatus.FINISHED, + 100, + COMPLETED_WITH_ERROR, + RESULT_WAAS_NOT_FOUND_ERROR, ) else: - Jobs.update( - job=job, - status=JobStatus.FINISHED, - progress=100, - status_text="Сleaning completed.", - result=f"{match.group(0)} have been cleared", + return ( + JobStatus.FINISHED, + 100, + CLEAR_COMPLETED, + f"{match.group(0)} have been cleared", ) def stream_process( - process, + stream, package_equal_to_percent, - job: Job, + set_job_status, ): - go = process.poll() is None percent = 0 - for line in process.stdout: + for line in stream: if "deleting '/nix/store/" in line: percent += package_equal_to_percent - Jobs.update( - job=job, + set_job_status( status=JobStatus.RUNNING, progress=int(percent), status_text="Сleaning...", ) elif "store paths deleted," in line: - parse_line(line, job) - - return go + status = parse_line(line) + set_job_status( + status=status[0], + progress=status[1], + status_text=status[2], + result=status[3], + ) + + +def get_dead_packages(output): + dead = len(re.findall("/nix/store/", output)) + percent = None + if dead != 0: + percent = 100 / dead + return dead, percent -@huey.task() def nix_collect_garbage( - job: Job, + job, + jobs=Jobs, run_nix_store=run_nix_store_print_dead, run_nix_collect=run_nix_collect_garbage, + set_job_status=None, ): # innocent as a pure function + set_job_status = set_job_status or set_job_status_wrapper(jobs, job) - Jobs.update( - job=job, + set_job_status( status=JobStatus.RUNNING, progress=0, status_text="Сalculate the number of dead packages...", ) - output = run_nix_store() - - dead_packages = len(re.findall("/nix/store/", output.decode("utf-8"))) + dead_packages, package_equal_to_percent = get_dead_packages(run_nix_store()) if dead_packages == 0: - Jobs.update( - job=job, + set_job_status( status=JobStatus.FINISHED, progress=100, status_text="Nothing to clear", result="System is clear", ) + return - package_equal_to_percent = 100 / dead_packages - - Jobs.update( - job=job, + set_job_status( status=JobStatus.RUNNING, progress=0, status_text=f"Found {dead_packages} packages to remove!", ) - stream_process(run_nix_collect, package_equal_to_percent, job) + stream_process(run_nix_collect(), package_equal_to_percent, set_job_status) diff --git a/tests/test_graphql/test_nix_collect_garbage.py b/tests/test_graphql/test_nix_collect_garbage.py index a14dcb3..b903027 100644 --- a/tests/test_graphql/test_nix_collect_garbage.py +++ b/tests/test_graphql/test_nix_collect_garbage.py @@ -3,30 +3,146 @@ # pylint: disable=missing-function-docstring import pytest +from selfprivacy_api.jobs import JobStatus -from selfprivacy_api.jobs.nix_collect_garbage import nix_collect_garbage +from selfprivacy_api.jobs.nix_collect_garbage import ( + get_dead_packages, + nix_collect_garbage, + parse_line, + CLEAR_COMPLETED, + COMPLETED_WITH_ERROR, + stream_process, + RESULT_WAAS_NOT_FOUND_ERROR, +) +output_print_dead = """ +finding garbage collector roots... +determining live/dead paths... +/nix/store/02k8pmw00p7p7mf2dg3n057771w7liia-python3.10-cchardet-2.1.7 +/nix/store/03vc6dznx8njbvyd3gfhfa4n5j4lvhbl-python3.10-async-timeout-4.0.2 +/nix/store/03ybv2dvfk7c3cpb527y5kzf6i35ch41-python3.10-pycparser-2.21 +/nix/store/04dn9slfqwhqisn1j3jv531lms9w5wlj-python3.10-hypothesis-6.50.1.drv +/nix/store/04hhx2z1iyi3b48hxykiw1g03lp46jk7-python-remove-bin-bytecode-hook +""" - - created_at: datetime.datetime - updated_at: datetime.datetime - uid: UUID - type_id: str - name: str - description: str - status: JobStatus +output_collect_garbage = """ +removing old generations of profile /nix/var/nix/profiles/per-user/def/channels +finding garbage collector roots... +deleting garbage... +deleting '/nix/store/02k8pmw00p7p7mf2dg3n057771w7liia-python3.10-cchardet-2.1.7' +deleting '/nix/store/03vc6dznx8njbvyd3gfhfa4n5j4lvhbl-python3.10-async-timeout-4.0.2' +deleting '/nix/store/03ybv2dvfk7c3cpb527y5kzf6i35ch41-python3.10-pycparser-2.21' +deleting '/nix/store/04dn9slfqwhqisn1j3jv531lms9w5wlj-python3.10-hypothesis-6.50.1.drv' +deleting '/nix/store/04hhx2z1iyi3b48hxykiw1g03lp46jk7-python-remove-bin-bytecode-hook' +deleting unused links... +note: currently hard linking saves -0.00 MiB +190 store paths deleted, 425.51 MiB freed +""" -def test_nix_collect_garbage(job( - created_at = "2019-12-04", - updated_at = "2019-12-04", - uid = UUID, - type_id = "typeid", - name = "name", - description: "desc", - status = status(CREATED = "CREATED"), -)): +def test_parse_line(): + txt = "190 store paths deleted, 425.51 MiB freed" + output = ( + JobStatus.FINISHED, + 100, + CLEAR_COMPLETED, + "425.51 MiB have been cleared", + ) + assert parse_line(txt) == output - assert nix_collect_garbage() is not None + +def test_parse_line_with_blank_line(): + txt = "" + output = ( + JobStatus.FINISHED, + 100, + COMPLETED_WITH_ERROR, + RESULT_WAAS_NOT_FOUND_ERROR, + ) + assert parse_line(txt) == output + + +def test_get_dead_packages(): + assert get_dead_packages(output_print_dead) == (5, 20.0) + + +def test_get_dead_packages_zero(): + assert get_dead_packages("") == (0, None) + + +def test_stream_process(): + log_event = [] + reference = [ + (JobStatus.RUNNING, 20, "Сleaning...", ""), + (JobStatus.RUNNING, 40, "Сleaning...", ""), + (JobStatus.RUNNING, 60, "Сleaning...", ""), + (JobStatus.RUNNING, 80, "Сleaning...", ""), + (JobStatus.RUNNING, 100, "Сleaning...", ""), + ( + JobStatus.FINISHED, + 100, + "Сleaning completed.", + "425.51 MiB have been cleared", + ), + ] + + def set_job_status(status, progress, status_text, result=""): + log_event.append((status, progress, status_text, result)) + + stream_process(output_collect_garbage.split("\n"), 20.0, set_job_status) + assert log_event == reference + + +def test_nix_collect_garbage(): + log_event = [] + reference = [ + (JobStatus.RUNNING, 0, "Сalculate the number of dead packages...", ""), + (JobStatus.RUNNING, 0, "Found 5 packages to remove!", ""), + (JobStatus.RUNNING, 20, "Сleaning...", ""), + (JobStatus.RUNNING, 40, "Сleaning...", ""), + (JobStatus.RUNNING, 60, "Сleaning...", ""), + (JobStatus.RUNNING, 80, "Сleaning...", ""), + (JobStatus.RUNNING, 100, "Сleaning...", ""), + ( + JobStatus.FINISHED, + 100, + "Сleaning completed.", + "425.51 MiB have been cleared", + ), + ] + + def set_job_status(status="", progress="", status_text="", result=""): + log_event.append((status, progress, status_text, result)) + + nix_collect_garbage( + None, + None, + lambda: output_print_dead, + lambda: output_collect_garbage.split("\n"), + set_job_status, + ) + + assert log_event == reference + + +def test_nix_collect_garbage_zero_trash(): + log_event = [] + reference = [ + (JobStatus.RUNNING, 0, "Сalculate the number of dead packages...", ""), + (JobStatus.FINISHED, 100, "Nothing to clear", "System is clear"), + ] + + def set_job_status(status="", progress="", status_text="", result=""): + log_event.append((status, progress, status_text, result)) + + nix_collect_garbage( + None, + None, + lambda: "", + lambda: output_collect_garbage.split("\n"), + set_job_status, + ) + + assert log_event == reference From af0129bdfe1b1846271a5e8230e43694e3bee880 Mon Sep 17 00:00:00 2001 From: def Date: Tue, 27 Dec 2022 04:15:02 +0400 Subject: [PATCH 17/50] fix: types, add tests --- selfprivacy_api/jobs/nix_collect_garbage.py | 7 +++-- selfprivacy_api/task_registry.py | 1 + .../test_graphql/test_nix_collect_garbage.py | 27 +++++++++++++++++-- 3 files changed, 31 insertions(+), 4 deletions(-) diff --git a/selfprivacy_api/jobs/nix_collect_garbage.py b/selfprivacy_api/jobs/nix_collect_garbage.py index 32de683..587da65 100644 --- a/selfprivacy_api/jobs/nix_collect_garbage.py +++ b/selfprivacy_api/jobs/nix_collect_garbage.py @@ -2,10 +2,11 @@ import re import subprocess from selfprivacy_api.jobs import JobStatus, Jobs +from selfprivacy_api.utils.huey import huey COMPLETED_WITH_ERROR = "Completed with an error" -RESULT_WAAS_NOT_FOUND_ERROR = "We are sorry, result was not found :(" +RESULT_WAS_NOT_FOUND_ERROR = "We are sorry, result was not found :(" CLEAR_COMPLETED = "Сleaning completed." @@ -46,7 +47,7 @@ def parse_line(line): JobStatus.FINISHED, 100, COMPLETED_WITH_ERROR, - RESULT_WAAS_NOT_FOUND_ERROR, + RESULT_WAS_NOT_FOUND_ERROR, ) else: @@ -72,6 +73,7 @@ def stream_process( set_job_status( status=JobStatus.RUNNING, progress=int(percent), + progress=int(percent), status_text="Сleaning...", ) @@ -93,6 +95,7 @@ def get_dead_packages(output): return dead, percent +@huey.task() def nix_collect_garbage( job, jobs=Jobs, diff --git a/selfprivacy_api/task_registry.py b/selfprivacy_api/task_registry.py index 82eaf06..f9f92e1 100644 --- a/selfprivacy_api/task_registry.py +++ b/selfprivacy_api/task_registry.py @@ -2,3 +2,4 @@ from selfprivacy_api.utils.huey import huey from selfprivacy_api.jobs.test import test_job from selfprivacy_api.restic_controller.tasks import * from selfprivacy_api.services.generic_service_mover import move_service +from selfprivacy_api.jobs.nix_collect_garbage import nix_collect_garbage diff --git a/tests/test_graphql/test_nix_collect_garbage.py b/tests/test_graphql/test_nix_collect_garbage.py index b903027..4305f5c 100644 --- a/tests/test_graphql/test_nix_collect_garbage.py +++ b/tests/test_graphql/test_nix_collect_garbage.py @@ -4,6 +4,11 @@ import pytest from selfprivacy_api.jobs import JobStatus +from selfprivacy_api.graphql import schema +import asyncio +import strawberry + +# from selfprivacy_api.graphql.schema import Subscription from selfprivacy_api.jobs.nix_collect_garbage import ( get_dead_packages, @@ -12,7 +17,7 @@ from selfprivacy_api.jobs.nix_collect_garbage import ( CLEAR_COMPLETED, COMPLETED_WITH_ERROR, stream_process, - RESULT_WAAS_NOT_FOUND_ERROR, + RESULT_WAS_NOT_FOUND_ERROR, ) @@ -59,7 +64,7 @@ def test_parse_line_with_blank_line(): JobStatus.FINISHED, 100, COMPLETED_WITH_ERROR, - RESULT_WAAS_NOT_FOUND_ERROR, + RESULT_WAS_NOT_FOUND_ERROR, ) assert parse_line(txt) == output @@ -146,3 +151,21 @@ def test_nix_collect_garbage_zero_trash(): ) assert log_event == reference + + +@pytest.mark.asyncio +async def test_graphql_nix_collect_garbage(): + query = """ + subscription { + nixCollectGarbage() + } + """ + + schema_for_garbage = strawberry.Schema( + query=schema.Query, mutation=schema.Mutation, subscription=schema.Subscription + ) + + sub = await schema_for_garbage.subscribe(query) + for result in sub: + assert not result.errors + assert result.data == {} From 03b5a6b5e4ee2fc775cdda64c1b7d1a7cce9eba5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=92=D0=B0=D1=88=C3=90dettlaff?= Date: Thu, 23 Mar 2023 20:49:30 +0300 Subject: [PATCH 18/50] fix: percentage --- dump.rdb | Bin 0 -> 88 bytes selfprivacy_api/jobs/nix_collect_garbage.py | 12 ++--- setup.py | 0 shell.nix | 45 ++---------------- .../test_graphql/test_nix_collect_garbage.py | 29 +++++------ 5 files changed, 23 insertions(+), 63 deletions(-) create mode 100644 dump.rdb mode change 100755 => 100644 setup.py diff --git a/dump.rdb b/dump.rdb new file mode 100644 index 0000000000000000000000000000000000000000..2f3b490816640dcfd897e08c8eb137f27ec5a6e2 GIT binary patch literal 88 zcmWG?b@2=~FfcUu#aWb^l3A=uO-d|IJ;3n)^rLMb4lXTO2>_mAByj)$ literal 0 HcmV?d00001 diff --git a/selfprivacy_api/jobs/nix_collect_garbage.py b/selfprivacy_api/jobs/nix_collect_garbage.py index 587da65..3d13382 100644 --- a/selfprivacy_api/jobs/nix_collect_garbage.py +++ b/selfprivacy_api/jobs/nix_collect_garbage.py @@ -61,19 +61,19 @@ def parse_line(line): def stream_process( stream, - package_equal_to_percent, + total_dead_packages, set_job_status, ): - percent = 0 + completed_packages = 0 for line in stream: if "deleting '/nix/store/" in line: - percent += package_equal_to_percent + completed_packages += 1 + percent = int((completed_packages / total_dead_packages) * 100) set_job_status( status=JobStatus.RUNNING, - progress=int(percent), - progress=int(percent), + progress=percent, status_text="Сleaning...", ) @@ -95,7 +95,7 @@ def get_dead_packages(output): return dead, percent -@huey.task() +# @huey.task() # ломает все к фигам def nix_collect_garbage( job, jobs=Jobs, diff --git a/setup.py b/setup.py old mode 100755 new mode 100644 diff --git a/shell.nix b/shell.nix index 668da69..b620ee2 100644 --- a/shell.nix +++ b/shell.nix @@ -1,6 +1,6 @@ -{ pkgs ? import { } }: +{ pkgs ? import { } }: let - sp-python = pkgs.python39.withPackages (p: with p; [ + sp-python = pkgs.python310.withPackages (p: with p; [ setuptools portalocker pytz @@ -19,45 +19,7 @@ let fastapi uvicorn redis - (buildPythonPackage rec { - pname = "strawberry-graphql"; - version = "0.123.0"; - format = "pyproject"; - patches = [ - ./strawberry-graphql.patch - ]; - propagatedBuildInputs = [ - typing-extensions - python-multipart - python-dateutil - # flask - pydantic - pygments - poetry - # flask-cors - (buildPythonPackage rec { - pname = "graphql-core"; - version = "3.2.0"; - format = "setuptools"; - src = fetchPypi { - inherit pname version; - sha256 = "sha256-huKgvgCL/eGe94OI3opyWh2UKpGQykMcJKYIN5c4A84="; - }; - checkInputs = [ - pytest-asyncio - pytest-benchmark - pytestCheckHook - ]; - pythonImportsCheck = [ - "graphql" - ]; - }) - ]; - src = fetchPypi { - inherit pname version; - sha256 = "KsmZ5Xv8tUg6yBxieAEtvoKoRG60VS+iVGV0X6oCExo="; - }; - }) + strawberry-graphql ]); in pkgs.mkShell { @@ -65,6 +27,7 @@ pkgs.mkShell { sp-python pkgs.black pkgs.redis + pkgs.restic ]; shellHook = '' PYTHONPATH=${sp-python}/${sp-python.sitePackages} diff --git a/tests/test_graphql/test_nix_collect_garbage.py b/tests/test_graphql/test_nix_collect_garbage.py index 4305f5c..0db620a 100644 --- a/tests/test_graphql/test_nix_collect_garbage.py +++ b/tests/test_graphql/test_nix_collect_garbage.py @@ -96,26 +96,21 @@ def test_stream_process(): def set_job_status(status, progress, status_text, result=""): log_event.append((status, progress, status_text, result)) - stream_process(output_collect_garbage.split("\n"), 20.0, set_job_status) + stream_process(output_collect_garbage.split("\n"), 5, set_job_status) assert log_event == reference def test_nix_collect_garbage(): log_event = [] reference = [ - (JobStatus.RUNNING, 0, "Сalculate the number of dead packages...", ""), - (JobStatus.RUNNING, 0, "Found 5 packages to remove!", ""), - (JobStatus.RUNNING, 20, "Сleaning...", ""), - (JobStatus.RUNNING, 40, "Сleaning...", ""), - (JobStatus.RUNNING, 60, "Сleaning...", ""), - (JobStatus.RUNNING, 80, "Сleaning...", ""), - (JobStatus.RUNNING, 100, "Сleaning...", ""), - ( - JobStatus.FINISHED, - 100, - "Сleaning completed.", - "425.51 MiB have been cleared", - ), + (JobStatus.RUNNING, 0, 'Сalculate the number of dead packages...', ''), + (JobStatus.RUNNING, 0, 'Found 5 packages to remove!', ''), + (JobStatus.RUNNING, 5, 'Сleaning...', ''), + (JobStatus.RUNNING, 10, 'Сleaning...', ''), + (JobStatus.RUNNING, 15, 'Сleaning...', ''), + (JobStatus.RUNNING, 20, 'Сleaning...', ''), + (JobStatus.RUNNING, 25, 'Сleaning...', ''), + (JobStatus.FINISHED, 100, 'Сleaning completed.', '425.51 MiB have been cleared'), ] def set_job_status(status="", progress="", status_text="", result=""): @@ -128,6 +123,8 @@ def test_nix_collect_garbage(): lambda: output_collect_garbage.split("\n"), set_job_status, ) + print("log_event:", log_event) + print("reference:", reference) assert log_event == reference @@ -152,7 +149,7 @@ def test_nix_collect_garbage_zero_trash(): assert log_event == reference - +# андр констракнш @pytest.mark.asyncio async def test_graphql_nix_collect_garbage(): query = """ @@ -166,6 +163,6 @@ async def test_graphql_nix_collect_garbage(): ) sub = await schema_for_garbage.subscribe(query) - for result in sub: + async for result in sub: assert not result.errors assert result.data == {} From 96c013c4e774b92ccd07cee60a7beae95c08e3bf Mon Sep 17 00:00:00 2001 From: dettlaff Date: Mon, 3 Apr 2023 17:41:41 +0300 Subject: [PATCH 19/50] refactor: delete purest of nix_collect_garbage() --- .gitignore | 3 +++ selfprivacy_api/jobs/nix_collect_garbage.py | 13 +++++-------- 2 files changed, 8 insertions(+), 8 deletions(-) mode change 100755 => 100644 .gitignore diff --git a/.gitignore b/.gitignore old mode 100755 new mode 100644 index 7941396..48a9439 --- a/.gitignore +++ b/.gitignore @@ -147,3 +147,6 @@ cython_debug/ # End of https://www.toptal.com/developers/gitignore/api/flask *.db + +# Redis db +*.rdb \ No newline at end of file diff --git a/selfprivacy_api/jobs/nix_collect_garbage.py b/selfprivacy_api/jobs/nix_collect_garbage.py index 3d13382..ec894fd 100644 --- a/selfprivacy_api/jobs/nix_collect_garbage.py +++ b/selfprivacy_api/jobs/nix_collect_garbage.py @@ -95,15 +95,12 @@ def get_dead_packages(output): return dead, percent -# @huey.task() # ломает все к фигам +# @huey.task() def nix_collect_garbage( job, jobs=Jobs, - run_nix_store=run_nix_store_print_dead, - run_nix_collect=run_nix_collect_garbage, - set_job_status=None, -): # innocent as a pure function - set_job_status = set_job_status or set_job_status_wrapper(jobs, job) +): + set_job_status = set_job_status_wrapper(jobs, job) set_job_status( status=JobStatus.RUNNING, @@ -111,7 +108,7 @@ def nix_collect_garbage( status_text="Сalculate the number of dead packages...", ) - dead_packages, package_equal_to_percent = get_dead_packages(run_nix_store()) + dead_packages, package_equal_to_percent = get_dead_packages(run_nix_store_print_dead()) if dead_packages == 0: set_job_status( @@ -128,4 +125,4 @@ def nix_collect_garbage( status_text=f"Found {dead_packages} packages to remove!", ) - stream_process(run_nix_collect(), package_equal_to_percent, set_job_status) + stream_process(run_nix_collect_garbage(), package_equal_to_percent, set_job_status) From 28701e16082e23a50283b9b73ea1791e2c49fa94 Mon Sep 17 00:00:00 2001 From: dettlaff Date: Wed, 5 Apr 2023 13:49:56 +0300 Subject: [PATCH 20/50] fix: do a redis reset() --- dump.rdb | Bin 88 -> 88 bytes selfprivacy_api/jobs/nix_collect_garbage.py | 7 +- .../test_graphql/test_nix_collect_garbage.py | 126 ++++++++++++------ 3 files changed, 86 insertions(+), 47 deletions(-) diff --git a/dump.rdb b/dump.rdb index 2f3b490816640dcfd897e08c8eb137f27ec5a6e2..7a274dbe6fa11a2889c1664f1b0cbee5dde9742c 100644 GIT binary patch delta 45 zcma!um|&n$&#ImBi=(tSHAOc!HTRH$CNIM;j>P;l-K50g)B_Cvryq~qzCu^z4FGn_ B5$ON` delta 45 zcma!um|&n$mn)OCL B6mI|k diff --git a/selfprivacy_api/jobs/nix_collect_garbage.py b/selfprivacy_api/jobs/nix_collect_garbage.py index ec894fd..82e00e2 100644 --- a/selfprivacy_api/jobs/nix_collect_garbage.py +++ b/selfprivacy_api/jobs/nix_collect_garbage.py @@ -98,9 +98,8 @@ def get_dead_packages(output): # @huey.task() def nix_collect_garbage( job, - jobs=Jobs, ): - set_job_status = set_job_status_wrapper(jobs, job) + set_job_status = set_job_status_wrapper(Jobs, job) set_job_status( status=JobStatus.RUNNING, @@ -108,7 +107,9 @@ def nix_collect_garbage( status_text="Сalculate the number of dead packages...", ) - dead_packages, package_equal_to_percent = get_dead_packages(run_nix_store_print_dead()) + dead_packages, package_equal_to_percent = get_dead_packages( + run_nix_store_print_dead() + ) if dead_packages == 0: set_job_status( diff --git a/tests/test_graphql/test_nix_collect_garbage.py b/tests/test_graphql/test_nix_collect_garbage.py index 0db620a..98848b4 100644 --- a/tests/test_graphql/test_nix_collect_garbage.py +++ b/tests/test_graphql/test_nix_collect_garbage.py @@ -3,11 +3,11 @@ # pylint: disable=missing-function-docstring import pytest -from selfprivacy_api.jobs import JobStatus -from selfprivacy_api.graphql import schema -import asyncio import strawberry +from selfprivacy_api.jobs import JobStatus, Jobs +from selfprivacy_api.graphql import schema + # from selfprivacy_api.graphql.schema import Subscription from selfprivacy_api.jobs.nix_collect_garbage import ( @@ -21,7 +21,7 @@ from selfprivacy_api.jobs.nix_collect_garbage import ( ) -output_print_dead = """ +OUTPUT_PRINT_DEAD = """ finding garbage collector roots... determining live/dead paths... /nix/store/02k8pmw00p7p7mf2dg3n057771w7liia-python3.10-cchardet-2.1.7 @@ -32,7 +32,7 @@ determining live/dead paths... """ -output_collect_garbage = """ +OUTPUT_COLLECT_GARBAGE = """ removing old generations of profile /nix/var/nix/profiles/per-user/def/channels finding garbage collector roots... deleting garbage... @@ -46,8 +46,53 @@ note: currently hard linking saves -0.00 MiB 190 store paths deleted, 425.51 MiB freed """ +log_event = [] + + +def set_job_status(status="", progress="", status_text="", result=""): + log_event.append((status, progress, status_text, result)) + + +@pytest.fixture +def mock_set_job_status(mocker): + mock = mocker.patch( + "selfprivacy_api.jobs.nix_collect_garbage.set_job_status_wrapper", + autospec=True, + return_value=set_job_status, + ) + return mock + + +@pytest.fixture +def mock_run_nix_collect_garbage(mocker): + mock = mocker.patch( + "selfprivacy_api.jobs.nix_collect_garbage.run_nix_collect_garbage", + autospec=True, + return_value=OUTPUT_COLLECT_GARBAGE.split("\n"), + ) + return mock + + +@pytest.fixture +def mock_run_nix_store_print_dead(mocker): + mock = mocker.patch( + "selfprivacy_api.jobs.nix_collect_garbage.run_nix_store_print_dead", + autospec=True, + return_value="", + ) + return mock + + +@pytest.fixture +def job_reset(): + Jobs.reset() + + +# --- + + +def test_parse_line(job_reset): -def test_parse_line(): txt = "190 store paths deleted, 425.51 MiB freed" output = ( JobStatus.FINISHED, @@ -58,7 +103,7 @@ def test_parse_line(): assert parse_line(txt) == output -def test_parse_line_with_blank_line(): +def test_parse_line_with_blank_line(job_reset): txt = "" output = ( JobStatus.FINISHED, @@ -69,11 +114,11 @@ def test_parse_line_with_blank_line(): assert parse_line(txt) == output -def test_get_dead_packages(): - assert get_dead_packages(output_print_dead) == (5, 20.0) +def test_get_dead_packages(job_resetм): + assert get_dead_packages(OUTPUT_PRINT_DEAD) == (5, 20.0) -def test_get_dead_packages_zero(): +def test_get_dead_packages_zero(job_reset): assert get_dead_packages("") == (0, None) @@ -96,59 +141,52 @@ def test_stream_process(): def set_job_status(status, progress, status_text, result=""): log_event.append((status, progress, status_text, result)) - stream_process(output_collect_garbage.split("\n"), 5, set_job_status) + stream_process(OUTPUT_COLLECT_GARBAGE.split("\n"), 5, set_job_status) assert log_event == reference -def test_nix_collect_garbage(): +def test_nix_collect_garbage( + mock_set_job_status, mock_run_nix_collect_garbage, job_reset +): log_event = [] reference = [ - (JobStatus.RUNNING, 0, 'Сalculate the number of dead packages...', ''), - (JobStatus.RUNNING, 0, 'Found 5 packages to remove!', ''), - (JobStatus.RUNNING, 5, 'Сleaning...', ''), - (JobStatus.RUNNING, 10, 'Сleaning...', ''), - (JobStatus.RUNNING, 15, 'Сleaning...', ''), - (JobStatus.RUNNING, 20, 'Сleaning...', ''), - (JobStatus.RUNNING, 25, 'Сleaning...', ''), - (JobStatus.FINISHED, 100, 'Сleaning completed.', '425.51 MiB have been cleared'), + (JobStatus.RUNNING, 0, "Сalculate the number of dead packages...", ""), + (JobStatus.RUNNING, 0, "Found 5 packages to remove!", ""), + (JobStatus.RUNNING, 5, "Сleaning...", ""), + (JobStatus.RUNNING, 10, "Сleaning...", ""), + (JobStatus.RUNNING, 15, "Сleaning...", ""), + (JobStatus.RUNNING, 20, "Сleaning...", ""), + (JobStatus.RUNNING, 25, "Сleaning...", ""), + ( + JobStatus.FINISHED, + 100, + "Сleaning completed.", + "425.51 MiB have been cleared", + ), ] - def set_job_status(status="", progress="", status_text="", result=""): - log_event.append((status, progress, status_text, result)) - - nix_collect_garbage( - None, - None, - lambda: output_print_dead, - lambda: output_collect_garbage.split("\n"), - set_job_status, - ) - print("log_event:", log_event) - print("reference:", reference) + nix_collect_garbage(None) assert log_event == reference -def test_nix_collect_garbage_zero_trash(): - log_event = [] +def test_nix_collect_garbage_zero_trash( + mock_set_job_status, + mock_run_nix_collect_garbage, + mock_run_nix_store_print_dead, + job_reset, +): + reference = [ (JobStatus.RUNNING, 0, "Сalculate the number of dead packages...", ""), (JobStatus.FINISHED, 100, "Nothing to clear", "System is clear"), ] - def set_job_status(status="", progress="", status_text="", result=""): - log_event.append((status, progress, status_text, result)) - - nix_collect_garbage( - None, - None, - lambda: "", - lambda: output_collect_garbage.split("\n"), - set_job_status, - ) + nix_collect_garbage(None) assert log_event == reference + # андр констракнш @pytest.mark.asyncio async def test_graphql_nix_collect_garbage(): From d6c4d458f78d7d87615fbc9ce351ead9e2d4dc71 Mon Sep 17 00:00:00 2001 From: dettlaff Date: Wed, 12 Apr 2023 04:59:40 +0400 Subject: [PATCH 21/50] tests: fix some tests --- selfprivacy_api/jobs/nix_collect_garbage.py | 2 +- .../test_graphql/test_nix_collect_garbage.py | 43 ++++++++++++++++--- 2 files changed, 39 insertions(+), 6 deletions(-) diff --git a/selfprivacy_api/jobs/nix_collect_garbage.py b/selfprivacy_api/jobs/nix_collect_garbage.py index 82e00e2..2a26e52 100644 --- a/selfprivacy_api/jobs/nix_collect_garbage.py +++ b/selfprivacy_api/jobs/nix_collect_garbage.py @@ -95,7 +95,7 @@ def get_dead_packages(output): return dead, percent -# @huey.task() +@huey.task() def nix_collect_garbage( job, ): diff --git a/tests/test_graphql/test_nix_collect_garbage.py b/tests/test_graphql/test_nix_collect_garbage.py index 98848b4..0d6adbe 100644 --- a/tests/test_graphql/test_nix_collect_garbage.py +++ b/tests/test_graphql/test_nix_collect_garbage.py @@ -46,6 +46,21 @@ note: currently hard linking saves -0.00 MiB 190 store paths deleted, 425.51 MiB freed """ +OUTPUT_COLLECT_GARBAGE_ZERO_TRASH = """ +removing old generations of profile /nix/var/nix/profiles/per-user/def/profile +removing old generations of profile /nix/var/nix/profiles/per-user/def/channels +finding garbage collector roots... +deleting garbage... +deleting unused links... +note: currently hard linking saves 0.00 MiB +0 store paths deleted, 0.00 MiB freed +""" + +OUTPUT_RUN_NIX_STORE_PRINT_DEAD_ZERO_TRASH = """ +finding garbage collector roots... +determining live/dead paths... +""" + log_event = [] @@ -53,6 +68,15 @@ def set_job_status(status="", progress="", status_text="", result=""): log_event.append((status, progress, status_text, result)) +@pytest.fixture +def mock_run_nix_store_print_dead_zero_trash(mocker): + mock = mocker.patch( + "selfprivacy_api.jobs.nix_collect_garbage.run_nix_store_print_dead", + autospec=True, + return_value=OUTPUT_RUN_NIX_STORE_PRINT_DEAD_ZERO_TRASH.split("\n"), + ) + + @pytest.fixture def mock_set_job_status(mocker): mock = mocker.patch( @@ -73,6 +97,16 @@ def mock_run_nix_collect_garbage(mocker): return mock +@pytest.fixture +def mock_run_nix_collect_garbage_zero_trash(mocker): + mock = mocker.patch( + "selfprivacy_api.jobs.nix_collect_garbage.run_nix_collect_garbage", + autospec=True, + return_value=OUTPUT_COLLECT_GARBAGE_ZERO_TRASH.split("\n"), + ) + return mock + + @pytest.fixture def mock_run_nix_store_print_dead(mocker): mock = mocker.patch( @@ -92,7 +126,6 @@ def job_reset(): def test_parse_line(job_reset): - txt = "190 store paths deleted, 425.51 MiB freed" output = ( JobStatus.FINISHED, @@ -114,7 +147,7 @@ def test_parse_line_with_blank_line(job_reset): assert parse_line(txt) == output -def test_get_dead_packages(job_resetм): +def test_get_dead_packages(job_reset): assert get_dead_packages(OUTPUT_PRINT_DEAD) == (5, 20.0) @@ -122,7 +155,7 @@ def test_get_dead_packages_zero(job_reset): assert get_dead_packages("") == (0, None) -def test_stream_process(): +def test_stream_process(job_reset): log_event = [] reference = [ (JobStatus.RUNNING, 20, "Сleaning...", ""), @@ -172,11 +205,11 @@ def test_nix_collect_garbage( def test_nix_collect_garbage_zero_trash( mock_set_job_status, - mock_run_nix_collect_garbage, + mock_run_nix_collect_garbage_zero_trash, mock_run_nix_store_print_dead, job_reset, + mock_run_nix_store_print_dead_zero_trash, ): - reference = [ (JobStatus.RUNNING, 0, "Сalculate the number of dead packages...", ""), (JobStatus.FINISHED, 100, "Nothing to clear", "System is clear"), From 83cf7cbd6f2a2b5b43e9fbfe746134e101acf172 Mon Sep 17 00:00:00 2001 From: dettlaff Date: Sun, 18 Jun 2023 08:37:27 +0400 Subject: [PATCH 22/50] fix: jobs --- .../graphql/mutations/system_mutations.py | 14 +++++ selfprivacy_api/jobs/migrate_to_binds.py | 4 +- selfprivacy_api/jobs/nix_collect_garbage.py | 60 +++++++++---------- .../check_for_failed_binds_migration.py | 8 +-- shell.nix | 1 + .../test_graphql/test_nix_collect_garbage.py | 2 + 6 files changed, 51 insertions(+), 38 deletions(-) diff --git a/selfprivacy_api/graphql/mutations/system_mutations.py b/selfprivacy_api/graphql/mutations/system_mutations.py index daada17..c483168 100644 --- a/selfprivacy_api/graphql/mutations/system_mutations.py +++ b/selfprivacy_api/graphql/mutations/system_mutations.py @@ -6,9 +6,12 @@ from selfprivacy_api.graphql import IsAuthenticated from selfprivacy_api.graphql.mutations.mutation_interface import ( GenericMutationReturn, MutationReturnInterface, + GenericJobButationReturn, ) import selfprivacy_api.actions.system as system_actions +from selfprivacy_api.graphql.common_types.jobs import job_to_api_job +from selfprivacy_api.jobs.nix_collect_garbage import start_nix_collect_garbage @strawberry.type @@ -126,3 +129,14 @@ class SystemMutations: message=f"Failed to pull repository changes:\n{result.data}", code=500, ) + + @strawberry.mutation(permission_classes=[IsAuthenticated]) + def nix_collect_garbage(self) -> GenericJobButationReturn: + job = start_nix_collect_garbage() + + return GenericJobButationReturn( + success=True, + code=200, + message="Cleaning started..,", + job=job_to_api_job(job), + ) diff --git a/selfprivacy_api/jobs/migrate_to_binds.py b/selfprivacy_api/jobs/migrate_to_binds.py index ce7229d..3250c9a 100644 --- a/selfprivacy_api/jobs/migrate_to_binds.py +++ b/selfprivacy_api/jobs/migrate_to_binds.py @@ -67,8 +67,8 @@ def move_folder( try: data_path.mkdir(mode=0o750, parents=True, exist_ok=True) - except Exception as e: - print(f"Error creating data path: {e}") + except Exception as error: + print(f"Error creating data path: {error}") return try: diff --git a/selfprivacy_api/jobs/nix_collect_garbage.py b/selfprivacy_api/jobs/nix_collect_garbage.py index 2a26e52..850764d 100644 --- a/selfprivacy_api/jobs/nix_collect_garbage.py +++ b/selfprivacy_api/jobs/nix_collect_garbage.py @@ -1,8 +1,10 @@ import re import subprocess -from selfprivacy_api.jobs import JobStatus, Jobs -from selfprivacy_api.utils.huey import huey +from selfprivacy_api.jobs import JobStatus, Jobs, Job + + +# from selfprivacy_api.utils.huey import huey COMPLETED_WITH_ERROR = "Completed with an error" @@ -22,25 +24,9 @@ def run_nix_collect_garbage(): ).stdout -def set_job_status_wrapper(Jobs, job): - def set_job_status(status, progress, status_text, result="Default result"): - Jobs.update( - job=job, - status=status, - progress=progress, - status_text=status_text, - result=result, - ) - - return set_job_status - - def parse_line(line): pattern = re.compile(r"[+-]?\d+\.\d+ \w+(?= freed)") - match = re.search( - pattern, - line, - ) + match = re.search(pattern, line) if match is None: return ( @@ -60,9 +46,9 @@ def parse_line(line): def stream_process( + job, stream, total_dead_packages, - set_job_status, ): completed_packages = 0 @@ -71,7 +57,8 @@ def stream_process( completed_packages += 1 percent = int((completed_packages / total_dead_packages) * 100) - set_job_status( + Jobs.update( + job=job, status=JobStatus.RUNNING, progress=percent, status_text="Сleaning...", @@ -79,7 +66,8 @@ def stream_process( elif "store paths deleted," in line: status = parse_line(line) - set_job_status( + Jobs.update( + job=job, status=status[0], progress=status[1], status_text=status[2], @@ -95,13 +83,9 @@ def get_dead_packages(output): return dead, percent -@huey.task() -def nix_collect_garbage( - job, -): - set_job_status = set_job_status_wrapper(Jobs, job) - - set_job_status( +def calculate_and_clear_dead_packages(job: Jobs): + Jobs.update( + job=job, status=JobStatus.RUNNING, progress=0, status_text="Сalculate the number of dead packages...", @@ -112,7 +96,8 @@ def nix_collect_garbage( ) if dead_packages == 0: - set_job_status( + Jobs.update( + job=job, status=JobStatus.FINISHED, progress=100, status_text="Nothing to clear", @@ -120,10 +105,21 @@ def nix_collect_garbage( ) return - set_job_status( + Jobs.update( + job=job, status=JobStatus.RUNNING, progress=0, status_text=f"Found {dead_packages} packages to remove!", ) - stream_process(run_nix_collect_garbage(), package_equal_to_percent, set_job_status) + stream_process(job, run_nix_collect_garbage(), package_equal_to_percent) + + +def start_nix_collect_garbage() -> Job: + job = Jobs.add( + type_id="maintenance.collect_nix_garbage", + name="Collect garbage", + description=".", + ) + calculate_and_clear_dead_packages(job=job) + return job diff --git a/selfprivacy_api/migrations/check_for_failed_binds_migration.py b/selfprivacy_api/migrations/check_for_failed_binds_migration.py index 41d56b2..85f73ec 100644 --- a/selfprivacy_api/migrations/check_for_failed_binds_migration.py +++ b/selfprivacy_api/migrations/check_for_failed_binds_migration.py @@ -25,8 +25,8 @@ class CheckForFailedBindsMigration(Migration): ): return True return False - except Exception as e: - print(e) + except Exception as error: + print(error) return False def migrate(self): @@ -43,6 +43,6 @@ class CheckForFailedBindsMigration(Migration): with WriteUserData() as userdata: userdata["useBinds"] = False print("Done") - except Exception as e: - print(e) + except Exception as error: + print(error) print("Error mounting volume") diff --git a/shell.nix b/shell.nix index b620ee2..68cee10 100644 --- a/shell.nix +++ b/shell.nix @@ -5,6 +5,7 @@ let portalocker pytz pytest + pytest-asyncio pytest-mock pytest-datadir huey diff --git a/tests/test_graphql/test_nix_collect_garbage.py b/tests/test_graphql/test_nix_collect_garbage.py index 0d6adbe..cdd2a70 100644 --- a/tests/test_graphql/test_nix_collect_garbage.py +++ b/tests/test_graphql/test_nix_collect_garbage.py @@ -20,6 +20,8 @@ from selfprivacy_api.jobs.nix_collect_garbage import ( RESULT_WAS_NOT_FOUND_ERROR, ) +pytest_plugins = ("pytest_asyncio",) + OUTPUT_PRINT_DEAD = """ finding garbage collector roots... From 79ef4c93103fbd59e2d9ee8addb75c72913e845a Mon Sep 17 00:00:00 2001 From: dettlaff Date: Tue, 20 Jun 2023 01:25:04 +0400 Subject: [PATCH 23/50] fix: task registry --- selfprivacy_api/jobs/nix_collect_garbage.py | 6 +++--- selfprivacy_api/task_registry.py | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/selfprivacy_api/jobs/nix_collect_garbage.py b/selfprivacy_api/jobs/nix_collect_garbage.py index 850764d..8fa40bf 100644 --- a/selfprivacy_api/jobs/nix_collect_garbage.py +++ b/selfprivacy_api/jobs/nix_collect_garbage.py @@ -1,12 +1,11 @@ import re import subprocess +from selfprivacy_api.utils.huey import huey + from selfprivacy_api.jobs import JobStatus, Jobs, Job -# from selfprivacy_api.utils.huey import huey - - COMPLETED_WITH_ERROR = "Completed with an error" RESULT_WAS_NOT_FOUND_ERROR = "We are sorry, result was not found :(" CLEAR_COMPLETED = "Сleaning completed." @@ -83,6 +82,7 @@ def get_dead_packages(output): return dead, percent +@huey.task() def calculate_and_clear_dead_packages(job: Jobs): Jobs.update( job=job, diff --git a/selfprivacy_api/task_registry.py b/selfprivacy_api/task_registry.py index f9f92e1..a4eb212 100644 --- a/selfprivacy_api/task_registry.py +++ b/selfprivacy_api/task_registry.py @@ -2,4 +2,4 @@ from selfprivacy_api.utils.huey import huey from selfprivacy_api.jobs.test import test_job from selfprivacy_api.restic_controller.tasks import * from selfprivacy_api.services.generic_service_mover import move_service -from selfprivacy_api.jobs.nix_collect_garbage import nix_collect_garbage +from selfprivacy_api.jobs.nix_collect_garbage import calculate_and_clear_dead_packages From 3d4caaf4ce05d4bda1ba35cce76f36c97dd65740 Mon Sep 17 00:00:00 2001 From: dettlaff Date: Tue, 20 Jun 2023 23:25:54 +0400 Subject: [PATCH 24/50] fix: encoding --- selfprivacy_api/jobs/nix_collect_garbage.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/selfprivacy_api/jobs/nix_collect_garbage.py b/selfprivacy_api/jobs/nix_collect_garbage.py index 8fa40bf..417110a 100644 --- a/selfprivacy_api/jobs/nix_collect_garbage.py +++ b/selfprivacy_api/jobs/nix_collect_garbage.py @@ -52,6 +52,8 @@ def stream_process( completed_packages = 0 for line in stream: + line = line.decode("utf-8") + if "deleting '/nix/store/" in line: completed_packages += 1 percent = int((completed_packages / total_dead_packages) * 100) From 993ea93ee4ccf8b4839cc7c5c4e13eb70be9c6cf Mon Sep 17 00:00:00 2001 From: dettlaff Date: Wed, 21 Jun 2023 10:52:31 +0400 Subject: [PATCH 25/50] fix: delete old gen --- selfprivacy_api/jobs/nix_collect_garbage.py | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/selfprivacy_api/jobs/nix_collect_garbage.py b/selfprivacy_api/jobs/nix_collect_garbage.py index 417110a..8cc3e66 100644 --- a/selfprivacy_api/jobs/nix_collect_garbage.py +++ b/selfprivacy_api/jobs/nix_collect_garbage.py @@ -1,4 +1,5 @@ import re +import os import subprocess from selfprivacy_api.utils.huey import huey @@ -8,10 +9,12 @@ from selfprivacy_api.jobs import JobStatus, Jobs, Job COMPLETED_WITH_ERROR = "Completed with an error" RESULT_WAS_NOT_FOUND_ERROR = "We are sorry, result was not found :(" -CLEAR_COMPLETED = "Сleaning completed." +CLEAR_COMPLETED = "Cleaning completed." def run_nix_store_print_dead(): + os.system("nix-env -p /nix/var/nix/profiles/system --delete-generations old") + return subprocess.check_output(["nix-store", "--gc", "--print-dead"]).decode( "utf-8" ) @@ -19,7 +22,7 @@ def run_nix_store_print_dead(): def run_nix_collect_garbage(): return subprocess.Popen( - ["nix-collect-garbage", "-d"], stdout=subprocess.PIPE, stderr=subprocess.STDOUT + ["nix-store", "--gc"], stdout=subprocess.PIPE, stderr=subprocess.STDOUT ).stdout @@ -62,7 +65,7 @@ def stream_process( job=job, status=JobStatus.RUNNING, progress=percent, - status_text="Сleaning...", + status_text="Cleaning...", ) elif "store paths deleted," in line: @@ -78,7 +81,7 @@ def stream_process( def get_dead_packages(output): dead = len(re.findall("/nix/store/", output)) - percent = None + percent = 0 if dead != 0: percent = 100 / dead return dead, percent @@ -90,7 +93,7 @@ def calculate_and_clear_dead_packages(job: Jobs): job=job, status=JobStatus.RUNNING, progress=0, - status_text="Сalculate the number of dead packages...", + status_text="Calculate the number of dead packages...", ) dead_packages, package_equal_to_percent = get_dead_packages( @@ -121,7 +124,7 @@ def start_nix_collect_garbage() -> Job: job = Jobs.add( type_id="maintenance.collect_nix_garbage", name="Collect garbage", - description=".", + description="Cleaning up unused packages", ) calculate_and_clear_dead_packages(job=job) return job From 19005158a8635884fba4ddfa1139ca139b67bda7 Mon Sep 17 00:00:00 2001 From: dettlaff Date: Mon, 26 Jun 2023 23:33:41 +0400 Subject: [PATCH 26/50] fix: add timewait --- selfprivacy_api/jobs/nix_collect_garbage.py | 21 ++++++++++----------- 1 file changed, 10 insertions(+), 11 deletions(-) diff --git a/selfprivacy_api/jobs/nix_collect_garbage.py b/selfprivacy_api/jobs/nix_collect_garbage.py index 8cc3e66..b9401b2 100644 --- a/selfprivacy_api/jobs/nix_collect_garbage.py +++ b/selfprivacy_api/jobs/nix_collect_garbage.py @@ -47,12 +47,9 @@ def parse_line(line): ) -def stream_process( - job, - stream, - total_dead_packages, -): +def stream_process(job, stream, total_dead_packages): completed_packages = 0 + prev_progress = 0 for line in stream: line = line.decode("utf-8") @@ -61,12 +58,14 @@ def stream_process( completed_packages += 1 percent = int((completed_packages / total_dead_packages) * 100) - Jobs.update( - job=job, - status=JobStatus.RUNNING, - progress=percent, - status_text="Cleaning...", - ) + if percent - prev_progress >= 5: + Jobs.update( + job=job, + status=JobStatus.RUNNING, + progress=percent, + status_text="Cleaning...", + ) + prev_progress = percent elif "store paths deleted," in line: status = parse_line(line) From 66970e6375c65c4e672427403689b7c2df30602f Mon Sep 17 00:00:00 2001 From: dettlaff Date: Fri, 30 Jun 2023 20:51:06 +0400 Subject: [PATCH 27/50] fix: stupid russian C --- .../test_graphql/test_nix_collect_garbage.py | 30 +++++++++---------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/tests/test_graphql/test_nix_collect_garbage.py b/tests/test_graphql/test_nix_collect_garbage.py index 151327c..9a2c4a0 100644 --- a/tests/test_graphql/test_nix_collect_garbage.py +++ b/tests/test_graphql/test_nix_collect_garbage.py @@ -160,15 +160,15 @@ def test_get_dead_packages_zero(job_reset): def test_stream_process(job_reset): log_event = [] reference = [ - (JobStatus.RUNNING, 20, "Сleaning...", ""), - (JobStatus.RUNNING, 40, "Сleaning...", ""), - (JobStatus.RUNNING, 60, "Сleaning...", ""), - (JobStatus.RUNNING, 80, "Сleaning...", ""), - (JobStatus.RUNNING, 100, "Сleaning...", ""), + (JobStatus.RUNNING, 20, "Cleaning...", ""), + (JobStatus.RUNNING, 40, "Cleaning...", ""), + (JobStatus.RUNNING, 60, "Cleaning...", ""), + (JobStatus.RUNNING, 80, "Cleaning...", ""), + (JobStatus.RUNNING, 100, "Cleaning...", ""), ( JobStatus.FINISHED, 100, - "Сleaning completed.", + "Cleaning completed.", "425.51 MiB have been cleared", ), ] @@ -185,17 +185,17 @@ def test_nix_collect_garbage( ): log_event = [] reference = [ - (JobStatus.RUNNING, 0, "Сalculate the number of dead packages...", ""), + (JobStatus.RUNNING, 0, "Calculate the number of dead packages...", ""), (JobStatus.RUNNING, 0, "Found 5 packages to remove!", ""), - (JobStatus.RUNNING, 5, "Сleaning...", ""), - (JobStatus.RUNNING, 10, "Сleaning...", ""), - (JobStatus.RUNNING, 15, "Сleaning...", ""), - (JobStatus.RUNNING, 20, "Сleaning...", ""), - (JobStatus.RUNNING, 25, "Сleaning...", ""), + (JobStatus.RUNNING, 5, "Cleaning...", ""), + (JobStatus.RUNNING, 10, "Cleaning...", ""), + (JobStatus.RUNNING, 15, "Cleaning...", ""), + (JobStatus.RUNNING, 20, "Cleaning...", ""), + (JobStatus.RUNNING, 25, "Cleaning...", ""), ( JobStatus.FINISHED, 100, - "Сleaning completed.", + "Cleaning completed.", "425.51 MiB have been cleared", ), ] @@ -213,7 +213,7 @@ def test_nix_collect_garbage_zero_trash( mock_run_nix_store_print_dead_zero_trash, ): reference = [ - (JobStatus.RUNNING, 0, "Сalculate the number of dead packages...", ""), + (JobStatus.RUNNING, 0, "Calculate the number of dead packages...", ""), (JobStatus.FINISHED, 100, "Nothing to clear", "System is clear"), ] @@ -222,7 +222,7 @@ def test_nix_collect_garbage_zero_trash( assert log_event == reference -# андр констракнш +# андр конcтракнш @pytest.mark.asyncio async def test_graphql_nix_collect_garbage(): query = """ From 7c68f05040cd78f27a0856db1f7c3a72a17f6768 Mon Sep 17 00:00:00 2001 From: dettlaff Date: Thu, 3 Aug 2023 05:47:57 +0400 Subject: [PATCH 28/50] tests: delete old, fix --- dump.rdb | Bin 89 -> 89 bytes .../test_graphql/test_nix_collect_garbage.py | 161 ++++-------------- 2 files changed, 29 insertions(+), 132 deletions(-) diff --git a/dump.rdb b/dump.rdb index 390bec997b2e3719704dd13d59d827fac314f842..0caa6188be39d199135bd5eb2d2dbc60770b9627 100644 GIT binary patch delta 68 zcmV-K0K5NLS&$k(G5QK}Wn^h{En;bObHG6Q1!Ht+ZDqn-0?TCj2z7I1WG!uFZNe}B a5CHlJVQ*$FVqtS-zySZZlmHfy;^{)GrW^(U delta 68 zcmV-K0K5NLS&$k(F!~B|Wn^h{En;bObHG6Q1!Ht+ZDqpJ;frMY2z7I1WG!uFZNhMC a4*>cIVQ*$FVqtS-zySYFim8nyFP-q^i5;{6 diff --git a/tests/test_graphql/test_nix_collect_garbage.py b/tests/test_graphql/test_nix_collect_garbage.py index 9a2c4a0..f795ca2 100644 --- a/tests/test_graphql/test_nix_collect_garbage.py +++ b/tests/test_graphql/test_nix_collect_garbage.py @@ -8,17 +8,15 @@ import strawberry from selfprivacy_api.jobs import JobStatus, Jobs from selfprivacy_api.graphql import schema -# from selfprivacy_api.graphql.schema import Subscription -# from selfprivacy_api.jobs.nix_collect_garbage import ( -# get_dead_packages, -# nix_collect_garbage, -# parse_line, -# CLEAR_COMPLETED, -# COMPLETED_WITH_ERROR, -# stream_process, -# RESULT_WAS_NOT_FOUND_ERROR, -# ) +from selfprivacy_api.jobs.nix_collect_garbage import ( + get_dead_packages, + parse_line, + CLEAR_COMPLETED, + COMPLETED_WITH_ERROR, + stream_process, + RESULT_WAS_NOT_FOUND_ERROR, +) pytest_plugins = ("pytest_asyncio",) @@ -66,19 +64,6 @@ determining live/dead paths... log_event = [] -def set_job_status(status="", progress="", status_text="", result=""): - log_event.append((status, progress, status_text, result)) - - -@pytest.fixture -def mock_run_nix_store_print_dead_zero_trash(mocker): - mock = mocker.patch( - "selfprivacy_api.jobs.nix_collect_garbage.run_nix_store_print_dead", - autospec=True, - return_value=OUTPUT_RUN_NIX_STORE_PRINT_DEAD_ZERO_TRASH.split("\n"), - ) - - @pytest.fixture def mock_set_job_status(mocker): mock = mocker.patch( @@ -89,36 +74,6 @@ def mock_set_job_status(mocker): return mock -@pytest.fixture -def mock_run_nix_collect_garbage(mocker): - mock = mocker.patch( - "selfprivacy_api.jobs.nix_collect_garbage.run_nix_collect_garbage", - autospec=True, - return_value=OUTPUT_COLLECT_GARBAGE.split("\n"), - ) - return mock - - -@pytest.fixture -def mock_run_nix_collect_garbage_zero_trash(mocker): - mock = mocker.patch( - "selfprivacy_api.jobs.nix_collect_garbage.run_nix_collect_garbage", - autospec=True, - return_value=OUTPUT_COLLECT_GARBAGE_ZERO_TRASH.split("\n"), - ) - return mock - - -@pytest.fixture -def mock_run_nix_store_print_dead(mocker): - mock = mocker.patch( - "selfprivacy_api.jobs.nix_collect_garbage.run_nix_store_print_dead", - autospec=True, - return_value="", - ) - return mock - - @pytest.fixture def job_reset(): Jobs.reset() @@ -154,88 +109,30 @@ def test_get_dead_packages(job_reset): def test_get_dead_packages_zero(job_reset): - assert get_dead_packages("") == (0, None) + assert get_dead_packages("") == (0, 0) -def test_stream_process(job_reset): - log_event = [] - reference = [ - (JobStatus.RUNNING, 20, "Cleaning...", ""), - (JobStatus.RUNNING, 40, "Cleaning...", ""), - (JobStatus.RUNNING, 60, "Cleaning...", ""), - (JobStatus.RUNNING, 80, "Cleaning...", ""), - (JobStatus.RUNNING, 100, "Cleaning...", ""), - ( - JobStatus.FINISHED, - 100, - "Cleaning completed.", - "425.51 MiB have been cleared", - ), - ] - - def set_job_status(status, progress, status_text, result=""): - log_event.append((status, progress, status_text, result)) - - stream_process(OUTPUT_COLLECT_GARBAGE.split("\n"), 5, set_job_status) - assert log_event == reference +RUN_NIX_COLLECT_GARBAGE_QUERY = """ +mutation system { + nixCollectGarbage { + success + message + code + } +} +""" -def test_nix_collect_garbage( - mock_set_job_status, mock_run_nix_collect_garbage, job_reset -): - log_event = [] - reference = [ - (JobStatus.RUNNING, 0, "Calculate the number of dead packages...", ""), - (JobStatus.RUNNING, 0, "Found 5 packages to remove!", ""), - (JobStatus.RUNNING, 5, "Cleaning...", ""), - (JobStatus.RUNNING, 10, "Cleaning...", ""), - (JobStatus.RUNNING, 15, "Cleaning...", ""), - (JobStatus.RUNNING, 20, "Cleaning...", ""), - (JobStatus.RUNNING, 25, "Cleaning...", ""), - ( - JobStatus.FINISHED, - 100, - "Cleaning completed.", - "425.51 MiB have been cleared", - ), - ] - - nix_collect_garbage(None) - - assert log_event == reference - - -def test_nix_collect_garbage_zero_trash( - mock_set_job_status, - mock_run_nix_collect_garbage_zero_trash, - mock_run_nix_store_print_dead, - job_reset, - mock_run_nix_store_print_dead_zero_trash, -): - reference = [ - (JobStatus.RUNNING, 0, "Calculate the number of dead packages...", ""), - (JobStatus.FINISHED, 100, "Nothing to clear", "System is clear"), - ] - - nix_collect_garbage(None) - - assert log_event == reference - - -# андр конcтракнш -@pytest.mark.asyncio -async def test_graphql_nix_collect_garbage(): - query = """ - subscription { - nixCollectGarbage() - } - """ - - schema_for_garbage = strawberry.Schema( - query=schema.Query, mutation=schema.Mutation, subscription=schema.Subscription +def test_graphql_nix_collect_garbage(authorized_client): + response = authorized_client.post( + "/graphql", + json={ + "query": RUN_NIX_COLLECT_GARBAGE_QUERY, + }, ) - sub = await schema_for_garbage.subscribe(query) - async for result in sub: - assert not result.errors - assert result.data == {} + assert response.status_code == 200 + assert response.json().get("data") is not None + assert response.json()["data"]["nixCollectGarbage"]["success"] is True + assert response.json()["data"]["nixCollectGarbage"]["message"] is not None + assert response.json()["data"]["nixCollectGarbage"]["code"] == 200 From 1640ed44f6843e52d32cefcdbeec5d91543b0443 Mon Sep 17 00:00:00 2001 From: dettlaff Date: Mon, 7 Aug 2023 06:24:48 +0400 Subject: [PATCH 29/50] tests: add success check --- dump.rdb | Bin 89 -> 476 bytes tests/test_graphql/test_nix_collect_garbage.py | 1 + 2 files changed, 1 insertion(+) diff --git a/dump.rdb b/dump.rdb index 0caa6188be39d199135bd5eb2d2dbc60770b9627..5198f51641b5291db4f7760ebe55b3f31fe30e47 100644 GIT binary patch delta 436 zcmZwC%}x_R902fLu~M*xqJ*H;>|Bg^*k*U9-IiXI@DaE)Byi!hPG|ltgWZ|U%xuD* z+VBmHfwLau;K^t3VC3#m6XSsg@ClHJoQ)6gJN(vv4xRcvy7`q|=2H#KjUoPW7LQq% zc_Q0xi)i?4{jq-A_SO3L?O#W>Wfh-q%9d_~EMTmKiCY5}xRpw~?vfeiyFjb79r_i( zxPCsnFq=8q4VtocQqFFF_@8om|GK9KCQBgVEdM?xTo?xg5fsW)cDq1LB=}a;A*w|> zfOZ9m;a2WsuK&nFXOm8Gz(7;Q6T_t_yqoI+5nOaonxItN6mtwEguX^ECiLTdZHP&A zZ}@O=xw*W$v@kz1=#P&^2DVJ$v?TIo1V*v0*4W7V?n2e=lN25m05-gvcQ6)r|s`qRZa sE|>N|DXH%4Hl={vK6-O6?V90})l3_R5!d~XcJcZ;d-CbQ{I?(BU)_t5!T=}({z&(i&GCU{NFx>L2Tlqmrekj CloE~r diff --git a/tests/test_graphql/test_nix_collect_garbage.py b/tests/test_graphql/test_nix_collect_garbage.py index f795ca2..900014a 100644 --- a/tests/test_graphql/test_nix_collect_garbage.py +++ b/tests/test_graphql/test_nix_collect_garbage.py @@ -136,3 +136,4 @@ def test_graphql_nix_collect_garbage(authorized_client): assert response.json()["data"]["nixCollectGarbage"]["success"] is True assert response.json()["data"]["nixCollectGarbage"]["message"] is not None assert response.json()["data"]["nixCollectGarbage"]["code"] == 200 + assert response.json()["data"]["nixCollectGarbage"]["success"] == True From a316f8b9105d740822136c3873ade9539320d419 Mon Sep 17 00:00:00 2001 From: dettlaff Date: Mon, 7 Aug 2023 16:56:35 +0400 Subject: [PATCH 30/50] tests: fix --- dump.rdb | Bin 476 -> 633 bytes .../graphql/mutations/system_mutations.py | 6 +++--- .../test_graphql/test_nix_collect_garbage.py | 20 ++++++++++-------- 3 files changed, 14 insertions(+), 12 deletions(-) diff --git a/dump.rdb b/dump.rdb index 5198f51641b5291db4f7760ebe55b3f31fe30e47..ddf1da50f459311aab5195b1e1cc6dc3c2cc6d4f 100644 GIT binary patch literal 633 zcmcJM&u-H|5XLuZ98#c?B~Bor2Cm5Q+9XZu0}2vFsstRM;#h0!orIO`U2As}lv`Eh zz`uv!9e9DLPXGt5T#-0(G>EWPVu=^vw)>4{<~QH>{P@wy3rYi8?J8tkdvjEsQJ?mM zpt`zI5^nTrTsf)X1zvvn{+U(XnMURnSX`d}bV=0(Iq?$ExFUbZuTIO|D%Mvx__Ag0r1BjkxBW(@V-Y6JvurK!f0?1g-89tFdU5z?@=02dS~t4ZFc9NjWr<6 ztSk18e`fw~26D4u;DVc)eS3_KR;{+%6iX%NoS|~Mwa11_F@EW*nec6p$`2@|ez~Zz zJ)0!MN%IT+T8s-h$2hUAnU1$3V=l@ZsJ{(oCKTqickPS|)_#2Qyqar)r(!*729Vh!Ax delta 437 zcmZwC%}x_R902fLp;EAhqQ-!Q>|Bg`*k*U9-IiXE@DaH*BzW<(PG|ltgWZ|U%xuD* z+Q>HzfwLau;K^t3VDRoy661jf@C8I8ayCA|@9_IM{C()e@6nAf>@uHfU~UZY*=am( zUFL~wyDg&Oll8~?W!t}BoWK5YU|Ux4$%bs{M#uujT9~*sV1ZkywCgUJWxflvO534d z0gUUP=a=R)C%Zg{d@caKMr}wXWxM#8iBF^&fV#0-SKoCKpOl7wV)I@@BMjfJB zqyuPMkQi>{ju-k5EOdI`DGnHDs(50!6ot17T_A#s4oVZ0itA#Ip@h&^Xzre#-qnVf zRQE>qmsgrAYtNPzM+g0hvBa^Ju(U_Wc0Y!qTA41=s5@SFOUu*lPcA uv5w279Z*WDo7+t(Ah(N-Zl_%{a=eyl12N*d|G_R^U1c*L?=F7*7XAhG GenericJobButationReturn: + def nix_collect_garbage(self) -> GenericJobMutationReturn: job = start_nix_collect_garbage() - return GenericJobButationReturn( + return GenericJobMutationReturn( success=True, code=200, message="Cleaning started..,", diff --git a/tests/test_graphql/test_nix_collect_garbage.py b/tests/test_graphql/test_nix_collect_garbage.py index 900014a..703dbed 100644 --- a/tests/test_graphql/test_nix_collect_garbage.py +++ b/tests/test_graphql/test_nix_collect_garbage.py @@ -113,11 +113,13 @@ def test_get_dead_packages_zero(job_reset): RUN_NIX_COLLECT_GARBAGE_QUERY = """ -mutation system { - nixCollectGarbage { - success - message - code +mutation CollectGarbage { + system { + nixCollectGarbage { + success + message + code + } } } """ @@ -133,7 +135,7 @@ def test_graphql_nix_collect_garbage(authorized_client): assert response.status_code == 200 assert response.json().get("data") is not None - assert response.json()["data"]["nixCollectGarbage"]["success"] is True - assert response.json()["data"]["nixCollectGarbage"]["message"] is not None - assert response.json()["data"]["nixCollectGarbage"]["code"] == 200 - assert response.json()["data"]["nixCollectGarbage"]["success"] == True + assert response.json()["data"]["system"]["nixCollectGarbage"]["success"] is True + assert response.json()["data"]["system"]["nixCollectGarbage"]["message"] is not None + assert response.json()["data"]["system"]["nixCollectGarbage"]["success"] == True + assert response.json()["data"]["system"]["nixCollectGarbage"]["code"] == 200 From be821fe1e0af84dbff018af38729497ea16b9c1d Mon Sep 17 00:00:00 2001 From: def Date: Mon, 7 Aug 2023 15:58:45 +0300 Subject: [PATCH 31/50] Delete 'dump.rdb' --- dump.rdb | Bin 633 -> 0 bytes 1 file changed, 0 insertions(+), 0 deletions(-) delete mode 100644 dump.rdb diff --git a/dump.rdb b/dump.rdb deleted file mode 100644 index ddf1da50f459311aab5195b1e1cc6dc3c2cc6d4f..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 633 zcmcJM&u-H|5XLuZ98#c?B~Bor2Cm5Q+9XZu0}2vFsstRM;#h0!orIO`U2As}lv`Eh zz`uv!9e9DLPXGt5T#-0(G>EWPVu=^vw)>4{<~QH>{P@wy3rYi8?J8tkdvjEsQJ?mM zpt`zI5^nTrTsf)X1zvvn{+U(XnMURnSX`d}bV=0(Iq?$ExFUbZuTIO|D%Mvx__Ag0r1BjkxBW(@V-Y6JvurK!f0?1g-89tFdU5z?@=02dS~t4ZFc9NjWr<6 ztSk18e`fw~26D4u;DVc)eS3_KR;{+%6iX%NoS|~Mwa11_F@EW*nec6p$`2@|ez~Zz zJ)0!MN%IT+T8s-h$2hUAnU1$3V=l@ZsJ{(oCKTqickPS|)_#2Qyqar)r(!*729Vh!Ax From aadf0cb1a31cb6252cbdb4943eebbeea5d028c8b Mon Sep 17 00:00:00 2001 From: def Date: Mon, 7 Aug 2023 16:00:01 +0300 Subject: [PATCH 32/50] Delete 'zsh-config.nix' --- zsh-config.nix | 32 -------------------------------- 1 file changed, 32 deletions(-) delete mode 100644 zsh-config.nix diff --git a/zsh-config.nix b/zsh-config.nix deleted file mode 100644 index 8d00e92..0000000 --- a/zsh-config.nix +++ /dev/null @@ -1,32 +0,0 @@ -{ pkgs }: - -let - zshWithPlugins = pkgs.zsh.overrideAttrs (oldAttrs: { - postInstall = oldAttrs.postInstall + '' - # Create symlinks for plugins in $out/share - mkdir -p $out/share/zsh/plugins - ln -s ${pkgs.zsh-autosuggestions}/share/zsh-autosuggestions $out/share/zsh/plugins/autosuggestions - ln -s ${pkgs.zsh-syntax-highlighting}/share/zsh-syntax-highlighting $out/share/zsh/plugins/syntax-highlighting - ''; - }); -in -{ - shell = zshWithPlugins; - shellSetup = '' - # create .zshrc file if it doesnt exist - if [ ! -f ~/.zshrc ]; then - echo "Creating a default .zshrc file..." - touch ~/.zshrc - fi - - # add zsh-autosuggestions and zsh-syntax-highlighting in zshrc - if ! grep -q "source .*zsh-autosuggestions.zsh" ~/.zshrc; then - echo "Adding zsh-autosuggestions plugin to .zshrc..." - echo "source ${zshWithPlugins}/share/zsh/plugins/autosuggestions/zsh-autosuggestions.zsh" >> ~/.zshrc - fi - if ! grep -q "source .*zsh-syntax-highlighting.zsh" ~/.zshrc; then - echo "Adding zsh-syntax-highlighting plugin to .zshrc..." - echo "source ${zshWithPlugins}/share/zsh/plugins/syntax-highlighting/zsh-syntax-highlighting.zsh" >> ~/.zshrc - fi - ''; -} From 446220a9c5cdeaad4ef1b4ffad3df6a27b4bae3b Mon Sep 17 00:00:00 2001 From: dettlaff Date: Mon, 7 Aug 2023 17:38:44 +0400 Subject: [PATCH 33/50] fix: replace os to subprocess --- selfprivacy_api/jobs/nix_collect_garbage.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/selfprivacy_api/jobs/nix_collect_garbage.py b/selfprivacy_api/jobs/nix_collect_garbage.py index b9401b2..9b14084 100644 --- a/selfprivacy_api/jobs/nix_collect_garbage.py +++ b/selfprivacy_api/jobs/nix_collect_garbage.py @@ -1,5 +1,4 @@ import re -import os import subprocess from selfprivacy_api.utils.huey import huey @@ -13,7 +12,7 @@ CLEAR_COMPLETED = "Cleaning completed." def run_nix_store_print_dead(): - os.system("nix-env -p /nix/var/nix/profiles/system --delete-generations old") + subprocess.run(["nix-env", "-p", "/nix/var/nix/profiles/system", "--delete-generations old"], check=False) return subprocess.check_output(["nix-store", "--gc", "--print-dead"]).decode( "utf-8" From 5c42d7a907e018045612cec82a68af2befa5b54a Mon Sep 17 00:00:00 2001 From: dettlaff Date: Thu, 12 Oct 2023 02:01:31 +0400 Subject: [PATCH 34/50] refactor: problems from pr --- .../graphql/mutations/system_mutations.py | 2 +- selfprivacy_api/jobs/nix_collect_garbage.py | 24 ++++++++++++------- .../test_graphql/test_nix_collect_garbage.py | 15 ++---------- 3 files changed, 18 insertions(+), 23 deletions(-) diff --git a/selfprivacy_api/graphql/mutations/system_mutations.py b/selfprivacy_api/graphql/mutations/system_mutations.py index aea3bd5..f286828 100644 --- a/selfprivacy_api/graphql/mutations/system_mutations.py +++ b/selfprivacy_api/graphql/mutations/system_mutations.py @@ -137,6 +137,6 @@ class SystemMutations: return GenericJobMutationReturn( success=True, code=200, - message="Cleaning started..,", + message="Garbage collector started...", job=job_to_api_job(job), ) diff --git a/selfprivacy_api/jobs/nix_collect_garbage.py b/selfprivacy_api/jobs/nix_collect_garbage.py index 9b14084..cd6dfed 100644 --- a/selfprivacy_api/jobs/nix_collect_garbage.py +++ b/selfprivacy_api/jobs/nix_collect_garbage.py @@ -1,25 +1,31 @@ import re import subprocess +from subprocess import Popen, PIPE from selfprivacy_api.utils.huey import huey from selfprivacy_api.jobs import JobStatus, Jobs, Job -COMPLETED_WITH_ERROR = "Completed with an error" -RESULT_WAS_NOT_FOUND_ERROR = "We are sorry, result was not found :(" +COMPLETED_WITH_ERROR = ( + "We are sorry, сompleted with an error, report it to support chat" +) +RESULT_WAS_NOT_FOUND_ERROR = "We are sorry, garbage collection result was not found, something went wrong, report it to support chat :(" CLEAR_COMPLETED = "Cleaning completed." -def run_nix_store_print_dead(): - subprocess.run(["nix-env", "-p", "/nix/var/nix/profiles/system", "--delete-generations old"], check=False) +def run_nix_store_print_dead() -> PIPE: + subprocess.run( + ["nix-env", "-p", "/nix/var/nix/profiles/system", "--delete-generations old"], + check=False, + ) return subprocess.check_output(["nix-store", "--gc", "--print-dead"]).decode( "utf-8" ) -def run_nix_collect_garbage(): +def run_nix_collect_garbage() -> subprocess.Popen: return subprocess.Popen( ["nix-store", "--gc"], stdout=subprocess.PIPE, stderr=subprocess.STDOUT ).stdout @@ -46,7 +52,7 @@ def parse_line(line): ) -def stream_process(job, stream, total_dead_packages): +def process_stream(job, stream, total_dead_packages): completed_packages = 0 prev_progress = 0 @@ -86,7 +92,7 @@ def get_dead_packages(output): @huey.task() -def calculate_and_clear_dead_packages(job: Jobs): +def calculate_and_clear_dead_packages(job: Job): Jobs.update( job=job, status=JobStatus.RUNNING, @@ -94,7 +100,7 @@ def calculate_and_clear_dead_packages(job: Jobs): status_text="Calculate the number of dead packages...", ) - dead_packages, package_equal_to_percent = get_dead_packages( + dead_packages, package_equal_to_percent = get_dead_packages(stream_process run_nix_store_print_dead() ) @@ -115,7 +121,7 @@ def calculate_and_clear_dead_packages(job: Jobs): status_text=f"Found {dead_packages} packages to remove!", ) - stream_process(job, run_nix_collect_garbage(), package_equal_to_percent) + process_stream(job, run_nix_collect_garbage(), package_equal_to_percent) def start_nix_collect_garbage() -> Job: diff --git a/tests/test_graphql/test_nix_collect_garbage.py b/tests/test_graphql/test_nix_collect_garbage.py index 703dbed..340292f 100644 --- a/tests/test_graphql/test_nix_collect_garbage.py +++ b/tests/test_graphql/test_nix_collect_garbage.py @@ -64,16 +64,6 @@ determining live/dead paths... log_event = [] -@pytest.fixture -def mock_set_job_status(mocker): - mock = mocker.patch( - "selfprivacy_api.jobs.nix_collect_garbage.set_job_status_wrapper", - autospec=True, - return_value=set_job_status, - ) - return mock - - @pytest.fixture def job_reset(): Jobs.reset() @@ -118,7 +108,7 @@ mutation CollectGarbage { nixCollectGarbage { success message - code + code } } } @@ -135,7 +125,6 @@ def test_graphql_nix_collect_garbage(authorized_client): assert response.status_code == 200 assert response.json().get("data") is not None - assert response.json()["data"]["system"]["nixCollectGarbage"]["success"] is True assert response.json()["data"]["system"]["nixCollectGarbage"]["message"] is not None - assert response.json()["data"]["system"]["nixCollectGarbage"]["success"] == True + assert response.json()["data"]["system"]["nixCollectGarbage"]["success"] is True assert response.json()["data"]["system"]["nixCollectGarbage"]["code"] == 200 From 3fbc93c99838adfb5e058ef9e60d5c4f7ea1d6d5 Mon Sep 17 00:00:00 2001 From: dettlaff Date: Thu, 12 Oct 2023 02:09:13 +0400 Subject: [PATCH 35/50] tests: fix rename --- selfprivacy_api/jobs/nix_collect_garbage.py | 2 +- tests/test_graphql/test_nix_collect_garbage.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/selfprivacy_api/jobs/nix_collect_garbage.py b/selfprivacy_api/jobs/nix_collect_garbage.py index cd6dfed..18f2e6b 100644 --- a/selfprivacy_api/jobs/nix_collect_garbage.py +++ b/selfprivacy_api/jobs/nix_collect_garbage.py @@ -100,7 +100,7 @@ def calculate_and_clear_dead_packages(job: Job): status_text="Calculate the number of dead packages...", ) - dead_packages, package_equal_to_percent = get_dead_packages(stream_process + dead_packages, package_equal_to_percent = get_dead_packages( run_nix_store_print_dead() ) diff --git a/tests/test_graphql/test_nix_collect_garbage.py b/tests/test_graphql/test_nix_collect_garbage.py index 340292f..a23f7ea 100644 --- a/tests/test_graphql/test_nix_collect_garbage.py +++ b/tests/test_graphql/test_nix_collect_garbage.py @@ -14,7 +14,7 @@ from selfprivacy_api.jobs.nix_collect_garbage import ( parse_line, CLEAR_COMPLETED, COMPLETED_WITH_ERROR, - stream_process, + process_stream, RESULT_WAS_NOT_FOUND_ERROR, ) From f2ead0c77d290d058c50371057d7fdd190effdbd Mon Sep 17 00:00:00 2001 From: dettlaff Date: Thu, 12 Oct 2023 02:18:29 +0400 Subject: [PATCH 36/50] fix: typos --- selfprivacy_api/jobs/nix_collect_garbage.py | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/selfprivacy_api/jobs/nix_collect_garbage.py b/selfprivacy_api/jobs/nix_collect_garbage.py index 18f2e6b..f323214 100644 --- a/selfprivacy_api/jobs/nix_collect_garbage.py +++ b/selfprivacy_api/jobs/nix_collect_garbage.py @@ -32,6 +32,12 @@ def run_nix_collect_garbage() -> subprocess.Popen: def parse_line(line): + """ + We parse the string for the presence of a final line, + with the final amount of space cleared. + Simply put, we're just looking for a similar string: + "1537 store paths deleted, 339.84 MiB freed". + """ pattern = re.compile(r"[+-]?\d+\.\d+ \w+(?= freed)") match = re.search(pattern, line) @@ -65,7 +71,7 @@ def process_stream(job, stream, total_dead_packages): if percent - prev_progress >= 5: Jobs.update( - job=job, + job=Job, status=JobStatus.RUNNING, progress=percent, status_text="Cleaning...", @@ -75,7 +81,7 @@ def process_stream(job, stream, total_dead_packages): elif "store paths deleted," in line: status = parse_line(line) Jobs.update( - job=job, + job=Job, status=status[0], progress=status[1], status_text=status[2], @@ -94,7 +100,7 @@ def get_dead_packages(output): @huey.task() def calculate_and_clear_dead_packages(job: Job): Jobs.update( - job=job, + job=Job, status=JobStatus.RUNNING, progress=0, status_text="Calculate the number of dead packages...", @@ -106,7 +112,7 @@ def calculate_and_clear_dead_packages(job: Job): if dead_packages == 0: Jobs.update( - job=job, + job=Job, status=JobStatus.FINISHED, progress=100, status_text="Nothing to clear", @@ -115,7 +121,7 @@ def calculate_and_clear_dead_packages(job: Job): return Jobs.update( - job=job, + job=Job, status=JobStatus.RUNNING, progress=0, status_text=f"Found {dead_packages} packages to remove!", From 70a498d0e584575539b55ba3de0b74abd31d8ba9 Mon Sep 17 00:00:00 2001 From: dettlaff Date: Thu, 12 Oct 2023 02:26:27 +0400 Subject: [PATCH 37/50] fix: missclick with Job jobs --- selfprivacy_api/jobs/nix_collect_garbage.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/selfprivacy_api/jobs/nix_collect_garbage.py b/selfprivacy_api/jobs/nix_collect_garbage.py index f323214..a0d5d9a 100644 --- a/selfprivacy_api/jobs/nix_collect_garbage.py +++ b/selfprivacy_api/jobs/nix_collect_garbage.py @@ -98,7 +98,7 @@ def get_dead_packages(output): @huey.task() -def calculate_and_clear_dead_packages(job: Job): +def calculate_and_clear_dead_packages(job: Jobs): Jobs.update( job=Job, status=JobStatus.RUNNING, From 98eef8d08e6fd3dded59b9f71736abd4841db4ea Mon Sep 17 00:00:00 2001 From: dettlaff Date: Thu, 12 Oct 2023 02:40:47 +0400 Subject: [PATCH 38/50] fix: tests, jobs.error return --- selfprivacy_api/jobs/nix_collect_garbage.py | 4 ++-- tests/test_graphql/test_nix_collect_garbage.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/selfprivacy_api/jobs/nix_collect_garbage.py b/selfprivacy_api/jobs/nix_collect_garbage.py index a0d5d9a..a623f9f 100644 --- a/selfprivacy_api/jobs/nix_collect_garbage.py +++ b/selfprivacy_api/jobs/nix_collect_garbage.py @@ -43,7 +43,7 @@ def parse_line(line): if match is None: return ( - JobStatus.FINISHED, + JobStatus.ERROR, 100, COMPLETED_WITH_ERROR, RESULT_WAS_NOT_FOUND_ERROR, @@ -98,7 +98,7 @@ def get_dead_packages(output): @huey.task() -def calculate_and_clear_dead_packages(job: Jobs): +def calculate_and_clear_dead_packages(job: Job): Jobs.update( job=Job, status=JobStatus.RUNNING, diff --git a/tests/test_graphql/test_nix_collect_garbage.py b/tests/test_graphql/test_nix_collect_garbage.py index a23f7ea..db62ef1 100644 --- a/tests/test_graphql/test_nix_collect_garbage.py +++ b/tests/test_graphql/test_nix_collect_garbage.py @@ -86,7 +86,7 @@ def test_parse_line(job_reset): def test_parse_line_with_blank_line(job_reset): txt = "" output = ( - JobStatus.FINISHED, + JobStatus.ERROR, 100, COMPLETED_WITH_ERROR, RESULT_WAS_NOT_FOUND_ERROR, From eda5923cc677b424d74078e7b0f7ab82d154934d Mon Sep 17 00:00:00 2001 From: dettlaff Date: Thu, 12 Oct 2023 02:59:31 +0400 Subject: [PATCH 39/50] tests: fix query --- tests/test_graphql/test_nix_collect_garbage.py | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/tests/test_graphql/test_nix_collect_garbage.py b/tests/test_graphql/test_nix_collect_garbage.py index db62ef1..ea56d0f 100644 --- a/tests/test_graphql/test_nix_collect_garbage.py +++ b/tests/test_graphql/test_nix_collect_garbage.py @@ -108,7 +108,21 @@ mutation CollectGarbage { nixCollectGarbage { success message - code + code + job { + uid, + typeId, + name, + description, + status, + statusText, + progress, + createdAt, + updatedAt, + finishedAt, + error, + result, + } } } } @@ -128,3 +142,5 @@ def test_graphql_nix_collect_garbage(authorized_client): assert response.json()["data"]["system"]["nixCollectGarbage"]["message"] is not None assert response.json()["data"]["system"]["nixCollectGarbage"]["success"] is True assert response.json()["data"]["system"]["nixCollectGarbage"]["code"] == 200 + + assert response.json()["data"]["system"]["nixCollectGarbage"]["job"] is not None From c0f8b9026be3d781e487e443ecf01a3c9435b40f Mon Sep 17 00:00:00 2001 From: dettlaff Date: Mon, 16 Oct 2023 20:39:58 +0400 Subject: [PATCH 40/50] fix: corrected from comment from pr --- selfprivacy_api/jobs/nix_collect_garbage.py | 12 ++++++------ tests/test_graphql/test_nix_collect_garbage.py | 7 ++----- 2 files changed, 8 insertions(+), 11 deletions(-) diff --git a/selfprivacy_api/jobs/nix_collect_garbage.py b/selfprivacy_api/jobs/nix_collect_garbage.py index a623f9f..259af96 100644 --- a/selfprivacy_api/jobs/nix_collect_garbage.py +++ b/selfprivacy_api/jobs/nix_collect_garbage.py @@ -1,6 +1,7 @@ import re import subprocess from subprocess import Popen, PIPE +from typing import Tuple, Iterable, IO, Any, Optional from selfprivacy_api.utils.huey import huey @@ -14,7 +15,7 @@ RESULT_WAS_NOT_FOUND_ERROR = "We are sorry, garbage collection result was not fo CLEAR_COMPLETED = "Cleaning completed." -def run_nix_store_print_dead() -> PIPE: +def run_nix_store_print_dead() -> IO[Any]: subprocess.run( ["nix-env", "-p", "/nix/var/nix/profiles/system", "--delete-generations old"], check=False, @@ -24,14 +25,13 @@ def run_nix_store_print_dead() -> PIPE: "utf-8" ) - -def run_nix_collect_garbage() -> subprocess.Popen: +def run_nix_collect_garbage() -> Optional[IO[bytes]]: return subprocess.Popen( ["nix-store", "--gc"], stdout=subprocess.PIPE, stderr=subprocess.STDOUT ).stdout -def parse_line(line): +def parse_line(line: str): """ We parse the string for the presence of a final line, with the final amount of space cleared. @@ -58,7 +58,7 @@ def parse_line(line): ) -def process_stream(job, stream, total_dead_packages): +def process_stream(job: Job, stream: Iterable[bytes], total_dead_packages: int) -> None: completed_packages = 0 prev_progress = 0 @@ -89,7 +89,7 @@ def process_stream(job, stream, total_dead_packages): ) -def get_dead_packages(output): +def get_dead_packages(output) -> Tuple[int, int]: dead = len(re.findall("/nix/store/", output)) percent = 0 if dead != 0: diff --git a/tests/test_graphql/test_nix_collect_garbage.py b/tests/test_graphql/test_nix_collect_garbage.py index ea56d0f..e05a5dc 100644 --- a/tests/test_graphql/test_nix_collect_garbage.py +++ b/tests/test_graphql/test_nix_collect_garbage.py @@ -18,9 +18,6 @@ from selfprivacy_api.jobs.nix_collect_garbage import ( RESULT_WAS_NOT_FOUND_ERROR, ) -pytest_plugins = ("pytest_asyncio",) - - OUTPUT_PRINT_DEAD = """ finding garbage collector roots... determining live/dead paths... @@ -102,7 +99,7 @@ def test_get_dead_packages_zero(job_reset): assert get_dead_packages("") == (0, 0) -RUN_NIX_COLLECT_GARBAGE_QUERY = """ +RUN_NIX_COLLECT_GARBAGE_MUTATION = """ mutation CollectGarbage { system { nixCollectGarbage { @@ -133,7 +130,7 @@ def test_graphql_nix_collect_garbage(authorized_client): response = authorized_client.post( "/graphql", json={ - "query": RUN_NIX_COLLECT_GARBAGE_QUERY, + "query": RUN_NIX_COLLECT_GARBAGE_MUTATION, }, ) From 9114bc6ae03540ffe760a332ec9290d48525be88 Mon Sep 17 00:00:00 2001 From: dettlaff Date: Sun, 12 Nov 2023 01:13:14 +0400 Subject: [PATCH 41/50] fix: remove unused imports --- tests/test_graphql/test_nix_collect_garbage.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/tests/test_graphql/test_nix_collect_garbage.py b/tests/test_graphql/test_nix_collect_garbage.py index e05a5dc..a2422a7 100644 --- a/tests/test_graphql/test_nix_collect_garbage.py +++ b/tests/test_graphql/test_nix_collect_garbage.py @@ -6,15 +6,12 @@ import pytest import strawberry from selfprivacy_api.jobs import JobStatus, Jobs -from selfprivacy_api.graphql import schema - from selfprivacy_api.jobs.nix_collect_garbage import ( get_dead_packages, parse_line, CLEAR_COMPLETED, COMPLETED_WITH_ERROR, - process_stream, RESULT_WAS_NOT_FOUND_ERROR, ) From d31ad487eb81c71baee16754d9e1c829505062da Mon Sep 17 00:00:00 2001 From: dettlaff Date: Sun, 12 Nov 2023 01:16:04 +0400 Subject: [PATCH 42/50] fix: corrected from comment from pr --- selfprivacy_api/jobs/nix_collect_garbage.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/selfprivacy_api/jobs/nix_collect_garbage.py b/selfprivacy_api/jobs/nix_collect_garbage.py index 259af96..12bce80 100644 --- a/selfprivacy_api/jobs/nix_collect_garbage.py +++ b/selfprivacy_api/jobs/nix_collect_garbage.py @@ -1,6 +1,5 @@ import re import subprocess -from subprocess import Popen, PIPE from typing import Tuple, Iterable, IO, Any, Optional from selfprivacy_api.utils.huey import huey @@ -9,9 +8,9 @@ from selfprivacy_api.jobs import JobStatus, Jobs, Job COMPLETED_WITH_ERROR = ( - "We are sorry, сompleted with an error, report it to support chat" + "We are sorry, сompleted with an error, report it to support chat." ) -RESULT_WAS_NOT_FOUND_ERROR = "We are sorry, garbage collection result was not found, something went wrong, report it to support chat :(" +RESULT_WAS_NOT_FOUND_ERROR = "We are sorry, garbage collection result was not found, something went wrong, report it to support chat." CLEAR_COMPLETED = "Cleaning completed." @@ -25,6 +24,7 @@ def run_nix_store_print_dead() -> IO[Any]: "utf-8" ) + def run_nix_collect_garbage() -> Optional[IO[bytes]]: return subprocess.Popen( ["nix-store", "--gc"], stdout=subprocess.PIPE, stderr=subprocess.STDOUT From 3be5816f5113c45e9cfcb5d0636a07c1c847813c Mon Sep 17 00:00:00 2001 From: dettlaff Date: Sun, 12 Nov 2023 01:38:31 +0400 Subject: [PATCH 43/50] fix: del unused return from parse line --- selfprivacy_api/jobs/nix_collect_garbage.py | 8 ++------ tests/test_graphql/test_nix_collect_garbage.py | 3 --- 2 files changed, 2 insertions(+), 9 deletions(-) diff --git a/selfprivacy_api/jobs/nix_collect_garbage.py b/selfprivacy_api/jobs/nix_collect_garbage.py index 12bce80..f6ad9ad 100644 --- a/selfprivacy_api/jobs/nix_collect_garbage.py +++ b/selfprivacy_api/jobs/nix_collect_garbage.py @@ -44,7 +44,6 @@ def parse_line(line: str): if match is None: return ( JobStatus.ERROR, - 100, COMPLETED_WITH_ERROR, RESULT_WAS_NOT_FOUND_ERROR, ) @@ -52,7 +51,6 @@ def parse_line(line: str): else: return ( JobStatus.FINISHED, - 100, CLEAR_COMPLETED, f"{match.group(0)} have been cleared", ) @@ -83,9 +81,8 @@ def process_stream(job: Job, stream: Iterable[bytes], total_dead_packages: int) Jobs.update( job=Job, status=status[0], - progress=status[1], - status_text=status[2], - result=status[3], + status_text=status[1], + result=status[2], ) @@ -114,7 +111,6 @@ def calculate_and_clear_dead_packages(job: Job): Jobs.update( job=Job, status=JobStatus.FINISHED, - progress=100, status_text="Nothing to clear", result="System is clear", ) diff --git a/tests/test_graphql/test_nix_collect_garbage.py b/tests/test_graphql/test_nix_collect_garbage.py index a2422a7..1e3a9f0 100644 --- a/tests/test_graphql/test_nix_collect_garbage.py +++ b/tests/test_graphql/test_nix_collect_garbage.py @@ -3,7 +3,6 @@ # pylint: disable=missing-function-docstring import pytest -import strawberry from selfprivacy_api.jobs import JobStatus, Jobs @@ -70,7 +69,6 @@ def test_parse_line(job_reset): txt = "190 store paths deleted, 425.51 MiB freed" output = ( JobStatus.FINISHED, - 100, CLEAR_COMPLETED, "425.51 MiB have been cleared", ) @@ -81,7 +79,6 @@ def test_parse_line_with_blank_line(job_reset): txt = "" output = ( JobStatus.ERROR, - 100, COMPLETED_WITH_ERROR, RESULT_WAS_NOT_FOUND_ERROR, ) From 03e6d45279ef00423612a1352bdd5c5c17ad19be Mon Sep 17 00:00:00 2001 From: dettlaff Date: Wed, 15 Nov 2023 16:47:04 +0400 Subject: [PATCH 44/50] fix: final typos fix --- selfprivacy_api/jobs/nix_collect_garbage.py | 26 +++++++++++---------- 1 file changed, 14 insertions(+), 12 deletions(-) diff --git a/selfprivacy_api/jobs/nix_collect_garbage.py b/selfprivacy_api/jobs/nix_collect_garbage.py index f6ad9ad..5c4bcb4 100644 --- a/selfprivacy_api/jobs/nix_collect_garbage.py +++ b/selfprivacy_api/jobs/nix_collect_garbage.py @@ -1,6 +1,6 @@ import re import subprocess -from typing import Tuple, Iterable, IO, Any, Optional +from typing import Tuple, Iterable from selfprivacy_api.utils.huey import huey @@ -14,7 +14,7 @@ RESULT_WAS_NOT_FOUND_ERROR = "We are sorry, garbage collection result was not fo CLEAR_COMPLETED = "Cleaning completed." -def run_nix_store_print_dead() -> IO[Any]: +def run_nix_store_print_dead() -> str: subprocess.run( ["nix-env", "-p", "/nix/var/nix/profiles/system", "--delete-generations old"], check=False, @@ -25,10 +25,11 @@ def run_nix_store_print_dead() -> IO[Any]: ) -def run_nix_collect_garbage() -> Optional[IO[bytes]]: - return subprocess.Popen( +def run_nix_collect_garbage() -> Iterable[bytes]: + process = subprocess.Popen( ["nix-store", "--gc"], stdout=subprocess.PIPE, stderr=subprocess.STDOUT - ).stdout + ) + return process.stdout if process.stdout else iter([]) def parse_line(line: str): @@ -69,7 +70,7 @@ def process_stream(job: Job, stream: Iterable[bytes], total_dead_packages: int) if percent - prev_progress >= 5: Jobs.update( - job=Job, + job=job, status=JobStatus.RUNNING, progress=percent, status_text="Cleaning...", @@ -79,14 +80,14 @@ def process_stream(job: Job, stream: Iterable[bytes], total_dead_packages: int) elif "store paths deleted," in line: status = parse_line(line) Jobs.update( - job=Job, + job=job, status=status[0], status_text=status[1], result=status[2], ) -def get_dead_packages(output) -> Tuple[int, int]: +def get_dead_packages(output) -> Tuple[int, float]: dead = len(re.findall("/nix/store/", output)) percent = 0 if dead != 0: @@ -97,7 +98,7 @@ def get_dead_packages(output) -> Tuple[int, int]: @huey.task() def calculate_and_clear_dead_packages(job: Job): Jobs.update( - job=Job, + job=job, status=JobStatus.RUNNING, progress=0, status_text="Calculate the number of dead packages...", @@ -109,7 +110,7 @@ def calculate_and_clear_dead_packages(job: Job): if dead_packages == 0: Jobs.update( - job=Job, + job=job, status=JobStatus.FINISHED, status_text="Nothing to clear", result="System is clear", @@ -117,13 +118,14 @@ def calculate_and_clear_dead_packages(job: Job): return Jobs.update( - job=Job, + job=job, status=JobStatus.RUNNING, progress=0, status_text=f"Found {dead_packages} packages to remove!", ) - process_stream(job, run_nix_collect_garbage(), package_equal_to_percent) + stream = run_nix_collect_garbage() + process_stream(job, stream, dead_packages) def start_nix_collect_garbage() -> Job: From 372eadd3061152c391eae393d2c2b3e84cde6d3e Mon Sep 17 00:00:00 2001 From: dettlaff Date: Wed, 29 Nov 2023 13:02:59 +0400 Subject: [PATCH 45/50] tests: fix test_parse_line --- selfprivacy_api/jobs/nix_collect_garbage.py | 36 +++++++-------- .../test_graphql/test_nix_collect_garbage.py | 46 ++++++++++--------- 2 files changed, 42 insertions(+), 40 deletions(-) diff --git a/selfprivacy_api/jobs/nix_collect_garbage.py b/selfprivacy_api/jobs/nix_collect_garbage.py index 5c4bcb4..be11f4c 100644 --- a/selfprivacy_api/jobs/nix_collect_garbage.py +++ b/selfprivacy_api/jobs/nix_collect_garbage.py @@ -8,10 +8,11 @@ from selfprivacy_api.jobs import JobStatus, Jobs, Job COMPLETED_WITH_ERROR = ( - "We are sorry, сompleted with an error, report it to support chat." + "Error occured, please report this to the support chat." ) -RESULT_WAS_NOT_FOUND_ERROR = "We are sorry, garbage collection result was not found, something went wrong, report it to support chat." -CLEAR_COMPLETED = "Cleaning completed." +RESULT_WAS_NOT_FOUND_ERROR = "We are sorry, garbage collection result was not found. " \ + "Something went wrong, please report this to the support chat." +CLEAR_COMPLETED = "Garbage collection completed." def run_nix_store_print_dead() -> str: @@ -32,7 +33,7 @@ def run_nix_collect_garbage() -> Iterable[bytes]: return process.stdout if process.stdout else iter([]) -def parse_line(line: str): +def parse_line(job: Job, line: str) -> Job: """ We parse the string for the presence of a final line, with the final amount of space cleared. @@ -43,18 +44,21 @@ def parse_line(line: str): match = re.search(pattern, line) if match is None: - return ( - JobStatus.ERROR, - COMPLETED_WITH_ERROR, - RESULT_WAS_NOT_FOUND_ERROR, + Jobs.update( + job=job, + status=JobStatus.ERROR, + status_text=COMPLETED_WITH_ERROR, + error=RESULT_WAS_NOT_FOUND_ERROR, ) else: - return ( - JobStatus.FINISHED, - CLEAR_COMPLETED, - f"{match.group(0)} have been cleared", + Jobs.update( + job=job, + status=JobStatus.FINISHED, + status_text=CLEAR_COMPLETED, + result=f"{match.group(0)} have been cleared", ) + return job def process_stream(job: Job, stream: Iterable[bytes], total_dead_packages: int) -> None: @@ -78,13 +82,7 @@ def process_stream(job: Job, stream: Iterable[bytes], total_dead_packages: int) prev_progress = percent elif "store paths deleted," in line: - status = parse_line(line) - Jobs.update( - job=job, - status=status[0], - status_text=status[1], - result=status[2], - ) + parse_line(job, line) def get_dead_packages(output) -> Tuple[int, float]: diff --git a/tests/test_graphql/test_nix_collect_garbage.py b/tests/test_graphql/test_nix_collect_garbage.py index 1e3a9f0..b3b7b53 100644 --- a/tests/test_graphql/test_nix_collect_garbage.py +++ b/tests/test_graphql/test_nix_collect_garbage.py @@ -4,7 +4,7 @@ import pytest -from selfprivacy_api.jobs import JobStatus, Jobs +from selfprivacy_api.jobs import JobStatus, Jobs, Job from selfprivacy_api.jobs.nix_collect_garbage import ( get_dead_packages, @@ -57,39 +57,43 @@ determining live/dead paths... log_event = [] -@pytest.fixture -def job_reset(): - Jobs.reset() - - # --- -def test_parse_line(job_reset): - txt = "190 store paths deleted, 425.51 MiB freed" - output = ( - JobStatus.FINISHED, - CLEAR_COMPLETED, - "425.51 MiB have been cleared", +def test_parse_line(): + txt = "note: currently hard linking saves -0.00 MiB 190 store paths deleted, 425.51 MiB freed" + + job = Jobs.add( + name="name", + type_id="parse_line", + description="description", ) - assert parse_line(txt) == output + + output = parse_line(job, txt) + assert output.result == '425.51 MiB have been cleared' + assert output.status == JobStatus.FINISHED + assert output.error is None -def test_parse_line_with_blank_line(job_reset): +def test_parse_line_with_blank_line(): txt = "" - output = ( - JobStatus.ERROR, - COMPLETED_WITH_ERROR, - RESULT_WAS_NOT_FOUND_ERROR, + job = Jobs.add( + name="name", + type_id="parse_line", + description="description", ) - assert parse_line(txt) == output + + output = parse_line(job, txt) + assert output.error == RESULT_WAS_NOT_FOUND_ERROR + assert output.status_text == COMPLETED_WITH_ERROR + assert output.status == JobStatus.ERROR -def test_get_dead_packages(job_reset): +def test_get_dead_packages(): assert get_dead_packages(OUTPUT_PRINT_DEAD) == (5, 20.0) -def test_get_dead_packages_zero(job_reset): +def test_get_dead_packages_zero(): assert get_dead_packages("") == (0, 0) From bb1e1bcae0c522d4a598e430fd3866bd6d5098b9 Mon Sep 17 00:00:00 2001 From: dettlaff Date: Wed, 17 Jan 2024 19:26:10 +0400 Subject: [PATCH 46/50] fix: rename functions --- selfprivacy_api/jobs/nix_collect_garbage.py | 8 ++++---- selfprivacy_api/task_registry.py | 2 +- tests/test_graphql/test_nix_collect_garbage.py | 5 ----- 3 files changed, 5 insertions(+), 10 deletions(-) diff --git a/selfprivacy_api/jobs/nix_collect_garbage.py b/selfprivacy_api/jobs/nix_collect_garbage.py index be11f4c..84caf03 100644 --- a/selfprivacy_api/jobs/nix_collect_garbage.py +++ b/selfprivacy_api/jobs/nix_collect_garbage.py @@ -15,7 +15,7 @@ RESULT_WAS_NOT_FOUND_ERROR = "We are sorry, garbage collection result was not fo CLEAR_COMPLETED = "Garbage collection completed." -def run_nix_store_print_dead() -> str: +def delete_old_gens_and_print_dead() -> str: subprocess.run( ["nix-env", "-p", "/nix/var/nix/profiles/system", "--delete-generations old"], check=False, @@ -94,7 +94,7 @@ def get_dead_packages(output) -> Tuple[int, float]: @huey.task() -def calculate_and_clear_dead_packages(job: Job): +def calculate_and_clear_dead_paths(job: Job): Jobs.update( job=job, status=JobStatus.RUNNING, @@ -103,7 +103,7 @@ def calculate_and_clear_dead_packages(job: Job): ) dead_packages, package_equal_to_percent = get_dead_packages( - run_nix_store_print_dead() + delete_old_gens_and_print_dead() ) if dead_packages == 0: @@ -132,5 +132,5 @@ def start_nix_collect_garbage() -> Job: name="Collect garbage", description="Cleaning up unused packages", ) - calculate_and_clear_dead_packages(job=job) + calculate_and_clear_dead_paths(job=job) return job diff --git a/selfprivacy_api/task_registry.py b/selfprivacy_api/task_registry.py index ce51442..9c02cac 100644 --- a/selfprivacy_api/task_registry.py +++ b/selfprivacy_api/task_registry.py @@ -2,4 +2,4 @@ from selfprivacy_api.utils.huey import huey from selfprivacy_api.jobs.test import test_job from selfprivacy_api.backup.tasks import * from selfprivacy_api.services.generic_service_mover import move_service -from selfprivacy_api.jobs.nix_collect_garbage import calculate_and_clear_dead_packages +from selfprivacy_api.jobs.nix_collect_garbage import calculate_and_clear_dead_paths diff --git a/tests/test_graphql/test_nix_collect_garbage.py b/tests/test_graphql/test_nix_collect_garbage.py index b3b7b53..c4b44d7 100644 --- a/tests/test_graphql/test_nix_collect_garbage.py +++ b/tests/test_graphql/test_nix_collect_garbage.py @@ -49,11 +49,6 @@ note: currently hard linking saves 0.00 MiB 0 store paths deleted, 0.00 MiB freed """ -OUTPUT_RUN_NIX_STORE_PRINT_DEAD_ZERO_TRASH = """ -finding garbage collector roots... -determining live/dead paths... -""" - log_event = [] From 045ad30ec3742727ed4019db2986f5ad0c01dadb Mon Sep 17 00:00:00 2001 From: dettlaff Date: Tue, 30 Jan 2024 18:04:43 +0400 Subject: [PATCH 47/50] fix: add mock for nix collect garbage --- tests/test_graphql/test_nix_collect_garbage.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/tests/test_graphql/test_nix_collect_garbage.py b/tests/test_graphql/test_nix_collect_garbage.py index c4b44d7..547eb70 100644 --- a/tests/test_graphql/test_nix_collect_garbage.py +++ b/tests/test_graphql/test_nix_collect_garbage.py @@ -52,6 +52,11 @@ note: currently hard linking saves 0.00 MiB log_event = [] +@pytest.fixture +def mock_delete_old_gens_and_print_dead(mocker): + mock = mocker.patch("selfprivacy_api.jobs.nix_collect_garbage.delete_old_gens_and_print_dead", autospec=True, return_value=None) + return mock + # --- @@ -119,7 +124,7 @@ mutation CollectGarbage { """ -def test_graphql_nix_collect_garbage(authorized_client): +def test_graphql_nix_collect_garbage(authorized_client, mock_delete_old_gens_and_print_dead): response = authorized_client.post( "/graphql", json={ From eb1bc9d730883dafcecba27c7111e85f74f01932 Mon Sep 17 00:00:00 2001 From: dettlaff Date: Sat, 3 Feb 2024 23:10:01 +0400 Subject: [PATCH 48/50] fix: from review --- selfprivacy_api/jobs/nix_collect_garbage.py | 37 ++++++++++------ .../check_for_failed_binds_migration.py | 0 shell.nix | 0 .../test_graphql/test_nix_collect_garbage.py | 44 +++++++++++++------ 4 files changed, 55 insertions(+), 26 deletions(-) delete mode 100644 selfprivacy_api/migrations/check_for_failed_binds_migration.py delete mode 100644 shell.nix diff --git a/selfprivacy_api/jobs/nix_collect_garbage.py b/selfprivacy_api/jobs/nix_collect_garbage.py index 84caf03..20e3664 100644 --- a/selfprivacy_api/jobs/nix_collect_garbage.py +++ b/selfprivacy_api/jobs/nix_collect_garbage.py @@ -6,25 +6,30 @@ from selfprivacy_api.utils.huey import huey from selfprivacy_api.jobs import JobStatus, Jobs, Job +class ShellException(Exception): + """Custom exception for shell-related errors.""" + pass COMPLETED_WITH_ERROR = ( - "Error occured, please report this to the support chat." + "Error occurred, please report this to the support chat." ) RESULT_WAS_NOT_FOUND_ERROR = "We are sorry, garbage collection result was not found. " \ "Something went wrong, please report this to the support chat." CLEAR_COMPLETED = "Garbage collection completed." -def delete_old_gens_and_print_dead() -> str: +def delete_old_gens_and_return_dead_report() -> str: subprocess.run( ["nix-env", "-p", "/nix/var/nix/profiles/system", "--delete-generations old"], check=False, ) - return subprocess.check_output(["nix-store", "--gc", "--print-dead"]).decode( + result = subprocess.check_output(["nix-store", "--gc", "--print-dead"]).decode( "utf-8" ) + return " " if result is None else result + def run_nix_collect_garbage() -> Iterable[bytes]: process = subprocess.Popen( @@ -44,12 +49,7 @@ def parse_line(job: Job, line: str) -> Job: match = re.search(pattern, line) if match is None: - Jobs.update( - job=job, - status=JobStatus.ERROR, - status_text=COMPLETED_WITH_ERROR, - error=RESULT_WAS_NOT_FOUND_ERROR, - ) + raise ShellException("nix returned gibberish output") else: Jobs.update( @@ -103,17 +103,18 @@ def calculate_and_clear_dead_paths(job: Job): ) dead_packages, package_equal_to_percent = get_dead_packages( - delete_old_gens_and_print_dead() + delete_old_gens_and_return_dead_report() ) if dead_packages == 0: + Jobs.update( job=job, status=JobStatus.FINISHED, status_text="Nothing to clear", result="System is clear", ) - return + return True Jobs.update( job=job, @@ -123,7 +124,15 @@ def calculate_and_clear_dead_paths(job: Job): ) stream = run_nix_collect_garbage() - process_stream(job, stream, dead_packages) + try: + process_stream(job, stream, dead_packages) + except ShellException as error: + Jobs.update( + job=job, + status=JobStatus.ERROR, + status_text=COMPLETED_WITH_ERROR, + error=RESULT_WAS_NOT_FOUND_ERROR, + ) def start_nix_collect_garbage() -> Job: @@ -132,5 +141,7 @@ def start_nix_collect_garbage() -> Job: name="Collect garbage", description="Cleaning up unused packages", ) - calculate_and_clear_dead_paths(job=job) + task_handle = calculate_and_clear_dead_paths(job=job) + result = task_handle(blocking=True) + assert result return job diff --git a/selfprivacy_api/migrations/check_for_failed_binds_migration.py b/selfprivacy_api/migrations/check_for_failed_binds_migration.py deleted file mode 100644 index e69de29..0000000 diff --git a/shell.nix b/shell.nix deleted file mode 100644 index e69de29..0000000 diff --git a/tests/test_graphql/test_nix_collect_garbage.py b/tests/test_graphql/test_nix_collect_garbage.py index 547eb70..11eeb5a 100644 --- a/tests/test_graphql/test_nix_collect_garbage.py +++ b/tests/test_graphql/test_nix_collect_garbage.py @@ -5,6 +5,7 @@ import pytest from selfprivacy_api.jobs import JobStatus, Jobs, Job +from tests.test_graphql.common import get_data, assert_errorcode, assert_ok from selfprivacy_api.jobs.nix_collect_garbage import ( get_dead_packages, @@ -12,6 +13,7 @@ from selfprivacy_api.jobs.nix_collect_garbage import ( CLEAR_COMPLETED, COMPLETED_WITH_ERROR, RESULT_WAS_NOT_FOUND_ERROR, + ShellException, ) OUTPUT_PRINT_DEAD = """ @@ -53,10 +55,17 @@ log_event = [] @pytest.fixture -def mock_delete_old_gens_and_print_dead(mocker): - mock = mocker.patch("selfprivacy_api.jobs.nix_collect_garbage.delete_old_gens_and_print_dead", autospec=True, return_value=None) +def mock_delete_old_gens_and_return_dead_report(mocker): + mock = mocker.patch("selfprivacy_api.jobs.nix_collect_garbage.delete_old_gens_and_return_dead_report", autospec=True, return_value=OUTPUT_PRINT_DEAD) return mock + +@pytest.fixture +def mock_run_nix_collect_garbage_returns_gibberish(mocker): + mock = mocker.patch("selfprivacy_api.jobs.nix_collect_garbage.run_nix_collect_garbage", autospec=True, return_value=b" oiuojkhjkhjkhjkhkjh") + return mock + + # --- @@ -83,10 +92,9 @@ def test_parse_line_with_blank_line(): description="description", ) - output = parse_line(job, txt) - assert output.error == RESULT_WAS_NOT_FOUND_ERROR - assert output.status_text == COMPLETED_WITH_ERROR - assert output.status == JobStatus.ERROR + with pytest.raises(ShellException): + output = parse_line(job, txt) + def test_get_dead_packages(): @@ -124,7 +132,7 @@ mutation CollectGarbage { """ -def test_graphql_nix_collect_garbage(authorized_client, mock_delete_old_gens_and_print_dead): +def test_graphql_nix_collect_garbage(authorized_client, mock_delete_old_gens_and_return_dead_report): response = authorized_client.post( "/graphql", json={ @@ -132,10 +140,20 @@ def test_graphql_nix_collect_garbage(authorized_client, mock_delete_old_gens_and }, ) - assert response.status_code == 200 - assert response.json().get("data") is not None - assert response.json()["data"]["system"]["nixCollectGarbage"]["message"] is not None - assert response.json()["data"]["system"]["nixCollectGarbage"]["success"] is True - assert response.json()["data"]["system"]["nixCollectGarbage"]["code"] == 200 + output = get_data(response)["system"]["nixCollectGarbage"] + assert_ok(output) + assert output["job"] is not None - assert response.json()["data"]["system"]["nixCollectGarbage"]["job"] is not None + +def test_graphql_nix_collect_garbage_with_problems(authorized_client, mock_run_nix_collect_garbage_returns_gibberish): + response = authorized_client.post( + "/graphql", + json={ + "query": RUN_NIX_COLLECT_GARBAGE_MUTATION, + }, + ) + + output = get_data(response)["system"]["nixCollectGarbage"] + assert_ok(output) + assert output["job"] is not None + assert output["job"]["status"] == "ERROR" \ No newline at end of file From 29596a4f47b33cb85b0d1a60a9df868fce753deb Mon Sep 17 00:00:00 2001 From: dettlaff Date: Mon, 5 Feb 2024 13:46:38 +0400 Subject: [PATCH 49/50] docs: changed the description of the error class --- selfprivacy_api/jobs/nix_collect_garbage.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/selfprivacy_api/jobs/nix_collect_garbage.py b/selfprivacy_api/jobs/nix_collect_garbage.py index 20e3664..1c02831 100644 --- a/selfprivacy_api/jobs/nix_collect_garbage.py +++ b/selfprivacy_api/jobs/nix_collect_garbage.py @@ -7,8 +7,7 @@ from selfprivacy_api.utils.huey import huey from selfprivacy_api.jobs import JobStatus, Jobs, Job class ShellException(Exception): - """Custom exception for shell-related errors.""" - pass + """Shell-related errors""" COMPLETED_WITH_ERROR = ( "Error occurred, please report this to the support chat." From 567f09433611fff98778033f5b8038b5fb3ecce4 Mon Sep 17 00:00:00 2001 From: dettlaff Date: Thu, 8 Feb 2024 18:40:12 +0400 Subject: [PATCH 50/50] fix: frem review --- selfprivacy_api/jobs/nix_collect_garbage.py | 9 +++++--- .../test_graphql/test_nix_collect_garbage.py | 22 ++++++++----------- 2 files changed, 15 insertions(+), 16 deletions(-) diff --git a/selfprivacy_api/jobs/nix_collect_garbage.py b/selfprivacy_api/jobs/nix_collect_garbage.py index 1c02831..1220fce 100644 --- a/selfprivacy_api/jobs/nix_collect_garbage.py +++ b/selfprivacy_api/jobs/nix_collect_garbage.py @@ -6,9 +6,11 @@ from selfprivacy_api.utils.huey import huey from selfprivacy_api.jobs import JobStatus, Jobs, Job + class ShellException(Exception): """Shell-related errors""" + COMPLETED_WITH_ERROR = ( "Error occurred, please report this to the support chat." ) @@ -72,6 +74,7 @@ def process_stream(job: Job, stream: Iterable[bytes], total_dead_packages: int) percent = int((completed_packages / total_dead_packages) * 100) if percent - prev_progress >= 5: + Jobs.update( job=job, status=JobStatus.RUNNING, @@ -140,7 +143,7 @@ def start_nix_collect_garbage() -> Job: name="Collect garbage", description="Cleaning up unused packages", ) - task_handle = calculate_and_clear_dead_paths(job=job) - result = task_handle(blocking=True) - assert result + + calculate_and_clear_dead_paths(job=job) + return job diff --git a/tests/test_graphql/test_nix_collect_garbage.py b/tests/test_graphql/test_nix_collect_garbage.py index 11eeb5a..6f00f79 100644 --- a/tests/test_graphql/test_nix_collect_garbage.py +++ b/tests/test_graphql/test_nix_collect_garbage.py @@ -3,9 +3,10 @@ # pylint: disable=missing-function-docstring import pytest +from selfprivacy_api.utils.huey import huey from selfprivacy_api.jobs import JobStatus, Jobs, Job -from tests.test_graphql.common import get_data, assert_errorcode, assert_ok +from tests.test_graphql.common import get_data, assert_errorcode, assert_ok, assert_empty from selfprivacy_api.jobs.nix_collect_garbage import ( get_dead_packages, @@ -60,12 +61,6 @@ def mock_delete_old_gens_and_return_dead_report(mocker): return mock -@pytest.fixture -def mock_run_nix_collect_garbage_returns_gibberish(mocker): - mock = mocker.patch("selfprivacy_api.jobs.nix_collect_garbage.run_nix_collect_garbage", autospec=True, return_value=b" oiuojkhjkhjkhjkhkjh") - return mock - - # --- @@ -133,6 +128,7 @@ mutation CollectGarbage { def test_graphql_nix_collect_garbage(authorized_client, mock_delete_old_gens_and_return_dead_report): + assert huey.immediate is True response = authorized_client.post( "/graphql", json={ @@ -143,17 +139,17 @@ def test_graphql_nix_collect_garbage(authorized_client, mock_delete_old_gens_and output = get_data(response)["system"]["nixCollectGarbage"] assert_ok(output) assert output["job"] is not None + assert output["job"]["status"] == "FINISHED" + assert output["job"]["error"] is None -def test_graphql_nix_collect_garbage_with_problems(authorized_client, mock_run_nix_collect_garbage_returns_gibberish): - response = authorized_client.post( +def test_graphql_nix_collect_garbage_not_authorized_client(client, mock_delete_old_gens_and_return_dead_report): + assert huey.immediate is True + response = client.post( "/graphql", json={ "query": RUN_NIX_COLLECT_GARBAGE_MUTATION, }, ) - output = get_data(response)["system"]["nixCollectGarbage"] - assert_ok(output) - assert output["job"] is not None - assert output["job"]["status"] == "ERROR" \ No newline at end of file + assert_empty(response)