From 13d3261d36d878af138f5e411a28409ef183eb82 Mon Sep 17 00:00:00 2001 From: def Date: Thu, 24 Nov 2022 06:08:58 +0400 Subject: [PATCH 01/10] feat: add nix-collect-garbage job --- selfprivacy_api/jobs/nix_collect_garbage.py | 32 +++++++++++++++++++++ 1 file changed, 32 insertions(+) create mode 100644 selfprivacy_api/jobs/nix_collect_garbage.py diff --git a/selfprivacy_api/jobs/nix_collect_garbage.py b/selfprivacy_api/jobs/nix_collect_garbage.py new file mode 100644 index 0000000..18e2a63 --- /dev/null +++ b/selfprivacy_api/jobs/nix_collect_garbage.py @@ -0,0 +1,32 @@ +import re +import subprocess + +from selfprivacy_api.jobs import Job, JobStatus, Jobs +from selfprivacy_api.utils.huey import huey + + +@huey.task() +def nix_collect_garbage(job: Job): + + Jobs.update( + job=job, + status=JobStatus.RUNNING, + progress=0, + status_text="Start cleaning.", + ) + + output = subprocess.check_output("nix-collect-garbage -d") + + pat = re.compile(r"linking saves ([+-]?\d+\.\d+ \w+).+?([+-]?\d+\.\d+ \w+) freed") + match = re.search( + pat, + output, + ) + + Jobs.update( + job=job, + status=JobStatus.FINISHED, + progress=100, + status_text="Сleaning completed.", + result=f"Currently hard linking saves {match.group(1)}, {match.group(2)} freed", + ) From 6b93df9630fc9b3036d801802dd4785e152917de Mon Sep 17 00:00:00 2001 From: def Date: Thu, 24 Nov 2022 06:32:37 +0400 Subject: [PATCH 02/10] fix: subprocess.check_output --- selfprivacy_api/jobs/nix_collect_garbage.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/selfprivacy_api/jobs/nix_collect_garbage.py b/selfprivacy_api/jobs/nix_collect_garbage.py index 18e2a63..9bb6268 100644 --- a/selfprivacy_api/jobs/nix_collect_garbage.py +++ b/selfprivacy_api/jobs/nix_collect_garbage.py @@ -15,7 +15,7 @@ def nix_collect_garbage(job: Job): status_text="Start cleaning.", ) - output = subprocess.check_output("nix-collect-garbage -d") + output = subprocess.check_output(["nix-collect-garbage", "-d"]) pat = re.compile(r"linking saves ([+-]?\d+\.\d+ \w+).+?([+-]?\d+\.\d+ \w+) freed") match = re.search( From 4ce96c303d778bf865be2ac2094b73902bf7c1a2 Mon Sep 17 00:00:00 2001 From: def Date: Sat, 3 Dec 2022 22:27:10 +0400 Subject: [PATCH 03/10] feat: loading percentage --- selfprivacy_api/jobs/nix_collect_garbage.py | 71 +++++++++++++++++---- 1 file changed, 60 insertions(+), 11 deletions(-) diff --git a/selfprivacy_api/jobs/nix_collect_garbage.py b/selfprivacy_api/jobs/nix_collect_garbage.py index 9bb6268..2c81324 100644 --- a/selfprivacy_api/jobs/nix_collect_garbage.py +++ b/selfprivacy_api/jobs/nix_collect_garbage.py @@ -1,3 +1,4 @@ +from time import sleep import re import subprocess @@ -12,21 +13,69 @@ def nix_collect_garbage(job: Job): job=job, status=JobStatus.RUNNING, progress=0, - status_text="Start cleaning.", + status_text="Сalculate the number of dead packages...", ) - output = subprocess.check_output(["nix-collect-garbage", "-d"]) - - pat = re.compile(r"linking saves ([+-]?\d+\.\d+ \w+).+?([+-]?\d+\.\d+ \w+) freed") - match = re.search( - pat, - output, + output = subprocess.check_output( + ["nix-store --gc --print-dead", "--gc", "--print-dead"] ) + dead_packages = len(re.findall("/nix/store/", output.decode("utf-8"))) + package_equal_to_percent = 100 / dead_packages + Jobs.update( job=job, - status=JobStatus.FINISHED, - progress=100, - status_text="Сleaning completed.", - result=f"Currently hard linking saves {match.group(1)}, {match.group(2)} freed", + status=JobStatus.RUNNING, + progress=0, + status_text=f"Found {dead_packages} packages to remove!", ) + + def _parse_line(line): + pattern = re.compile(r"[+-]?\d+\.\d+ \w+ freed") + match = re.search( + pattern, + line, + ) + + if match is None: + Jobs.update( + job=job, + status=JobStatus.FINISHED, + progress=100, + status_text="Completed with an error", + result="We are sorry, result was not found :(", + ) + + else: + Jobs.update( + job=job, + status=JobStatus.FINISHED, + progress=100, + status_text="Сleaning completed.", + result=f"{match.group(0)} have been cleared", + ) + + def _stream_process(process): + go = process.poll() is None + percent = 0 + + for line in process.stdout: + if "deleting '/nix/store/" in line: + percent += package_equal_to_percent + + Jobs.update( + job=job, + status=JobStatus.RUNNING, + progress=int(percent), + status_text="Сleaning...", + ) + + elif "store paths deleted," in line: + _parse_line(line) + + return go + + process = subprocess.Popen( + ["nix-collect-garbage", "-d"], stdout=subprocess.PIPE, stderr=subprocess.STDOUT + ) + _stream_process(process) From 3ecd522bd2dd092ca4063ae2daeed37f32788f37 Mon Sep 17 00:00:00 2001 From: def Date: Sat, 3 Dec 2022 22:30:41 +0400 Subject: [PATCH 04/10] refactor: delete unnecessary import --- selfprivacy_api/jobs/nix_collect_garbage.py | 1 - 1 file changed, 1 deletion(-) diff --git a/selfprivacy_api/jobs/nix_collect_garbage.py b/selfprivacy_api/jobs/nix_collect_garbage.py index 2c81324..78d306e 100644 --- a/selfprivacy_api/jobs/nix_collect_garbage.py +++ b/selfprivacy_api/jobs/nix_collect_garbage.py @@ -1,4 +1,3 @@ -from time import sleep import re import subprocess From 510b94039e68372fce9e72329a3385616467bf60 Mon Sep 17 00:00:00 2001 From: def Date: Tue, 13 Dec 2022 05:44:52 +0400 Subject: [PATCH 05/10] refactor: nix-collect-garbage is now pure --- selfprivacy_api/jobs/nix_collect_garbage.py | 131 +++++++++++------- .../test_graphql/test_nix_collect_garbage.py | 32 +++++ 2 files changed, 110 insertions(+), 53 deletions(-) create mode 100644 tests/test_graphql/test_nix_collect_garbage.py diff --git a/selfprivacy_api/jobs/nix_collect_garbage.py b/selfprivacy_api/jobs/nix_collect_garbage.py index 78d306e..3d84ca3 100644 --- a/selfprivacy_api/jobs/nix_collect_garbage.py +++ b/selfprivacy_api/jobs/nix_collect_garbage.py @@ -5,8 +5,73 @@ from selfprivacy_api.jobs import Job, JobStatus, Jobs from selfprivacy_api.utils.huey import huey +def run_nix_store_print_dead(): + return subprocess.check_output(["nix-store", "--gc", "--print-dead"]) + + +def run_nix_collect_garbage(): + return subprocess.Popen( + ["nix-collect-garbage", "-d"], stdout=subprocess.PIPE, stderr=subprocess.STDOUT + ) + + +def parse_line(line, job: Job): + pattern = re.compile(r"[+-]?\d+\.\d+ \w+ freed") + match = re.search( + pattern, + line, + ) + + if match is None: + Jobs.update( + job=job, + status=JobStatus.FINISHED, + progress=100, + status_text="Completed with an error", + result="We are sorry, result was not found :(", + ) + + else: + Jobs.update( + job=job, + status=JobStatus.FINISHED, + progress=100, + status_text="Сleaning completed.", + result=f"{match.group(0)} have been cleared", + ) + + +def stream_process( + process, + package_equal_to_percent, + job: Job, +): + go = process.poll() is None + percent = 0 + + for line in process.stdout: + if "deleting '/nix/store/" in line: + percent += package_equal_to_percent + + Jobs.update( + job=job, + status=JobStatus.RUNNING, + progress=int(percent), + status_text="Сleaning...", + ) + + elif "store paths deleted," in line: + parse_line(line, job) + + return go + + @huey.task() -def nix_collect_garbage(job: Job): +def nix_collect_garbage( + job: Job, + run_nix_store=run_nix_store_print_dead, + run_nix_collect=run_nix_collect_garbage, +): # innocent as a pure function Jobs.update( job=job, @@ -15,11 +80,19 @@ def nix_collect_garbage(job: Job): status_text="Сalculate the number of dead packages...", ) - output = subprocess.check_output( - ["nix-store --gc --print-dead", "--gc", "--print-dead"] - ) + output = run_nix_store() dead_packages = len(re.findall("/nix/store/", output.decode("utf-8"))) + + if dead_packages == 0: + Jobs.update( + job=job, + status=JobStatus.FINISHED, + progress=100, + status_text="Nothing to clear", + result="System is clear", + ) + package_equal_to_percent = 100 / dead_packages Jobs.update( @@ -29,52 +102,4 @@ def nix_collect_garbage(job: Job): status_text=f"Found {dead_packages} packages to remove!", ) - def _parse_line(line): - pattern = re.compile(r"[+-]?\d+\.\d+ \w+ freed") - match = re.search( - pattern, - line, - ) - - if match is None: - Jobs.update( - job=job, - status=JobStatus.FINISHED, - progress=100, - status_text="Completed with an error", - result="We are sorry, result was not found :(", - ) - - else: - Jobs.update( - job=job, - status=JobStatus.FINISHED, - progress=100, - status_text="Сleaning completed.", - result=f"{match.group(0)} have been cleared", - ) - - def _stream_process(process): - go = process.poll() is None - percent = 0 - - for line in process.stdout: - if "deleting '/nix/store/" in line: - percent += package_equal_to_percent - - Jobs.update( - job=job, - status=JobStatus.RUNNING, - progress=int(percent), - status_text="Сleaning...", - ) - - elif "store paths deleted," in line: - _parse_line(line) - - return go - - process = subprocess.Popen( - ["nix-collect-garbage", "-d"], stdout=subprocess.PIPE, stderr=subprocess.STDOUT - ) - _stream_process(process) + stream_process(run_nix_collect, package_equal_to_percent, job) diff --git a/tests/test_graphql/test_nix_collect_garbage.py b/tests/test_graphql/test_nix_collect_garbage.py new file mode 100644 index 0000000..a14dcb3 --- /dev/null +++ b/tests/test_graphql/test_nix_collect_garbage.py @@ -0,0 +1,32 @@ +# pylint: disable=redefined-outer-name +# pylint: disable=unused-argument +# pylint: disable=missing-function-docstring + +import pytest + +from selfprivacy_api.jobs.nix_collect_garbage import nix_collect_garbage + + + + + + created_at: datetime.datetime + updated_at: datetime.datetime + uid: UUID + type_id: str + name: str + description: str + status: JobStatus + + +def test_nix_collect_garbage(job( + created_at = "2019-12-04", + updated_at = "2019-12-04", + uid = UUID, + type_id = "typeid", + name = "name", + description: "desc", + status = status(CREATED = "CREATED"), +)): + + assert nix_collect_garbage() is not None From 2340a0f8e90e5e135d6ec1c0b4d52de876ac15f7 Mon Sep 17 00:00:00 2001 From: def Date: Mon, 19 Dec 2022 06:27:44 +0400 Subject: [PATCH 06/10] test: fix nix collect garbage, add tests --- selfprivacy_api/jobs/nix_collect_garbage.py | 105 +++++++----- .../test_graphql/test_nix_collect_garbage.py | 154 +++++++++++++++--- 2 files changed, 199 insertions(+), 60 deletions(-) diff --git a/selfprivacy_api/jobs/nix_collect_garbage.py b/selfprivacy_api/jobs/nix_collect_garbage.py index 3d84ca3..32de683 100644 --- a/selfprivacy_api/jobs/nix_collect_garbage.py +++ b/selfprivacy_api/jobs/nix_collect_garbage.py @@ -1,105 +1,128 @@ import re import subprocess -from selfprivacy_api.jobs import Job, JobStatus, Jobs -from selfprivacy_api.utils.huey import huey +from selfprivacy_api.jobs import JobStatus, Jobs + + +COMPLETED_WITH_ERROR = "Completed with an error" +RESULT_WAAS_NOT_FOUND_ERROR = "We are sorry, result was not found :(" +CLEAR_COMPLETED = "Сleaning completed." def run_nix_store_print_dead(): - return subprocess.check_output(["nix-store", "--gc", "--print-dead"]) + return subprocess.check_output(["nix-store", "--gc", "--print-dead"]).decode( + "utf-8" + ) def run_nix_collect_garbage(): return subprocess.Popen( ["nix-collect-garbage", "-d"], stdout=subprocess.PIPE, stderr=subprocess.STDOUT - ) + ).stdout -def parse_line(line, job: Job): - pattern = re.compile(r"[+-]?\d+\.\d+ \w+ freed") +def set_job_status_wrapper(Jobs, job): + def set_job_status(status, progress, status_text, result="Default result"): + Jobs.update( + job=job, + status=status, + progress=progress, + status_text=status_text, + result=result, + ) + + return set_job_status + + +def parse_line(line): + pattern = re.compile(r"[+-]?\d+\.\d+ \w+(?= freed)") match = re.search( pattern, line, ) if match is None: - Jobs.update( - job=job, - status=JobStatus.FINISHED, - progress=100, - status_text="Completed with an error", - result="We are sorry, result was not found :(", + return ( + JobStatus.FINISHED, + 100, + COMPLETED_WITH_ERROR, + RESULT_WAAS_NOT_FOUND_ERROR, ) else: - Jobs.update( - job=job, - status=JobStatus.FINISHED, - progress=100, - status_text="Сleaning completed.", - result=f"{match.group(0)} have been cleared", + return ( + JobStatus.FINISHED, + 100, + CLEAR_COMPLETED, + f"{match.group(0)} have been cleared", ) def stream_process( - process, + stream, package_equal_to_percent, - job: Job, + set_job_status, ): - go = process.poll() is None percent = 0 - for line in process.stdout: + for line in stream: if "deleting '/nix/store/" in line: percent += package_equal_to_percent - Jobs.update( - job=job, + set_job_status( status=JobStatus.RUNNING, progress=int(percent), status_text="Сleaning...", ) elif "store paths deleted," in line: - parse_line(line, job) - - return go + status = parse_line(line) + set_job_status( + status=status[0], + progress=status[1], + status_text=status[2], + result=status[3], + ) + + +def get_dead_packages(output): + dead = len(re.findall("/nix/store/", output)) + percent = None + if dead != 0: + percent = 100 / dead + return dead, percent -@huey.task() def nix_collect_garbage( - job: Job, + job, + jobs=Jobs, run_nix_store=run_nix_store_print_dead, run_nix_collect=run_nix_collect_garbage, + set_job_status=None, ): # innocent as a pure function + set_job_status = set_job_status or set_job_status_wrapper(jobs, job) - Jobs.update( - job=job, + set_job_status( status=JobStatus.RUNNING, progress=0, status_text="Сalculate the number of dead packages...", ) - output = run_nix_store() - - dead_packages = len(re.findall("/nix/store/", output.decode("utf-8"))) + dead_packages, package_equal_to_percent = get_dead_packages(run_nix_store()) if dead_packages == 0: - Jobs.update( - job=job, + set_job_status( status=JobStatus.FINISHED, progress=100, status_text="Nothing to clear", result="System is clear", ) + return - package_equal_to_percent = 100 / dead_packages - - Jobs.update( - job=job, + set_job_status( status=JobStatus.RUNNING, progress=0, status_text=f"Found {dead_packages} packages to remove!", ) - stream_process(run_nix_collect, package_equal_to_percent, job) + stream_process(run_nix_collect(), package_equal_to_percent, set_job_status) diff --git a/tests/test_graphql/test_nix_collect_garbage.py b/tests/test_graphql/test_nix_collect_garbage.py index a14dcb3..b903027 100644 --- a/tests/test_graphql/test_nix_collect_garbage.py +++ b/tests/test_graphql/test_nix_collect_garbage.py @@ -3,30 +3,146 @@ # pylint: disable=missing-function-docstring import pytest +from selfprivacy_api.jobs import JobStatus -from selfprivacy_api.jobs.nix_collect_garbage import nix_collect_garbage +from selfprivacy_api.jobs.nix_collect_garbage import ( + get_dead_packages, + nix_collect_garbage, + parse_line, + CLEAR_COMPLETED, + COMPLETED_WITH_ERROR, + stream_process, + RESULT_WAAS_NOT_FOUND_ERROR, +) +output_print_dead = """ +finding garbage collector roots... +determining live/dead paths... +/nix/store/02k8pmw00p7p7mf2dg3n057771w7liia-python3.10-cchardet-2.1.7 +/nix/store/03vc6dznx8njbvyd3gfhfa4n5j4lvhbl-python3.10-async-timeout-4.0.2 +/nix/store/03ybv2dvfk7c3cpb527y5kzf6i35ch41-python3.10-pycparser-2.21 +/nix/store/04dn9slfqwhqisn1j3jv531lms9w5wlj-python3.10-hypothesis-6.50.1.drv +/nix/store/04hhx2z1iyi3b48hxykiw1g03lp46jk7-python-remove-bin-bytecode-hook +""" - - created_at: datetime.datetime - updated_at: datetime.datetime - uid: UUID - type_id: str - name: str - description: str - status: JobStatus +output_collect_garbage = """ +removing old generations of profile /nix/var/nix/profiles/per-user/def/channels +finding garbage collector roots... +deleting garbage... +deleting '/nix/store/02k8pmw00p7p7mf2dg3n057771w7liia-python3.10-cchardet-2.1.7' +deleting '/nix/store/03vc6dznx8njbvyd3gfhfa4n5j4lvhbl-python3.10-async-timeout-4.0.2' +deleting '/nix/store/03ybv2dvfk7c3cpb527y5kzf6i35ch41-python3.10-pycparser-2.21' +deleting '/nix/store/04dn9slfqwhqisn1j3jv531lms9w5wlj-python3.10-hypothesis-6.50.1.drv' +deleting '/nix/store/04hhx2z1iyi3b48hxykiw1g03lp46jk7-python-remove-bin-bytecode-hook' +deleting unused links... +note: currently hard linking saves -0.00 MiB +190 store paths deleted, 425.51 MiB freed +""" -def test_nix_collect_garbage(job( - created_at = "2019-12-04", - updated_at = "2019-12-04", - uid = UUID, - type_id = "typeid", - name = "name", - description: "desc", - status = status(CREATED = "CREATED"), -)): +def test_parse_line(): + txt = "190 store paths deleted, 425.51 MiB freed" + output = ( + JobStatus.FINISHED, + 100, + CLEAR_COMPLETED, + "425.51 MiB have been cleared", + ) + assert parse_line(txt) == output - assert nix_collect_garbage() is not None + +def test_parse_line_with_blank_line(): + txt = "" + output = ( + JobStatus.FINISHED, + 100, + COMPLETED_WITH_ERROR, + RESULT_WAAS_NOT_FOUND_ERROR, + ) + assert parse_line(txt) == output + + +def test_get_dead_packages(): + assert get_dead_packages(output_print_dead) == (5, 20.0) + + +def test_get_dead_packages_zero(): + assert get_dead_packages("") == (0, None) + + +def test_stream_process(): + log_event = [] + reference = [ + (JobStatus.RUNNING, 20, "Сleaning...", ""), + (JobStatus.RUNNING, 40, "Сleaning...", ""), + (JobStatus.RUNNING, 60, "Сleaning...", ""), + (JobStatus.RUNNING, 80, "Сleaning...", ""), + (JobStatus.RUNNING, 100, "Сleaning...", ""), + ( + JobStatus.FINISHED, + 100, + "Сleaning completed.", + "425.51 MiB have been cleared", + ), + ] + + def set_job_status(status, progress, status_text, result=""): + log_event.append((status, progress, status_text, result)) + + stream_process(output_collect_garbage.split("\n"), 20.0, set_job_status) + assert log_event == reference + + +def test_nix_collect_garbage(): + log_event = [] + reference = [ + (JobStatus.RUNNING, 0, "Сalculate the number of dead packages...", ""), + (JobStatus.RUNNING, 0, "Found 5 packages to remove!", ""), + (JobStatus.RUNNING, 20, "Сleaning...", ""), + (JobStatus.RUNNING, 40, "Сleaning...", ""), + (JobStatus.RUNNING, 60, "Сleaning...", ""), + (JobStatus.RUNNING, 80, "Сleaning...", ""), + (JobStatus.RUNNING, 100, "Сleaning...", ""), + ( + JobStatus.FINISHED, + 100, + "Сleaning completed.", + "425.51 MiB have been cleared", + ), + ] + + def set_job_status(status="", progress="", status_text="", result=""): + log_event.append((status, progress, status_text, result)) + + nix_collect_garbage( + None, + None, + lambda: output_print_dead, + lambda: output_collect_garbage.split("\n"), + set_job_status, + ) + + assert log_event == reference + + +def test_nix_collect_garbage_zero_trash(): + log_event = [] + reference = [ + (JobStatus.RUNNING, 0, "Сalculate the number of dead packages...", ""), + (JobStatus.FINISHED, 100, "Nothing to clear", "System is clear"), + ] + + def set_job_status(status="", progress="", status_text="", result=""): + log_event.append((status, progress, status_text, result)) + + nix_collect_garbage( + None, + None, + lambda: "", + lambda: output_collect_garbage.split("\n"), + set_job_status, + ) + + assert log_event == reference From fcdd61006b8ea509bd6363c58947ae605a784ec9 Mon Sep 17 00:00:00 2001 From: def Date: Tue, 27 Dec 2022 04:15:02 +0400 Subject: [PATCH 07/10] fix: types, add tests --- selfprivacy_api/jobs/nix_collect_garbage.py | 7 +++-- selfprivacy_api/task_registry.py | 1 + .../test_graphql/test_nix_collect_garbage.py | 27 +++++++++++++++++-- 3 files changed, 31 insertions(+), 4 deletions(-) diff --git a/selfprivacy_api/jobs/nix_collect_garbage.py b/selfprivacy_api/jobs/nix_collect_garbage.py index 32de683..587da65 100644 --- a/selfprivacy_api/jobs/nix_collect_garbage.py +++ b/selfprivacy_api/jobs/nix_collect_garbage.py @@ -2,10 +2,11 @@ import re import subprocess from selfprivacy_api.jobs import JobStatus, Jobs +from selfprivacy_api.utils.huey import huey COMPLETED_WITH_ERROR = "Completed with an error" -RESULT_WAAS_NOT_FOUND_ERROR = "We are sorry, result was not found :(" +RESULT_WAS_NOT_FOUND_ERROR = "We are sorry, result was not found :(" CLEAR_COMPLETED = "Сleaning completed." @@ -46,7 +47,7 @@ def parse_line(line): JobStatus.FINISHED, 100, COMPLETED_WITH_ERROR, - RESULT_WAAS_NOT_FOUND_ERROR, + RESULT_WAS_NOT_FOUND_ERROR, ) else: @@ -72,6 +73,7 @@ def stream_process( set_job_status( status=JobStatus.RUNNING, progress=int(percent), + progress=int(percent), status_text="Сleaning...", ) @@ -93,6 +95,7 @@ def get_dead_packages(output): return dead, percent +@huey.task() def nix_collect_garbage( job, jobs=Jobs, diff --git a/selfprivacy_api/task_registry.py b/selfprivacy_api/task_registry.py index 82eaf06..f9f92e1 100644 --- a/selfprivacy_api/task_registry.py +++ b/selfprivacy_api/task_registry.py @@ -2,3 +2,4 @@ from selfprivacy_api.utils.huey import huey from selfprivacy_api.jobs.test import test_job from selfprivacy_api.restic_controller.tasks import * from selfprivacy_api.services.generic_service_mover import move_service +from selfprivacy_api.jobs.nix_collect_garbage import nix_collect_garbage diff --git a/tests/test_graphql/test_nix_collect_garbage.py b/tests/test_graphql/test_nix_collect_garbage.py index b903027..4305f5c 100644 --- a/tests/test_graphql/test_nix_collect_garbage.py +++ b/tests/test_graphql/test_nix_collect_garbage.py @@ -4,6 +4,11 @@ import pytest from selfprivacy_api.jobs import JobStatus +from selfprivacy_api.graphql import schema +import asyncio +import strawberry + +# from selfprivacy_api.graphql.schema import Subscription from selfprivacy_api.jobs.nix_collect_garbage import ( get_dead_packages, @@ -12,7 +17,7 @@ from selfprivacy_api.jobs.nix_collect_garbage import ( CLEAR_COMPLETED, COMPLETED_WITH_ERROR, stream_process, - RESULT_WAAS_NOT_FOUND_ERROR, + RESULT_WAS_NOT_FOUND_ERROR, ) @@ -59,7 +64,7 @@ def test_parse_line_with_blank_line(): JobStatus.FINISHED, 100, COMPLETED_WITH_ERROR, - RESULT_WAAS_NOT_FOUND_ERROR, + RESULT_WAS_NOT_FOUND_ERROR, ) assert parse_line(txt) == output @@ -146,3 +151,21 @@ def test_nix_collect_garbage_zero_trash(): ) assert log_event == reference + + +@pytest.mark.asyncio +async def test_graphql_nix_collect_garbage(): + query = """ + subscription { + nixCollectGarbage() + } + """ + + schema_for_garbage = strawberry.Schema( + query=schema.Query, mutation=schema.Mutation, subscription=schema.Subscription + ) + + sub = await schema_for_garbage.subscribe(query) + for result in sub: + assert not result.errors + assert result.data == {} From 88d66fea8cbc302984f31386c84acea50bf55a80 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=92=D0=B0=D1=88=C3=90dettlaff?= Date: Thu, 23 Mar 2023 20:49:30 +0300 Subject: [PATCH 08/10] fix: percentage --- dump.rdb | Bin 0 -> 88 bytes selfprivacy_api/jobs/nix_collect_garbage.py | 12 ++-- setup.py | 0 shell.nix | 52 ++++-------------- .../test_graphql/test_nix_collect_garbage.py | 29 +++++----- 5 files changed, 30 insertions(+), 63 deletions(-) create mode 100644 dump.rdb mode change 100755 => 100644 setup.py diff --git a/dump.rdb b/dump.rdb new file mode 100644 index 0000000000000000000000000000000000000000..2f3b490816640dcfd897e08c8eb137f27ec5a6e2 GIT binary patch literal 88 zcmWG?b@2=~FfcUu#aWb^l3A=uO-d|IJ;3n)^rLMb4lXTO2>_mAByj)$ literal 0 HcmV?d00001 diff --git a/selfprivacy_api/jobs/nix_collect_garbage.py b/selfprivacy_api/jobs/nix_collect_garbage.py index 587da65..3d13382 100644 --- a/selfprivacy_api/jobs/nix_collect_garbage.py +++ b/selfprivacy_api/jobs/nix_collect_garbage.py @@ -61,19 +61,19 @@ def parse_line(line): def stream_process( stream, - package_equal_to_percent, + total_dead_packages, set_job_status, ): - percent = 0 + completed_packages = 0 for line in stream: if "deleting '/nix/store/" in line: - percent += package_equal_to_percent + completed_packages += 1 + percent = int((completed_packages / total_dead_packages) * 100) set_job_status( status=JobStatus.RUNNING, - progress=int(percent), - progress=int(percent), + progress=percent, status_text="Сleaning...", ) @@ -95,7 +95,7 @@ def get_dead_packages(output): return dead, percent -@huey.task() +# @huey.task() # ломает все к фигам def nix_collect_garbage( job, jobs=Jobs, diff --git a/setup.py b/setup.py old mode 100755 new mode 100644 diff --git a/shell.nix b/shell.nix index 0ccb99d..b620ee2 100644 --- a/shell.nix +++ b/shell.nix @@ -1,6 +1,6 @@ -{ pkgs ? import { } }: +{ pkgs ? import { } }: let - sp-python = pkgs.python39.withPackages (p: with p; [ + sp-python = pkgs.python310.withPackages (p: with p; [ setuptools portalocker pytz @@ -18,54 +18,24 @@ let black fastapi uvicorn - (buildPythonPackage rec { - pname = "strawberry-graphql"; - version = "0.123.0"; - format = "pyproject"; - patches = [ - ./strawberry-graphql.patch - ]; - propagatedBuildInputs = [ - typing-extensions - python-multipart - python-dateutil - # flask - pydantic - pygments - poetry - # flask-cors - (buildPythonPackage rec { - pname = "graphql-core"; - version = "3.2.0"; - format = "setuptools"; - src = fetchPypi { - inherit pname version; - sha256 = "sha256-huKgvgCL/eGe94OI3opyWh2UKpGQykMcJKYIN5c4A84="; - }; - checkInputs = [ - pytest-asyncio - pytest-benchmark - pytestCheckHook - ]; - pythonImportsCheck = [ - "graphql" - ]; - }) - ]; - src = fetchPypi { - inherit pname version; - sha256 = "KsmZ5Xv8tUg6yBxieAEtvoKoRG60VS+iVGV0X6oCExo="; - }; - }) + redis + strawberry-graphql ]); in pkgs.mkShell { buildInputs = [ sp-python pkgs.black + pkgs.redis + pkgs.restic ]; shellHook = '' PYTHONPATH=${sp-python}/${sp-python.sitePackages} + # envs set with export and as attributes are treated differently. + # for example. printenv will not fetch the value of an attribute. + export USE_REDIS_PORT=6379 + pkill redis-server + redis-server --bind 127.0.0.1 --port $USE_REDIS_PORT >/dev/null & # maybe set more env-vars ''; } diff --git a/tests/test_graphql/test_nix_collect_garbage.py b/tests/test_graphql/test_nix_collect_garbage.py index 4305f5c..0db620a 100644 --- a/tests/test_graphql/test_nix_collect_garbage.py +++ b/tests/test_graphql/test_nix_collect_garbage.py @@ -96,26 +96,21 @@ def test_stream_process(): def set_job_status(status, progress, status_text, result=""): log_event.append((status, progress, status_text, result)) - stream_process(output_collect_garbage.split("\n"), 20.0, set_job_status) + stream_process(output_collect_garbage.split("\n"), 5, set_job_status) assert log_event == reference def test_nix_collect_garbage(): log_event = [] reference = [ - (JobStatus.RUNNING, 0, "Сalculate the number of dead packages...", ""), - (JobStatus.RUNNING, 0, "Found 5 packages to remove!", ""), - (JobStatus.RUNNING, 20, "Сleaning...", ""), - (JobStatus.RUNNING, 40, "Сleaning...", ""), - (JobStatus.RUNNING, 60, "Сleaning...", ""), - (JobStatus.RUNNING, 80, "Сleaning...", ""), - (JobStatus.RUNNING, 100, "Сleaning...", ""), - ( - JobStatus.FINISHED, - 100, - "Сleaning completed.", - "425.51 MiB have been cleared", - ), + (JobStatus.RUNNING, 0, 'Сalculate the number of dead packages...', ''), + (JobStatus.RUNNING, 0, 'Found 5 packages to remove!', ''), + (JobStatus.RUNNING, 5, 'Сleaning...', ''), + (JobStatus.RUNNING, 10, 'Сleaning...', ''), + (JobStatus.RUNNING, 15, 'Сleaning...', ''), + (JobStatus.RUNNING, 20, 'Сleaning...', ''), + (JobStatus.RUNNING, 25, 'Сleaning...', ''), + (JobStatus.FINISHED, 100, 'Сleaning completed.', '425.51 MiB have been cleared'), ] def set_job_status(status="", progress="", status_text="", result=""): @@ -128,6 +123,8 @@ def test_nix_collect_garbage(): lambda: output_collect_garbage.split("\n"), set_job_status, ) + print("log_event:", log_event) + print("reference:", reference) assert log_event == reference @@ -152,7 +149,7 @@ def test_nix_collect_garbage_zero_trash(): assert log_event == reference - +# андр констракнш @pytest.mark.asyncio async def test_graphql_nix_collect_garbage(): query = """ @@ -166,6 +163,6 @@ async def test_graphql_nix_collect_garbage(): ) sub = await schema_for_garbage.subscribe(query) - for result in sub: + async for result in sub: assert not result.errors assert result.data == {} From cf06d4c8d545badcfffaf392f7d466d1359c64a5 Mon Sep 17 00:00:00 2001 From: dettlaff Date: Mon, 3 Apr 2023 17:41:41 +0300 Subject: [PATCH 09/10] refactor: delete purest of nix_collect_garbage() --- .gitignore | 3 +++ selfprivacy_api/jobs/nix_collect_garbage.py | 13 +++++-------- 2 files changed, 8 insertions(+), 8 deletions(-) mode change 100755 => 100644 .gitignore diff --git a/.gitignore b/.gitignore old mode 100755 new mode 100644 index 7941396..48a9439 --- a/.gitignore +++ b/.gitignore @@ -147,3 +147,6 @@ cython_debug/ # End of https://www.toptal.com/developers/gitignore/api/flask *.db + +# Redis db +*.rdb \ No newline at end of file diff --git a/selfprivacy_api/jobs/nix_collect_garbage.py b/selfprivacy_api/jobs/nix_collect_garbage.py index 3d13382..ec894fd 100644 --- a/selfprivacy_api/jobs/nix_collect_garbage.py +++ b/selfprivacy_api/jobs/nix_collect_garbage.py @@ -95,15 +95,12 @@ def get_dead_packages(output): return dead, percent -# @huey.task() # ломает все к фигам +# @huey.task() def nix_collect_garbage( job, jobs=Jobs, - run_nix_store=run_nix_store_print_dead, - run_nix_collect=run_nix_collect_garbage, - set_job_status=None, -): # innocent as a pure function - set_job_status = set_job_status or set_job_status_wrapper(jobs, job) +): + set_job_status = set_job_status_wrapper(jobs, job) set_job_status( status=JobStatus.RUNNING, @@ -111,7 +108,7 @@ def nix_collect_garbage( status_text="Сalculate the number of dead packages...", ) - dead_packages, package_equal_to_percent = get_dead_packages(run_nix_store()) + dead_packages, package_equal_to_percent = get_dead_packages(run_nix_store_print_dead()) if dead_packages == 0: set_job_status( @@ -128,4 +125,4 @@ def nix_collect_garbage( status_text=f"Found {dead_packages} packages to remove!", ) - stream_process(run_nix_collect(), package_equal_to_percent, set_job_status) + stream_process(run_nix_collect_garbage(), package_equal_to_percent, set_job_status) From a67f3e822adf6a0a03d8ca6a8a206b158e7525f3 Mon Sep 17 00:00:00 2001 From: dettlaff Date: Wed, 5 Apr 2023 13:49:56 +0300 Subject: [PATCH 10/10] fix: do a redis reset() --- dump.rdb | Bin 88 -> 88 bytes selfprivacy_api/jobs/nix_collect_garbage.py | 7 +- .../test_graphql/test_nix_collect_garbage.py | 126 ++++++++++++------ 3 files changed, 86 insertions(+), 47 deletions(-) diff --git a/dump.rdb b/dump.rdb index 2f3b490816640dcfd897e08c8eb137f27ec5a6e2..7a274dbe6fa11a2889c1664f1b0cbee5dde9742c 100644 GIT binary patch delta 45 zcma!um|&n$&#ImBi=(tSHAOc!HTRH$CNIM;j>P;l-K50g)B_Cvryq~qzCu^z4FGn_ B5$ON` delta 45 zcma!um|&n$mn)OCL B6mI|k diff --git a/selfprivacy_api/jobs/nix_collect_garbage.py b/selfprivacy_api/jobs/nix_collect_garbage.py index ec894fd..82e00e2 100644 --- a/selfprivacy_api/jobs/nix_collect_garbage.py +++ b/selfprivacy_api/jobs/nix_collect_garbage.py @@ -98,9 +98,8 @@ def get_dead_packages(output): # @huey.task() def nix_collect_garbage( job, - jobs=Jobs, ): - set_job_status = set_job_status_wrapper(jobs, job) + set_job_status = set_job_status_wrapper(Jobs, job) set_job_status( status=JobStatus.RUNNING, @@ -108,7 +107,9 @@ def nix_collect_garbage( status_text="Сalculate the number of dead packages...", ) - dead_packages, package_equal_to_percent = get_dead_packages(run_nix_store_print_dead()) + dead_packages, package_equal_to_percent = get_dead_packages( + run_nix_store_print_dead() + ) if dead_packages == 0: set_job_status( diff --git a/tests/test_graphql/test_nix_collect_garbage.py b/tests/test_graphql/test_nix_collect_garbage.py index 0db620a..98848b4 100644 --- a/tests/test_graphql/test_nix_collect_garbage.py +++ b/tests/test_graphql/test_nix_collect_garbage.py @@ -3,11 +3,11 @@ # pylint: disable=missing-function-docstring import pytest -from selfprivacy_api.jobs import JobStatus -from selfprivacy_api.graphql import schema -import asyncio import strawberry +from selfprivacy_api.jobs import JobStatus, Jobs +from selfprivacy_api.graphql import schema + # from selfprivacy_api.graphql.schema import Subscription from selfprivacy_api.jobs.nix_collect_garbage import ( @@ -21,7 +21,7 @@ from selfprivacy_api.jobs.nix_collect_garbage import ( ) -output_print_dead = """ +OUTPUT_PRINT_DEAD = """ finding garbage collector roots... determining live/dead paths... /nix/store/02k8pmw00p7p7mf2dg3n057771w7liia-python3.10-cchardet-2.1.7 @@ -32,7 +32,7 @@ determining live/dead paths... """ -output_collect_garbage = """ +OUTPUT_COLLECT_GARBAGE = """ removing old generations of profile /nix/var/nix/profiles/per-user/def/channels finding garbage collector roots... deleting garbage... @@ -46,8 +46,53 @@ note: currently hard linking saves -0.00 MiB 190 store paths deleted, 425.51 MiB freed """ +log_event = [] + + +def set_job_status(status="", progress="", status_text="", result=""): + log_event.append((status, progress, status_text, result)) + + +@pytest.fixture +def mock_set_job_status(mocker): + mock = mocker.patch( + "selfprivacy_api.jobs.nix_collect_garbage.set_job_status_wrapper", + autospec=True, + return_value=set_job_status, + ) + return mock + + +@pytest.fixture +def mock_run_nix_collect_garbage(mocker): + mock = mocker.patch( + "selfprivacy_api.jobs.nix_collect_garbage.run_nix_collect_garbage", + autospec=True, + return_value=OUTPUT_COLLECT_GARBAGE.split("\n"), + ) + return mock + + +@pytest.fixture +def mock_run_nix_store_print_dead(mocker): + mock = mocker.patch( + "selfprivacy_api.jobs.nix_collect_garbage.run_nix_store_print_dead", + autospec=True, + return_value="", + ) + return mock + + +@pytest.fixture +def job_reset(): + Jobs.reset() + + +# --- + + +def test_parse_line(job_reset): -def test_parse_line(): txt = "190 store paths deleted, 425.51 MiB freed" output = ( JobStatus.FINISHED, @@ -58,7 +103,7 @@ def test_parse_line(): assert parse_line(txt) == output -def test_parse_line_with_blank_line(): +def test_parse_line_with_blank_line(job_reset): txt = "" output = ( JobStatus.FINISHED, @@ -69,11 +114,11 @@ def test_parse_line_with_blank_line(): assert parse_line(txt) == output -def test_get_dead_packages(): - assert get_dead_packages(output_print_dead) == (5, 20.0) +def test_get_dead_packages(job_resetм): + assert get_dead_packages(OUTPUT_PRINT_DEAD) == (5, 20.0) -def test_get_dead_packages_zero(): +def test_get_dead_packages_zero(job_reset): assert get_dead_packages("") == (0, None) @@ -96,59 +141,52 @@ def test_stream_process(): def set_job_status(status, progress, status_text, result=""): log_event.append((status, progress, status_text, result)) - stream_process(output_collect_garbage.split("\n"), 5, set_job_status) + stream_process(OUTPUT_COLLECT_GARBAGE.split("\n"), 5, set_job_status) assert log_event == reference -def test_nix_collect_garbage(): +def test_nix_collect_garbage( + mock_set_job_status, mock_run_nix_collect_garbage, job_reset +): log_event = [] reference = [ - (JobStatus.RUNNING, 0, 'Сalculate the number of dead packages...', ''), - (JobStatus.RUNNING, 0, 'Found 5 packages to remove!', ''), - (JobStatus.RUNNING, 5, 'Сleaning...', ''), - (JobStatus.RUNNING, 10, 'Сleaning...', ''), - (JobStatus.RUNNING, 15, 'Сleaning...', ''), - (JobStatus.RUNNING, 20, 'Сleaning...', ''), - (JobStatus.RUNNING, 25, 'Сleaning...', ''), - (JobStatus.FINISHED, 100, 'Сleaning completed.', '425.51 MiB have been cleared'), + (JobStatus.RUNNING, 0, "Сalculate the number of dead packages...", ""), + (JobStatus.RUNNING, 0, "Found 5 packages to remove!", ""), + (JobStatus.RUNNING, 5, "Сleaning...", ""), + (JobStatus.RUNNING, 10, "Сleaning...", ""), + (JobStatus.RUNNING, 15, "Сleaning...", ""), + (JobStatus.RUNNING, 20, "Сleaning...", ""), + (JobStatus.RUNNING, 25, "Сleaning...", ""), + ( + JobStatus.FINISHED, + 100, + "Сleaning completed.", + "425.51 MiB have been cleared", + ), ] - def set_job_status(status="", progress="", status_text="", result=""): - log_event.append((status, progress, status_text, result)) - - nix_collect_garbage( - None, - None, - lambda: output_print_dead, - lambda: output_collect_garbage.split("\n"), - set_job_status, - ) - print("log_event:", log_event) - print("reference:", reference) + nix_collect_garbage(None) assert log_event == reference -def test_nix_collect_garbage_zero_trash(): - log_event = [] +def test_nix_collect_garbage_zero_trash( + mock_set_job_status, + mock_run_nix_collect_garbage, + mock_run_nix_store_print_dead, + job_reset, +): + reference = [ (JobStatus.RUNNING, 0, "Сalculate the number of dead packages...", ""), (JobStatus.FINISHED, 100, "Nothing to clear", "System is clear"), ] - def set_job_status(status="", progress="", status_text="", result=""): - log_event.append((status, progress, status_text, result)) - - nix_collect_garbage( - None, - None, - lambda: "", - lambda: output_collect_garbage.split("\n"), - set_job_status, - ) + nix_collect_garbage(None) assert log_event == reference + # андр констракнш @pytest.mark.asyncio async def test_graphql_nix_collect_garbage():