selfprivacy-rest-api/selfprivacy_api/jobs/nix_collect_garbage.py

150 lines
3.9 KiB
Python
Raw Normal View History

2022-11-24 02:08:58 +00:00
import re
import subprocess
2023-11-15 12:47:04 +00:00
from typing import Tuple, Iterable
2022-11-24 02:08:58 +00:00
2023-06-19 21:25:04 +00:00
from selfprivacy_api.utils.huey import huey
2023-06-18 04:37:27 +00:00
2023-06-19 21:25:04 +00:00
from selfprivacy_api.jobs import JobStatus, Jobs, Job
2024-02-08 14:40:12 +00:00
2024-02-03 19:10:01 +00:00
class ShellException(Exception):
"""Shell-related errors"""
2024-02-08 14:40:12 +00:00
2023-10-11 22:01:31 +00:00
COMPLETED_WITH_ERROR = (
2024-02-03 19:10:01 +00:00
"Error occurred, please report this to the support chat."
2023-10-11 22:01:31 +00:00
)
2023-11-29 09:02:59 +00:00
RESULT_WAS_NOT_FOUND_ERROR = "We are sorry, garbage collection result was not found. " \
"Something went wrong, please report this to the support chat."
CLEAR_COMPLETED = "Garbage collection completed."
2022-11-24 02:08:58 +00:00
2024-02-03 19:10:01 +00:00
def delete_old_gens_and_return_dead_report() -> str:
2023-10-11 22:01:31 +00:00
subprocess.run(
["nix-env", "-p", "/nix/var/nix/profiles/system", "--delete-generations old"],
check=False,
)
2023-06-21 06:52:31 +00:00
2024-02-03 19:10:01 +00:00
result = subprocess.check_output(["nix-store", "--gc", "--print-dead"]).decode(
"utf-8"
)
2022-11-24 02:08:58 +00:00
2024-02-03 19:10:01 +00:00
return " " if result is None else result
2023-11-11 21:16:04 +00:00
2023-11-15 12:47:04 +00:00
def run_nix_collect_garbage() -> Iterable[bytes]:
process = subprocess.Popen(
2023-06-21 06:52:31 +00:00
["nix-store", "--gc"], stdout=subprocess.PIPE, stderr=subprocess.STDOUT
2023-11-15 12:47:04 +00:00
)
return process.stdout if process.stdout else iter([])
2023-11-29 09:02:59 +00:00
def parse_line(job: Job, line: str) -> Job:
2023-10-11 22:18:29 +00:00
"""
We parse the string for the presence of a final line,
with the final amount of space cleared.
Simply put, we're just looking for a similar string:
"1537 store paths deleted, 339.84 MiB freed".
"""
pattern = re.compile(r"[+-]?\d+\.\d+ \w+(?= freed)")
2023-06-18 04:37:27 +00:00
match = re.search(pattern, line)
2022-12-03 18:27:10 +00:00
if match is None:
2024-02-03 19:10:01 +00:00
raise ShellException("nix returned gibberish output")
2022-12-03 18:27:10 +00:00
else:
2023-11-29 09:02:59 +00:00
Jobs.update(
job=job,
status=JobStatus.FINISHED,
status_text=CLEAR_COMPLETED,
result=f"{match.group(0)} have been cleared",
)
2023-11-29 09:02:59 +00:00
return job
2023-10-16 16:39:58 +00:00
def process_stream(job: Job, stream: Iterable[bytes], total_dead_packages: int) -> None:
2023-03-23 17:49:30 +00:00
completed_packages = 0
2023-06-26 19:33:41 +00:00
prev_progress = 0
for line in stream:
2023-06-20 19:25:54 +00:00
line = line.decode("utf-8")
if "deleting '/nix/store/" in line:
2023-03-23 17:49:30 +00:00
completed_packages += 1
percent = int((completed_packages / total_dead_packages) * 100)
2022-12-03 18:27:10 +00:00
2023-06-26 19:33:41 +00:00
if percent - prev_progress >= 5:
2024-02-08 14:40:12 +00:00
2023-06-26 19:33:41 +00:00
Jobs.update(
2023-11-15 12:47:04 +00:00
job=job,
2023-06-26 19:33:41 +00:00
status=JobStatus.RUNNING,
progress=percent,
status_text="Cleaning...",
)
prev_progress = percent
2022-12-03 18:27:10 +00:00
elif "store paths deleted," in line:
2023-11-29 09:02:59 +00:00
parse_line(job, line)
2022-12-03 18:27:10 +00:00
2023-11-15 12:47:04 +00:00
def get_dead_packages(output) -> Tuple[int, float]:
dead = len(re.findall("/nix/store/", output))
2023-06-21 06:52:31 +00:00
percent = 0
if dead != 0:
percent = 100 / dead
return dead, percent
2022-12-03 18:27:10 +00:00
2023-06-19 21:25:04 +00:00
@huey.task()
2024-01-17 15:26:10 +00:00
def calculate_and_clear_dead_paths(job: Job):
2023-06-18 04:37:27 +00:00
Jobs.update(
2023-11-15 12:47:04 +00:00
job=job,
status=JobStatus.RUNNING,
progress=0,
2023-06-21 06:52:31 +00:00
status_text="Calculate the number of dead packages...",
)
2022-12-03 18:27:10 +00:00
2023-10-11 22:09:13 +00:00
dead_packages, package_equal_to_percent = get_dead_packages(
2024-02-03 19:10:01 +00:00
delete_old_gens_and_return_dead_report()
2023-04-05 10:49:56 +00:00
)
if dead_packages == 0:
2024-02-03 19:10:01 +00:00
2023-06-18 04:37:27 +00:00
Jobs.update(
2023-11-15 12:47:04 +00:00
job=job,
status=JobStatus.FINISHED,
status_text="Nothing to clear",
result="System is clear",
)
2024-02-03 19:10:01 +00:00
return True
2023-06-18 04:37:27 +00:00
Jobs.update(
2023-11-15 12:47:04 +00:00
job=job,
status=JobStatus.RUNNING,
progress=0,
status_text=f"Found {dead_packages} packages to remove!",
2022-11-24 02:08:58 +00:00
)
2023-11-15 12:47:04 +00:00
stream = run_nix_collect_garbage()
2024-02-03 19:10:01 +00:00
try:
process_stream(job, stream, dead_packages)
except ShellException as error:
Jobs.update(
job=job,
status=JobStatus.ERROR,
status_text=COMPLETED_WITH_ERROR,
error=RESULT_WAS_NOT_FOUND_ERROR,
)
2023-06-18 04:37:27 +00:00
def start_nix_collect_garbage() -> Job:
job = Jobs.add(
type_id="maintenance.collect_nix_garbage",
name="Collect garbage",
2023-06-21 06:52:31 +00:00
description="Cleaning up unused packages",
2023-06-18 04:37:27 +00:00
)
2024-02-08 14:40:12 +00:00
calculate_and_clear_dead_paths(job=job)
2023-06-18 04:37:27 +00:00
return job