selfprivacy-rest-api/selfprivacy_api/jobs/upgrade_system.py

121 lines
4 KiB
Python

"""
A task to start the system upgrade or rebuild by starting a systemd unit.
After starting, track the status of the systemd unit and update the Job
status accordingly.
"""
import subprocess
from selfprivacy_api.utils.huey import huey
from selfprivacy_api.jobs import JobStatus, Jobs, Job
import time
@huey.task()
def rebuild_system_task(job: Job, upgrade: bool = False):
"""Rebuild the system"""
try:
if upgrade:
command = ["systemctl", "start", "sp-nixos-upgrade.service"]
else:
command = ["systemctl", "start", "sp-nixos-rebuild.service"]
subprocess.run(
command,
check=True,
start_new_session=True,
shell=False,
)
Jobs.update(
job=job,
status=JobStatus.RUNNING,
status_text="Rebuilding the system...",
)
# Get current time to handle timeout
start_time = time.time()
# Wait for the systemd unit to start
while True:
try:
status = subprocess.run(
["systemctl", "is-active", "selfprivacy-upgrade"],
check=True,
capture_output=True,
text=True,
)
if status.stdout.strip() == "active":
log_line = subprocess.run(
[
"journalctl",
"-u",
"selfprivacy-upgrade",
"-n",
"1",
"-o",
"cat",
],
check=True,
capture_output=True,
text=True,
).stdout.strip()
Jobs.update(
job=job,
status=JobStatus.RUNNING,
status_text=f"Rebuilding the system... Latest log line: {log_line}",
)
break
# Timeount after 5 minutes
if time.time() - start_time > 300:
Jobs.update(
job=job,
status=JobStatus.ERROR,
error="System rebuild timed out.",
)
return
time.sleep(1)
except subprocess.CalledProcessError:
pass
Jobs.update(
job=job,
status=JobStatus.RUNNING,
status_text="Rebuilding the system...",
)
# Wait for the systemd unit to finish
while True:
try:
status = subprocess.run(
["systemctl", "is-active", "selfprivacy-upgrade"],
check=True,
capture_output=True,
text=True,
)
if status.stdout.strip() == "inactive":
Jobs.update(
job=job,
status=JobStatus.FINISHED,
result="System rebuilt.",
progress=100,
)
elif status.stdout.strip() == "failed":
Jobs.update(
job=job,
status=JobStatus.ERROR,
error="System rebuild failed.",
)
break
# Timeout of 60 minutes
if time.time() - start_time > 3600:
Jobs.update(
job=job,
status=JobStatus.ERROR,
error="System rebuild timed out.",
)
break
except subprocess.CalledProcessError:
pass
time.sleep(5)
except subprocess.CalledProcessError as e:
Jobs.update(
job=job,
status=JobStatus.ERROR,
status_text=str(e),
)