mirror of
https://git.selfprivacy.org/SelfPrivacy/selfprivacy-rest-api.git
synced 2024-11-25 13:31:27 +00:00
feature(backups): manual autobackup -> total backup
This commit is contained in:
parent
ee06d68047
commit
5ea000baab
|
@ -76,18 +76,34 @@ def complain_about_service_operation_running(service: Service) -> str:
|
|||
|
||||
|
||||
def add_total_restore_job() -> Job:
|
||||
for service in ServiceManager.get_all_services():
|
||||
if (
|
||||
not isinstance(service, ServiceManager)
|
||||
and is_something_running_for(service) is True
|
||||
):
|
||||
complain_about_service_operation_running(service)
|
||||
for service in ServiceManager.get_enabled_services():
|
||||
ensure_nothing_runs_for(service)
|
||||
|
||||
display_name = service.get_display_name()
|
||||
job = Jobs.add(
|
||||
type_id="backups.total_restore",
|
||||
name=f"Restore {display_name}",
|
||||
description="restoring all the services",
|
||||
name=f"Total restore",
|
||||
description="restoring all enabled services",
|
||||
)
|
||||
return job
|
||||
|
||||
|
||||
def ensure_nothing_runs_for(service: Service):
|
||||
if (
|
||||
# TODO: try removing the exception. Why would we have it?
|
||||
not isinstance(service, ServiceManager)
|
||||
and is_something_running_for(service) is True
|
||||
):
|
||||
complain_about_service_operation_running(service)
|
||||
|
||||
|
||||
def add_total_backup_job() -> Job:
|
||||
for service in ServiceManager.get_enabled_services():
|
||||
ensure_nothing_runs_for(service)
|
||||
|
||||
job = Jobs.add(
|
||||
type_id="backups.total_backup",
|
||||
name=f"Total backup",
|
||||
description="Backing up all the enabled services",
|
||||
)
|
||||
return job
|
||||
|
||||
|
|
|
@ -3,6 +3,7 @@ The tasks module contains the worker tasks that are used to back up and restore
|
|||
"""
|
||||
|
||||
from datetime import datetime, timezone
|
||||
from typing import List
|
||||
|
||||
from selfprivacy_api.graphql.common_types.backup import (
|
||||
RestoreStrategy,
|
||||
|
@ -13,7 +14,7 @@ from selfprivacy_api.models.backup.snapshot import Snapshot
|
|||
from selfprivacy_api.utils.huey import huey
|
||||
from huey import crontab
|
||||
|
||||
from selfprivacy_api.services import ServiceManager
|
||||
from selfprivacy_api.services import ServiceManager, Service
|
||||
from selfprivacy_api.backup import Backups
|
||||
from selfprivacy_api.backup.jobs import add_autobackup_job, add_total_restore_job
|
||||
from selfprivacy_api.jobs import Jobs, JobStatus, Job
|
||||
|
@ -34,6 +35,14 @@ def validate_datetime(dt: datetime) -> bool:
|
|||
return Backups.is_time_to_backup(dt)
|
||||
|
||||
|
||||
def report_job_error(error: Exception, job: Job):
|
||||
Jobs.update(
|
||||
job,
|
||||
status=JobStatus.ERROR,
|
||||
error=type(error).__name__ + ": " + str(error),
|
||||
)
|
||||
|
||||
|
||||
# huey tasks need to return something
|
||||
@huey.task()
|
||||
def start_backup(service_id: str, reason: BackupReason = BackupReason.EXPLICIT) -> bool:
|
||||
|
@ -75,12 +84,70 @@ def restore_snapshot(
|
|||
return True
|
||||
|
||||
|
||||
@huey.task()
|
||||
def full_restore(job: Job) -> bool:
|
||||
do_full_restore(job)
|
||||
return True
|
||||
|
||||
|
||||
@huey.periodic_task(validate_datetime=validate_datetime)
|
||||
def automatic_backup() -> None:
|
||||
"""
|
||||
The worker periodic task that starts the automatic backup process.
|
||||
"""
|
||||
do_autobackup()
|
||||
|
||||
|
||||
@huey.task()
|
||||
def total_backup(job: Job) -> bool:
|
||||
do_total_backup(job)
|
||||
return True
|
||||
|
||||
|
||||
@huey.periodic_task(crontab(hour="*/" + str(SNAPSHOT_CACHE_TTL_HOURS)))
|
||||
def reload_snapshot_cache():
|
||||
Backups.force_snapshot_cache_reload()
|
||||
|
||||
|
||||
def back_up_multiple(
|
||||
job: Job,
|
||||
services_to_back_up: List[Service],
|
||||
reason: BackupReason = BackupReason.EXPLICIT,
|
||||
):
|
||||
if services_to_back_up == []:
|
||||
return
|
||||
|
||||
progress_per_service = 100 // len(services_to_back_up)
|
||||
progress = 0
|
||||
Jobs.update(job, JobStatus.RUNNING, progress=progress)
|
||||
|
||||
for service in services_to_back_up:
|
||||
try:
|
||||
Backups.back_up(service, reason)
|
||||
except Exception as error:
|
||||
report_job_error(error, job)
|
||||
raise error
|
||||
progress = progress + progress_per_service
|
||||
Jobs.update(job, JobStatus.RUNNING, progress=progress)
|
||||
|
||||
|
||||
|
||||
def do_total_backup(job: Job) -> None:
|
||||
"""
|
||||
Body of total backup task, broken out to test it
|
||||
"""
|
||||
back_up_multiple(job, ServiceManager.get_enabled_services())
|
||||
|
||||
Jobs.update(job, JobStatus.FINISHED)
|
||||
|
||||
|
||||
def do_autobackup() -> None:
|
||||
"""
|
||||
Body of autobackup task, broken out to test it
|
||||
For some reason, we cannot launch periodic huey tasks
|
||||
inside tests
|
||||
"""
|
||||
|
||||
time = datetime.now(timezone.utc)
|
||||
backups_were_disabled = Backups.autobackup_period_minutes() is None
|
||||
|
||||
|
@ -93,22 +160,7 @@ def do_autobackup() -> None:
|
|||
return
|
||||
job = add_autobackup_job(services_to_back_up)
|
||||
|
||||
progress_per_service = 100 // len(services_to_back_up)
|
||||
progress = 0
|
||||
Jobs.update(job, JobStatus.RUNNING, progress=progress)
|
||||
|
||||
for service in services_to_back_up:
|
||||
try:
|
||||
Backups.back_up(service, BackupReason.AUTO)
|
||||
except Exception as error:
|
||||
Jobs.update(
|
||||
job,
|
||||
status=JobStatus.ERROR,
|
||||
error=type(error).__name__ + ": " + str(error),
|
||||
)
|
||||
raise error
|
||||
progress = progress + progress_per_service
|
||||
Jobs.update(job, JobStatus.RUNNING, progress=progress)
|
||||
back_up_multiple(job, services_to_back_up, BackupReason.AUTO)
|
||||
|
||||
if backups_were_disabled:
|
||||
Backups.set_autobackup_period_minutes(0)
|
||||
|
@ -183,35 +235,3 @@ def do_full_restore(job: Job) -> None:
|
|||
rebuild_job = add_rebuild_job()
|
||||
rebuild_system(rebuild_job)
|
||||
Jobs.update(job, JobStatus.FINISHED)
|
||||
|
||||
|
||||
def report_job_error(error: Exception, job: Job):
|
||||
Jobs.update(
|
||||
job,
|
||||
status=JobStatus.ERROR,
|
||||
error=type(error).__name__ + ": " + str(error),
|
||||
)
|
||||
|
||||
|
||||
@huey.task()
|
||||
def full_restore(job: Job) -> bool:
|
||||
do_full_restore(job)
|
||||
return True
|
||||
|
||||
|
||||
@huey.periodic_task(validate_datetime=validate_datetime)
|
||||
def automatic_backup() -> None:
|
||||
"""
|
||||
The worker periodic task that starts the automatic backup process.
|
||||
"""
|
||||
|
||||
|
||||
@huey.task()
|
||||
def trigger_autobackup() -> bool:
|
||||
do_autobackup()
|
||||
return True
|
||||
|
||||
|
||||
@huey.periodic_task(crontab(hour="*/" + str(SNAPSHOT_CACHE_TTL_HOURS)))
|
||||
def reload_snapshot_cache():
|
||||
Backups.force_snapshot_cache_reload()
|
||||
|
|
|
@ -1,6 +1,8 @@
|
|||
import typing
|
||||
import strawberry
|
||||
|
||||
from selfprivacy_api.utils.graphql import api_job_mutation_error
|
||||
|
||||
from selfprivacy_api.jobs import Jobs
|
||||
|
||||
from selfprivacy_api.graphql import IsAuthenticated
|
||||
|
@ -25,12 +27,13 @@ from selfprivacy_api.backup.tasks import (
|
|||
restore_snapshot,
|
||||
prune_autobackup_snapshots,
|
||||
full_restore,
|
||||
trigger_autobackup,
|
||||
total_backup,
|
||||
)
|
||||
from selfprivacy_api.backup.jobs import (
|
||||
add_backup_job,
|
||||
add_restore_job,
|
||||
add_total_restore_job,
|
||||
add_total_backup_job,
|
||||
)
|
||||
from selfprivacy_api.backup.local_secret import LocalBackupSecret
|
||||
|
||||
|
@ -173,19 +176,22 @@ class BackupMutations:
|
|||
)
|
||||
|
||||
@strawberry.mutation(permission_classes=[IsAuthenticated])
|
||||
def manual_autobackup(self) -> GenericMutationReturn:
|
||||
"""Induce autobackup to back up all the services at once
|
||||
Useful when migrating but a bit of a hack
|
||||
def total_backup(self) -> GenericJobMutationReturn:
|
||||
"""Back up all the enabled services at once
|
||||
Useful when migrating
|
||||
"""
|
||||
|
||||
# This cannot give us a job, unfortunately
|
||||
# TODO: We need to pass it
|
||||
trigger_autobackup()
|
||||
try:
|
||||
job = add_total_backup_job()
|
||||
total_backup(job)
|
||||
except Exception as error:
|
||||
return api_job_mutation_error(error)
|
||||
|
||||
return GenericMutationReturn(
|
||||
return GenericJobMutationReturn(
|
||||
success=True,
|
||||
code=200,
|
||||
message="Backup task queued",
|
||||
message="Total backup task queued",
|
||||
job=job_to_api_job(job),
|
||||
)
|
||||
|
||||
@strawberry.mutation(permission_classes=[IsAuthenticated])
|
||||
|
|
12
selfprivacy_api/utils/graphql.py
Normal file
12
selfprivacy_api/utils/graphql.py
Normal file
|
@ -0,0 +1,12 @@
|
|||
from selfprivacy_api.graphql.mutations.mutation_interface import (
|
||||
GenericJobMutationReturn,
|
||||
)
|
||||
|
||||
|
||||
def api_job_mutation_error(error: Exception, code: int = 400):
|
||||
return GenericJobMutationReturn(
|
||||
success=False,
|
||||
code=code,
|
||||
message=str(error),
|
||||
job=None,
|
||||
)
|
|
@ -36,10 +36,10 @@ mutation TestSnapshotsReload {
|
|||
}
|
||||
"""
|
||||
|
||||
API_MANUAL_AUTOBACKUP = """
|
||||
API_TOTAL_BACKUP = """
|
||||
mutation TestForcedAutobackup {
|
||||
backup {
|
||||
manualAutobackup{
|
||||
totalBackup{
|
||||
success
|
||||
message
|
||||
code
|
||||
|
@ -293,11 +293,11 @@ def api_reload_snapshots(authorized_client):
|
|||
return response
|
||||
|
||||
|
||||
def api_manual_autobackup(authorized_client):
|
||||
def api_total_backup(authorized_client):
|
||||
response = authorized_client.post(
|
||||
"/graphql",
|
||||
json={
|
||||
"query": API_MANUAL_AUTOBACKUP,
|
||||
"query": API_TOTAL_BACKUP,
|
||||
"variables": {},
|
||||
},
|
||||
)
|
||||
|
@ -616,9 +616,9 @@ def test_induce_autobackup_if_dir_exists(
|
|||
# mkdir(CONFIG_STASH_DIR)
|
||||
dummy_service = only_dummy_service_and_api
|
||||
|
||||
response = api_manual_autobackup(authorized_client)
|
||||
response = api_total_backup(authorized_client)
|
||||
# raise ValueError(get_data(response))
|
||||
data = get_data(response)["backup"]["manualAutobackup"]
|
||||
data = get_data(response)["backup"]["totalBackup"]
|
||||
assert_ok(data)
|
||||
|
||||
snaps = api_snapshots(authorized_client)
|
||||
|
@ -628,9 +628,9 @@ def test_induce_autobackup_if_dir_exists(
|
|||
def test_induce_autobackup(authorized_client, only_dummy_service_and_api, backups):
|
||||
dummy_service = only_dummy_service_and_api
|
||||
|
||||
response = api_manual_autobackup(authorized_client)
|
||||
response = api_total_backup(authorized_client)
|
||||
# raise ValueError(get_data(response))
|
||||
data = get_data(response)["backup"]["manualAutobackup"]
|
||||
data = get_data(response)["backup"]["totalBackup"]
|
||||
assert_ok(data)
|
||||
|
||||
snaps = api_snapshots(authorized_client)
|
||||
|
@ -677,7 +677,7 @@ def test_forget_nonexistent_snapshot(authorized_client, dummy_service, backups):
|
|||
|
||||
|
||||
def test_last_slice(authorized_client, only_dummy_service_and_api, backups):
|
||||
api_manual_autobackup(authorized_client)
|
||||
api_total_backup(authorized_client)
|
||||
snaps = api_last_slice(authorized_client)
|
||||
|
||||
assert len(snaps) == 2
|
||||
|
|
Loading…
Reference in a new issue