feature(backups): manual autobackup -> total backup

selfprivacy_api/backup/jobs.py

@@ -76,18 +76,34 @@ def complain_about_service_operation_running(service: Service) -> str:
 
 
 def add_total_restore_job() -> Job:
-    for service in ServiceManager.get_all_services():
+    for service in ServiceManager.get_enabled_services():
-        if (
+        ensure_nothing_runs_for(service)
-            not isinstance(service, ServiceManager)
-            and is_something_running_for(service) is True
-        ):
-            complain_about_service_operation_running(service)
 
-    display_name = service.get_display_name()
     job = Jobs.add(
         type_id="backups.total_restore",
-        name=f"Restore {display_name}",
+        name=f"Total restore",
-        description="restoring all the services",
+        description="restoring all enabled services",
+    )
+    return job
+
+
+def ensure_nothing_runs_for(service: Service):
+    if (
+        # TODO: try removing the exception. Why would we have it?
+        not isinstance(service, ServiceManager)
+        and is_something_running_for(service) is True
+    ):
+        complain_about_service_operation_running(service)
+
+
+def add_total_backup_job() -> Job:
+    for service in ServiceManager.get_enabled_services():
+        ensure_nothing_runs_for(service)
+
+    job = Jobs.add(
+        type_id="backups.total_backup",
+        name=f"Total backup",
+        description="Backing up all the enabled services",
     )
     return job
 

selfprivacy_api/backup/tasks.py

@@ -3,6 +3,7 @@ The tasks module contains the worker tasks that are used to back up and restore
 """
 
 from datetime import datetime, timezone
+from typing import List
 
 from selfprivacy_api.graphql.common_types.backup import (
     RestoreStrategy,
@@ -13,7 +14,7 @@ from selfprivacy_api.models.backup.snapshot import Snapshot
 from selfprivacy_api.utils.huey import huey
 from huey import crontab
 
-from selfprivacy_api.services import ServiceManager
+from selfprivacy_api.services import ServiceManager, Service
 from selfprivacy_api.backup import Backups
 from selfprivacy_api.backup.jobs import add_autobackup_job, add_total_restore_job
 from selfprivacy_api.jobs import Jobs, JobStatus, Job
@@ -34,6 +35,14 @@ def validate_datetime(dt: datetime) -> bool:
     return Backups.is_time_to_backup(dt)
 
 
+def report_job_error(error: Exception, job: Job):
+    Jobs.update(
+        job,
+        status=JobStatus.ERROR,
+        error=type(error).__name__ + ": " + str(error),
+    )
+
+
 # huey tasks need to return something
 @huey.task()
 def start_backup(service_id: str, reason: BackupReason = BackupReason.EXPLICIT) -> bool:
@@ -75,12 +84,70 @@ def restore_snapshot(
     return True
 
 
+@huey.task()
+def full_restore(job: Job) -> bool:
+    do_full_restore(job)
+    return True
+
+
+@huey.periodic_task(validate_datetime=validate_datetime)
+def automatic_backup() -> None:
+    """
+    The worker periodic task that starts the automatic backup process.
+    """
+    do_autobackup()
+
+
+@huey.task()
+def total_backup(job: Job) -> bool:
+    do_total_backup(job)
+    return True
+
+
+@huey.periodic_task(crontab(hour="*/" + str(SNAPSHOT_CACHE_TTL_HOURS)))
+def reload_snapshot_cache():
+    Backups.force_snapshot_cache_reload()
+
+
+def back_up_multiple(
+    job: Job,
+    services_to_back_up: List[Service],
+    reason: BackupReason = BackupReason.EXPLICIT,
+):
+    if services_to_back_up == []:
+        return
+
+    progress_per_service = 100 // len(services_to_back_up)
+    progress = 0
+    Jobs.update(job, JobStatus.RUNNING, progress=progress)
+
+    for service in services_to_back_up:
+        try:
+            Backups.back_up(service, reason)
+        except Exception as error:
+            report_job_error(error, job)
+            raise error
+        progress = progress + progress_per_service
+        Jobs.update(job, JobStatus.RUNNING, progress=progress)
+
+
+
+def do_total_backup(job: Job) -> None:
+    """
+    Body of total backup task, broken out to test it
+    """
+    back_up_multiple(job, ServiceManager.get_enabled_services())
+
+    Jobs.update(job, JobStatus.FINISHED)
+
+
 def do_autobackup() -> None:
     """
     Body of autobackup task, broken out to test it
     For some reason, we cannot launch periodic huey tasks
     inside tests
     """
+
     time = datetime.now(timezone.utc)
     backups_were_disabled = Backups.autobackup_period_minutes() is None
 
@@ -93,22 +160,7 @@ def do_autobackup() -> None:
         return
     job = add_autobackup_job(services_to_back_up)
 
-    progress_per_service = 100 // len(services_to_back_up)
+    back_up_multiple(job, services_to_back_up, BackupReason.AUTO)
-    progress = 0
-    Jobs.update(job, JobStatus.RUNNING, progress=progress)
-
-    for service in services_to_back_up:
-        try:
-            Backups.back_up(service, BackupReason.AUTO)
-        except Exception as error:
-            Jobs.update(
-                job,
-                status=JobStatus.ERROR,
-                error=type(error).__name__ + ": " + str(error),
-            )
-            raise error
-        progress = progress + progress_per_service
-        Jobs.update(job, JobStatus.RUNNING, progress=progress)
 
     if backups_were_disabled:
         Backups.set_autobackup_period_minutes(0)
@@ -183,35 +235,3 @@ def do_full_restore(job: Job) -> None:
     rebuild_job = add_rebuild_job()
     rebuild_system(rebuild_job)
     Jobs.update(job, JobStatus.FINISHED)
-
-
-def report_job_error(error: Exception, job: Job):
-    Jobs.update(
-        job,
-        status=JobStatus.ERROR,
-        error=type(error).__name__ + ": " + str(error),
-    )
-
-
-@huey.task()
-def full_restore(job: Job) -> bool:
-    do_full_restore(job)
-    return True
-
-
-@huey.periodic_task(validate_datetime=validate_datetime)
-def automatic_backup() -> None:
-    """
-    The worker periodic task that starts the automatic backup process.
-    """
-
-
-@huey.task()
-def trigger_autobackup() -> bool:
-    do_autobackup()
-    return True
-
-
-@huey.periodic_task(crontab(hour="*/" + str(SNAPSHOT_CACHE_TTL_HOURS)))
-def reload_snapshot_cache():
-    Backups.force_snapshot_cache_reload()

selfprivacy_api/graphql/mutations/backup_mutations.py

@@ -1,6 +1,8 @@
 import typing
 import strawberry
 
+from selfprivacy_api.utils.graphql import api_job_mutation_error
+
 from selfprivacy_api.jobs import Jobs
 
 from selfprivacy_api.graphql import IsAuthenticated
@@ -25,12 +27,13 @@ from selfprivacy_api.backup.tasks import (
     restore_snapshot,
     prune_autobackup_snapshots,
     full_restore,
-    trigger_autobackup,
+    total_backup,
 )
 from selfprivacy_api.backup.jobs import (
     add_backup_job,
     add_restore_job,
     add_total_restore_job,
+    add_total_backup_job,
 )
 from selfprivacy_api.backup.local_secret import LocalBackupSecret
 
@@ -173,19 +176,22 @@ class BackupMutations:
         )
 
     @strawberry.mutation(permission_classes=[IsAuthenticated])
-    def manual_autobackup(self) -> GenericMutationReturn:
+    def total_backup(self) -> GenericJobMutationReturn:
-        """Induce autobackup to back up all the services at once
+        """Back up all the enabled services at once
-        Useful when migrating but a bit of a hack
+        Useful when migrating
         """
 
-        # This cannot give us a job, unfortunately
+        try:
-        # TODO: We need to pass it
+            job = add_total_backup_job()
-        trigger_autobackup()
+            total_backup(job)
+        except Exception as error:
+            return api_job_mutation_error(error)
 
-        return GenericMutationReturn(
+        return GenericJobMutationReturn(
             success=True,
             code=200,
-            message="Backup task queued",
+            message="Total backup task queued",
+            job=job_to_api_job(job),
         )
 
     @strawberry.mutation(permission_classes=[IsAuthenticated])

selfprivacy_api/utils/graphql.py

@@ -0,0 +1,12 @@
+from selfprivacy_api.graphql.mutations.mutation_interface import (
+    GenericJobMutationReturn,
+)
+
+
+def api_job_mutation_error(error: Exception, code: int = 400):
+    return GenericJobMutationReturn(
+        success=False,
+        code=code,
+        message=str(error),
+        job=None,
+    )

tests/test_graphql/test_api_backup.py

@@ -36,10 +36,10 @@ mutation TestSnapshotsReload {
 }
 """
 
-API_MANUAL_AUTOBACKUP = """
+API_TOTAL_BACKUP = """
 mutation TestForcedAutobackup {
     backup {
-         manualAutobackup{
+         totalBackup{
             success
             message
             code
@@ -293,11 +293,11 @@ def api_reload_snapshots(authorized_client):
     return response
 
 
-def api_manual_autobackup(authorized_client):
+def api_total_backup(authorized_client):
     response = authorized_client.post(
         "/graphql",
         json={
-            "query": API_MANUAL_AUTOBACKUP,
+            "query": API_TOTAL_BACKUP,
             "variables": {},
         },
     )
@@ -616,9 +616,9 @@ def test_induce_autobackup_if_dir_exists(
     # mkdir(CONFIG_STASH_DIR)
     dummy_service = only_dummy_service_and_api
 
-    response = api_manual_autobackup(authorized_client)
+    response = api_total_backup(authorized_client)
     # raise ValueError(get_data(response))
-    data = get_data(response)["backup"]["manualAutobackup"]
+    data = get_data(response)["backup"]["totalBackup"]
     assert_ok(data)
 
     snaps = api_snapshots(authorized_client)
@@ -628,9 +628,9 @@ def test_induce_autobackup_if_dir_exists(
 def test_induce_autobackup(authorized_client, only_dummy_service_and_api, backups):
     dummy_service = only_dummy_service_and_api
 
-    response = api_manual_autobackup(authorized_client)
+    response = api_total_backup(authorized_client)
     # raise ValueError(get_data(response))
-    data = get_data(response)["backup"]["manualAutobackup"]
+    data = get_data(response)["backup"]["totalBackup"]
     assert_ok(data)
 
     snaps = api_snapshots(authorized_client)
@@ -677,7 +677,7 @@ def test_forget_nonexistent_snapshot(authorized_client, dummy_service, backups):
 
 
 def test_last_slice(authorized_client, only_dummy_service_and_api, backups):
-    api_manual_autobackup(authorized_client)
+    api_total_backup(authorized_client)
     snaps = api_last_slice(authorized_client)
 
     assert len(snaps) == 2