From cfa7f4ae59b23a279efa597d05453fcf11ac67bf Mon Sep 17 00:00:00 2001 From: Houkime <> Date: Wed, 26 Jul 2023 16:45:08 +0000 Subject: [PATCH] feature(backups): add full repo erasure capability --- selfprivacy_api/backup/__init__.py | 8 ++++ selfprivacy_api/backup/backuppers/__init__.py | 5 +++ .../backup/backuppers/none_backupper.py | 4 ++ .../backup/backuppers/restic_backupper.py | 44 +++++++++++++++---- selfprivacy_api/backup/storage.py | 13 ++++-- tests/test_graphql/test_backup.py | 13 ++++++ 6 files changed, 74 insertions(+), 13 deletions(-) diff --git a/selfprivacy_api/backup/__init__.py b/selfprivacy_api/backup/__init__.py index 725904e..c28c01f 100644 --- a/selfprivacy_api/backup/__init__.py +++ b/selfprivacy_api/backup/__init__.py @@ -236,6 +236,14 @@ class Backups: Backups.provider().backupper.init() Storage.mark_as_init() + @staticmethod + def erase_repo() -> None: + """ + Completely empties the remote + """ + Backups.provider().backupper.erase_repo() + Storage.mark_as_uninitted() + @staticmethod def is_initted() -> bool: """ diff --git a/selfprivacy_api/backup/backuppers/__init__.py b/selfprivacy_api/backup/backuppers/__init__.py index ea2350b..ccf78b9 100644 --- a/selfprivacy_api/backup/backuppers/__init__.py +++ b/selfprivacy_api/backup/backuppers/__init__.py @@ -36,6 +36,11 @@ class AbstractBackupper(ABC): """Initialize the repository""" raise NotImplementedError + @abstractmethod + def erase_repo(self) -> None: + """Completely empties the remote""" + raise NotImplementedError + @abstractmethod def restore_from_backup( self, diff --git a/selfprivacy_api/backup/backuppers/none_backupper.py b/selfprivacy_api/backup/backuppers/none_backupper.py index d9edaeb..87e43c5 100644 --- a/selfprivacy_api/backup/backuppers/none_backupper.py +++ b/selfprivacy_api/backup/backuppers/none_backupper.py @@ -23,6 +23,10 @@ class NoneBackupper(AbstractBackupper): def init(self): raise NotImplementedError + def erase_repo(self) -> None: + """Completely empties the remote""" + raise NotImplementedError + def restore_from_backup(self, snapshot_id: str, folders: List[str], verify=True): """Restore a target folder using a snapshot""" raise NotImplementedError diff --git a/selfprivacy_api/backup/backuppers/restic_backupper.py b/selfprivacy_api/backup/backuppers/restic_backupper.py index e98c4c3..816bebf 100644 --- a/selfprivacy_api/backup/backuppers/restic_backupper.py +++ b/selfprivacy_api/backup/backuppers/restic_backupper.py @@ -40,20 +40,25 @@ class ResticBackupper(AbstractBackupper): def restic_repo(self) -> str: # https://restic.readthedocs.io/en/latest/030_preparing_a_new_repo.html#other-services-via-rclone # https://forum.rclone.org/t/can-rclone-be-run-solely-with-command-line-options-no-config-no-env-vars/6314/5 - return f"rclone:{self.storage_type}{self.repo}" + return f"rclone:{self.rclone_repo()}" + + def rclone_repo(self) -> str: + return f"{self.storage_type}{self.repo}" def rclone_args(self): - return "rclone.args=serve restic --stdio " + self.backend_rclone_args() + return "rclone.args=serve restic --stdio " + " ".join( + self.backend_rclone_args() + ) - def backend_rclone_args(self) -> str: - acc_arg = "" - key_arg = "" + def backend_rclone_args(self) -> list[str]: + args = [] if self.account != "": - acc_arg = f"{self.login_flag} {self.account}" + acc_args = [self.login_flag, self.account] + args.extend(acc_args) if self.key != "": - key_arg = f"{self.key_flag} {self.key}" - - return f"{acc_arg} {key_arg}" + key_args = [self.key_flag, self.key] + args.extend(key_args) + return args def _password_command(self): return f"echo {LocalBackupSecret.get()}" @@ -79,6 +84,27 @@ class ResticBackupper(AbstractBackupper): command.extend(ResticBackupper.__flatten_list(args)) return command + def erase_repo(self) -> None: + """Fully erases repo on remote, can be reinitted again""" + command = [ + "rclone", + "purge", + self.rclone_repo(), + ] + backend_args = self.backend_rclone_args() + if backend_args: + command.extend(backend_args) + + with subprocess.Popen(command, stdout=subprocess.PIPE, shell=False) as handle: + output = handle.communicate()[0].decode("utf-8") + if handle.returncode != 0: + raise ValueError( + "purge exited with errorcode", + handle.returncode, + ":", + output, + ) + def mount_repo(self, mount_directory): mount_command = self.restic_command("mount", mount_directory) mount_command.insert(0, "nohup") diff --git a/selfprivacy_api/backup/storage.py b/selfprivacy_api/backup/storage.py index f7384a0..d46f584 100644 --- a/selfprivacy_api/backup/storage.py +++ b/selfprivacy_api/backup/storage.py @@ -21,7 +21,7 @@ REDIS_SNAPSHOT_CACHE_EXPIRE_SECONDS = 24 * 60 * 60 # one day REDIS_SNAPSHOTS_PREFIX = "backups:snapshots:" REDIS_LAST_BACKUP_PREFIX = "backups:last-backed-up:" -REDIS_INITTED_CACHE_PREFIX = "backups:initted_services:" +REDIS_INITTED_CACHE = "backups:repo_initted" REDIS_PROVIDER_KEY = "backups:provider" REDIS_AUTOBACKUP_PERIOD_KEY = "backups:autobackup_period" @@ -38,9 +38,9 @@ class Storage: """Deletes all backup related data from redis""" redis.delete(REDIS_PROVIDER_KEY) redis.delete(REDIS_AUTOBACKUP_PERIOD_KEY) + redis.delete(REDIS_INITTED_CACHE) prefixes_to_clean = [ - REDIS_INITTED_CACHE_PREFIX, REDIS_SNAPSHOTS_PREFIX, REDIS_LAST_BACKUP_PREFIX, ] @@ -162,11 +162,16 @@ class Storage: @staticmethod def has_init_mark() -> bool: """Returns True if the repository was initialized""" - if redis.exists(REDIS_INITTED_CACHE_PREFIX): + if redis.exists(REDIS_INITTED_CACHE): return True return False @staticmethod def mark_as_init(): """Marks the repository as initialized""" - redis.set(REDIS_INITTED_CACHE_PREFIX, 1) + redis.set(REDIS_INITTED_CACHE, 1) + + @staticmethod + def mark_as_uninitted(): + """Marks the repository as initialized""" + redis.delete(REDIS_INITTED_CACHE) diff --git a/tests/test_graphql/test_backup.py b/tests/test_graphql/test_backup.py index fc42ca2..e85d1de 100644 --- a/tests/test_graphql/test_backup.py +++ b/tests/test_graphql/test_backup.py @@ -222,6 +222,19 @@ def test_file_backend_init(file_backup): file_backup.backupper.init() +def test_reinit_after_purge(backups): + assert Backups.is_initted() is True + + Backups.erase_repo() + assert Backups.is_initted() is False + with pytest.raises(ValueError): + Backups.get_all_snapshots() + + Backups.init_repo() + assert Backups.is_initted() is True + assert len(Backups.get_all_snapshots()) == 0 + + def test_backup_simple_file(raw_dummy_service, file_backup): # temporarily incomplete service = raw_dummy_service