feature(backups): batched removals of snapshots

This commit is contained in:
Houkime 2023-11-17 14:36:11 +00:00 committed by Inex Code
parent 564fedf806
commit 409d4fc2bb
4 changed files with 35 additions and 11 deletions

View file

@ -397,9 +397,8 @@ class Backups:
# TODO: Can be optimized since there is forgetting of an array in one restic op # TODO: Can be optimized since there is forgetting of an array in one restic op
# but most of the time this will be only one snap to forget. # but most of the time this will be only one snap to forget.
for snap in auto_snaps: deletable_snaps = [snap for snap in auto_snaps if snap not in new_snaplist]
if snap not in new_snaplist: Backups.forget_snapshots(deletable_snaps)
Backups.forget_snapshot(snap)
@staticmethod @staticmethod
def _standardize_quotas(i: int) -> int: def _standardize_quotas(i: int) -> int:
@ -606,6 +605,19 @@ class Backups:
return snap return snap
@staticmethod
def forget_snapshots(snapshots: List[Snapshot]) -> None:
"""
Deletes a batch of snapshots from the repo and from cache
Optimized
"""
ids = [snapshot.id for snapshot in snapshots]
Backups.provider().backupper.forget_snapshots(ids)
# less critical
for snapshot in snapshots:
Storage.delete_cached_snapshot(snapshot)
@staticmethod @staticmethod
def forget_snapshot(snapshot: Snapshot) -> None: def forget_snapshot(snapshot: Snapshot) -> None:
"""Deletes a snapshot from the repo and from cache""" """Deletes a snapshot from the repo and from cache"""

View file

@ -66,3 +66,8 @@ class AbstractBackupper(ABC):
def forget_snapshot(self, snapshot_id) -> None: def forget_snapshot(self, snapshot_id) -> None:
"""Forget a snapshot""" """Forget a snapshot"""
raise NotImplementedError raise NotImplementedError
@abstractmethod
def forget_snapshots(self, snapshot_ids: List[str]) -> None:
"""Maybe optimized deletion of a batch of snapshots, just cycling if unsupported"""
raise NotImplementedError

View file

@ -39,4 +39,7 @@ class NoneBackupper(AbstractBackupper):
raise NotImplementedError raise NotImplementedError
def forget_snapshot(self, snapshot_id): def forget_snapshot(self, snapshot_id):
raise NotImplementedError raise NotImplementedError("forget_snapshot")
def forget_snapshots(self, snapshots):
raise NotImplementedError("forget_snapshots")

View file

@ -86,6 +86,10 @@ class ResticBackupper(AbstractBackupper):
return f"echo {LocalBackupSecret.get()}" return f"echo {LocalBackupSecret.get()}"
def restic_command(self, *args, tags: Optional[List[str]] = None) -> List[str]: def restic_command(self, *args, tags: Optional[List[str]] = None) -> List[str]:
"""
Construct a restic command against the currently configured repo
Can support [nested] arrays as arguments, will flatten them into the final commmand
"""
if tags is None: if tags is None:
tags = [] tags = []
@ -384,15 +388,15 @@ class ResticBackupper(AbstractBackupper):
output, output,
) )
def forget_snapshot(self, snapshot_id: str) -> None:
self.forget_snapshots([snapshot_id])
@unlocked_repo @unlocked_repo
def forget_snapshot(self, snapshot_id) -> None: def forget_snapshots(self, snapshot_ids: List[str]) -> None:
""" # in case the backupper program supports batching, otherwise implement it by cycling
Either removes snapshot or marks it for deletion later,
depending on server settings
"""
forget_command = self.restic_command( forget_command = self.restic_command(
"forget", "forget",
snapshot_id, [snapshot_ids],
# TODO: prune should be done in a separate process # TODO: prune should be done in a separate process
"--prune", "--prune",
) )
@ -414,7 +418,7 @@ class ResticBackupper(AbstractBackupper):
if "no matching ID found" in err: if "no matching ID found" in err:
raise ValueError( raise ValueError(
"trying to delete, but no such snapshot: ", snapshot_id "trying to delete, but no such snapshot(s): ", snapshot_ids
) )
assert ( assert (