From 027a37bb47040266edd09d2cab9469dad6ea3a6f Mon Sep 17 00:00:00 2001 From: Houkime <> Date: Mon, 21 Aug 2023 11:11:56 +0000 Subject: [PATCH 01/13] feature(backup): remember the reason for making a snapshot --- selfprivacy_api/backup/__init__.py | 15 ++-- selfprivacy_api/backup/backuppers/__init__.py | 8 +- .../backup/backuppers/none_backupper.py | 5 +- .../backup/backuppers/restic_backupper.py | 81 ++++++++++--------- selfprivacy_api/backup/tasks.py | 10 ++- .../graphql/common_types/backup.py | 7 ++ selfprivacy_api/models/backup/snapshot.py | 3 + tests/test_graphql/test_backup.py | 7 +- 8 files changed, 85 insertions(+), 51 deletions(-) diff --git a/selfprivacy_api/backup/__init__.py b/selfprivacy_api/backup/__init__.py index c28c01f..3b141fa 100644 --- a/selfprivacy_api/backup/__init__.py +++ b/selfprivacy_api/backup/__init__.py @@ -23,7 +23,7 @@ from selfprivacy_api.jobs import Jobs, JobStatus, Job from selfprivacy_api.graphql.queries.providers import ( BackupProvider as BackupProviderEnum, ) -from selfprivacy_api.graphql.common_types.backup import RestoreStrategy +from selfprivacy_api.graphql.common_types.backup import RestoreStrategy, BackupReason from selfprivacy_api.models.backup.snapshot import Snapshot @@ -264,10 +264,12 @@ class Backups: # Backup @staticmethod - def back_up(service: Service) -> Snapshot: + def back_up( + service: Service, reason: BackupReason = BackupReason.EXPLICIT + ) -> Snapshot: """The top-level function to back up a service""" folders = service.get_folders() - tag = service.get_id() + service_name = service.get_id() job = get_backup_job(service) if job is None: @@ -278,9 +280,10 @@ class Backups: service.pre_backup() snapshot = Backups.provider().backupper.start_backup( folders, - tag, + service_name, + reason=reason, ) - Backups._store_last_snapshot(tag, snapshot) + Backups._store_last_snapshot(service_name, snapshot) service.post_restore() except Exception as error: Jobs.update(job, status=JobStatus.ERROR) @@ -306,7 +309,7 @@ class Backups: snapshot: Snapshot, job: Job, ) -> None: - failsafe_snapshot = Backups.back_up(service) + failsafe_snapshot = Backups.back_up(service, BackupReason.PRE_RESTORE) Jobs.update(job, status=JobStatus.RUNNING) try: diff --git a/selfprivacy_api/backup/backuppers/__init__.py b/selfprivacy_api/backup/backuppers/__init__.py index ccf78b9..0067a41 100644 --- a/selfprivacy_api/backup/backuppers/__init__.py +++ b/selfprivacy_api/backup/backuppers/__init__.py @@ -2,6 +2,7 @@ from abc import ABC, abstractmethod from typing import List from selfprivacy_api.models.backup.snapshot import Snapshot +from selfprivacy_api.graphql.common_types.backup import BackupReason class AbstractBackupper(ABC): @@ -22,7 +23,12 @@ class AbstractBackupper(ABC): raise NotImplementedError @abstractmethod - def start_backup(self, folders: List[str], tag: str) -> Snapshot: + def start_backup( + self, + folders: List[str], + service_name: str, + reason: BackupReason = BackupReason.EXPLICIT, + ) -> Snapshot: """Start a backup of the given folders""" raise NotImplementedError diff --git a/selfprivacy_api/backup/backuppers/none_backupper.py b/selfprivacy_api/backup/backuppers/none_backupper.py index 3f9f7fd..429d9ab 100644 --- a/selfprivacy_api/backup/backuppers/none_backupper.py +++ b/selfprivacy_api/backup/backuppers/none_backupper.py @@ -2,6 +2,7 @@ from typing import List from selfprivacy_api.models.backup.snapshot import Snapshot from selfprivacy_api.backup.backuppers import AbstractBackupper +from selfprivacy_api.graphql.common_types.backup import BackupReason class NoneBackupper(AbstractBackupper): @@ -13,7 +14,9 @@ class NoneBackupper(AbstractBackupper): def set_creds(self, account: str, key: str, repo: str): pass - def start_backup(self, folders: List[str], tag: str): + def start_backup( + self, folders: List[str], tag: str, reason: BackupReason = BackupReason.EXPLICIT + ): raise NotImplementedError def get_snapshots(self) -> List[Snapshot]: diff --git a/selfprivacy_api/backup/backuppers/restic_backupper.py b/selfprivacy_api/backup/backuppers/restic_backupper.py index 3a5fc49..f5467ff 100644 --- a/selfprivacy_api/backup/backuppers/restic_backupper.py +++ b/selfprivacy_api/backup/backuppers/restic_backupper.py @@ -12,6 +12,7 @@ from os.path import exists, join from os import listdir from time import sleep +from selfprivacy_api.graphql.common_types.backup import BackupReason from selfprivacy_api.backup.util import output_yielder, sync from selfprivacy_api.backup.backuppers import AbstractBackupper from selfprivacy_api.models.backup.snapshot import Snapshot @@ -84,7 +85,7 @@ class ResticBackupper(AbstractBackupper): def _password_command(self): return f"echo {LocalBackupSecret.get()}" - def restic_command(self, *args, tag: str = "") -> List[str]: + def restic_command(self, *args, tags: List[str] = []) -> List[str]: command = [ "restic", "-o", @@ -94,13 +95,14 @@ class ResticBackupper(AbstractBackupper): "--password-command", self._password_command(), ] - if tag != "": - command.extend( - [ - "--tag", - tag, - ] - ) + if tags != []: + for tag in tags: + command.extend( + [ + "--tag", + tag, + ] + ) if args: command.extend(ResticBackupper.__flatten_list(args)) return command @@ -164,7 +166,12 @@ class ResticBackupper(AbstractBackupper): return result @unlocked_repo - def start_backup(self, folders: List[str], tag: str) -> Snapshot: + def start_backup( + self, + folders: List[str], + service_name: str, + reason: BackupReason = BackupReason.EXPLICIT, + ) -> Snapshot: """ Start backup with restic """ @@ -173,33 +180,35 @@ class ResticBackupper(AbstractBackupper): # of a string and an array of strings assert not isinstance(folders, str) + tags = [service_name, reason.value] + backup_command = self.restic_command( "backup", "--json", folders, - tag=tag, + tags=tags, ) - messages = [] - - service = get_service_by_id(tag) + service = get_service_by_id(service_name) if service is None: - raise ValueError("No service with id ", tag) - + raise ValueError("No service with id ", service_name) job = get_backup_job(service) + + messages = [] output = [] try: for raw_message in output_yielder(backup_command): output.append(raw_message) - message = self.parse_message( - raw_message, - job, - ) + message = self.parse_message(raw_message, job) messages.append(message) - return ResticBackupper._snapshot_from_backup_messages( - messages, - tag, + id = ResticBackupper._snapshot_id_from_backup_messages(messages) + return Snapshot( + created_at=datetime.datetime.now(datetime.timezone.utc), + id=id, + service_name=service_name, + reason=reason, ) + except ValueError as error: raise ValueError( "Could not create a snapshot: ", @@ -210,13 +219,13 @@ class ResticBackupper(AbstractBackupper): ) from error @staticmethod - def _snapshot_from_backup_messages(messages, repo_name) -> Snapshot: + def _snapshot_id_from_backup_messages(messages) -> Snapshot: for message in messages: if message["message_type"] == "summary": - return ResticBackupper._snapshot_from_fresh_summary( - message, - repo_name, - ) + # There is a discrepancy between versions of restic/rclone + # Some report short_id in this field and some full + return message["snapshot_id"][0:SHORT_ID_LEN] + raise ValueError("no summary message in restic json output") def parse_message(self, raw_message_line: str, job=None) -> dict: @@ -232,16 +241,6 @@ class ResticBackupper(AbstractBackupper): ) return message - @staticmethod - def _snapshot_from_fresh_summary(message: dict, repo_name) -> Snapshot: - return Snapshot( - # There is a discrepancy between versions of restic/rclone - # Some report short_id in this field and some full - id=message["snapshot_id"][0:SHORT_ID_LEN], - created_at=datetime.datetime.now(datetime.timezone.utc), - service_name=repo_name, - ) - def init(self) -> None: init_command = self.restic_command( "init", @@ -475,11 +474,19 @@ class ResticBackupper(AbstractBackupper): def get_snapshots(self) -> List[Snapshot]: """Get all snapshots from the repo""" snapshots = [] + for restic_snapshot in self._load_snapshots(): + # Compatibility with previous snaps: + if len(restic_snapshot["tags"]) == 1: + reason = BackupReason.EXPLICIT + else: + reason = restic_snapshot["tags"][1] + snapshot = Snapshot( id=restic_snapshot["short_id"], created_at=restic_snapshot["time"], service_name=restic_snapshot["tags"][0], + reason=reason, ) snapshots.append(snapshot) diff --git a/selfprivacy_api/backup/tasks.py b/selfprivacy_api/backup/tasks.py index db350d4..546b27c 100644 --- a/selfprivacy_api/backup/tasks.py +++ b/selfprivacy_api/backup/tasks.py @@ -3,7 +3,7 @@ The tasks module contains the worker tasks that are used to back up and restore """ from datetime import datetime, timezone -from selfprivacy_api.graphql.common_types.backup import RestoreStrategy +from selfprivacy_api.graphql.common_types.backup import RestoreStrategy, BackupReason from selfprivacy_api.models.backup.snapshot import Snapshot from selfprivacy_api.utils.huey import huey @@ -22,11 +22,13 @@ def validate_datetime(dt: datetime) -> bool: # huey tasks need to return something @huey.task() -def start_backup(service: Service) -> bool: +def start_backup( + service: Service, reason: BackupReason = BackupReason.EXPLICIT +) -> bool: """ The worker task that starts the backup process. """ - Backups.back_up(service) + Backups.back_up(service, reason) return True @@ -49,4 +51,4 @@ def automatic_backup(): """ time = datetime.utcnow().replace(tzinfo=timezone.utc) for service in Backups.services_to_back_up(time): - start_backup(service) + start_backup(service, BackupReason.AUTO) diff --git a/selfprivacy_api/graphql/common_types/backup.py b/selfprivacy_api/graphql/common_types/backup.py index 992363b..9eaef12 100644 --- a/selfprivacy_api/graphql/common_types/backup.py +++ b/selfprivacy_api/graphql/common_types/backup.py @@ -8,3 +8,10 @@ from enum import Enum class RestoreStrategy(Enum): INPLACE = "INPLACE" DOWNLOAD_VERIFY_OVERWRITE = "DOWNLOAD_VERIFY_OVERWRITE" + + +@strawberry.enum +class BackupReason(Enum): + EXPLICIT = "EXPLICIT" + AUTO = "AUTO" + PRE_RESTORE = "PRE_RESTORE" diff --git a/selfprivacy_api/models/backup/snapshot.py b/selfprivacy_api/models/backup/snapshot.py index 9893f03..28ad661 100644 --- a/selfprivacy_api/models/backup/snapshot.py +++ b/selfprivacy_api/models/backup/snapshot.py @@ -1,8 +1,11 @@ import datetime from pydantic import BaseModel +from selfprivacy_api.graphql.common_types.backup import BackupReason + class Snapshot(BaseModel): id: str service_name: str created_at: datetime.datetime + reason: BackupReason diff --git a/tests/test_graphql/test_backup.py b/tests/test_graphql/test_backup.py index 1990ef7..fdb8497 100644 --- a/tests/test_graphql/test_backup.py +++ b/tests/test_graphql/test_backup.py @@ -14,7 +14,7 @@ from selfprivacy_api.services import Service, get_all_services from selfprivacy_api.services import get_service_by_id from selfprivacy_api.services.test_service import DummyService from selfprivacy_api.graphql.queries.providers import BackupProvider -from selfprivacy_api.graphql.common_types.backup import RestoreStrategy +from selfprivacy_api.graphql.common_types.backup import RestoreStrategy, BackupReason from selfprivacy_api.jobs import Jobs, JobStatus from selfprivacy_api.models.backup.snapshot import Snapshot @@ -428,7 +428,10 @@ def test_forget_snapshot(backups, dummy_service): def test_forget_nonexistent_snapshot(backups, dummy_service): bogus = Snapshot( - id="gibberjibber", service_name="nohoho", created_at=datetime.now(timezone.utc) + id="gibberjibber", + service_name="nohoho", + created_at=datetime.now(timezone.utc), + reason=BackupReason.EXPLICIT, ) with pytest.raises(ValueError): Backups.forget_snapshot(bogus) From 1b9761293cea920a3e39ce606173412eba30758f Mon Sep 17 00:00:00 2001 From: Houkime <> Date: Mon, 21 Aug 2023 11:30:35 +0000 Subject: [PATCH 02/13] test(backup): test reasons --- tests/test_graphql/test_backup.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/tests/test_graphql/test_backup.py b/tests/test_graphql/test_backup.py index fdb8497..16933b8 100644 --- a/tests/test_graphql/test_backup.py +++ b/tests/test_graphql/test_backup.py @@ -286,6 +286,16 @@ def test_backup_returns_snapshot(backups, dummy_service): assert Backups.get_snapshot_by_id(snapshot.id) is not None assert snapshot.service_name == name assert snapshot.created_at is not None + assert snapshot.reason == BackupReason.EXPLICIT + + +def test_backup_reasons(backups, dummy_service): + snap = Backups.back_up(dummy_service, BackupReason.AUTO) + assert snap.reason == BackupReason.AUTO + + Backups.force_snapshot_cache_reload() + snaps = Backups.get_snapshots(dummy_service) + assert snaps[0].reason == BackupReason.AUTO def folder_files(folder): @@ -495,6 +505,8 @@ def test_restore_snapshot_task( snaps = Backups.get_snapshots(dummy_service) if restore_strategy == RestoreStrategy.INPLACE: assert len(snaps) == 2 + reasons = [snap.reason for snap in snaps] + assert BackupReason.PRE_RESTORE in reasons else: assert len(snaps) == 1 From 30b62c351aba37c60edb81aa8f219c1dad2fd6a4 Mon Sep 17 00:00:00 2001 From: Houkime <> Date: Mon, 21 Aug 2023 11:31:29 +0000 Subject: [PATCH 03/13] feature(redis): compatibility with str enums --- selfprivacy_api/utils/redis_model_storage.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/selfprivacy_api/utils/redis_model_storage.py b/selfprivacy_api/utils/redis_model_storage.py index 51faff7..06dfe8c 100644 --- a/selfprivacy_api/utils/redis_model_storage.py +++ b/selfprivacy_api/utils/redis_model_storage.py @@ -1,11 +1,14 @@ from datetime import datetime from typing import Optional +from enum import Enum def store_model_as_hash(redis, redis_key, model): for key, value in model.dict().items(): if isinstance(value, datetime): value = value.isoformat() + if isinstance(value, Enum): + value = value.value redis.hset(redis_key, key, str(value)) From b2c7e8b73a3cf59ddd67206da6314e120f3de7ee Mon Sep 17 00:00:00 2001 From: Houkime <> Date: Mon, 21 Aug 2023 12:45:31 +0000 Subject: [PATCH 04/13] feature(backups): caps for autobackups --- selfprivacy_api/backup/__init__.py | 37 ++++++++++++++++++++++++++++++ selfprivacy_api/backup/storage.py | 13 +++++++++++ tests/test_graphql/test_backup.py | 25 ++++++++++++++++++++ 3 files changed, 75 insertions(+) diff --git a/selfprivacy_api/backup/__init__.py b/selfprivacy_api/backup/__init__.py index 3b141fa..b16f089 100644 --- a/selfprivacy_api/backup/__init__.py +++ b/selfprivacy_api/backup/__init__.py @@ -283,7 +283,10 @@ class Backups: service_name, reason=reason, ) + Backups._store_last_snapshot(service_name, snapshot) + if reason == BackupReason.AUTO: + Backups._prune_auto_snaps(service) service.post_restore() except Exception as error: Jobs.update(job, status=JobStatus.ERROR) @@ -292,6 +295,40 @@ class Backups: Jobs.update(job, status=JobStatus.FINISHED) return snapshot + @staticmethod + def _auto_snaps(service): + return [ + snap + for snap in Backups.get_snapshots(service) + if snap.reason == BackupReason.AUTO + ] + + @staticmethod + def _prune_auto_snaps(service) -> None: + max = Backups.max_auto_snapshots() + if max == -1: + return + + auto_snaps = Backups._auto_snaps(service) + if len(auto_snaps) > max: + n_to_kill = len(auto_snaps) - max + sorted_snaps = sorted(auto_snaps, key=lambda s: s.created_at) + snaps_to_kill = sorted_snaps[:n_to_kill] + for snap in snaps_to_kill: + Backups.forget_snapshot(snap) + + @staticmethod + def set_max_auto_snapshots(value: int) -> None: + """everything <=0 means unlimited""" + if value <= 0: + value = -1 + Storage.set_max_auto_snapshots(value) + + @staticmethod + def max_auto_snapshots() -> int: + """-1 means unlimited""" + return Storage.max_auto_snapshots() + # Restoring @staticmethod diff --git a/selfprivacy_api/backup/storage.py b/selfprivacy_api/backup/storage.py index d46f584..1a0091f 100644 --- a/selfprivacy_api/backup/storage.py +++ b/selfprivacy_api/backup/storage.py @@ -26,6 +26,7 @@ REDIS_INITTED_CACHE = "backups:repo_initted" REDIS_PROVIDER_KEY = "backups:provider" REDIS_AUTOBACKUP_PERIOD_KEY = "backups:autobackup_period" +REDIS_AUTOBACKUP_MAX_KEY = "backups:autobackup_cap" redis = RedisPool().get_connection() @@ -39,6 +40,7 @@ class Storage: redis.delete(REDIS_PROVIDER_KEY) redis.delete(REDIS_AUTOBACKUP_PERIOD_KEY) redis.delete(REDIS_INITTED_CACHE) + redis.delete(REDIS_AUTOBACKUP_MAX_KEY) prefixes_to_clean = [ REDIS_SNAPSHOTS_PREFIX, @@ -175,3 +177,14 @@ class Storage: def mark_as_uninitted(): """Marks the repository as initialized""" redis.delete(REDIS_INITTED_CACHE) + + @staticmethod + def set_max_auto_snapshots(value: int): + redis.set(REDIS_AUTOBACKUP_MAX_KEY, value) + + @staticmethod + def max_auto_snapshots(): + if redis.exists(REDIS_AUTOBACKUP_MAX_KEY): + return int(redis.get(REDIS_AUTOBACKUP_MAX_KEY)) + else: + return -1 diff --git a/tests/test_graphql/test_backup.py b/tests/test_graphql/test_backup.py index 16933b8..781468a 100644 --- a/tests/test_graphql/test_backup.py +++ b/tests/test_graphql/test_backup.py @@ -298,6 +298,31 @@ def test_backup_reasons(backups, dummy_service): assert snaps[0].reason == BackupReason.AUTO +def test_too_many_auto(backups, dummy_service): + assert Backups.max_auto_snapshots() == -1 + Backups.set_max_auto_snapshots(2) + assert Backups.max_auto_snapshots() == 2 + + snap = Backups.back_up(dummy_service, BackupReason.AUTO) + assert len(Backups.get_snapshots(dummy_service)) == 1 + snap2 = Backups.back_up(dummy_service, BackupReason.AUTO) + assert len(Backups.get_snapshots(dummy_service)) == 2 + snap3 = Backups.back_up(dummy_service, BackupReason.AUTO) + assert len(Backups.get_snapshots(dummy_service)) == 2 + + snaps = Backups.get_snapshots(dummy_service) + + assert snap2 in snaps + assert snap3 in snaps + assert snap not in snaps + + Backups.set_max_auto_snapshots(-1) + snap4 = Backups.back_up(dummy_service, BackupReason.AUTO) + snaps = Backups.get_snapshots(dummy_service) + assert len(snaps) == 3 + assert snap4 in snaps + + def folder_files(folder): return [ path.join(folder, filename) From 9207f5385ca2f23f4ef240c80246499930d0afd9 Mon Sep 17 00:00:00 2001 From: Houkime <> Date: Mon, 28 Aug 2023 17:02:45 +0000 Subject: [PATCH 05/13] feature(backups): actual finegrained quotas --- selfprivacy_api/backup/__init__.py | 97 +++++++- selfprivacy_api/backup/storage.py | 26 ++ .../graphql/common_types/backup.py | 14 ++ tests/test_graphql/test_backup.py | 232 ++++++++++++++++-- 4 files changed, 340 insertions(+), 29 deletions(-) diff --git a/selfprivacy_api/backup/__init__.py b/selfprivacy_api/backup/__init__.py index b16f089..73f74a9 100644 --- a/selfprivacy_api/backup/__init__.py +++ b/selfprivacy_api/backup/__init__.py @@ -23,7 +23,18 @@ from selfprivacy_api.jobs import Jobs, JobStatus, Job from selfprivacy_api.graphql.queries.providers import ( BackupProvider as BackupProviderEnum, ) -from selfprivacy_api.graphql.common_types.backup import RestoreStrategy, BackupReason +from selfprivacy_api.graphql.common_types.backup import ( + RestoreStrategy, + BackupReason, + AutobackupQuotas, +) +from selfprivacy_api.backup.time import ( + same_day, + same_month, + same_week, + same_year, + same_lifetime_of_the_universe, +) from selfprivacy_api.models.backup.snapshot import Snapshot @@ -303,20 +314,88 @@ class Backups: if snap.reason == BackupReason.AUTO ] + @staticmethod + def add_snap_but_with_quotas( + new_snap: Snapshot, snaps: List[Snapshot], quotas: AutobackupQuotas + ) -> None: + quotas_map = { + same_day: quotas.daily, + same_week: quotas.weekly, + same_month: quotas.monthly, + same_year: quotas.yearly, + same_lifetime_of_the_universe: quotas.total, + } + + snaps.append(new_snap) + + for is_same_period, quota in quotas_map.items(): + if quota <= 0: + continue + + cohort = [ + snap + for snap in snaps + if is_same_period(snap.created_at, new_snap.created_at) + ] + sorted_cohort = sorted(cohort, key=lambda s: s.created_at) + n_to_kill = len(cohort) - quota + if n_to_kill > 0: + snaps_to_kill = sorted_cohort[:n_to_kill] + for snap in snaps_to_kill: + snaps.remove(snap) + + @staticmethod + def _prune_snaps_with_quotas(snapshots: List[Snapshot]) -> List[Snapshot]: + # Function broken out for testability + sorted_snaps = sorted(snapshots, key=lambda s: s.created_at) + quotas = Backups.autobackup_quotas() + + new_snaplist: List[Snapshot] = [] + for snap in sorted_snaps: + Backups.add_snap_but_with_quotas(snap, new_snaplist, quotas) + + return new_snaplist + @staticmethod def _prune_auto_snaps(service) -> None: - max = Backups.max_auto_snapshots() - if max == -1: - return + # Not very testable by itself, so most testing is going on Backups._prune_snaps_with_quotas + # We can still test total limits and, say, daily limits auto_snaps = Backups._auto_snaps(service) - if len(auto_snaps) > max: - n_to_kill = len(auto_snaps) - max - sorted_snaps = sorted(auto_snaps, key=lambda s: s.created_at) - snaps_to_kill = sorted_snaps[:n_to_kill] - for snap in snaps_to_kill: + new_snaplist = Backups._prune_snaps_with_quotas(auto_snaps) + + # TODO: Can be optimized since there is forgetting of an array in one restic op + # but most of the time this will be only one snap to forget. + for snap in auto_snaps: + if snap not in new_snaplist: Backups.forget_snapshot(snap) + @staticmethod + def _standardize_quotas(i: int) -> int: + if i <= 0: + i = -1 + return i + + @staticmethod + def autobackup_quotas() -> AutobackupQuotas: + """everything <=0 means unlimited""" + + return Storage.autobackup_quotas() + + @staticmethod + def set_autobackup_quotas(quotas: AutobackupQuotas) -> None: + """everything <=0 means unlimited""" + + Storage.set_autobackup_quotas( + AutobackupQuotas( + daily=Backups._standardize_quotas(quotas.daily), + weekly=Backups._standardize_quotas(quotas.weekly), + monthly=Backups._standardize_quotas(quotas.monthly), + yearly=Backups._standardize_quotas(quotas.yearly), + total=Backups._standardize_quotas(quotas.total), + ) + ) + @staticmethod def set_max_auto_snapshots(value: int) -> None: """everything <=0 means unlimited""" diff --git a/selfprivacy_api/backup/storage.py b/selfprivacy_api/backup/storage.py index 1a0091f..38fc3a2 100644 --- a/selfprivacy_api/backup/storage.py +++ b/selfprivacy_api/backup/storage.py @@ -6,6 +6,10 @@ from datetime import datetime from selfprivacy_api.models.backup.snapshot import Snapshot from selfprivacy_api.models.backup.provider import BackupProviderModel +from selfprivacy_api.graphql.common_types.backup import ( + AutobackupQuotas, + _AutobackupQuotas, +) from selfprivacy_api.utils.redis_pool import RedisPool from selfprivacy_api.utils.redis_model_storage import ( @@ -27,6 +31,7 @@ REDIS_PROVIDER_KEY = "backups:provider" REDIS_AUTOBACKUP_PERIOD_KEY = "backups:autobackup_period" REDIS_AUTOBACKUP_MAX_KEY = "backups:autobackup_cap" +REDIS_AUTOBACKUP_QUOTAS_KEY = "backups:autobackup_quotas_key" redis = RedisPool().get_connection() @@ -41,6 +46,7 @@ class Storage: redis.delete(REDIS_AUTOBACKUP_PERIOD_KEY) redis.delete(REDIS_INITTED_CACHE) redis.delete(REDIS_AUTOBACKUP_MAX_KEY) + redis.delete(REDIS_AUTOBACKUP_QUOTAS_KEY) prefixes_to_clean = [ REDIS_SNAPSHOTS_PREFIX, @@ -178,6 +184,26 @@ class Storage: """Marks the repository as initialized""" redis.delete(REDIS_INITTED_CACHE) + @staticmethod + def set_autobackup_quotas(quotas: AutobackupQuotas) -> None: + store_model_as_hash(redis, REDIS_AUTOBACKUP_QUOTAS_KEY, quotas.to_pydantic()) + + @staticmethod + def autobackup_quotas() -> AutobackupQuotas: + quotas_model = hash_as_model( + redis, REDIS_AUTOBACKUP_QUOTAS_KEY, _AutobackupQuotas + ) + if quotas_model is None: + unlimited_quotas = AutobackupQuotas( + daily=-1, + weekly=-1, + monthly=-1, + yearly=-1, + total=-1, + ) + return unlimited_quotas + return AutobackupQuotas.from_pydantic(quotas_model) + @staticmethod def set_max_auto_snapshots(value: int): redis.set(REDIS_AUTOBACKUP_MAX_KEY, value) diff --git a/selfprivacy_api/graphql/common_types/backup.py b/selfprivacy_api/graphql/common_types/backup.py index 9eaef12..3d5b5aa 100644 --- a/selfprivacy_api/graphql/common_types/backup.py +++ b/selfprivacy_api/graphql/common_types/backup.py @@ -2,6 +2,7 @@ # pylint: disable=too-few-public-methods import strawberry from enum import Enum +from pydantic import BaseModel @strawberry.enum @@ -15,3 +16,16 @@ class BackupReason(Enum): EXPLICIT = "EXPLICIT" AUTO = "AUTO" PRE_RESTORE = "PRE_RESTORE" + + +class _AutobackupQuotas(BaseModel): + daily: int + weekly: int + monthly: int + yearly: int + total: int + + +@strawberry.experimental.pydantic.type(model=_AutobackupQuotas, all_fields=True) +class AutobackupQuotas: + pass diff --git a/tests/test_graphql/test_backup.py b/tests/test_graphql/test_backup.py index 781468a..3314597 100644 --- a/tests/test_graphql/test_backup.py +++ b/tests/test_graphql/test_backup.py @@ -5,8 +5,12 @@ from os import makedirs from os import remove from os import listdir from os import urandom -from datetime import datetime, timedelta, timezone +from datetime import datetime, timedelta, timezone, date, time from subprocess import Popen +from copy import copy + +import secrets + import selfprivacy_api.services as services from selfprivacy_api.services import Service, get_all_services @@ -19,6 +23,8 @@ from selfprivacy_api.jobs import Jobs, JobStatus from selfprivacy_api.models.backup.snapshot import Snapshot +from selfprivacy_api.graphql.common_types.backup import AutobackupQuotas + from selfprivacy_api.backup import Backups, BACKUP_PROVIDER_ENVS import selfprivacy_api.backup.providers as providers from selfprivacy_api.backup.providers import AbstractBackupProvider @@ -298,29 +304,215 @@ def test_backup_reasons(backups, dummy_service): assert snaps[0].reason == BackupReason.AUTO -def test_too_many_auto(backups, dummy_service): - assert Backups.max_auto_snapshots() == -1 - Backups.set_max_auto_snapshots(2) - assert Backups.max_auto_snapshots() == 2 +unlimited_quotas = AutobackupQuotas( + daily=-1, + weekly=-1, + monthly=-1, + yearly=-1, + total=-1, +) - snap = Backups.back_up(dummy_service, BackupReason.AUTO) - assert len(Backups.get_snapshots(dummy_service)) == 1 - snap2 = Backups.back_up(dummy_service, BackupReason.AUTO) - assert len(Backups.get_snapshots(dummy_service)) == 2 - snap3 = Backups.back_up(dummy_service, BackupReason.AUTO) - assert len(Backups.get_snapshots(dummy_service)) == 2 - snaps = Backups.get_snapshots(dummy_service) +def test_get_empty_quotas(backups): + quotas = Backups.autobackup_quotas() + assert quotas is not None + assert quotas == unlimited_quotas - assert snap2 in snaps - assert snap3 in snaps - assert snap not in snaps - Backups.set_max_auto_snapshots(-1) - snap4 = Backups.back_up(dummy_service, BackupReason.AUTO) - snaps = Backups.get_snapshots(dummy_service) - assert len(snaps) == 3 - assert snap4 in snaps +def test_set_quotas(backups): + quotas = AutobackupQuotas( + daily=2343, + weekly=343, + monthly=0, + yearly=-34556, + total=563, + ) + Backups.set_autobackup_quotas(quotas) + assert Backups.autobackup_quotas() == AutobackupQuotas( + daily=2343, + weekly=343, + monthly=-1, + yearly=-1, + total=563, + ) + + +def dummy_snapshot(date: datetime): + return Snapshot( + id=str(hash(date)), + service_name="someservice", + created_at=date, + reason=BackupReason.EXPLICIT, + ) + + +def test_autobackup_snapshots_pruning(backups): + # Wednesday, fourth week + now = datetime(year=2023, month=1, day=25, hour=10) + + snaps = [ + dummy_snapshot(now - timedelta(days=365 * 2)), + dummy_snapshot(now - timedelta(days=20)), + dummy_snapshot(now - timedelta(days=2)), + dummy_snapshot(now - timedelta(days=1, hours=3)), + dummy_snapshot(now - timedelta(days=1, hours=2)), + dummy_snapshot(now - timedelta(days=1)), + dummy_snapshot(now - timedelta(hours=2)), + dummy_snapshot(now - timedelta(minutes=5)), + dummy_snapshot(now), + ] + old_len = len(snaps) + + quotas = copy(unlimited_quotas) + Backups.set_autobackup_quotas(quotas) + assert Backups._prune_snaps_with_quotas(snaps) == snaps + + quotas = copy(unlimited_quotas) + quotas.daily = 2 + Backups.set_autobackup_quotas(quotas) + + pruned_snaps = Backups._prune_snaps_with_quotas(snaps) + assert pruned_snaps == [ + dummy_snapshot(now - timedelta(days=365 * 2)), + dummy_snapshot(now - timedelta(days=20)), + dummy_snapshot(now - timedelta(days=2)), + dummy_snapshot(now - timedelta(days=1, hours=2)), + dummy_snapshot(now - timedelta(days=1)), + dummy_snapshot(now - timedelta(minutes=5)), + dummy_snapshot(now), + ] + + # checking that this function does not mutate the argument + assert snaps != pruned_snaps + assert len(snaps) == old_len + + quotas = copy(unlimited_quotas) + quotas.weekly = 4 + Backups.set_autobackup_quotas(quotas) + + pruned_snaps = Backups._prune_snaps_with_quotas(snaps) + assert pruned_snaps == [ + dummy_snapshot(now - timedelta(days=365 * 2)), + dummy_snapshot(now - timedelta(days=20)), + dummy_snapshot(now - timedelta(days=1)), + dummy_snapshot(now - timedelta(hours=2)), + dummy_snapshot(now - timedelta(minutes=5)), + dummy_snapshot(now), + ] + + quotas = copy(unlimited_quotas) + quotas.monthly = 7 + Backups.set_autobackup_quotas(quotas) + + pruned_snaps = Backups._prune_snaps_with_quotas(snaps) + assert pruned_snaps == [ + dummy_snapshot(now - timedelta(days=365 * 2)), + dummy_snapshot(now - timedelta(days=2)), + dummy_snapshot(now - timedelta(days=1, hours=3)), + dummy_snapshot(now - timedelta(days=1, hours=2)), + dummy_snapshot(now - timedelta(days=1)), + dummy_snapshot(now - timedelta(hours=2)), + dummy_snapshot(now - timedelta(minutes=5)), + dummy_snapshot(now), + ] + + +def test_autobackup_snapshots_pruning_yearly(backups): + snaps = [ + dummy_snapshot(datetime(year=2023, month=2, day=1)), + dummy_snapshot(datetime(year=2023, month=3, day=1)), + dummy_snapshot(datetime(year=2023, month=4, day=1)), + dummy_snapshot(datetime(year=2055, month=3, day=1)), + ] + quotas = copy(unlimited_quotas) + quotas.yearly = 2 + Backups.set_autobackup_quotas(quotas) + + pruned_snaps = Backups._prune_snaps_with_quotas(snaps) + assert pruned_snaps == [ + dummy_snapshot(datetime(year=2023, month=3, day=1)), + dummy_snapshot(datetime(year=2023, month=4, day=1)), + dummy_snapshot(datetime(year=2055, month=3, day=1)), + ] + + +def test_autobackup_snapshots_pruning_bottleneck(backups): + now = datetime(year=2023, month=1, day=25, hour=10) + snaps = [ + dummy_snapshot(now - timedelta(hours=4)), + dummy_snapshot(now - timedelta(hours=3)), + dummy_snapshot(now - timedelta(hours=2)), + dummy_snapshot(now - timedelta(minutes=5)), + dummy_snapshot(now), + ] + + yearly_quota = copy(unlimited_quotas) + yearly_quota.yearly = 2 + + monthly_quota = copy(unlimited_quotas) + monthly_quota.monthly = 2 + + weekly_quota = copy(unlimited_quotas) + weekly_quota.weekly = 2 + + daily_quota = copy(unlimited_quotas) + daily_quota.daily = 2 + + total_quota = copy(unlimited_quotas) + total_quota.total = 2 + + for quota in [total_quota, yearly_quota, monthly_quota, weekly_quota, daily_quota]: + Backups.set_autobackup_quotas(quota) + pruned_snaps = Backups._prune_snaps_with_quotas(snaps) + assert pruned_snaps == [ + dummy_snapshot(now - timedelta(minutes=5)), + dummy_snapshot(now), + ] + + +def test_autobackup_snapshots_pruning_edgeweek(backups): + # jan 1 2023 is Sunday + snaps = [ + dummy_snapshot(datetime(year=2022, month=12, day=30)), + dummy_snapshot(datetime(year=2022, month=12, day=31)), + dummy_snapshot(datetime(year=2023, month=1, day=1)), + dummy_snapshot(datetime(year=2023, month=1, day=6)), + ] + quotas = copy(unlimited_quotas) + quotas.weekly = 2 + Backups.set_autobackup_quotas(quotas) + + pruned_snaps = Backups._prune_snaps_with_quotas(snaps) + assert pruned_snaps == [ + dummy_snapshot(datetime(year=2022, month=12, day=31)), + dummy_snapshot(datetime(year=2023, month=1, day=1)), + dummy_snapshot(datetime(year=2023, month=1, day=6)), + ] + + +# def test_too_many_auto(backups, dummy_service): +# assert Backups.autobackup_quotas() +# Backups.set_max_auto_snapshots(2) +# assert Backups.max_auto_snapshots() == 2 + +# snap = Backups.back_up(dummy_service, BackupReason.AUTO) +# assert len(Backups.get_snapshots(dummy_service)) == 1 +# snap2 = Backups.back_up(dummy_service, BackupReason.AUTO) +# assert len(Backups.get_snapshots(dummy_service)) == 2 +# snap3 = Backups.back_up(dummy_service, BackupReason.AUTO) +# assert len(Backups.get_snapshots(dummy_service)) == 2 + +# snaps = Backups.get_snapshots(dummy_service) + +# assert snap2 in snaps +# assert snap3 in snaps +# assert snap not in snaps + +# Backups.set_max_auto_snapshots(-1) +# snap4 = Backups.back_up(dummy_service, BackupReason.AUTO) +# snaps = Backups.get_snapshots(dummy_service) +# assert len(snaps) == 3 +# assert snap4 in snaps def folder_files(folder): From a75a102df6cb2846bde8ad9e013dab8f06648c28 Mon Sep 17 00:00:00 2001 From: Houkime <> Date: Mon, 28 Aug 2023 17:15:27 +0000 Subject: [PATCH 06/13] test(backups): test quotas with actual backups --- tests/test_graphql/test_backup.py | 58 +++++++++++++++++++++---------- 1 file changed, 39 insertions(+), 19 deletions(-) diff --git a/tests/test_graphql/test_backup.py b/tests/test_graphql/test_backup.py index 3314597..550c56b 100644 --- a/tests/test_graphql/test_backup.py +++ b/tests/test_graphql/test_backup.py @@ -490,29 +490,49 @@ def test_autobackup_snapshots_pruning_edgeweek(backups): ] -# def test_too_many_auto(backups, dummy_service): -# assert Backups.autobackup_quotas() -# Backups.set_max_auto_snapshots(2) -# assert Backups.max_auto_snapshots() == 2 +def test_too_many_auto(backups, dummy_service): + assert Backups.autobackup_quotas() + quota = copy(unlimited_quotas) + quota.total = 2 + Backups.set_autobackup_quotas(quota) + assert Backups.autobackup_quotas().total == 2 -# snap = Backups.back_up(dummy_service, BackupReason.AUTO) -# assert len(Backups.get_snapshots(dummy_service)) == 1 -# snap2 = Backups.back_up(dummy_service, BackupReason.AUTO) -# assert len(Backups.get_snapshots(dummy_service)) == 2 -# snap3 = Backups.back_up(dummy_service, BackupReason.AUTO) -# assert len(Backups.get_snapshots(dummy_service)) == 2 + snap = Backups.back_up(dummy_service, BackupReason.AUTO) + assert len(Backups.get_snapshots(dummy_service)) == 1 + snap2 = Backups.back_up(dummy_service, BackupReason.AUTO) + assert len(Backups.get_snapshots(dummy_service)) == 2 + snap3 = Backups.back_up(dummy_service, BackupReason.AUTO) + assert len(Backups.get_snapshots(dummy_service)) == 2 -# snaps = Backups.get_snapshots(dummy_service) + snaps = Backups.get_snapshots(dummy_service) + assert snap2 in snaps + assert snap3 in snaps + assert snap not in snaps -# assert snap2 in snaps -# assert snap3 in snaps -# assert snap not in snaps + quota.total = -1 + Backups.set_autobackup_quotas(quota) + snap4 = Backups.back_up(dummy_service, BackupReason.AUTO) -# Backups.set_max_auto_snapshots(-1) -# snap4 = Backups.back_up(dummy_service, BackupReason.AUTO) -# snaps = Backups.get_snapshots(dummy_service) -# assert len(snaps) == 3 -# assert snap4 in snaps + snaps = Backups.get_snapshots(dummy_service) + assert len(snaps) == 3 + assert snap4 in snaps + + # Retroactivity + quota.total = 1 + Backups.set_autobackup_quotas(quota) + snap5 = Backups.back_up(dummy_service, BackupReason.AUTO) + + snaps = Backups.get_snapshots(dummy_service) + assert len(snaps) == 1 + assert snap5 in snaps + + # Explicit snaps are not affected + snap6 = Backups.back_up(dummy_service, BackupReason.EXPLICIT) + + snaps = Backups.get_snapshots(dummy_service) + assert len(snaps) == 2 + assert snap5 in snaps + assert snap6 in snaps def folder_files(folder): From 1fc47b049daa262637c244e785de4e792fa2dd06 Mon Sep 17 00:00:00 2001 From: Houkime <> Date: Mon, 28 Aug 2023 17:23:21 +0000 Subject: [PATCH 07/13] refactor(backups): clean up caps code --- selfprivacy_api/backup/__init__.py | 12 ------------ selfprivacy_api/backup/storage.py | 13 ------------- 2 files changed, 25 deletions(-) diff --git a/selfprivacy_api/backup/__init__.py b/selfprivacy_api/backup/__init__.py index 73f74a9..336b705 100644 --- a/selfprivacy_api/backup/__init__.py +++ b/selfprivacy_api/backup/__init__.py @@ -396,18 +396,6 @@ class Backups: ) ) - @staticmethod - def set_max_auto_snapshots(value: int) -> None: - """everything <=0 means unlimited""" - if value <= 0: - value = -1 - Storage.set_max_auto_snapshots(value) - - @staticmethod - def max_auto_snapshots() -> int: - """-1 means unlimited""" - return Storage.max_auto_snapshots() - # Restoring @staticmethod diff --git a/selfprivacy_api/backup/storage.py b/selfprivacy_api/backup/storage.py index 38fc3a2..86b92f3 100644 --- a/selfprivacy_api/backup/storage.py +++ b/selfprivacy_api/backup/storage.py @@ -30,7 +30,6 @@ REDIS_INITTED_CACHE = "backups:repo_initted" REDIS_PROVIDER_KEY = "backups:provider" REDIS_AUTOBACKUP_PERIOD_KEY = "backups:autobackup_period" -REDIS_AUTOBACKUP_MAX_KEY = "backups:autobackup_cap" REDIS_AUTOBACKUP_QUOTAS_KEY = "backups:autobackup_quotas_key" redis = RedisPool().get_connection() @@ -45,7 +44,6 @@ class Storage: redis.delete(REDIS_PROVIDER_KEY) redis.delete(REDIS_AUTOBACKUP_PERIOD_KEY) redis.delete(REDIS_INITTED_CACHE) - redis.delete(REDIS_AUTOBACKUP_MAX_KEY) redis.delete(REDIS_AUTOBACKUP_QUOTAS_KEY) prefixes_to_clean = [ @@ -203,14 +201,3 @@ class Storage: ) return unlimited_quotas return AutobackupQuotas.from_pydantic(quotas_model) - - @staticmethod - def set_max_auto_snapshots(value: int): - redis.set(REDIS_AUTOBACKUP_MAX_KEY, value) - - @staticmethod - def max_auto_snapshots(): - if redis.exists(REDIS_AUTOBACKUP_MAX_KEY): - return int(redis.get(REDIS_AUTOBACKUP_MAX_KEY)) - else: - return -1 From 0c04975ea4013d39ba60f98eee8894e2cc1f2bbb Mon Sep 17 00:00:00 2001 From: Houkime <> Date: Mon, 28 Aug 2023 17:24:20 +0000 Subject: [PATCH 08/13] flx(backups): commit forgotten time.py --- selfprivacy_api/backup/time.py | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) create mode 100644 selfprivacy_api/backup/time.py diff --git a/selfprivacy_api/backup/time.py b/selfprivacy_api/backup/time.py new file mode 100644 index 0000000..aba12bd --- /dev/null +++ b/selfprivacy_api/backup/time.py @@ -0,0 +1,28 @@ +from datetime import datetime, timedelta, time + + +def same_day(a: datetime, b: datetime) -> bool: + return a.date() == b.date() + + +def same_week(a: datetime, b: datetime) -> bool: + # doing the hard way because weeks traverse the edges of years + zerobased_weekday = a.isoweekday() - 1 + start_of_day = datetime.combine(a.date(), time.min) + start_of_week = start_of_day - timedelta(days=zerobased_weekday) + end_of_week = start_of_week + timedelta(days=7) + + if b >= start_of_week and b <= end_of_week: + return True + return False + + +def same_month(a: datetime, b: datetime) -> bool: + return a.month == b.month and a.year == b.year + + +def same_year(a: datetime, b: datetime) -> bool: + return a.year == b.year + +def same_lifetime_of_the_universe(a: datetime, b: datetime) -> bool: + return True From 9fdc536f9fa5fce702b1fea0133ce0f73c497dcf Mon Sep 17 00:00:00 2001 From: Houkime <> Date: Mon, 28 Aug 2023 18:24:29 +0000 Subject: [PATCH 09/13] BROKEN(backups): hooking up quotas to API fails. AutobackupQuotas needs to be an input type, but if input type, it fails because it needs to be an Output type, which is not documented --- .../graphql/mutations/backup_mutations.py | 39 ++++++++++++++- selfprivacy_api/graphql/queries/backup.py | 4 ++ tests/test_graphql/test_api_backup.py | 50 +++++++++++++++++++ 3 files changed, 92 insertions(+), 1 deletion(-) diff --git a/selfprivacy_api/graphql/mutations/backup_mutations.py b/selfprivacy_api/graphql/mutations/backup_mutations.py index c022d57..babbcf8 100644 --- a/selfprivacy_api/graphql/mutations/backup_mutations.py +++ b/selfprivacy_api/graphql/mutations/backup_mutations.py @@ -11,7 +11,10 @@ from selfprivacy_api.graphql.queries.backup import BackupConfiguration from selfprivacy_api.graphql.queries.backup import Backup from selfprivacy_api.graphql.queries.providers import BackupProvider from selfprivacy_api.graphql.common_types.jobs import job_to_api_job -from selfprivacy_api.graphql.common_types.backup import RestoreStrategy +from selfprivacy_api.graphql.common_types.backup import ( + RestoreStrategy, + AutobackupQuotas, +) from selfprivacy_api.backup import Backups from selfprivacy_api.services import get_service_by_id @@ -33,6 +36,13 @@ class InitializeRepositoryInput: password: str +@strawberry.input +class SetAutobackupQuotasInput: + """A single field input to reuse AutobackupQuotas""" + + quotas: AutobackupQuotas + + @strawberry.type class GenericBackupConfigReturn(MutationReturnInterface): """Generic backup config return""" @@ -90,6 +100,33 @@ class BackupMutations: configuration=Backup().configuration(), ) + @strawberry.mutation(permission_classes=[IsAuthenticated]) + def set_autobackup_quotas( + self, quotas: SetAutobackupQuotasInput + ) -> GenericBackupConfigReturn: + """ + Set autobackup quotas. + Values <=0 for any timeframe mean no limits for that timeframe. + To disable autobackup use autobackup period setting, not this mutation. + """ + + try: + Backups.set_autobackup_quotas(quotas) + return GenericBackupConfigReturn( + success=True, + message="", + code=200, + configuration=Backup().configuration(), + ) + + except Exception as e: + return GenericBackupConfigReturn( + success=False, + message=str(e), + code=400, + configuration=Backup().configuration(), + ) + @strawberry.mutation(permission_classes=[IsAuthenticated]) def start_backup(self, service_id: str) -> GenericJobMutationReturn: """Start backup""" diff --git a/selfprivacy_api/graphql/queries/backup.py b/selfprivacy_api/graphql/queries/backup.py index 6535a88..e03215d 100644 --- a/selfprivacy_api/graphql/queries/backup.py +++ b/selfprivacy_api/graphql/queries/backup.py @@ -13,6 +13,7 @@ from selfprivacy_api.graphql.common_types.service import ( SnapshotInfo, service_to_graphql_service, ) +from selfprivacy_api.graphql.common_types.backup import AutobackupQuotas from selfprivacy_api.services import get_service_by_id @@ -26,6 +27,8 @@ class BackupConfiguration: is_initialized: bool # If none, autobackups are disabled autobackup_period: typing.Optional[int] + # None is equal to all quotas being unlimited (-1). Optional for compatibility reasons. + autobackup_quotas: typing.Optional[AutobackupQuotas] # Bucket name for Backblaze, path for some other providers location_name: typing.Optional[str] location_id: typing.Optional[str] @@ -42,6 +45,7 @@ class Backup: autobackup_period=Backups.autobackup_period_minutes(), location_name=Backups.provider().location, location_id=Backups.provider().repo_id, + autobackup_quotas=Backups.autobackup_quotas(), ) @strawberry.field diff --git a/tests/test_graphql/test_api_backup.py b/tests/test_graphql/test_api_backup.py index e53ce2a..9681e7b 100644 --- a/tests/test_graphql/test_api_backup.py +++ b/tests/test_graphql/test_api_backup.py @@ -4,6 +4,7 @@ from tests.common import generate_backup_query from selfprivacy_api.graphql.common_types.service import service_to_graphql_service +from selfprivacy_api.graphql.common_types.backup import AutobackupQuotas from selfprivacy_api.jobs import Jobs, JobStatus API_RELOAD_SNAPSHOTS = """ @@ -38,6 +39,28 @@ mutation TestAutobackupPeriod($period: Int) { } """ + +API_SET_AUTOBACKUP_QUOTAS_MUTATION = """ +mutation TestAutobackupQuotas($input: SetAutobackupQuotasInput!) { + backup { + setAutobackupQuotas(quotas: $input) { + success + message + code + configuration { + provider + encryptionKey + isInitialized + autobackupPeriod + locationName + locationId + autobackupQuotas + } + } + } +} +""" + API_REMOVE_REPOSITORY_MUTATION = """ mutation TestRemoveRepo { backup { @@ -177,6 +200,17 @@ def api_set_period(authorized_client, period): return response +def api_set_quotas(authorized_client, quotas): + response = authorized_client.post( + "/graphql", + json={ + "query": API_SET_AUTOBACKUP_QUOTAS_MUTATION, + "variables": {"input": {"quotas": quotas}}, + }, + ) + return response + + def api_remove(authorized_client): response = authorized_client.post( "/graphql", @@ -323,6 +357,22 @@ def test_remove(authorized_client, generic_userdata): assert configuration["isInitialized"] is False +def test_autobackup_quotas_nonzero(authorized_client): + quotas = AutobackupQuotas( + daily=2, + weekly=4, + monthly=13, + yearly=14, + total=3, + ) + response = api_set_quotas(authorized_client, quotas) + data = get_data(response)["backup"]["setAutobackupQuotas"] + assert_ok(data) + + configuration = data["configuration"] + assert configuration["autobackupQuotas"] == quotas + + def test_autobackup_period_nonzero(authorized_client): new_period = 11 response = api_set_period(authorized_client, new_period) From ad9384c850d0249db4f30bb58f472d8b3013a2ba Mon Sep 17 00:00:00 2001 From: Inex Code Date: Wed, 30 Aug 2023 12:03:19 +0300 Subject: [PATCH 10/13] fix(graphql): backup quotas field typing --- selfprivacy_api/backup/time.py | 1 + .../graphql/common_types/backup.py | 5 +++++ .../graphql/mutations/backup_mutations.py | 11 ++-------- selfprivacy_api/graphql/queries/backup.py | 2 +- tests/test_graphql/test_api_backup.py | 21 +++++++++++++------ 5 files changed, 24 insertions(+), 16 deletions(-) diff --git a/selfprivacy_api/backup/time.py b/selfprivacy_api/backup/time.py index aba12bd..9e34211 100644 --- a/selfprivacy_api/backup/time.py +++ b/selfprivacy_api/backup/time.py @@ -24,5 +24,6 @@ def same_month(a: datetime, b: datetime) -> bool: def same_year(a: datetime, b: datetime) -> bool: return a.year == b.year + def same_lifetime_of_the_universe(a: datetime, b: datetime) -> bool: return True diff --git a/selfprivacy_api/graphql/common_types/backup.py b/selfprivacy_api/graphql/common_types/backup.py index 3d5b5aa..cc03936 100644 --- a/selfprivacy_api/graphql/common_types/backup.py +++ b/selfprivacy_api/graphql/common_types/backup.py @@ -29,3 +29,8 @@ class _AutobackupQuotas(BaseModel): @strawberry.experimental.pydantic.type(model=_AutobackupQuotas, all_fields=True) class AutobackupQuotas: pass + + +@strawberry.experimental.pydantic.input(model=_AutobackupQuotas, all_fields=True) +class AutobackupQuotasInput: + pass diff --git a/selfprivacy_api/graphql/mutations/backup_mutations.py b/selfprivacy_api/graphql/mutations/backup_mutations.py index babbcf8..dcfebff 100644 --- a/selfprivacy_api/graphql/mutations/backup_mutations.py +++ b/selfprivacy_api/graphql/mutations/backup_mutations.py @@ -12,8 +12,8 @@ from selfprivacy_api.graphql.queries.backup import Backup from selfprivacy_api.graphql.queries.providers import BackupProvider from selfprivacy_api.graphql.common_types.jobs import job_to_api_job from selfprivacy_api.graphql.common_types.backup import ( + AutobackupQuotasInput, RestoreStrategy, - AutobackupQuotas, ) from selfprivacy_api.backup import Backups @@ -36,13 +36,6 @@ class InitializeRepositoryInput: password: str -@strawberry.input -class SetAutobackupQuotasInput: - """A single field input to reuse AutobackupQuotas""" - - quotas: AutobackupQuotas - - @strawberry.type class GenericBackupConfigReturn(MutationReturnInterface): """Generic backup config return""" @@ -102,7 +95,7 @@ class BackupMutations: @strawberry.mutation(permission_classes=[IsAuthenticated]) def set_autobackup_quotas( - self, quotas: SetAutobackupQuotasInput + self, quotas: AutobackupQuotasInput ) -> GenericBackupConfigReturn: """ Set autobackup quotas. diff --git a/selfprivacy_api/graphql/queries/backup.py b/selfprivacy_api/graphql/queries/backup.py index e03215d..6d47a8c 100644 --- a/selfprivacy_api/graphql/queries/backup.py +++ b/selfprivacy_api/graphql/queries/backup.py @@ -28,7 +28,7 @@ class BackupConfiguration: # If none, autobackups are disabled autobackup_period: typing.Optional[int] # None is equal to all quotas being unlimited (-1). Optional for compatibility reasons. - autobackup_quotas: typing.Optional[AutobackupQuotas] + autobackup_quotas: AutobackupQuotas # Bucket name for Backblaze, path for some other providers location_name: typing.Optional[str] location_id: typing.Optional[str] diff --git a/tests/test_graphql/test_api_backup.py b/tests/test_graphql/test_api_backup.py index 9681e7b..e8de4a1 100644 --- a/tests/test_graphql/test_api_backup.py +++ b/tests/test_graphql/test_api_backup.py @@ -4,7 +4,10 @@ from tests.common import generate_backup_query from selfprivacy_api.graphql.common_types.service import service_to_graphql_service -from selfprivacy_api.graphql.common_types.backup import AutobackupQuotas +from selfprivacy_api.graphql.common_types.backup import ( + _AutobackupQuotas, + AutobackupQuotas, +) from selfprivacy_api.jobs import Jobs, JobStatus API_RELOAD_SNAPSHOTS = """ @@ -41,7 +44,7 @@ mutation TestAutobackupPeriod($period: Int) { API_SET_AUTOBACKUP_QUOTAS_MUTATION = """ -mutation TestAutobackupQuotas($input: SetAutobackupQuotasInput!) { +mutation TestAutobackupQuotas($input: AutobackupQuotasInput!) { backup { setAutobackupQuotas(quotas: $input) { success @@ -54,7 +57,13 @@ mutation TestAutobackupQuotas($input: SetAutobackupQuotasInput!) { autobackupPeriod locationName locationId - autobackupQuotas + autobackupQuotas { + daily + weekly + monthly + yearly + total + } } } } @@ -200,12 +209,12 @@ def api_set_period(authorized_client, period): return response -def api_set_quotas(authorized_client, quotas): +def api_set_quotas(authorized_client, quotas: _AutobackupQuotas): response = authorized_client.post( "/graphql", json={ "query": API_SET_AUTOBACKUP_QUOTAS_MUTATION, - "variables": {"input": {"quotas": quotas}}, + "variables": {"input": quotas.dict()}, }, ) return response @@ -358,7 +367,7 @@ def test_remove(authorized_client, generic_userdata): def test_autobackup_quotas_nonzero(authorized_client): - quotas = AutobackupQuotas( + quotas = _AutobackupQuotas( daily=2, weekly=4, monthly=13, From 56be3d9c31b972c1bb6eb03e8fb2a624b84852d9 Mon Sep 17 00:00:00 2001 From: Houkime <> Date: Fri, 8 Sep 2023 16:22:53 +0000 Subject: [PATCH 11/13] fix(backup): trim auto-snapshots on setting the quotas --- selfprivacy_api/backup/__init__.py | 3 +++ tests/test_graphql/test_backup.py | 4 +++- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/selfprivacy_api/backup/__init__.py b/selfprivacy_api/backup/__init__.py index 336b705..7056071 100644 --- a/selfprivacy_api/backup/__init__.py +++ b/selfprivacy_api/backup/__init__.py @@ -396,6 +396,9 @@ class Backups: ) ) + for service in get_all_services(): + Backups._prune_auto_snaps(service) + # Restoring @staticmethod diff --git a/tests/test_graphql/test_backup.py b/tests/test_graphql/test_backup.py index 550c56b..5daae0c 100644 --- a/tests/test_graphql/test_backup.py +++ b/tests/test_graphql/test_backup.py @@ -520,8 +520,10 @@ def test_too_many_auto(backups, dummy_service): # Retroactivity quota.total = 1 Backups.set_autobackup_quotas(quota) - snap5 = Backups.back_up(dummy_service, BackupReason.AUTO) + snaps = Backups.get_snapshots(dummy_service) + assert len(snaps) == 1 + snap5 = Backups.back_up(dummy_service, BackupReason.AUTO) snaps = Backups.get_snapshots(dummy_service) assert len(snaps) == 1 assert snap5 in snaps From dedd6a9cc949fcd176ce941b5182611e4bc7d33c Mon Sep 17 00:00:00 2001 From: Inex Code Date: Sat, 9 Sep 2023 03:26:41 +0300 Subject: [PATCH 12/13] refactor(backups): use restic-like rotation policy --- selfprivacy_api/backup/__init__.py | 113 +++++--- .../backup/backuppers/restic_backupper.py | 21 +- selfprivacy_api/backup/storage.py | 4 +- selfprivacy_api/backup/time.py | 29 -- .../graphql/common_types/backup.py | 4 +- tests/test_graphql/test_api_backup.py | 4 +- tests/test_graphql/test_backup.py | 266 +++++++++++++----- 7 files changed, 278 insertions(+), 163 deletions(-) delete mode 100644 selfprivacy_api/backup/time.py diff --git a/selfprivacy_api/backup/__init__.py b/selfprivacy_api/backup/__init__.py index 7056071..dff4b3b 100644 --- a/selfprivacy_api/backup/__init__.py +++ b/selfprivacy_api/backup/__init__.py @@ -4,7 +4,7 @@ This module contains the controller class for backups. from datetime import datetime, timedelta import os from os import statvfs -from typing import List, Optional +from typing import Callable, List, Optional from selfprivacy_api.utils import ReadUserData, WriteUserData @@ -28,13 +28,7 @@ from selfprivacy_api.graphql.common_types.backup import ( BackupReason, AutobackupQuotas, ) -from selfprivacy_api.backup.time import ( - same_day, - same_month, - same_week, - same_year, - same_lifetime_of_the_universe, -) + from selfprivacy_api.models.backup.snapshot import Snapshot @@ -81,6 +75,24 @@ class NotDeadError(AssertionError): """ +class RotationBucket: + """ + Bucket object used for rotation. + Has the following mutable fields: + - the counter, int + - the lambda function which takes datetime and the int and returns the int + - the last, int + """ + + def __init__(self, counter: int, last: int, rotation_lambda): + self.counter: int = counter + self.last: int = last + self.rotation_lambda: Callable[[datetime, int], int] = rotation_lambda + + def __str__(self) -> str: + return f"Bucket(counter={self.counter}, last={self.last})" + + class Backups: """A stateless controller class for backups""" @@ -314,45 +326,54 @@ class Backups: if snap.reason == BackupReason.AUTO ] - @staticmethod - def add_snap_but_with_quotas( - new_snap: Snapshot, snaps: List[Snapshot], quotas: AutobackupQuotas - ) -> None: - quotas_map = { - same_day: quotas.daily, - same_week: quotas.weekly, - same_month: quotas.monthly, - same_year: quotas.yearly, - same_lifetime_of_the_universe: quotas.total, - } - - snaps.append(new_snap) - - for is_same_period, quota in quotas_map.items(): - if quota <= 0: - continue - - cohort = [ - snap - for snap in snaps - if is_same_period(snap.created_at, new_snap.created_at) - ] - sorted_cohort = sorted(cohort, key=lambda s: s.created_at) - n_to_kill = len(cohort) - quota - if n_to_kill > 0: - snaps_to_kill = sorted_cohort[:n_to_kill] - for snap in snaps_to_kill: - snaps.remove(snap) - @staticmethod def _prune_snaps_with_quotas(snapshots: List[Snapshot]) -> List[Snapshot]: # Function broken out for testability - sorted_snaps = sorted(snapshots, key=lambda s: s.created_at) - quotas = Backups.autobackup_quotas() + # Sorting newest first + sorted_snaps = sorted(snapshots, key=lambda s: s.created_at, reverse=True) + quotas: AutobackupQuotas = Backups.autobackup_quotas() + + buckets: list[RotationBucket] = [ + RotationBucket( + quotas.last, + -1, + lambda _, index: index, + ), + RotationBucket( + quotas.daily, + -1, + lambda date, _: date.year * 10000 + date.month * 100 + date.day, + ), + RotationBucket( + quotas.weekly, + -1, + lambda date, _: date.year * 100 + date.isocalendar()[1], + ), + RotationBucket( + quotas.monthly, + -1, + lambda date, _: date.year * 100 + date.month, + ), + RotationBucket( + quotas.yearly, + -1, + lambda date, _: date.year, + ), + ] new_snaplist: List[Snapshot] = [] - for snap in sorted_snaps: - Backups.add_snap_but_with_quotas(snap, new_snaplist, quotas) + for i, snap in enumerate(sorted_snaps): + keep_snap = False + for bucket in buckets: + if (bucket.counter > 0) or (bucket.counter == -1): + val = bucket.rotation_lambda(snap.created_at, i) + if (val != bucket.last) or (i == len(sorted_snaps) - 1): + bucket.last = val + if bucket.counter > 0: + bucket.counter -= 1 + if not keep_snap: + new_snaplist.append(snap) + keep_snap = True return new_snaplist @@ -372,27 +393,27 @@ class Backups: @staticmethod def _standardize_quotas(i: int) -> int: - if i <= 0: + if i <= -1: i = -1 return i @staticmethod def autobackup_quotas() -> AutobackupQuotas: - """everything <=0 means unlimited""" + """0 means do not keep, -1 means unlimited""" return Storage.autobackup_quotas() @staticmethod def set_autobackup_quotas(quotas: AutobackupQuotas) -> None: - """everything <=0 means unlimited""" + """0 means do not keep, -1 means unlimited""" Storage.set_autobackup_quotas( AutobackupQuotas( + last=Backups._standardize_quotas(quotas.last), daily=Backups._standardize_quotas(quotas.daily), weekly=Backups._standardize_quotas(quotas.weekly), monthly=Backups._standardize_quotas(quotas.monthly), yearly=Backups._standardize_quotas(quotas.yearly), - total=Backups._standardize_quotas(quotas.total), ) ) diff --git a/selfprivacy_api/backup/backuppers/restic_backupper.py b/selfprivacy_api/backup/backuppers/restic_backupper.py index f5467ff..b6c643b 100644 --- a/selfprivacy_api/backup/backuppers/restic_backupper.py +++ b/selfprivacy_api/backup/backuppers/restic_backupper.py @@ -5,7 +5,7 @@ import json import datetime import tempfile -from typing import List, TypeVar, Callable +from typing import List, Optional, TypeVar, Callable from collections.abc import Iterable from json.decoder import JSONDecodeError from os.path import exists, join @@ -33,12 +33,12 @@ def unlocked_repo(func: T) -> T: def inner(self: ResticBackupper, *args, **kwargs): try: return func(self, *args, **kwargs) - except Exception as e: - if "unable to create lock" in str(e): + except Exception as error: + if "unable to create lock" in str(error): self.unlock() return func(self, *args, **kwargs) else: - raise e + raise error # Above, we manually guarantee that the type returned is compatible. return inner # type: ignore @@ -85,7 +85,10 @@ class ResticBackupper(AbstractBackupper): def _password_command(self): return f"echo {LocalBackupSecret.get()}" - def restic_command(self, *args, tags: List[str] = []) -> List[str]: + def restic_command(self, *args, tags: Optional[List[str]] = None) -> List[str]: + if tags is None: + tags = [] + command = [ "restic", "-o", @@ -219,7 +222,7 @@ class ResticBackupper(AbstractBackupper): ) from error @staticmethod - def _snapshot_id_from_backup_messages(messages) -> Snapshot: + def _snapshot_id_from_backup_messages(messages) -> str: for message in messages: if message["message_type"] == "summary": # There is a discrepancy between versions of restic/rclone @@ -317,8 +320,8 @@ class ResticBackupper(AbstractBackupper): break if "unable" in line: raise ValueError(line) - except Exception as e: - raise ValueError("could not lock repository") from e + except Exception as error: + raise ValueError("could not lock repository") from error @unlocked_repo def restored_size(self, snapshot_id: str) -> int: @@ -415,6 +418,8 @@ class ResticBackupper(AbstractBackupper): forget_command = self.restic_command( "forget", snapshot_id, + # TODO: prune should be done in a separate process + "--prune", ) with subprocess.Popen( diff --git a/selfprivacy_api/backup/storage.py b/selfprivacy_api/backup/storage.py index 86b92f3..ddfd176 100644 --- a/selfprivacy_api/backup/storage.py +++ b/selfprivacy_api/backup/storage.py @@ -193,11 +193,11 @@ class Storage: ) if quotas_model is None: unlimited_quotas = AutobackupQuotas( + last=-1, daily=-1, weekly=-1, monthly=-1, yearly=-1, - total=-1, ) return unlimited_quotas - return AutobackupQuotas.from_pydantic(quotas_model) + return AutobackupQuotas.from_pydantic(quotas_model) # pylint: disable=no-member diff --git a/selfprivacy_api/backup/time.py b/selfprivacy_api/backup/time.py deleted file mode 100644 index 9e34211..0000000 --- a/selfprivacy_api/backup/time.py +++ /dev/null @@ -1,29 +0,0 @@ -from datetime import datetime, timedelta, time - - -def same_day(a: datetime, b: datetime) -> bool: - return a.date() == b.date() - - -def same_week(a: datetime, b: datetime) -> bool: - # doing the hard way because weeks traverse the edges of years - zerobased_weekday = a.isoweekday() - 1 - start_of_day = datetime.combine(a.date(), time.min) - start_of_week = start_of_day - timedelta(days=zerobased_weekday) - end_of_week = start_of_week + timedelta(days=7) - - if b >= start_of_week and b <= end_of_week: - return True - return False - - -def same_month(a: datetime, b: datetime) -> bool: - return a.month == b.month and a.year == b.year - - -def same_year(a: datetime, b: datetime) -> bool: - return a.year == b.year - - -def same_lifetime_of_the_universe(a: datetime, b: datetime) -> bool: - return True diff --git a/selfprivacy_api/graphql/common_types/backup.py b/selfprivacy_api/graphql/common_types/backup.py index cc03936..953009d 100644 --- a/selfprivacy_api/graphql/common_types/backup.py +++ b/selfprivacy_api/graphql/common_types/backup.py @@ -1,7 +1,7 @@ """Backup""" # pylint: disable=too-few-public-methods -import strawberry from enum import Enum +import strawberry from pydantic import BaseModel @@ -19,11 +19,11 @@ class BackupReason(Enum): class _AutobackupQuotas(BaseModel): + last: int daily: int weekly: int monthly: int yearly: int - total: int @strawberry.experimental.pydantic.type(model=_AutobackupQuotas, all_fields=True) diff --git a/tests/test_graphql/test_api_backup.py b/tests/test_graphql/test_api_backup.py index e8de4a1..14410e3 100644 --- a/tests/test_graphql/test_api_backup.py +++ b/tests/test_graphql/test_api_backup.py @@ -58,11 +58,11 @@ mutation TestAutobackupQuotas($input: AutobackupQuotasInput!) { locationName locationId autobackupQuotas { + last daily weekly monthly yearly - total } } } @@ -368,11 +368,11 @@ def test_remove(authorized_client, generic_userdata): def test_autobackup_quotas_nonzero(authorized_client): quotas = _AutobackupQuotas( + last=3, daily=2, weekly=4, monthly=13, yearly=14, - total=3, ) response = api_set_quotas(authorized_client, quotas) data = get_data(response)["backup"]["setAutobackupQuotas"] diff --git a/tests/test_graphql/test_backup.py b/tests/test_graphql/test_backup.py index 5daae0c..edef6d0 100644 --- a/tests/test_graphql/test_backup.py +++ b/tests/test_graphql/test_backup.py @@ -305,11 +305,19 @@ def test_backup_reasons(backups, dummy_service): unlimited_quotas = AutobackupQuotas( + last=-1, daily=-1, weekly=-1, monthly=-1, yearly=-1, - total=-1, +) + +zero_quotas = AutobackupQuotas( + last=0, + daily=0, + weekly=0, + monthly=0, + yearly=0, ) @@ -321,20 +329,66 @@ def test_get_empty_quotas(backups): def test_set_quotas(backups): quotas = AutobackupQuotas( + last=3, daily=2343, weekly=343, monthly=0, yearly=-34556, - total=563, ) Backups.set_autobackup_quotas(quotas) assert Backups.autobackup_quotas() == AutobackupQuotas( + last=3, daily=2343, weekly=343, + monthly=0, + yearly=-1, + ) + + +def test_set_zero_quotas(backups): + quotas = AutobackupQuotas( + last=0, + daily=0, + weekly=0, + monthly=0, + yearly=0, + ) + Backups.set_autobackup_quotas(quotas) + assert Backups.autobackup_quotas() == zero_quotas + + +def test_set_unlimited_quotas(backups): + quotas = AutobackupQuotas( + last=-1, + daily=-1, + weekly=-1, monthly=-1, yearly=-1, - total=563, ) + Backups.set_autobackup_quotas(quotas) + assert Backups.autobackup_quotas() == unlimited_quotas + + +def test_set_zero_quotas_after_unlimited(backups): + quotas = AutobackupQuotas( + last=-1, + daily=-1, + weekly=-1, + monthly=-1, + yearly=-1, + ) + Backups.set_autobackup_quotas(quotas) + assert Backups.autobackup_quotas() == unlimited_quotas + + quotas = AutobackupQuotas( + last=0, + daily=0, + weekly=0, + monthly=0, + yearly=0, + ) + Backups.set_autobackup_quotas(quotas) + assert Backups.autobackup_quotas() == zero_quotas def dummy_snapshot(date: datetime): @@ -351,15 +405,24 @@ def test_autobackup_snapshots_pruning(backups): now = datetime(year=2023, month=1, day=25, hour=10) snaps = [ - dummy_snapshot(now - timedelta(days=365 * 2)), - dummy_snapshot(now - timedelta(days=20)), - dummy_snapshot(now - timedelta(days=2)), - dummy_snapshot(now - timedelta(days=1, hours=3)), - dummy_snapshot(now - timedelta(days=1, hours=2)), - dummy_snapshot(now - timedelta(days=1)), - dummy_snapshot(now - timedelta(hours=2)), - dummy_snapshot(now - timedelta(minutes=5)), dummy_snapshot(now), + dummy_snapshot(now - timedelta(minutes=5)), + dummy_snapshot(now - timedelta(hours=2)), + dummy_snapshot(now - timedelta(hours=5)), + dummy_snapshot(now - timedelta(days=1)), + dummy_snapshot(now - timedelta(days=1, hours=2)), + dummy_snapshot(now - timedelta(days=1, hours=3)), + dummy_snapshot(now - timedelta(days=2)), + dummy_snapshot(now - timedelta(days=7)), + dummy_snapshot(now - timedelta(days=12)), + dummy_snapshot(now - timedelta(days=23)), + dummy_snapshot(now - timedelta(days=28)), + dummy_snapshot(now - timedelta(days=32)), + dummy_snapshot(now - timedelta(days=47)), + dummy_snapshot(now - timedelta(days=64)), + dummy_snapshot(now - timedelta(days=84)), + dummy_snapshot(now - timedelta(days=104)), + dummy_snapshot(now - timedelta(days=365 * 2)), ] old_len = len(snaps) @@ -367,135 +430,190 @@ def test_autobackup_snapshots_pruning(backups): Backups.set_autobackup_quotas(quotas) assert Backups._prune_snaps_with_quotas(snaps) == snaps - quotas = copy(unlimited_quotas) + quotas = copy(zero_quotas) + quotas.last = 2 quotas.daily = 2 Backups.set_autobackup_quotas(quotas) - pruned_snaps = Backups._prune_snaps_with_quotas(snaps) - assert pruned_snaps == [ - dummy_snapshot(now - timedelta(days=365 * 2)), - dummy_snapshot(now - timedelta(days=20)), - dummy_snapshot(now - timedelta(days=2)), - dummy_snapshot(now - timedelta(days=1, hours=2)), - dummy_snapshot(now - timedelta(days=1)), - dummy_snapshot(now - timedelta(minutes=5)), + snaps_to_keep = Backups._prune_snaps_with_quotas(snaps) + assert snaps_to_keep == [ dummy_snapshot(now), + dummy_snapshot(now - timedelta(minutes=5)), + # dummy_snapshot(now - timedelta(hours=2)), + # dummy_snapshot(now - timedelta(hours=5)), + dummy_snapshot(now - timedelta(days=1)), + # dummy_snapshot(now - timedelta(days=1, hours=2)), + # dummy_snapshot(now - timedelta(days=1, hours=3)), + # dummy_snapshot(now - timedelta(days=2)), + # dummy_snapshot(now - timedelta(days=7)), + # dummy_snapshot(now - timedelta(days=12)), + # dummy_snapshot(now - timedelta(days=23)), + # dummy_snapshot(now - timedelta(days=28)), + # dummy_snapshot(now - timedelta(days=32)), + # dummy_snapshot(now - timedelta(days=47)), + # dummy_snapshot(now - timedelta(days=64)), + # dummy_snapshot(now - timedelta(days=84)), + # dummy_snapshot(now - timedelta(days=104)), + # dummy_snapshot(now - timedelta(days=365 * 2)), ] # checking that this function does not mutate the argument - assert snaps != pruned_snaps + assert snaps != snaps_to_keep assert len(snaps) == old_len - quotas = copy(unlimited_quotas) + quotas = copy(zero_quotas) quotas.weekly = 4 Backups.set_autobackup_quotas(quotas) - pruned_snaps = Backups._prune_snaps_with_quotas(snaps) - assert pruned_snaps == [ - dummy_snapshot(now - timedelta(days=365 * 2)), - dummy_snapshot(now - timedelta(days=20)), - dummy_snapshot(now - timedelta(days=1)), - dummy_snapshot(now - timedelta(hours=2)), - dummy_snapshot(now - timedelta(minutes=5)), + snaps_to_keep = Backups._prune_snaps_with_quotas(snaps) + assert snaps_to_keep == [ dummy_snapshot(now), + # dummy_snapshot(now - timedelta(minutes=5)), + # dummy_snapshot(now - timedelta(hours=2)), + # dummy_snapshot(now - timedelta(hours=5)), + # dummy_snapshot(now - timedelta(days=1)), + # dummy_snapshot(now - timedelta(days=1, hours=2)), + # dummy_snapshot(now - timedelta(days=1, hours=3)), + # dummy_snapshot(now - timedelta(days=2)), + dummy_snapshot(now - timedelta(days=7)), + dummy_snapshot(now - timedelta(days=12)), + dummy_snapshot(now - timedelta(days=23)), + # dummy_snapshot(now - timedelta(days=28)), + # dummy_snapshot(now - timedelta(days=32)), + # dummy_snapshot(now - timedelta(days=47)), + # dummy_snapshot(now - timedelta(days=64)), + # dummy_snapshot(now - timedelta(days=84)), + # dummy_snapshot(now - timedelta(days=104)), + # dummy_snapshot(now - timedelta(days=365 * 2)), ] - quotas = copy(unlimited_quotas) + quotas = copy(zero_quotas) quotas.monthly = 7 Backups.set_autobackup_quotas(quotas) - pruned_snaps = Backups._prune_snaps_with_quotas(snaps) - assert pruned_snaps == [ - dummy_snapshot(now - timedelta(days=365 * 2)), - dummy_snapshot(now - timedelta(days=2)), - dummy_snapshot(now - timedelta(days=1, hours=3)), - dummy_snapshot(now - timedelta(days=1, hours=2)), - dummy_snapshot(now - timedelta(days=1)), - dummy_snapshot(now - timedelta(hours=2)), - dummy_snapshot(now - timedelta(minutes=5)), + snaps_to_keep = Backups._prune_snaps_with_quotas(snaps) + assert snaps_to_keep == [ dummy_snapshot(now), + # dummy_snapshot(now - timedelta(minutes=5)), + # dummy_snapshot(now - timedelta(hours=2)), + # dummy_snapshot(now - timedelta(hours=5)), + # dummy_snapshot(now - timedelta(days=1)), + # dummy_snapshot(now - timedelta(days=1, hours=2)), + # dummy_snapshot(now - timedelta(days=1, hours=3)), + # dummy_snapshot(now - timedelta(days=2)), + # dummy_snapshot(now - timedelta(days=7)), + # dummy_snapshot(now - timedelta(days=12)), + # dummy_snapshot(now - timedelta(days=23)), + dummy_snapshot(now - timedelta(days=28)), + # dummy_snapshot(now - timedelta(days=32)), + # dummy_snapshot(now - timedelta(days=47)), + dummy_snapshot(now - timedelta(days=64)), + # dummy_snapshot(now - timedelta(days=84)), + dummy_snapshot(now - timedelta(days=104)), + dummy_snapshot(now - timedelta(days=365 * 2)), ] def test_autobackup_snapshots_pruning_yearly(backups): snaps = [ - dummy_snapshot(datetime(year=2023, month=2, day=1)), - dummy_snapshot(datetime(year=2023, month=3, day=1)), - dummy_snapshot(datetime(year=2023, month=4, day=1)), dummy_snapshot(datetime(year=2055, month=3, day=1)), + dummy_snapshot(datetime(year=2055, month=2, day=1)), + dummy_snapshot(datetime(year=2023, month=4, day=1)), + dummy_snapshot(datetime(year=2023, month=3, day=1)), + dummy_snapshot(datetime(year=2023, month=2, day=1)), + dummy_snapshot(datetime(year=2021, month=2, day=1)), ] - quotas = copy(unlimited_quotas) + quotas = copy(zero_quotas) quotas.yearly = 2 Backups.set_autobackup_quotas(quotas) - pruned_snaps = Backups._prune_snaps_with_quotas(snaps) - assert pruned_snaps == [ - dummy_snapshot(datetime(year=2023, month=3, day=1)), - dummy_snapshot(datetime(year=2023, month=4, day=1)), + snaps_to_keep = Backups._prune_snaps_with_quotas(snaps) + assert snaps_to_keep == [ dummy_snapshot(datetime(year=2055, month=3, day=1)), + dummy_snapshot(datetime(year=2023, month=4, day=1)), ] def test_autobackup_snapshots_pruning_bottleneck(backups): now = datetime(year=2023, month=1, day=25, hour=10) snaps = [ - dummy_snapshot(now - timedelta(hours=4)), - dummy_snapshot(now - timedelta(hours=3)), - dummy_snapshot(now - timedelta(hours=2)), - dummy_snapshot(now - timedelta(minutes=5)), dummy_snapshot(now), + dummy_snapshot(now - timedelta(minutes=5)), + dummy_snapshot(now - timedelta(hours=2)), + dummy_snapshot(now - timedelta(hours=3)), + dummy_snapshot(now - timedelta(hours=4)), ] - yearly_quota = copy(unlimited_quotas) + yearly_quota = copy(zero_quotas) yearly_quota.yearly = 2 - monthly_quota = copy(unlimited_quotas) + monthly_quota = copy(zero_quotas) monthly_quota.monthly = 2 - weekly_quota = copy(unlimited_quotas) + weekly_quota = copy(zero_quotas) weekly_quota.weekly = 2 - daily_quota = copy(unlimited_quotas) + daily_quota = copy(zero_quotas) daily_quota.daily = 2 - total_quota = copy(unlimited_quotas) - total_quota.total = 2 + last_quota = copy(zero_quotas) + last_quota.last = 1 + last_quota.yearly = 2 - for quota in [total_quota, yearly_quota, monthly_quota, weekly_quota, daily_quota]: + for quota in [last_quota, yearly_quota, monthly_quota, weekly_quota, daily_quota]: + print(quota) Backups.set_autobackup_quotas(quota) - pruned_snaps = Backups._prune_snaps_with_quotas(snaps) - assert pruned_snaps == [ - dummy_snapshot(now - timedelta(minutes=5)), + snaps_to_keep = Backups._prune_snaps_with_quotas(snaps) + assert snaps_to_keep == [ dummy_snapshot(now), + # If there is a vacant quota, we should keep the last snapshot even if it doesn't fit + dummy_snapshot(now - timedelta(hours=4)), ] def test_autobackup_snapshots_pruning_edgeweek(backups): # jan 1 2023 is Sunday snaps = [ - dummy_snapshot(datetime(year=2022, month=12, day=30)), - dummy_snapshot(datetime(year=2022, month=12, day=31)), - dummy_snapshot(datetime(year=2023, month=1, day=1)), dummy_snapshot(datetime(year=2023, month=1, day=6)), + dummy_snapshot(datetime(year=2023, month=1, day=1)), + dummy_snapshot(datetime(year=2022, month=12, day=31)), + dummy_snapshot(datetime(year=2022, month=12, day=30)), ] - quotas = copy(unlimited_quotas) + quotas = copy(zero_quotas) quotas.weekly = 2 Backups.set_autobackup_quotas(quotas) - pruned_snaps = Backups._prune_snaps_with_quotas(snaps) - assert pruned_snaps == [ - dummy_snapshot(datetime(year=2022, month=12, day=31)), - dummy_snapshot(datetime(year=2023, month=1, day=1)), + snaps_to_keep = Backups._prune_snaps_with_quotas(snaps) + assert snaps_to_keep == [ dummy_snapshot(datetime(year=2023, month=1, day=6)), + dummy_snapshot(datetime(year=2023, month=1, day=1)), + ] + + +def test_autobackup_snapshots_pruning_big_gap(backups): + snaps = [ + dummy_snapshot(datetime(year=2023, month=1, day=6)), + dummy_snapshot(datetime(year=2023, month=1, day=2)), + dummy_snapshot(datetime(year=2022, month=10, day=31)), + dummy_snapshot(datetime(year=2022, month=10, day=30)), + ] + quotas = copy(zero_quotas) + quotas.weekly = 2 + Backups.set_autobackup_quotas(quotas) + + snaps_to_keep = Backups._prune_snaps_with_quotas(snaps) + assert snaps_to_keep == [ + dummy_snapshot(datetime(year=2023, month=1, day=6)), + dummy_snapshot(datetime(year=2022, month=10, day=31)), ] def test_too_many_auto(backups, dummy_service): assert Backups.autobackup_quotas() - quota = copy(unlimited_quotas) - quota.total = 2 + quota = copy(zero_quotas) + quota.last = 2 Backups.set_autobackup_quotas(quota) - assert Backups.autobackup_quotas().total == 2 + assert Backups.autobackup_quotas().last == 2 snap = Backups.back_up(dummy_service, BackupReason.AUTO) assert len(Backups.get_snapshots(dummy_service)) == 1 @@ -509,7 +627,7 @@ def test_too_many_auto(backups, dummy_service): assert snap3 in snaps assert snap not in snaps - quota.total = -1 + quota.last = -1 Backups.set_autobackup_quotas(quota) snap4 = Backups.back_up(dummy_service, BackupReason.AUTO) @@ -518,7 +636,7 @@ def test_too_many_auto(backups, dummy_service): assert snap4 in snaps # Retroactivity - quota.total = 1 + quota.last = 1 Backups.set_autobackup_quotas(quota) snaps = Backups.get_snapshots(dummy_service) assert len(snaps) == 1 From 450a998ea638fc572027fc8433326ceaa496fc62 Mon Sep 17 00:00:00 2001 From: Inex Code Date: Sat, 9 Sep 2023 03:32:57 +0300 Subject: [PATCH 13/13] chore:bump version --- selfprivacy_api/dependencies.py | 2 +- setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/selfprivacy_api/dependencies.py b/selfprivacy_api/dependencies.py index fb974e8..9e144fd 100644 --- a/selfprivacy_api/dependencies.py +++ b/selfprivacy_api/dependencies.py @@ -27,4 +27,4 @@ async def get_token_header( def get_api_version() -> str: """Get API version""" - return "2.3.0" + return "2.4.0" diff --git a/setup.py b/setup.py index 684f54f..5ce3947 100755 --- a/setup.py +++ b/setup.py @@ -2,7 +2,7 @@ from setuptools import setup, find_packages setup( name="selfprivacy_api", - version="2.3.0", + version="2.4.0", packages=find_packages(), scripts=[ "selfprivacy_api/app.py",