from typing import List, Optional from datetime import datetime, timezone, timedelta from selfprivacy_api.models.backup.snapshot import Snapshot from selfprivacy_api.models.backup.provider import BackupProviderModel from selfprivacy_api.utils.singleton_metaclass import SingletonMetaclass from selfprivacy_api.utils import ReadUserData from selfprivacy_api.utils.redis_pool import RedisPool from selfprivacy_api.utils.redis_model_storage import store_model_as_hash, hash_as_model from selfprivacy_api.services import get_service_by_id from selfprivacy_api.services.service import Service from selfprivacy_api.backup.providers.provider import AbstractBackupProvider from selfprivacy_api.backup.providers import get_provider, get_kind from selfprivacy_api.graphql.queries.providers import BackupProvider # a hack to store file path. REDIS_SNAPSHOT_CACHE_EXPIRE_SECONDS = 24 * 60 * 60 # one day REDIS_AUTOBACKUP_ENABLED_PREFIX = "backup:autobackup:services:" REDIS_SNAPSHOTS_PREFIX = "backups:snapshots:" REDIS_LAST_BACKUP_PREFIX = "backups:last-backed-up:" REDIS_INITTED_CACHE_PREFIX = "backups:initted_services:" REDIS_REPO_PATH_KEY = "backups:test_repo_path" REDIS_PROVIDER_KEY = "backups:provider" REDIS_AUTOBACKUP_PERIOD_KEY = "backups:autobackup_period" redis = RedisPool().get_connection() # Singleton has a property of being persistent between tests. # I don't know what to do with this yet # class Backups(metaclass=SingletonMetaclass): class Backups: """A singleton controller for backups""" provider: AbstractBackupProvider @staticmethod def set_localfile_repo(file_path: str): ProviderClass = get_provider(BackupProvider.FILE) provider = ProviderClass(file_path) redis.set(REDIS_REPO_PATH_KEY, file_path) Backups.store_provider_redis(provider) @staticmethod def _redis_last_backup_key(service_id): return REDIS_LAST_BACKUP_PREFIX + service_id @staticmethod def _redis_snapshot_key(snapshot: Snapshot): return REDIS_SNAPSHOTS_PREFIX + snapshot.id @staticmethod def get_last_backed_up(service: Service) -> Optional[datetime]: """Get a timezone-aware time of the last backup of a service""" return Backups._get_last_backup_time_redis(service.get_id()) @staticmethod def _get_last_backup_time_redis(service_id: str) -> Optional[datetime]: key = Backups._redis_last_backup_key(service_id) if not redis.exists(key): return None snapshot = hash_as_model(redis, key, Snapshot) return snapshot.created_at @staticmethod def _store_last_snapshot(service_id: str, snapshot: Snapshot): # non-expiring timestamp of the last store_model_as_hash(redis, Backups._redis_last_backup_key(service_id), snapshot) # expiring cache entry Backups.cache_snapshot(snapshot) @staticmethod def cache_snapshot(snapshot: Snapshot): snapshot_key = Backups._redis_snapshot_key(snapshot) store_model_as_hash(redis, snapshot_key, snapshot) redis.expire(snapshot_key, REDIS_SNAPSHOT_CACHE_EXPIRE_SECONDS) @staticmethod def delete_cached_snapshot(snapshot: Snapshot): snapshot_key = Backups._redis_snapshot_key(snapshot) redis.delete(snapshot_key) @staticmethod def get_cached_snapshots() -> List[Snapshot]: keys = redis.keys(REDIS_SNAPSHOTS_PREFIX + "*") result = [] for key in keys: snapshot = hash_as_model(redis, key, Snapshot) result.append(snapshot) return result @staticmethod def get_cached_snapshots_service(service_id: str) -> List[Snapshot]: snapshots = Backups.get_cached_snapshots() return [snap for snap in snapshots if snap.service_name == service_id] @staticmethod def sync_service_snapshots(service_id: str, snapshots: List[Snapshot]): for snapshot in snapshots: if snapshot.service_name == service_id: Backups.cache_snapshot(snapshot) for snapshot in Backups.get_cached_snapshots_service(service_id): if snapshot.id not in [snap.id for snap in snapshots]: Backups.delete_cached_snapshot(snapshot) @staticmethod def _redis_autobackup_key(service_name: str) -> str: return REDIS_AUTOBACKUP_ENABLED_PREFIX + service_name @staticmethod def enable_autobackup(service: Service): redis.set(Backups._redis_autobackup_key(service.get_id()), 1) @staticmethod def is_time_to_backup(time: datetime) -> bool: """ Intended as a time validator for huey cron scheduler of automatic backups """ for key in redis.keys(REDIS_AUTOBACKUP_ENABLED_PREFIX + "*"): service_id = key.split(":")[-1] if Backups.is_time_to_backup_service(service_id, time): return True return False @staticmethod def is_time_to_backup_service(service_id: str, time: datetime): period = Backups.autobackup_period_minutes() if period is None: return False if not Backups._is_autobackup_enabled_by_name(service_id) is None: return False last_backup = Backups._get_last_backup_time_redis(service_id) if last_backup is None: return True # queue a backup immediately if there are no previous backups if time > last_backup + timedelta(minutes=period): return True return False @staticmethod def disable_autobackup(service: Service): """also see disable_all_autobackup()""" redis.delete(Backups._redis_autobackup_key(service.get_id())) @staticmethod def is_autobackup_enabled(service: Service) -> bool: return Backups._is_autobackup_enabled_by_name(service.get_id()) @staticmethod def _is_autobackup_enabled_by_name(service_name: str): return redis.exists(Backups._redis_autobackup_key(service_name)) @staticmethod def autobackup_period_minutes() -> Optional[int]: """None means autobackup is disabled""" if not redis.exists(REDIS_AUTOBACKUP_PERIOD_KEY): return None return int(redis.get(REDIS_AUTOBACKUP_PERIOD_KEY)) @staticmethod def set_autobackup_period_minutes(minutes: int): """ 0 and negative numbers are equivalent to disable. Setting to a positive number may result in a backup very soon if some services are not backed up. """ if minutes <= 0: Backups.disable_all_autobackup() return redis.set(REDIS_AUTOBACKUP_PERIOD_KEY, minutes) @staticmethod def disable_all_autobackup(): """disables all automatic backing up, but does not change per-service settings""" redis.delete(REDIS_AUTOBACKUP_PERIOD_KEY) @staticmethod def provider(): return Backups.lookup_provider() @staticmethod def set_provider(kind: str, login: str, key: str): provider = Backups.construct_provider(kind, login, key) Backups.store_provider_redis(provider) @staticmethod def construct_provider(kind: str, login: str, key: str): provider_class = get_provider(BackupProvider[kind]) if kind == "FILE": path = redis.get(REDIS_REPO_PATH_KEY) return provider_class(path) return provider_class(login=login, key=key) @staticmethod def store_provider_redis(provider: AbstractBackupProvider): store_model_as_hash( redis, REDIS_PROVIDER_KEY, BackupProviderModel( kind=get_kind(provider), login=provider.login, key=provider.key ), ) @staticmethod def load_provider_redis() -> AbstractBackupProvider: provider_model = hash_as_model(redis, REDIS_PROVIDER_KEY, BackupProviderModel) if provider_model is None: return None return Backups.construct_provider( provider_model.kind, provider_model.login, provider_model.key ) @staticmethod def reset(): redis.delete(REDIS_PROVIDER_KEY) redis.delete(REDIS_REPO_PATH_KEY) redis.delete(REDIS_AUTOBACKUP_PERIOD_KEY) prefixes_to_clean = [ REDIS_INITTED_CACHE_PREFIX, REDIS_SNAPSHOTS_PREFIX, REDIS_LAST_BACKUP_PREFIX, REDIS_AUTOBACKUP_ENABLED_PREFIX, ] for prefix in prefixes_to_clean: for key in redis.keys(prefix + "*"): redis.delete(key) @staticmethod def lookup_provider() -> AbstractBackupProvider: redis_provider = Backups.load_provider_redis() if redis_provider is not None: return redis_provider json_provider = Backups.load_provider_json() if json_provider is not None: Backups.store_provider_redis(json_provider) return json_provider memory_provider = Backups.construct_provider("MEMORY", login="", key="") Backups.store_provider_redis(memory_provider) return memory_provider @staticmethod def load_provider_json() -> AbstractBackupProvider: with ReadUserData() as user_data: account = "" key = "" if "backup" not in user_data.keys(): if "backblaze" in user_data.keys(): account = user_data["backblaze"]["accountId"] key = user_data["backblaze"]["accountKey"] provider_string = "BACKBLAZE" return Backups.construct_provider( kind=provider_string, login=account, key=key ) return None account = user_data["backup"]["accountId"] key = user_data["backup"]["accountKey"] provider_string = user_data["backup"]["provider"] return Backups.construct_provider( kind=provider_string, login=account, key=key ) @staticmethod def back_up(service: Service): """The top-level function to back up a service""" folder = service.get_location() repo_name = service.get_id() service.pre_backup() snapshot = Backups.provider().backuper.start_backup(folder, repo_name) Backups._store_last_snapshot(repo_name, snapshot) service.post_restore() @staticmethod def init_repo(service: Service): repo_name = service.get_id() Backups.provider().backuper.init(repo_name) Backups._redis_mark_as_init(service) @staticmethod def _has_redis_init_mark(service: Service) -> bool: repo_name = service.get_id() if redis.exists(REDIS_INITTED_CACHE_PREFIX + repo_name): return True return False @staticmethod def _redis_mark_as_init(service: Service): repo_name = service.get_id() redis.set(REDIS_INITTED_CACHE_PREFIX + repo_name, 1) @staticmethod def is_initted(service: Service) -> bool: repo_name = service.get_id() if Backups._has_redis_init_mark(service): return True initted = Backups.provider().backuper.is_initted(repo_name) if initted: Backups._redis_mark_as_init(service) return True return False @staticmethod def get_snapshots(service: Service) -> List[Snapshot]: service_id = service.get_id() cached_snapshots = Backups.get_cached_snapshots_service(service_id) if cached_snapshots != []: return cached_snapshots # TODO: the oldest snapshots will get expired faster than the new ones. # How to detect that the end is missing? upstream_snapshots = Backups.provider().backuper.get_snapshots(service_id) Backups.sync_service_snapshots(service_id, upstream_snapshots) return upstream_snapshots @staticmethod def restore_service_from_snapshot(service: Service, snapshot_id: str): repo_name = service.get_id() folder = service.get_location() Backups.provider().backuper.restore_from_backup(repo_name, snapshot_id, folder) # Our dummy service is not yet globally registered so this is not testable yet @staticmethod def restore_snapshot(snapshot: Snapshot): Backups.restore_service_from_snapshot( get_service_by_id(snapshot.service_name), snapshot.id ) @staticmethod def service_snapshot_size(service: Service, snapshot_id: str) -> float: repo_name = service.get_id() return Backups.provider().backuper.restored_size(repo_name, snapshot_id) # Our dummy service is not yet globally registered so this is not testable yet @staticmethod def snapshot_restored_size(snapshot: Snapshot) -> float: return Backups.service_snapshot_size( get_service_by_id(snapshot.service_name), snapshot.id )