mirror of
https://git.selfprivacy.org/SelfPrivacy/selfprivacy-rest-api.git
synced 2025-01-11 18:39:30 +00:00
Merge pull request 'Forgetting' (#46) from backups-forget into master
Reviewed-on: https://git.selfprivacy.org/SelfPrivacy/selfprivacy-rest-api/pulls/46 Reviewed-by: Inex Code <inex.code@selfprivacy.org>
This commit is contained in:
commit
36e915907f
|
@ -2,6 +2,7 @@
|
||||||
This module contains the controller class for backups.
|
This module contains the controller class for backups.
|
||||||
"""
|
"""
|
||||||
from datetime import datetime, timedelta
|
from datetime import datetime, timedelta
|
||||||
|
import os
|
||||||
from os import statvfs
|
from os import statvfs
|
||||||
from typing import List, Optional
|
from typing import List, Optional
|
||||||
|
|
||||||
|
@ -43,6 +44,13 @@ DEFAULT_JSON_PROVIDER = {
|
||||||
"bucket": "",
|
"bucket": "",
|
||||||
}
|
}
|
||||||
|
|
||||||
|
BACKUP_PROVIDER_ENVS = {
|
||||||
|
"kind": "BACKUP_KIND",
|
||||||
|
"login": "BACKUP_LOGIN",
|
||||||
|
"key": "BACKUP_KEY",
|
||||||
|
"location": "BACKUP_LOCATION",
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
class NotDeadError(AssertionError):
|
class NotDeadError(AssertionError):
|
||||||
"""
|
"""
|
||||||
|
@ -132,6 +140,24 @@ class Backups:
|
||||||
Storage.store_provider(none_provider)
|
Storage.store_provider(none_provider)
|
||||||
return none_provider
|
return none_provider
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def set_provider_from_envs():
|
||||||
|
for env in BACKUP_PROVIDER_ENVS.values():
|
||||||
|
if env not in os.environ.keys():
|
||||||
|
raise ValueError(
|
||||||
|
f"Cannot set backup provider from envs, there is no {env} set"
|
||||||
|
)
|
||||||
|
|
||||||
|
kind_str = os.environ[BACKUP_PROVIDER_ENVS["kind"]]
|
||||||
|
kind_enum = BackupProviderEnum[kind_str]
|
||||||
|
provider = Backups._construct_provider(
|
||||||
|
kind=kind_enum,
|
||||||
|
login=os.environ[BACKUP_PROVIDER_ENVS["login"]],
|
||||||
|
key=os.environ[BACKUP_PROVIDER_ENVS["key"]],
|
||||||
|
location=os.environ[BACKUP_PROVIDER_ENVS["location"]],
|
||||||
|
)
|
||||||
|
Storage.store_provider(provider)
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def _construct_provider(
|
def _construct_provider(
|
||||||
kind: BackupProviderEnum,
|
kind: BackupProviderEnum,
|
||||||
|
@ -210,6 +236,14 @@ class Backups:
|
||||||
Backups.provider().backupper.init()
|
Backups.provider().backupper.init()
|
||||||
Storage.mark_as_init()
|
Storage.mark_as_init()
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def erase_repo() -> None:
|
||||||
|
"""
|
||||||
|
Completely empties the remote
|
||||||
|
"""
|
||||||
|
Backups.provider().backupper.erase_repo()
|
||||||
|
Storage.mark_as_uninitted()
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def is_initted() -> bool:
|
def is_initted() -> bool:
|
||||||
"""
|
"""
|
||||||
|
@ -405,10 +439,18 @@ class Backups:
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def forget_snapshot(snapshot: Snapshot) -> None:
|
def forget_snapshot(snapshot: Snapshot) -> None:
|
||||||
"""Deletes a snapshot from the storage"""
|
"""Deletes a snapshot from the repo and from cache"""
|
||||||
Backups.provider().backupper.forget_snapshot(snapshot.id)
|
Backups.provider().backupper.forget_snapshot(snapshot.id)
|
||||||
Storage.delete_cached_snapshot(snapshot)
|
Storage.delete_cached_snapshot(snapshot)
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def forget_all_snapshots():
|
||||||
|
"""deliberately erase all snapshots we made"""
|
||||||
|
# there is no dedicated optimized command for this,
|
||||||
|
# but maybe we can have a multi-erase
|
||||||
|
for snapshot in Backups.get_all_snapshots():
|
||||||
|
Backups.forget_snapshot(snapshot)
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def force_snapshot_cache_reload() -> None:
|
def force_snapshot_cache_reload() -> None:
|
||||||
"""
|
"""
|
||||||
|
|
|
@ -36,6 +36,11 @@ class AbstractBackupper(ABC):
|
||||||
"""Initialize the repository"""
|
"""Initialize the repository"""
|
||||||
raise NotImplementedError
|
raise NotImplementedError
|
||||||
|
|
||||||
|
@abstractmethod
|
||||||
|
def erase_repo(self) -> None:
|
||||||
|
"""Completely empties the remote"""
|
||||||
|
raise NotImplementedError
|
||||||
|
|
||||||
@abstractmethod
|
@abstractmethod
|
||||||
def restore_from_backup(
|
def restore_from_backup(
|
||||||
self,
|
self,
|
||||||
|
|
|
@ -23,6 +23,11 @@ class NoneBackupper(AbstractBackupper):
|
||||||
def init(self):
|
def init(self):
|
||||||
raise NotImplementedError
|
raise NotImplementedError
|
||||||
|
|
||||||
|
def erase_repo(self) -> None:
|
||||||
|
"""Completely empties the remote"""
|
||||||
|
# this one is already empty
|
||||||
|
pass
|
||||||
|
|
||||||
def restore_from_backup(self, snapshot_id: str, folders: List[str], verify=True):
|
def restore_from_backup(self, snapshot_id: str, folders: List[str], verify=True):
|
||||||
"""Restore a target folder using a snapshot"""
|
"""Restore a target folder using a snapshot"""
|
||||||
raise NotImplementedError
|
raise NotImplementedError
|
||||||
|
|
|
@ -1,9 +1,11 @@
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
import subprocess
|
import subprocess
|
||||||
import json
|
import json
|
||||||
import datetime
|
import datetime
|
||||||
import tempfile
|
import tempfile
|
||||||
|
|
||||||
from typing import List
|
from typing import List, TypeVar, Callable
|
||||||
from collections.abc import Iterable
|
from collections.abc import Iterable
|
||||||
from json.decoder import JSONDecodeError
|
from json.decoder import JSONDecodeError
|
||||||
from os.path import exists, join
|
from os.path import exists, join
|
||||||
|
@ -21,6 +23,25 @@ from selfprivacy_api.backup.local_secret import LocalBackupSecret
|
||||||
|
|
||||||
SHORT_ID_LEN = 8
|
SHORT_ID_LEN = 8
|
||||||
|
|
||||||
|
T = TypeVar("T", bound=Callable)
|
||||||
|
|
||||||
|
|
||||||
|
def unlocked_repo(func: T) -> T:
|
||||||
|
"""unlock repo and retry if it appears to be locked"""
|
||||||
|
|
||||||
|
def inner(self: ResticBackupper, *args, **kwargs):
|
||||||
|
try:
|
||||||
|
return func(self, *args, **kwargs)
|
||||||
|
except Exception as e:
|
||||||
|
if "unable to create lock" in str(e):
|
||||||
|
self.unlock()
|
||||||
|
return func(self, *args, **kwargs)
|
||||||
|
else:
|
||||||
|
raise e
|
||||||
|
|
||||||
|
# Above, we manually guarantee that the type returned is compatible.
|
||||||
|
return inner # type: ignore
|
||||||
|
|
||||||
|
|
||||||
class ResticBackupper(AbstractBackupper):
|
class ResticBackupper(AbstractBackupper):
|
||||||
def __init__(self, login_flag: str, key_flag: str, storage_type: str) -> None:
|
def __init__(self, login_flag: str, key_flag: str, storage_type: str) -> None:
|
||||||
|
@ -40,20 +61,25 @@ class ResticBackupper(AbstractBackupper):
|
||||||
def restic_repo(self) -> str:
|
def restic_repo(self) -> str:
|
||||||
# https://restic.readthedocs.io/en/latest/030_preparing_a_new_repo.html#other-services-via-rclone
|
# https://restic.readthedocs.io/en/latest/030_preparing_a_new_repo.html#other-services-via-rclone
|
||||||
# https://forum.rclone.org/t/can-rclone-be-run-solely-with-command-line-options-no-config-no-env-vars/6314/5
|
# https://forum.rclone.org/t/can-rclone-be-run-solely-with-command-line-options-no-config-no-env-vars/6314/5
|
||||||
return f"rclone:{self.storage_type}{self.repo}"
|
return f"rclone:{self.rclone_repo()}"
|
||||||
|
|
||||||
|
def rclone_repo(self) -> str:
|
||||||
|
return f"{self.storage_type}{self.repo}"
|
||||||
|
|
||||||
def rclone_args(self):
|
def rclone_args(self):
|
||||||
return "rclone.args=serve restic --stdio " + self.backend_rclone_args()
|
return "rclone.args=serve restic --stdio " + " ".join(
|
||||||
|
self.backend_rclone_args()
|
||||||
|
)
|
||||||
|
|
||||||
def backend_rclone_args(self) -> str:
|
def backend_rclone_args(self) -> list[str]:
|
||||||
acc_arg = ""
|
args = []
|
||||||
key_arg = ""
|
|
||||||
if self.account != "":
|
if self.account != "":
|
||||||
acc_arg = f"{self.login_flag} {self.account}"
|
acc_args = [self.login_flag, self.account]
|
||||||
|
args.extend(acc_args)
|
||||||
if self.key != "":
|
if self.key != "":
|
||||||
key_arg = f"{self.key_flag} {self.key}"
|
key_args = [self.key_flag, self.key]
|
||||||
|
args.extend(key_args)
|
||||||
return f"{acc_arg} {key_arg}"
|
return args
|
||||||
|
|
||||||
def _password_command(self):
|
def _password_command(self):
|
||||||
return f"echo {LocalBackupSecret.get()}"
|
return f"echo {LocalBackupSecret.get()}"
|
||||||
|
@ -79,6 +105,27 @@ class ResticBackupper(AbstractBackupper):
|
||||||
command.extend(ResticBackupper.__flatten_list(args))
|
command.extend(ResticBackupper.__flatten_list(args))
|
||||||
return command
|
return command
|
||||||
|
|
||||||
|
def erase_repo(self) -> None:
|
||||||
|
"""Fully erases repo on remote, can be reinitted again"""
|
||||||
|
command = [
|
||||||
|
"rclone",
|
||||||
|
"purge",
|
||||||
|
self.rclone_repo(),
|
||||||
|
]
|
||||||
|
backend_args = self.backend_rclone_args()
|
||||||
|
if backend_args:
|
||||||
|
command.extend(backend_args)
|
||||||
|
|
||||||
|
with subprocess.Popen(command, stdout=subprocess.PIPE, shell=False) as handle:
|
||||||
|
output = handle.communicate()[0].decode("utf-8")
|
||||||
|
if handle.returncode != 0:
|
||||||
|
raise ValueError(
|
||||||
|
"purge exited with errorcode",
|
||||||
|
handle.returncode,
|
||||||
|
":",
|
||||||
|
output,
|
||||||
|
)
|
||||||
|
|
||||||
def mount_repo(self, mount_directory):
|
def mount_repo(self, mount_directory):
|
||||||
mount_command = self.restic_command("mount", mount_directory)
|
mount_command = self.restic_command("mount", mount_directory)
|
||||||
mount_command.insert(0, "nohup")
|
mount_command.insert(0, "nohup")
|
||||||
|
@ -116,6 +163,7 @@ class ResticBackupper(AbstractBackupper):
|
||||||
result.append(item)
|
result.append(item)
|
||||||
return result
|
return result
|
||||||
|
|
||||||
|
@unlocked_repo
|
||||||
def start_backup(self, folders: List[str], tag: str) -> Snapshot:
|
def start_backup(self, folders: List[str], tag: str) -> Snapshot:
|
||||||
"""
|
"""
|
||||||
Start backup with restic
|
Start backup with restic
|
||||||
|
@ -139,8 +187,10 @@ class ResticBackupper(AbstractBackupper):
|
||||||
raise ValueError("No service with id ", tag)
|
raise ValueError("No service with id ", tag)
|
||||||
|
|
||||||
job = get_backup_job(service)
|
job = get_backup_job(service)
|
||||||
|
output = []
|
||||||
try:
|
try:
|
||||||
for raw_message in output_yielder(backup_command):
|
for raw_message in output_yielder(backup_command):
|
||||||
|
output.append(raw_message)
|
||||||
message = self.parse_message(
|
message = self.parse_message(
|
||||||
raw_message,
|
raw_message,
|
||||||
job,
|
job,
|
||||||
|
@ -151,7 +201,13 @@ class ResticBackupper(AbstractBackupper):
|
||||||
tag,
|
tag,
|
||||||
)
|
)
|
||||||
except ValueError as error:
|
except ValueError as error:
|
||||||
raise ValueError("Could not create a snapshot: ", messages) from error
|
raise ValueError(
|
||||||
|
"Could not create a snapshot: ",
|
||||||
|
str(error),
|
||||||
|
output,
|
||||||
|
"parsed messages:",
|
||||||
|
messages,
|
||||||
|
) from error
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def _snapshot_from_backup_messages(messages, repo_name) -> Snapshot:
|
def _snapshot_from_backup_messages(messages, repo_name) -> Snapshot:
|
||||||
|
@ -200,23 +256,72 @@ class ResticBackupper(AbstractBackupper):
|
||||||
if "created restic repository" not in output:
|
if "created restic repository" not in output:
|
||||||
raise ValueError("cannot init a repo: " + output)
|
raise ValueError("cannot init a repo: " + output)
|
||||||
|
|
||||||
|
@unlocked_repo
|
||||||
def is_initted(self) -> bool:
|
def is_initted(self) -> bool:
|
||||||
command = self.restic_command(
|
command = self.restic_command(
|
||||||
"check",
|
"check",
|
||||||
"--json",
|
|
||||||
)
|
)
|
||||||
|
|
||||||
with subprocess.Popen(
|
with subprocess.Popen(
|
||||||
command,
|
command,
|
||||||
stdout=subprocess.PIPE,
|
stdout=subprocess.PIPE,
|
||||||
shell=False,
|
shell=False,
|
||||||
|
stderr=subprocess.STDOUT,
|
||||||
) as handle:
|
) as handle:
|
||||||
output = handle.communicate()[0].decode("utf-8")
|
output = handle.communicate()[0].decode("utf-8")
|
||||||
if not ResticBackupper.has_json(output):
|
if handle.returncode != 0:
|
||||||
|
if "unable to create lock" in output:
|
||||||
|
raise ValueError("Stale lock detected: ", output)
|
||||||
return False
|
return False
|
||||||
# raise NotImplementedError("error(big): " + output)
|
|
||||||
return True
|
return True
|
||||||
|
|
||||||
|
def unlock(self) -> None:
|
||||||
|
"""Remove stale locks."""
|
||||||
|
command = self.restic_command(
|
||||||
|
"unlock",
|
||||||
|
)
|
||||||
|
|
||||||
|
with subprocess.Popen(
|
||||||
|
command,
|
||||||
|
stdout=subprocess.PIPE,
|
||||||
|
shell=False,
|
||||||
|
stderr=subprocess.STDOUT,
|
||||||
|
) as handle:
|
||||||
|
# communication forces to complete and for returncode to get defined
|
||||||
|
output = handle.communicate()[0].decode("utf-8")
|
||||||
|
if handle.returncode != 0:
|
||||||
|
raise ValueError("cannot unlock the backup repository: ", output)
|
||||||
|
|
||||||
|
def lock(self) -> None:
|
||||||
|
"""
|
||||||
|
Introduce a stale lock.
|
||||||
|
Mainly for testing purposes.
|
||||||
|
Double lock is supposed to fail
|
||||||
|
"""
|
||||||
|
command = self.restic_command(
|
||||||
|
"check",
|
||||||
|
)
|
||||||
|
|
||||||
|
# using temporary cache in /run/user/1000/restic-check-cache-817079729
|
||||||
|
# repository 9639c714 opened (repository version 2) successfully, password is correct
|
||||||
|
# created new cache in /run/user/1000/restic-check-cache-817079729
|
||||||
|
# create exclusive lock for repository
|
||||||
|
# load indexes
|
||||||
|
# check all packs
|
||||||
|
# check snapshots, trees and blobs
|
||||||
|
# [0:00] 100.00% 1 / 1 snapshots
|
||||||
|
# no errors were found
|
||||||
|
|
||||||
|
try:
|
||||||
|
for line in output_yielder(command):
|
||||||
|
if "indexes" in line:
|
||||||
|
break
|
||||||
|
if "unable" in line:
|
||||||
|
raise ValueError(line)
|
||||||
|
except Exception as e:
|
||||||
|
raise ValueError("could not lock repository") from e
|
||||||
|
|
||||||
|
@unlocked_repo
|
||||||
def restored_size(self, snapshot_id: str) -> int:
|
def restored_size(self, snapshot_id: str) -> int:
|
||||||
"""
|
"""
|
||||||
Size of a snapshot
|
Size of a snapshot
|
||||||
|
@ -230,6 +335,7 @@ class ResticBackupper(AbstractBackupper):
|
||||||
with subprocess.Popen(
|
with subprocess.Popen(
|
||||||
command,
|
command,
|
||||||
stdout=subprocess.PIPE,
|
stdout=subprocess.PIPE,
|
||||||
|
stderr=subprocess.STDOUT,
|
||||||
shell=False,
|
shell=False,
|
||||||
) as handle:
|
) as handle:
|
||||||
output = handle.communicate()[0].decode("utf-8")
|
output = handle.communicate()[0].decode("utf-8")
|
||||||
|
@ -239,6 +345,7 @@ class ResticBackupper(AbstractBackupper):
|
||||||
except ValueError as error:
|
except ValueError as error:
|
||||||
raise ValueError("cannot restore a snapshot: " + output) from error
|
raise ValueError("cannot restore a snapshot: " + output) from error
|
||||||
|
|
||||||
|
@unlocked_repo
|
||||||
def restore_from_backup(
|
def restore_from_backup(
|
||||||
self,
|
self,
|
||||||
snapshot_id,
|
snapshot_id,
|
||||||
|
@ -277,7 +384,10 @@ class ResticBackupper(AbstractBackupper):
|
||||||
)
|
)
|
||||||
|
|
||||||
with subprocess.Popen(
|
with subprocess.Popen(
|
||||||
restore_command, stdout=subprocess.PIPE, shell=False
|
restore_command,
|
||||||
|
stdout=subprocess.PIPE,
|
||||||
|
stderr=subprocess.STDOUT,
|
||||||
|
shell=False,
|
||||||
) as handle:
|
) as handle:
|
||||||
|
|
||||||
# for some reason restore does not support
|
# for some reason restore does not support
|
||||||
|
@ -297,6 +407,7 @@ class ResticBackupper(AbstractBackupper):
|
||||||
output,
|
output,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
@unlocked_repo
|
||||||
def forget_snapshot(self, snapshot_id) -> None:
|
def forget_snapshot(self, snapshot_id) -> None:
|
||||||
"""
|
"""
|
||||||
Either removes snapshot or marks it for deletion later,
|
Either removes snapshot or marks it for deletion later,
|
||||||
|
@ -332,10 +443,7 @@ class ResticBackupper(AbstractBackupper):
|
||||||
) # none should be impossible after communicate
|
) # none should be impossible after communicate
|
||||||
if handle.returncode != 0:
|
if handle.returncode != 0:
|
||||||
raise ValueError(
|
raise ValueError(
|
||||||
"forget exited with errorcode",
|
"forget exited with errorcode", handle.returncode, ":", output, err
|
||||||
handle.returncode,
|
|
||||||
":",
|
|
||||||
output,
|
|
||||||
)
|
)
|
||||||
|
|
||||||
def _load_snapshots(self) -> object:
|
def _load_snapshots(self) -> object:
|
||||||
|
@ -361,8 +469,9 @@ class ResticBackupper(AbstractBackupper):
|
||||||
try:
|
try:
|
||||||
return ResticBackupper.parse_json_output(output)
|
return ResticBackupper.parse_json_output(output)
|
||||||
except ValueError as error:
|
except ValueError as error:
|
||||||
raise ValueError("Cannot load snapshots: ") from error
|
raise ValueError("Cannot load snapshots: ", output) from error
|
||||||
|
|
||||||
|
@unlocked_repo
|
||||||
def get_snapshots(self) -> List[Snapshot]:
|
def get_snapshots(self) -> List[Snapshot]:
|
||||||
"""Get all snapshots from the repo"""
|
"""Get all snapshots from the repo"""
|
||||||
snapshots = []
|
snapshots = []
|
||||||
|
|
|
@ -21,7 +21,7 @@ REDIS_SNAPSHOT_CACHE_EXPIRE_SECONDS = 24 * 60 * 60 # one day
|
||||||
|
|
||||||
REDIS_SNAPSHOTS_PREFIX = "backups:snapshots:"
|
REDIS_SNAPSHOTS_PREFIX = "backups:snapshots:"
|
||||||
REDIS_LAST_BACKUP_PREFIX = "backups:last-backed-up:"
|
REDIS_LAST_BACKUP_PREFIX = "backups:last-backed-up:"
|
||||||
REDIS_INITTED_CACHE_PREFIX = "backups:initted_services:"
|
REDIS_INITTED_CACHE = "backups:repo_initted"
|
||||||
|
|
||||||
REDIS_PROVIDER_KEY = "backups:provider"
|
REDIS_PROVIDER_KEY = "backups:provider"
|
||||||
REDIS_AUTOBACKUP_PERIOD_KEY = "backups:autobackup_period"
|
REDIS_AUTOBACKUP_PERIOD_KEY = "backups:autobackup_period"
|
||||||
|
@ -38,9 +38,9 @@ class Storage:
|
||||||
"""Deletes all backup related data from redis"""
|
"""Deletes all backup related data from redis"""
|
||||||
redis.delete(REDIS_PROVIDER_KEY)
|
redis.delete(REDIS_PROVIDER_KEY)
|
||||||
redis.delete(REDIS_AUTOBACKUP_PERIOD_KEY)
|
redis.delete(REDIS_AUTOBACKUP_PERIOD_KEY)
|
||||||
|
redis.delete(REDIS_INITTED_CACHE)
|
||||||
|
|
||||||
prefixes_to_clean = [
|
prefixes_to_clean = [
|
||||||
REDIS_INITTED_CACHE_PREFIX,
|
|
||||||
REDIS_SNAPSHOTS_PREFIX,
|
REDIS_SNAPSHOTS_PREFIX,
|
||||||
REDIS_LAST_BACKUP_PREFIX,
|
REDIS_LAST_BACKUP_PREFIX,
|
||||||
]
|
]
|
||||||
|
@ -162,11 +162,16 @@ class Storage:
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def has_init_mark() -> bool:
|
def has_init_mark() -> bool:
|
||||||
"""Returns True if the repository was initialized"""
|
"""Returns True if the repository was initialized"""
|
||||||
if redis.exists(REDIS_INITTED_CACHE_PREFIX):
|
if redis.exists(REDIS_INITTED_CACHE):
|
||||||
return True
|
return True
|
||||||
return False
|
return False
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def mark_as_init():
|
def mark_as_init():
|
||||||
"""Marks the repository as initialized"""
|
"""Marks the repository as initialized"""
|
||||||
redis.set(REDIS_INITTED_CACHE_PREFIX, 1)
|
redis.set(REDIS_INITTED_CACHE, 1)
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def mark_as_uninitted():
|
||||||
|
"""Marks the repository as initialized"""
|
||||||
|
redis.delete(REDIS_INITTED_CACHE)
|
||||||
|
|
|
@ -1,8 +1,10 @@
|
||||||
import subprocess
|
import subprocess
|
||||||
from os.path import exists
|
from os.path import exists
|
||||||
|
from typing import Generator
|
||||||
|
|
||||||
|
|
||||||
def output_yielder(command):
|
def output_yielder(command) -> Generator[str, None, None]:
|
||||||
|
"""Note: If you break during iteration, it kills the process"""
|
||||||
with subprocess.Popen(
|
with subprocess.Popen(
|
||||||
command,
|
command,
|
||||||
shell=False,
|
shell=False,
|
||||||
|
@ -10,9 +12,15 @@ def output_yielder(command):
|
||||||
stderr=subprocess.STDOUT,
|
stderr=subprocess.STDOUT,
|
||||||
universal_newlines=True,
|
universal_newlines=True,
|
||||||
) as handle:
|
) as handle:
|
||||||
for line in iter(handle.stdout.readline, ""):
|
if handle is None or handle.stdout is None:
|
||||||
if "NOTICE:" not in line:
|
raise ValueError("could not run command: ", command)
|
||||||
yield line
|
|
||||||
|
try:
|
||||||
|
for line in iter(handle.stdout.readline, ""):
|
||||||
|
if "NOTICE:" not in line:
|
||||||
|
yield line
|
||||||
|
except GeneratorExit:
|
||||||
|
handle.kill()
|
||||||
|
|
||||||
|
|
||||||
def sync(src_path: str, dest_path: str):
|
def sync(src_path: str, dest_path: str):
|
||||||
|
|
|
@ -27,4 +27,4 @@ async def get_token_header(
|
||||||
|
|
||||||
def get_api_version() -> str:
|
def get_api_version() -> str:
|
||||||
"""Get API version"""
|
"""Get API version"""
|
||||||
return "2.2.1"
|
return "2.3.0"
|
||||||
|
|
|
@ -157,6 +157,35 @@ class BackupMutations:
|
||||||
job=job_to_api_job(job),
|
job=job_to_api_job(job),
|
||||||
)
|
)
|
||||||
|
|
||||||
|
@strawberry.mutation(permission_classes=[IsAuthenticated])
|
||||||
|
def forget_snapshot(self, snapshot_id: str) -> GenericMutationReturn:
|
||||||
|
"""Forget a snapshot.
|
||||||
|
Makes it inaccessible from the server.
|
||||||
|
After some time, the data (encrypted) will not be recoverable
|
||||||
|
from the backup server too, but not immediately"""
|
||||||
|
|
||||||
|
snap = Backups.get_snapshot_by_id(snapshot_id)
|
||||||
|
if snap is None:
|
||||||
|
return GenericMutationReturn(
|
||||||
|
success=False,
|
||||||
|
code=404,
|
||||||
|
message=f"snapshot {snapshot_id} not found",
|
||||||
|
)
|
||||||
|
|
||||||
|
try:
|
||||||
|
Backups.forget_snapshot(snap)
|
||||||
|
return GenericMutationReturn(
|
||||||
|
success=True,
|
||||||
|
code=200,
|
||||||
|
message="",
|
||||||
|
)
|
||||||
|
except Exception as error:
|
||||||
|
return GenericMutationReturn(
|
||||||
|
success=False,
|
||||||
|
code=400,
|
||||||
|
message=str(error),
|
||||||
|
)
|
||||||
|
|
||||||
@strawberry.mutation(permission_classes=[IsAuthenticated])
|
@strawberry.mutation(permission_classes=[IsAuthenticated])
|
||||||
def force_snapshots_reload(self) -> GenericMutationReturn:
|
def force_snapshots_reload(self) -> GenericMutationReturn:
|
||||||
"""Force snapshots reload"""
|
"""Force snapshots reload"""
|
||||||
|
|
2
setup.py
2
setup.py
|
@ -2,7 +2,7 @@ from setuptools import setup, find_packages
|
||||||
|
|
||||||
setup(
|
setup(
|
||||||
name="selfprivacy_api",
|
name="selfprivacy_api",
|
||||||
version="2.2.1",
|
version="2.3.0",
|
||||||
packages=find_packages(),
|
packages=find_packages(),
|
||||||
scripts=[
|
scripts=[
|
||||||
"selfprivacy_api/app.py",
|
"selfprivacy_api/app.py",
|
||||||
|
|
|
@ -94,6 +94,18 @@ mutation TestRestoreService($snapshot_id: String!) {
|
||||||
}
|
}
|
||||||
"""
|
"""
|
||||||
|
|
||||||
|
API_FORGET_MUTATION = """
|
||||||
|
mutation TestForgetSnapshot($snapshot_id: String!) {
|
||||||
|
backup {
|
||||||
|
forgetSnapshot(snapshotId: $snapshot_id) {
|
||||||
|
success
|
||||||
|
message
|
||||||
|
code
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
"""
|
||||||
|
|
||||||
API_SNAPSHOTS_QUERY = """
|
API_SNAPSHOTS_QUERY = """
|
||||||
allSnapshots {
|
allSnapshots {
|
||||||
id
|
id
|
||||||
|
@ -143,6 +155,17 @@ def api_backup(authorized_client, service):
|
||||||
return response
|
return response
|
||||||
|
|
||||||
|
|
||||||
|
def api_forget(authorized_client, snapshot_id):
|
||||||
|
response = authorized_client.post(
|
||||||
|
"/graphql",
|
||||||
|
json={
|
||||||
|
"query": API_FORGET_MUTATION,
|
||||||
|
"variables": {"snapshot_id": snapshot_id},
|
||||||
|
},
|
||||||
|
)
|
||||||
|
return response
|
||||||
|
|
||||||
|
|
||||||
def api_set_period(authorized_client, period):
|
def api_set_period(authorized_client, period):
|
||||||
response = authorized_client.post(
|
response = authorized_client.post(
|
||||||
"/graphql",
|
"/graphql",
|
||||||
|
@ -370,3 +393,30 @@ def test_reload_snapshots(authorized_client, dummy_service):
|
||||||
|
|
||||||
snaps = api_snapshots(authorized_client)
|
snaps = api_snapshots(authorized_client)
|
||||||
assert len(snaps) == 1
|
assert len(snaps) == 1
|
||||||
|
|
||||||
|
|
||||||
|
def test_forget_snapshot(authorized_client, dummy_service):
|
||||||
|
response = api_backup(authorized_client, dummy_service)
|
||||||
|
data = get_data(response)["backup"]["startBackup"]
|
||||||
|
|
||||||
|
snaps = api_snapshots(authorized_client)
|
||||||
|
assert len(snaps) == 1
|
||||||
|
|
||||||
|
response = api_forget(authorized_client, snaps[0]["id"])
|
||||||
|
data = get_data(response)["backup"]["forgetSnapshot"]
|
||||||
|
assert_ok(data)
|
||||||
|
|
||||||
|
snaps = api_snapshots(authorized_client)
|
||||||
|
assert len(snaps) == 0
|
||||||
|
|
||||||
|
|
||||||
|
def test_forget_nonexistent_snapshot(authorized_client, dummy_service):
|
||||||
|
snaps = api_snapshots(authorized_client)
|
||||||
|
assert len(snaps) == 0
|
||||||
|
response = api_forget(authorized_client, "898798uekiodpjoiweoiwuoeirueor")
|
||||||
|
data = get_data(response)["backup"]["forgetSnapshot"]
|
||||||
|
assert data["code"] == 404
|
||||||
|
assert data["success"] is False
|
||||||
|
|
||||||
|
snaps = api_snapshots(authorized_client)
|
||||||
|
assert len(snaps) == 0
|
||||||
|
|
|
@ -1,4 +1,5 @@
|
||||||
import pytest
|
import pytest
|
||||||
|
import os
|
||||||
import os.path as path
|
import os.path as path
|
||||||
from os import makedirs
|
from os import makedirs
|
||||||
from os import remove
|
from os import remove
|
||||||
|
@ -18,10 +19,11 @@ from selfprivacy_api.jobs import Jobs, JobStatus
|
||||||
|
|
||||||
from selfprivacy_api.models.backup.snapshot import Snapshot
|
from selfprivacy_api.models.backup.snapshot import Snapshot
|
||||||
|
|
||||||
from selfprivacy_api.backup import Backups
|
from selfprivacy_api.backup import Backups, BACKUP_PROVIDER_ENVS
|
||||||
import selfprivacy_api.backup.providers as providers
|
import selfprivacy_api.backup.providers as providers
|
||||||
from selfprivacy_api.backup.providers import AbstractBackupProvider
|
from selfprivacy_api.backup.providers import AbstractBackupProvider
|
||||||
from selfprivacy_api.backup.providers.backblaze import Backblaze
|
from selfprivacy_api.backup.providers.backblaze import Backblaze
|
||||||
|
from selfprivacy_api.backup.providers.none import NoBackups
|
||||||
from selfprivacy_api.backup.util import sync
|
from selfprivacy_api.backup.util import sync
|
||||||
from selfprivacy_api.backup.backuppers.restic_backupper import ResticBackupper
|
from selfprivacy_api.backup.backuppers.restic_backupper import ResticBackupper
|
||||||
from selfprivacy_api.backup.jobs import add_backup_job, add_restore_job
|
from selfprivacy_api.backup.jobs import add_backup_job, add_restore_job
|
||||||
|
@ -37,14 +39,34 @@ TESTFILE_2_BODY = "testissimo!"
|
||||||
REPO_NAME = "test_backup"
|
REPO_NAME = "test_backup"
|
||||||
|
|
||||||
|
|
||||||
@pytest.fixture(scope="function")
|
def prepare_localfile_backups(temp_dir):
|
||||||
def backups(tmpdir):
|
test_repo_path = path.join(temp_dir, "totallyunrelated")
|
||||||
Backups.reset()
|
assert not path.exists(test_repo_path)
|
||||||
|
|
||||||
test_repo_path = path.join(tmpdir, "totallyunrelated")
|
|
||||||
Backups.set_localfile_repo(test_repo_path)
|
Backups.set_localfile_repo(test_repo_path)
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture(scope="function")
|
||||||
|
def backups_local(tmpdir):
|
||||||
|
Backups.reset()
|
||||||
|
prepare_localfile_backups(tmpdir)
|
||||||
Jobs.reset()
|
Jobs.reset()
|
||||||
|
Backups.init_repo()
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture(scope="function")
|
||||||
|
def backups(tmpdir):
|
||||||
|
# for those tests that are supposed to pass with any repo
|
||||||
|
Backups.reset()
|
||||||
|
if BACKUP_PROVIDER_ENVS["kind"] in os.environ.keys():
|
||||||
|
Backups.set_provider_from_envs()
|
||||||
|
else:
|
||||||
|
prepare_localfile_backups(tmpdir)
|
||||||
|
Jobs.reset()
|
||||||
|
# assert not repo_path
|
||||||
|
|
||||||
|
Backups.init_repo()
|
||||||
|
yield
|
||||||
|
Backups.erase_repo()
|
||||||
|
|
||||||
|
|
||||||
@pytest.fixture()
|
@pytest.fixture()
|
||||||
|
@ -80,11 +102,6 @@ def raw_dummy_service(tmpdir):
|
||||||
@pytest.fixture()
|
@pytest.fixture()
|
||||||
def dummy_service(tmpdir, backups, raw_dummy_service) -> Service:
|
def dummy_service(tmpdir, backups, raw_dummy_service) -> Service:
|
||||||
service = raw_dummy_service
|
service = raw_dummy_service
|
||||||
repo_path = path.join(tmpdir, "test_repo")
|
|
||||||
assert not path.exists(repo_path)
|
|
||||||
# assert not repo_path
|
|
||||||
|
|
||||||
Backups.init_repo()
|
|
||||||
|
|
||||||
# register our service
|
# register our service
|
||||||
services.services.append(service)
|
services.services.append(service)
|
||||||
|
@ -129,6 +146,53 @@ def test_config_load(generic_userdata):
|
||||||
assert provider.backupper.key == "KEY"
|
assert provider.backupper.key == "KEY"
|
||||||
|
|
||||||
|
|
||||||
|
def test_reset_sets_to_none1():
|
||||||
|
Backups.reset()
|
||||||
|
provider = Backups.provider()
|
||||||
|
assert provider is not None
|
||||||
|
assert isinstance(provider, NoBackups)
|
||||||
|
|
||||||
|
|
||||||
|
def test_reset_sets_to_none2(backups):
|
||||||
|
# now with something set up first^^^
|
||||||
|
Backups.reset()
|
||||||
|
provider = Backups.provider()
|
||||||
|
assert provider is not None
|
||||||
|
assert isinstance(provider, NoBackups)
|
||||||
|
|
||||||
|
|
||||||
|
def test_setting_from_envs(tmpdir):
|
||||||
|
Backups.reset()
|
||||||
|
environment_stash = {}
|
||||||
|
if BACKUP_PROVIDER_ENVS["kind"] in os.environ.keys():
|
||||||
|
# we are running under special envs, stash them before rewriting them
|
||||||
|
for key in BACKUP_PROVIDER_ENVS.values():
|
||||||
|
environment_stash[key] = os.environ[key]
|
||||||
|
|
||||||
|
os.environ[BACKUP_PROVIDER_ENVS["kind"]] = "BACKBLAZE"
|
||||||
|
os.environ[BACKUP_PROVIDER_ENVS["login"]] = "ID"
|
||||||
|
os.environ[BACKUP_PROVIDER_ENVS["key"]] = "KEY"
|
||||||
|
os.environ[BACKUP_PROVIDER_ENVS["location"]] = "selfprivacy"
|
||||||
|
Backups.set_provider_from_envs()
|
||||||
|
provider = Backups.provider()
|
||||||
|
|
||||||
|
assert provider is not None
|
||||||
|
assert isinstance(provider, Backblaze)
|
||||||
|
assert provider.login == "ID"
|
||||||
|
assert provider.key == "KEY"
|
||||||
|
assert provider.location == "selfprivacy"
|
||||||
|
|
||||||
|
assert provider.backupper.account == "ID"
|
||||||
|
assert provider.backupper.key == "KEY"
|
||||||
|
|
||||||
|
if environment_stash != {}:
|
||||||
|
for key in BACKUP_PROVIDER_ENVS.values():
|
||||||
|
os.environ[key] = environment_stash[key]
|
||||||
|
else:
|
||||||
|
for key in BACKUP_PROVIDER_ENVS.values():
|
||||||
|
del os.environ[key]
|
||||||
|
|
||||||
|
|
||||||
def test_json_reset(generic_userdata):
|
def test_json_reset(generic_userdata):
|
||||||
Backups.reset(reset_json=False)
|
Backups.reset(reset_json=False)
|
||||||
provider = Backups.provider()
|
provider = Backups.provider()
|
||||||
|
@ -158,6 +222,19 @@ def test_file_backend_init(file_backup):
|
||||||
file_backup.backupper.init()
|
file_backup.backupper.init()
|
||||||
|
|
||||||
|
|
||||||
|
def test_reinit_after_purge(backups):
|
||||||
|
assert Backups.is_initted() is True
|
||||||
|
|
||||||
|
Backups.erase_repo()
|
||||||
|
assert Backups.is_initted() is False
|
||||||
|
with pytest.raises(ValueError):
|
||||||
|
Backups.get_all_snapshots()
|
||||||
|
|
||||||
|
Backups.init_repo()
|
||||||
|
assert Backups.is_initted() is True
|
||||||
|
assert len(Backups.get_all_snapshots()) == 0
|
||||||
|
|
||||||
|
|
||||||
def test_backup_simple_file(raw_dummy_service, file_backup):
|
def test_backup_simple_file(raw_dummy_service, file_backup):
|
||||||
# temporarily incomplete
|
# temporarily incomplete
|
||||||
service = raw_dummy_service
|
service = raw_dummy_service
|
||||||
|
@ -258,9 +335,12 @@ def test_sizing(backups, dummy_service):
|
||||||
assert size > 0
|
assert size > 0
|
||||||
|
|
||||||
|
|
||||||
def test_init_tracking(backups, raw_dummy_service):
|
def test_init_tracking(backups, tmpdir):
|
||||||
|
assert Backups.is_initted() is True
|
||||||
|
Backups.reset()
|
||||||
assert Backups.is_initted() is False
|
assert Backups.is_initted() is False
|
||||||
|
separate_dir = tmpdir / "out_of_the_way"
|
||||||
|
prepare_localfile_backups(separate_dir)
|
||||||
Backups.init_repo()
|
Backups.init_repo()
|
||||||
|
|
||||||
assert Backups.is_initted() is True
|
assert Backups.is_initted() is True
|
||||||
|
@ -552,8 +632,38 @@ def test_snapshots_caching(backups, dummy_service):
|
||||||
assert len(cached_snapshots) == 1
|
assert len(cached_snapshots) == 1
|
||||||
|
|
||||||
|
|
||||||
|
def lowlevel_forget(snapshot_id):
|
||||||
|
Backups.provider().backupper.forget_snapshot(snapshot_id)
|
||||||
|
|
||||||
|
|
||||||
|
# Storage
|
||||||
|
def test_snapshots_cache_invalidation(backups, dummy_service):
|
||||||
|
Backups.back_up(dummy_service)
|
||||||
|
cached_snapshots = Storage.get_cached_snapshots()
|
||||||
|
assert len(cached_snapshots) == 1
|
||||||
|
|
||||||
|
Storage.invalidate_snapshot_storage()
|
||||||
|
cached_snapshots = Storage.get_cached_snapshots()
|
||||||
|
assert len(cached_snapshots) == 0
|
||||||
|
|
||||||
|
Backups.force_snapshot_cache_reload()
|
||||||
|
cached_snapshots = Storage.get_cached_snapshots()
|
||||||
|
assert len(cached_snapshots) == 1
|
||||||
|
snap = cached_snapshots[0]
|
||||||
|
|
||||||
|
lowlevel_forget(snap.id)
|
||||||
|
cached_snapshots = Storage.get_cached_snapshots()
|
||||||
|
assert len(cached_snapshots) == 1
|
||||||
|
|
||||||
|
Backups.force_snapshot_cache_reload()
|
||||||
|
cached_snapshots = Storage.get_cached_snapshots()
|
||||||
|
assert len(cached_snapshots) == 0
|
||||||
|
|
||||||
|
|
||||||
# Storage
|
# Storage
|
||||||
def test_init_tracking_caching(backups, raw_dummy_service):
|
def test_init_tracking_caching(backups, raw_dummy_service):
|
||||||
|
assert Storage.has_init_mark() is True
|
||||||
|
Backups.reset()
|
||||||
assert Storage.has_init_mark() is False
|
assert Storage.has_init_mark() is False
|
||||||
|
|
||||||
Storage.mark_as_init()
|
Storage.mark_as_init()
|
||||||
|
@ -563,7 +673,12 @@ def test_init_tracking_caching(backups, raw_dummy_service):
|
||||||
|
|
||||||
|
|
||||||
# Storage
|
# Storage
|
||||||
def test_init_tracking_caching2(backups, raw_dummy_service):
|
def test_init_tracking_caching2(backups, tmpdir):
|
||||||
|
assert Storage.has_init_mark() is True
|
||||||
|
Backups.reset()
|
||||||
|
assert Storage.has_init_mark() is False
|
||||||
|
separate_dir = tmpdir / "out_of_the_way"
|
||||||
|
prepare_localfile_backups(separate_dir)
|
||||||
assert Storage.has_init_mark() is False
|
assert Storage.has_init_mark() is False
|
||||||
|
|
||||||
Backups.init_repo()
|
Backups.init_repo()
|
||||||
|
@ -643,3 +758,51 @@ def test_move_blocks_backups(backups, dummy_service, restore_strategy):
|
||||||
|
|
||||||
with pytest.raises(ValueError):
|
with pytest.raises(ValueError):
|
||||||
Backups.restore_snapshot(snap, restore_strategy)
|
Backups.restore_snapshot(snap, restore_strategy)
|
||||||
|
|
||||||
|
|
||||||
|
def test_double_lock_unlock(backups, dummy_service):
|
||||||
|
# notice that introducing stale locks is only safe for other tests if we erase repo in between
|
||||||
|
# which we do at the time of writing this test
|
||||||
|
|
||||||
|
Backups.provider().backupper.lock()
|
||||||
|
with pytest.raises(ValueError):
|
||||||
|
Backups.provider().backupper.lock()
|
||||||
|
|
||||||
|
Backups.provider().backupper.unlock()
|
||||||
|
Backups.provider().backupper.lock()
|
||||||
|
|
||||||
|
Backups.provider().backupper.unlock()
|
||||||
|
Backups.provider().backupper.unlock()
|
||||||
|
|
||||||
|
|
||||||
|
def test_operations_while_locked(backups, dummy_service):
|
||||||
|
# Stale lock prevention test
|
||||||
|
|
||||||
|
# consider making it fully at the level of backupper?
|
||||||
|
# because this is where prevention lives?
|
||||||
|
# Backups singleton is here only so that we can run this against B2, S3 and whatever
|
||||||
|
# But maybe it is not necessary (if restic treats them uniformly enough)
|
||||||
|
|
||||||
|
Backups.provider().backupper.lock()
|
||||||
|
snap = Backups.back_up(dummy_service)
|
||||||
|
assert snap is not None
|
||||||
|
|
||||||
|
Backups.provider().backupper.lock()
|
||||||
|
# using lowlevel to make sure no caching interferes
|
||||||
|
assert Backups.provider().backupper.is_initted() is True
|
||||||
|
|
||||||
|
Backups.provider().backupper.lock()
|
||||||
|
assert Backups.snapshot_restored_size(snap.id) > 0
|
||||||
|
|
||||||
|
Backups.provider().backupper.lock()
|
||||||
|
Backups.restore_snapshot(snap)
|
||||||
|
|
||||||
|
Backups.provider().backupper.lock()
|
||||||
|
Backups.forget_snapshot(snap)
|
||||||
|
|
||||||
|
Backups.provider().backupper.lock()
|
||||||
|
assert Backups.provider().backupper.get_snapshots() == []
|
||||||
|
|
||||||
|
# check that no locks were left
|
||||||
|
Backups.provider().backupper.lock()
|
||||||
|
Backups.provider().backupper.unlock()
|
||||||
|
|
Loading…
Reference in a new issue