mirror of
https://git.selfprivacy.org/SelfPrivacy/selfprivacy-rest-api.git
synced 2025-01-11 18:39:30 +00:00
refactoring(backups): backuper -> backupper
This commit is contained in:
parent
c94b4d07bf
commit
4b1594ca22
|
@ -287,7 +287,7 @@ class Backups:
|
||||||
|
|
||||||
try:
|
try:
|
||||||
service.pre_backup()
|
service.pre_backup()
|
||||||
snapshot = Backups.provider().backuper.start_backup(
|
snapshot = Backups.provider().backupper.start_backup(
|
||||||
folders,
|
folders,
|
||||||
tag,
|
tag,
|
||||||
)
|
)
|
||||||
|
@ -302,7 +302,7 @@ class Backups:
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def init_repo():
|
def init_repo():
|
||||||
Backups.provider().backuper.init()
|
Backups.provider().backupper.init()
|
||||||
Storage.mark_as_init()
|
Storage.mark_as_init()
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
|
@ -310,7 +310,7 @@ class Backups:
|
||||||
if Storage.has_init_mark():
|
if Storage.has_init_mark():
|
||||||
return True
|
return True
|
||||||
|
|
||||||
initted = Backups.provider().backuper.is_initted()
|
initted = Backups.provider().backupper.is_initted()
|
||||||
if initted:
|
if initted:
|
||||||
Storage.mark_as_init()
|
Storage.mark_as_init()
|
||||||
return True
|
return True
|
||||||
|
@ -336,7 +336,7 @@ class Backups:
|
||||||
# TODO: the oldest snapshots will get expired faster than the new ones.
|
# TODO: the oldest snapshots will get expired faster than the new ones.
|
||||||
# How to detect that the end is missing?
|
# How to detect that the end is missing?
|
||||||
|
|
||||||
upstream_snapshots = Backups.provider().backuper.get_snapshots()
|
upstream_snapshots = Backups.provider().backupper.get_snapshots()
|
||||||
Backups.sync_all_snapshots()
|
Backups.sync_all_snapshots()
|
||||||
return upstream_snapshots
|
return upstream_snapshots
|
||||||
|
|
||||||
|
@ -358,7 +358,7 @@ class Backups:
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def sync_all_snapshots():
|
def sync_all_snapshots():
|
||||||
upstream_snapshots = Backups.provider().backuper.get_snapshots()
|
upstream_snapshots = Backups.provider().backupper.get_snapshots()
|
||||||
Storage.invalidate_snapshot_storage()
|
Storage.invalidate_snapshot_storage()
|
||||||
for snapshot in upstream_snapshots:
|
for snapshot in upstream_snapshots:
|
||||||
Storage.cache_snapshot(snapshot)
|
Storage.cache_snapshot(snapshot)
|
||||||
|
@ -368,7 +368,7 @@ class Backups:
|
||||||
def restore_service_from_snapshot(service: Service, snapshot_id: str):
|
def restore_service_from_snapshot(service: Service, snapshot_id: str):
|
||||||
folders = service.get_folders()
|
folders = service.get_folders()
|
||||||
|
|
||||||
Backups.provider().backuper.restore_from_backup(
|
Backups.provider().backupper.restore_from_backup(
|
||||||
snapshot_id,
|
snapshot_id,
|
||||||
folders,
|
folders,
|
||||||
)
|
)
|
||||||
|
@ -427,7 +427,7 @@ class Backups:
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def service_snapshot_size(snapshot_id: str) -> int:
|
def service_snapshot_size(snapshot_id: str) -> int:
|
||||||
return Backups.provider().backuper.restored_size(
|
return Backups.provider().backupper.restored_size(
|
||||||
snapshot_id,
|
snapshot_id,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
|
@ -4,7 +4,7 @@ from typing import List
|
||||||
from selfprivacy_api.models.backup.snapshot import Snapshot
|
from selfprivacy_api.models.backup.snapshot import Snapshot
|
||||||
|
|
||||||
|
|
||||||
class AbstractBackuper(ABC):
|
class AbstractBackupper(ABC):
|
||||||
def __init__(self):
|
def __init__(self):
|
||||||
pass
|
pass
|
||||||
|
|
||||||
|
|
|
@ -1,10 +1,10 @@
|
||||||
from typing import List
|
from typing import List
|
||||||
|
|
||||||
from selfprivacy_api.models.backup.snapshot import Snapshot
|
from selfprivacy_api.models.backup.snapshot import Snapshot
|
||||||
from selfprivacy_api.backup.backuppers import AbstractBackuper
|
from selfprivacy_api.backup.backuppers import AbstractBackupper
|
||||||
|
|
||||||
|
|
||||||
class NoneBackupper(AbstractBackuper):
|
class NoneBackupper(AbstractBackupper):
|
||||||
def is_initted(self, repo_name: str = "") -> bool:
|
def is_initted(self, repo_name: str = "") -> bool:
|
||||||
return False
|
return False
|
||||||
|
|
||||||
|
|
|
@ -6,7 +6,7 @@ from typing import List
|
||||||
from collections.abc import Iterable
|
from collections.abc import Iterable
|
||||||
from json.decoder import JSONDecodeError
|
from json.decoder import JSONDecodeError
|
||||||
|
|
||||||
from selfprivacy_api.backup.backuppers import AbstractBackuper
|
from selfprivacy_api.backup.backuppers import AbstractBackupper
|
||||||
from selfprivacy_api.models.backup.snapshot import Snapshot
|
from selfprivacy_api.models.backup.snapshot import Snapshot
|
||||||
from selfprivacy_api.backup.jobs import get_backup_job
|
from selfprivacy_api.backup.jobs import get_backup_job
|
||||||
from selfprivacy_api.services import get_service_by_id
|
from selfprivacy_api.services import get_service_by_id
|
||||||
|
@ -15,7 +15,7 @@ from selfprivacy_api.jobs import Jobs, JobStatus
|
||||||
from selfprivacy_api.backup.local_secret import LocalBackupSecret
|
from selfprivacy_api.backup.local_secret import LocalBackupSecret
|
||||||
|
|
||||||
|
|
||||||
class ResticBackuper(AbstractBackuper):
|
class ResticBackupper(AbstractBackupper):
|
||||||
def __init__(self, login_flag: str, key_flag: str, type: str):
|
def __init__(self, login_flag: str, key_flag: str, type: str):
|
||||||
self.login_flag = login_flag
|
self.login_flag = login_flag
|
||||||
self.key_flag = key_flag
|
self.key_flag = key_flag
|
||||||
|
@ -68,7 +68,7 @@ class ResticBackuper(AbstractBackuper):
|
||||||
]
|
]
|
||||||
)
|
)
|
||||||
if args != []:
|
if args != []:
|
||||||
command.extend(ResticBackuper.__flatten_list(args))
|
command.extend(ResticBackupper.__flatten_list(args))
|
||||||
return command
|
return command
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
|
@ -77,7 +77,7 @@ class ResticBackuper(AbstractBackuper):
|
||||||
result = []
|
result = []
|
||||||
for item in list:
|
for item in list:
|
||||||
if isinstance(item, Iterable) and not isinstance(item, str):
|
if isinstance(item, Iterable) and not isinstance(item, str):
|
||||||
result.extend(ResticBackuper.__flatten_list(item))
|
result.extend(ResticBackupper.__flatten_list(item))
|
||||||
continue
|
continue
|
||||||
result.append(item)
|
result.append(item)
|
||||||
return result
|
return result
|
||||||
|
@ -113,10 +113,10 @@ class ResticBackuper(AbstractBackuper):
|
||||||
messages = []
|
messages = []
|
||||||
job = get_backup_job(get_service_by_id(tag))
|
job = get_backup_job(get_service_by_id(tag))
|
||||||
try:
|
try:
|
||||||
for raw_message in ResticBackuper.output_yielder(backup_command):
|
for raw_message in ResticBackupper.output_yielder(backup_command):
|
||||||
message = self.parse_message(raw_message, job)
|
message = self.parse_message(raw_message, job)
|
||||||
messages.append(message)
|
messages.append(message)
|
||||||
return ResticBackuper._snapshot_from_backup_messages(messages, tag)
|
return ResticBackupper._snapshot_from_backup_messages(messages, tag)
|
||||||
except ValueError as e:
|
except ValueError as e:
|
||||||
raise ValueError("could not create a snapshot: ", messages) from e
|
raise ValueError("could not create a snapshot: ", messages) from e
|
||||||
|
|
||||||
|
@ -124,11 +124,11 @@ class ResticBackuper(AbstractBackuper):
|
||||||
def _snapshot_from_backup_messages(messages, repo_name) -> Snapshot:
|
def _snapshot_from_backup_messages(messages, repo_name) -> Snapshot:
|
||||||
for message in messages:
|
for message in messages:
|
||||||
if message["message_type"] == "summary":
|
if message["message_type"] == "summary":
|
||||||
return ResticBackuper._snapshot_from_fresh_summary(message, repo_name)
|
return ResticBackupper._snapshot_from_fresh_summary(message, repo_name)
|
||||||
raise ValueError("no summary message in restic json output")
|
raise ValueError("no summary message in restic json output")
|
||||||
|
|
||||||
def parse_message(self, raw_message, job=None) -> object:
|
def parse_message(self, raw_message, job=None) -> object:
|
||||||
message = ResticBackuper.parse_json_output(raw_message)
|
message = ResticBackupper.parse_json_output(raw_message)
|
||||||
if message["message_type"] == "status":
|
if message["message_type"] == "status":
|
||||||
if job is not None: # only update status if we run under some job
|
if job is not None: # only update status if we run under some job
|
||||||
Jobs.update(
|
Jobs.update(
|
||||||
|
@ -168,7 +168,7 @@ class ResticBackuper(AbstractBackuper):
|
||||||
|
|
||||||
with subprocess.Popen(command, stdout=subprocess.PIPE, shell=False) as handle:
|
with subprocess.Popen(command, stdout=subprocess.PIPE, shell=False) as handle:
|
||||||
output = handle.communicate()[0].decode("utf-8")
|
output = handle.communicate()[0].decode("utf-8")
|
||||||
if not ResticBackuper.has_json(output):
|
if not ResticBackupper.has_json(output):
|
||||||
return False
|
return False
|
||||||
# raise NotImplementedError("error(big): " + output)
|
# raise NotImplementedError("error(big): " + output)
|
||||||
return True
|
return True
|
||||||
|
@ -190,7 +190,7 @@ class ResticBackuper(AbstractBackuper):
|
||||||
) as handle:
|
) as handle:
|
||||||
output = handle.communicate()[0].decode("utf-8")
|
output = handle.communicate()[0].decode("utf-8")
|
||||||
try:
|
try:
|
||||||
parsed_output = ResticBackuper.parse_json_output(output)
|
parsed_output = ResticBackupper.parse_json_output(output)
|
||||||
return parsed_output["total_size"]
|
return parsed_output["total_size"]
|
||||||
except ValueError as e:
|
except ValueError as e:
|
||||||
raise ValueError("cannot restore a snapshot: " + output) from e
|
raise ValueError("cannot restore a snapshot: " + output) from e
|
||||||
|
@ -239,7 +239,7 @@ class ResticBackuper(AbstractBackuper):
|
||||||
if "Is there a repository at the following location?" in output:
|
if "Is there a repository at the following location?" in output:
|
||||||
raise ValueError("No repository! : " + output)
|
raise ValueError("No repository! : " + output)
|
||||||
try:
|
try:
|
||||||
return ResticBackuper.parse_json_output(output)
|
return ResticBackupper.parse_json_output(output)
|
||||||
except ValueError as e:
|
except ValueError as e:
|
||||||
raise ValueError("Cannot load snapshots: ") from e
|
raise ValueError("Cannot load snapshots: ") from e
|
||||||
|
|
||||||
|
@ -258,7 +258,7 @@ class ResticBackuper(AbstractBackuper):
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def parse_json_output(output: str) -> object:
|
def parse_json_output(output: str) -> object:
|
||||||
starting_index = ResticBackuper.json_start(output)
|
starting_index = ResticBackupper.json_start(output)
|
||||||
|
|
||||||
if starting_index == -1:
|
if starting_index == -1:
|
||||||
raise ValueError("There is no json in the restic output : " + output)
|
raise ValueError("There is no json in the restic output : " + output)
|
||||||
|
@ -292,6 +292,6 @@ class ResticBackuper(AbstractBackuper):
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def has_json(output: str) -> bool:
|
def has_json(output: str) -> bool:
|
||||||
if ResticBackuper.json_start(output) == -1:
|
if ResticBackupper.json_start(output) == -1:
|
||||||
return False
|
return False
|
||||||
return True
|
return True
|
||||||
|
|
|
@ -1,11 +1,11 @@
|
||||||
from .provider import AbstractBackupProvider
|
from .provider import AbstractBackupProvider
|
||||||
from selfprivacy_api.backup.backuppers.restic_backupper import ResticBackuper
|
from selfprivacy_api.backup.backuppers.restic_backupper import ResticBackupper
|
||||||
from selfprivacy_api.graphql.queries.providers import (
|
from selfprivacy_api.graphql.queries.providers import (
|
||||||
BackupProvider as BackupProviderEnum,
|
BackupProvider as BackupProviderEnum,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
class Backblaze(AbstractBackupProvider):
|
class Backblaze(AbstractBackupProvider):
|
||||||
backuper = ResticBackuper("--b2-account", "--b2-key", ":b2:")
|
backupper = ResticBackupper("--b2-account", "--b2-key", ":b2:")
|
||||||
|
|
||||||
name = BackupProviderEnum.BACKBLAZE
|
name = BackupProviderEnum.BACKBLAZE
|
||||||
|
|
|
@ -1,11 +1,11 @@
|
||||||
from .provider import AbstractBackupProvider
|
from .provider import AbstractBackupProvider
|
||||||
from selfprivacy_api.backup.backuppers.restic_backupper import ResticBackuper
|
from selfprivacy_api.backup.backuppers.restic_backupper import ResticBackupper
|
||||||
from selfprivacy_api.graphql.queries.providers import (
|
from selfprivacy_api.graphql.queries.providers import (
|
||||||
BackupProvider as BackupProviderEnum,
|
BackupProvider as BackupProviderEnum,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
class LocalFileBackup(AbstractBackupProvider):
|
class LocalFileBackup(AbstractBackupProvider):
|
||||||
backuper = ResticBackuper("", "", ":local:")
|
backupper = ResticBackupper("", "", ":local:")
|
||||||
|
|
||||||
name = BackupProviderEnum.FILE
|
name = BackupProviderEnum.FILE
|
||||||
|
|
|
@ -1,11 +1,11 @@
|
||||||
from .provider import AbstractBackupProvider
|
from .provider import AbstractBackupProvider
|
||||||
from selfprivacy_api.backup.backuppers.restic_backupper import ResticBackuper
|
from selfprivacy_api.backup.backuppers.restic_backupper import ResticBackupper
|
||||||
from selfprivacy_api.graphql.queries.providers import (
|
from selfprivacy_api.graphql.queries.providers import (
|
||||||
BackupProvider as BackupProviderEnum,
|
BackupProvider as BackupProviderEnum,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
class InMemoryBackup(AbstractBackupProvider):
|
class InMemoryBackup(AbstractBackupProvider):
|
||||||
backuper = ResticBackuper("", "", ":memory:")
|
backupper = ResticBackupper("", "", ":memory:")
|
||||||
|
|
||||||
name = BackupProviderEnum.MEMORY
|
name = BackupProviderEnum.MEMORY
|
||||||
|
|
|
@ -6,6 +6,6 @@ from selfprivacy_api.graphql.queries.providers import (
|
||||||
|
|
||||||
|
|
||||||
class NoBackups(AbstractBackupProvider):
|
class NoBackups(AbstractBackupProvider):
|
||||||
backuper = NoneBackupper()
|
backupper = NoneBackupper()
|
||||||
|
|
||||||
name = BackupProviderEnum.NONE
|
name = BackupProviderEnum.NONE
|
||||||
|
|
|
@ -4,19 +4,19 @@ It assumes that while some providers are supported via restic/rclone, others
|
||||||
may require different backends
|
may require different backends
|
||||||
"""
|
"""
|
||||||
from abc import ABC, abstractmethod
|
from abc import ABC, abstractmethod
|
||||||
from selfprivacy_api.backup.backuppers import AbstractBackuper
|
from selfprivacy_api.backup.backuppers import AbstractBackupper
|
||||||
from selfprivacy_api.graphql.queries.providers import (
|
from selfprivacy_api.graphql.queries.providers import (
|
||||||
BackupProvider as BackupProviderEnum,
|
BackupProvider as BackupProviderEnum,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
class AbstractBackupProvider(ABC):
|
class AbstractBackupProvider(ABC):
|
||||||
backuper: AbstractBackuper
|
backupper: AbstractBackupper
|
||||||
|
|
||||||
name: BackupProviderEnum
|
name: BackupProviderEnum
|
||||||
|
|
||||||
def __init__(self, login="", key="", location="", repo_id=""):
|
def __init__(self, login="", key="", location="", repo_id=""):
|
||||||
self.backuper.set_creds(login, key, location)
|
self.backupper.set_creds(login, key, location)
|
||||||
self.login = login
|
self.login = login
|
||||||
self.key = key
|
self.key = key
|
||||||
self.location = location
|
self.location = location
|
||||||
|
|
|
@ -111,8 +111,8 @@ def test_config_load(generic_userdata):
|
||||||
assert provider.key == "KEY"
|
assert provider.key == "KEY"
|
||||||
assert provider.location == "selfprivacy"
|
assert provider.location == "selfprivacy"
|
||||||
|
|
||||||
assert provider.backuper.account == "ID"
|
assert provider.backupper.account == "ID"
|
||||||
assert provider.backuper.key == "KEY"
|
assert provider.backupper.key == "KEY"
|
||||||
|
|
||||||
|
|
||||||
def test_json_reset(generic_userdata):
|
def test_json_reset(generic_userdata):
|
||||||
|
@ -141,7 +141,7 @@ def test_select_backend():
|
||||||
|
|
||||||
|
|
||||||
def test_file_backend_init(file_backup):
|
def test_file_backend_init(file_backup):
|
||||||
file_backup.backuper.init()
|
file_backup.backupper.init()
|
||||||
|
|
||||||
|
|
||||||
def test_backup_simple_file(raw_dummy_service, file_backup):
|
def test_backup_simple_file(raw_dummy_service, file_backup):
|
||||||
|
@ -151,7 +151,7 @@ def test_backup_simple_file(raw_dummy_service, file_backup):
|
||||||
assert file_backup is not None
|
assert file_backup is not None
|
||||||
|
|
||||||
name = service.get_id()
|
name = service.get_id()
|
||||||
file_backup.backuper.init()
|
file_backup.backupper.init()
|
||||||
|
|
||||||
|
|
||||||
def test_backup_service(dummy_service, backups):
|
def test_backup_service(dummy_service, backups):
|
||||||
|
@ -172,7 +172,7 @@ def test_backup_service(dummy_service, backups):
|
||||||
|
|
||||||
def test_no_repo(memory_backup):
|
def test_no_repo(memory_backup):
|
||||||
with pytest.raises(ValueError):
|
with pytest.raises(ValueError):
|
||||||
assert memory_backup.backuper.get_snapshots() == []
|
assert memory_backup.backupper.get_snapshots() == []
|
||||||
|
|
||||||
|
|
||||||
def test_one_snapshot(backups, dummy_service):
|
def test_one_snapshot(backups, dummy_service):
|
||||||
|
@ -188,7 +188,7 @@ def test_backup_returns_snapshot(backups, dummy_service):
|
||||||
service_folders = dummy_service.get_folders()
|
service_folders = dummy_service.get_folders()
|
||||||
provider = Backups.provider()
|
provider = Backups.provider()
|
||||||
name = dummy_service.get_id()
|
name = dummy_service.get_id()
|
||||||
snapshot = provider.backuper.start_backup(service_folders, name)
|
snapshot = provider.backupper.start_backup(service_folders, name)
|
||||||
|
|
||||||
assert snapshot.id is not None
|
assert snapshot.id is not None
|
||||||
assert snapshot.service_name == name
|
assert snapshot.service_name == name
|
||||||
|
|
Loading…
Reference in a new issue