fix(backups): robustness against stale locks: backing up

This commit is contained in:
Houkime 2023-08-09 13:47:18 +00:00
parent 752a0b807e
commit eca4b26a31
2 changed files with 38 additions and 2 deletions

View file

@ -1,9 +1,11 @@
from __future__ import annotations
import subprocess import subprocess
import json import json
import datetime import datetime
import tempfile import tempfile
from typing import List from typing import List, TypeVar, Callable
from collections.abc import Iterable from collections.abc import Iterable
from json.decoder import JSONDecodeError from json.decoder import JSONDecodeError
from os.path import exists, join from os.path import exists, join
@ -21,6 +23,25 @@ from selfprivacy_api.backup.local_secret import LocalBackupSecret
SHORT_ID_LEN = 8 SHORT_ID_LEN = 8
T = TypeVar("T", bound=Callable)
def unlocked_repo(func: T) -> T:
"""unlock repo and retry if it appears to be locked"""
def inner(self: ResticBackupper, *args, **kwargs):
try:
return func(self, *args, **kwargs)
except Exception as e:
if "unable to create lock" in str(e):
self.unlock()
return func(self, *args, **kwargs)
else:
raise e
# Above, we manually guarantee that the type returned is compatible.
return inner # type: ignore
class ResticBackupper(AbstractBackupper): class ResticBackupper(AbstractBackupper):
def __init__(self, login_flag: str, key_flag: str, storage_type: str) -> None: def __init__(self, login_flag: str, key_flag: str, storage_type: str) -> None:
@ -142,6 +163,7 @@ class ResticBackupper(AbstractBackupper):
result.append(item) result.append(item)
return result return result
@unlocked_repo
def start_backup(self, folders: List[str], tag: str) -> Snapshot: def start_backup(self, folders: List[str], tag: str) -> Snapshot:
""" """
Start backup with restic Start backup with restic
@ -165,8 +187,10 @@ class ResticBackupper(AbstractBackupper):
raise ValueError("No service with id ", tag) raise ValueError("No service with id ", tag)
job = get_backup_job(service) job = get_backup_job(service)
output = []
try: try:
for raw_message in output_yielder(backup_command): for raw_message in output_yielder(backup_command):
output.append(raw_message)
message = self.parse_message( message = self.parse_message(
raw_message, raw_message,
job, job,
@ -177,7 +201,13 @@ class ResticBackupper(AbstractBackupper):
tag, tag,
) )
except ValueError as error: except ValueError as error:
raise ValueError("Could not create a snapshot: ", messages) from error raise ValueError(
"Could not create a snapshot: ",
str(error),
output,
"parsed messages:",
messages,
) from error
@staticmethod @staticmethod
def _snapshot_from_backup_messages(messages, repo_name) -> Snapshot: def _snapshot_from_backup_messages(messages, repo_name) -> Snapshot:

View file

@ -773,3 +773,9 @@ def test_double_lock_unlock(backups, dummy_service):
Backups.provider().backupper.unlock() Backups.provider().backupper.unlock()
Backups.provider().backupper.unlock() Backups.provider().backupper.unlock()
def test_operations_while_locked(backups, dummy_service):
Backups.provider().backupper.lock()
snap = Backups.back_up(dummy_service)
assert snap is not None