diff --git a/selfprivacy_api/backup/backuppers/restic_backupper.py b/selfprivacy_api/backup/backuppers/restic_backupper.py index 9232f60..4412fd0 100644 --- a/selfprivacy_api/backup/backuppers/restic_backupper.py +++ b/selfprivacy_api/backup/backuppers/restic_backupper.py @@ -11,6 +11,7 @@ from json.decoder import JSONDecodeError from os.path import exists, join from os import mkdir from shutil import rmtree +from selfprivacy_api.utils.waitloop import wait_until_success from selfprivacy_api.graphql.common_types.backup import BackupReason from selfprivacy_api.backup.util import output_yielder, sync @@ -23,6 +24,7 @@ from selfprivacy_api.jobs import Jobs, JobStatus, Job from selfprivacy_api.backup.local_secret import LocalBackupSecret SHORT_ID_LEN = 8 +FILESYSTEM_TIMEOUT_SEC = 60 T = TypeVar("T", bound=Callable) @@ -391,7 +393,9 @@ class ResticBackupper(AbstractBackupper): else: # attempting inplace restore for folder in folders: - rmtree(folder) + wait_until_success( + lambda: rmtree(folder), timeout_sec=FILESYSTEM_TIMEOUT_SEC + ) mkdir(folder) self._raw_verified_restore(snapshot_id, target="/") return diff --git a/selfprivacy_api/utils/waitloop.py b/selfprivacy_api/utils/waitloop.py index 9f71a37..f129142 100644 --- a/selfprivacy_api/utils/waitloop.py +++ b/selfprivacy_api/utils/waitloop.py @@ -1,20 +1,60 @@ from time import sleep from typing import Callable +from typing import Any from typing import Optional +MAX_TIMEOUT = 10e16 +DEFAULT_INTERVAL_SEC = 0.1 + def wait_until_true( readiness_checker: Callable[[], bool], *, - interval: float = 0.1, - timeout_sec: Optional[float] = None + interval: float = DEFAULT_INTERVAL_SEC, + timeout_sec: float = MAX_TIMEOUT ): elapsed = 0.0 - if timeout_sec is None: - timeout_sec = 10e16 while (not readiness_checker()) and elapsed < timeout_sec: sleep(interval) elapsed += interval if elapsed > timeout_sec: raise TimeoutError() + + +def is_fail(cal: Callable[[], Any]) -> Optional[Exception]: + try: + cal() + except Exception as e: + # We want it to be a logical True + assert e + return e + return None + + +def wait_until_success( + operation: Callable[[], Any], + *, + interval: float = DEFAULT_INTERVAL_SEC, + timeout_sec: float = MAX_TIMEOUT +): + + elapsed = 0.0 + error = is_fail(operation) + + while error and elapsed < timeout_sec: + sleep(interval) + elapsed += interval + error = is_fail(operation) + + if elapsed >= timeout_sec: + if isinstance(error, Exception): + raise TimeoutError( + "timed out on", + operation, + " with error: " + error.__class__.__name__ + ":" + str(error), + ) + else: + raise TimeoutError( + "timed out waiting for an operation to stop failing", operation + ) diff --git a/tests/test_utils.py b/tests/test_utils.py new file mode 100644 index 0000000..a985afe --- /dev/null +++ b/tests/test_utils.py @@ -0,0 +1,35 @@ +import pytest +from selfprivacy_api.utils.waitloop import wait_until_success + + +class Counter: + def __init__(self): + self.count = 0 + + def tick(self): + self.count += 1 + + def reset(self): + self.count = 0 + + +def failing_operation(c: Counter) -> str: + if c.count < 10: + c.tick() + raise ValueError("nooooope") + return "yeees" + + +def test_wait_until_success(): + counter = Counter() + + with pytest.raises(TimeoutError): + wait_until_success( + lambda: failing_operation(counter), interval=0.1, timeout_sec=0.5 + ) + + counter.reset() + + wait_until_success( + lambda: failing_operation(counter), interval=0.1, timeout_sec=1.1 + )