fix(backups): allow retrying when deleting service files

selfprivacy_api/backup/backuppers/restic_backupper.py

@@ -11,6 +11,7 @@ from json.decoder import JSONDecodeError
 from os.path import exists, join
 from os import mkdir
 from shutil import rmtree
+from selfprivacy_api.utils.waitloop import wait_until_success
 
 from selfprivacy_api.graphql.common_types.backup import BackupReason
 from selfprivacy_api.backup.util import output_yielder, sync
@@ -23,6 +24,7 @@ from selfprivacy_api.jobs import Jobs, JobStatus, Job
 from selfprivacy_api.backup.local_secret import LocalBackupSecret
 
 SHORT_ID_LEN = 8
+FILESYSTEM_TIMEOUT_SEC = 60
 
 T = TypeVar("T", bound=Callable)
 
@@ -391,7 +393,9 @@ class ResticBackupper(AbstractBackupper):
 
             else:  # attempting inplace restore
                 for folder in folders:
-                    rmtree(folder)
+                    wait_until_success(
+                        lambda: rmtree(folder), timeout_sec=FILESYSTEM_TIMEOUT_SEC
+                    )
                     mkdir(folder)
                 self._raw_verified_restore(snapshot_id, target="/")
                 return

selfprivacy_api/utils/waitloop.py

@@ -1,20 +1,60 @@
 from time import sleep
 from typing import Callable
+from typing import Any
 from typing import Optional
 
+MAX_TIMEOUT = 10e16
+DEFAULT_INTERVAL_SEC = 0.1
+
 
 def wait_until_true(
     readiness_checker: Callable[[], bool],
     *,
-    interval: float = 0.1,
+    interval: float = DEFAULT_INTERVAL_SEC,
-    timeout_sec: Optional[float] = None
+    timeout_sec: float = MAX_TIMEOUT
 ):
     elapsed = 0.0
-    if timeout_sec is None:
-        timeout_sec = 10e16
 
     while (not readiness_checker()) and elapsed < timeout_sec:
         sleep(interval)
         elapsed += interval
     if elapsed > timeout_sec:
         raise TimeoutError()
+
+
+def is_fail(cal: Callable[[], Any]) -> Optional[Exception]:
+    try:
+        cal()
+    except Exception as e:
+        # We want it to be a logical True
+        assert e
+        return e
+    return None
+
+
+def wait_until_success(
+    operation: Callable[[], Any],
+    *,
+    interval: float = DEFAULT_INTERVAL_SEC,
+    timeout_sec: float = MAX_TIMEOUT
+):
+
+    elapsed = 0.0
+    error = is_fail(operation)
+
+    while error and elapsed < timeout_sec:
+        sleep(interval)
+        elapsed += interval
+        error = is_fail(operation)
+
+    if elapsed >= timeout_sec:
+        if isinstance(error, Exception):
+            raise TimeoutError(
+                "timed out on",
+                operation,
+                " with error: " + error.__class__.__name__ + ":" + str(error),
+            )
+        else:
+            raise TimeoutError(
+                "timed out waiting for an operation to stop failing", operation
+            )

tests/test_utils.py

@@ -0,0 +1,35 @@
+import pytest
+from selfprivacy_api.utils.waitloop import wait_until_success
+
+
+class Counter:
+    def __init__(self):
+        self.count = 0
+
+    def tick(self):
+        self.count += 1
+
+    def reset(self):
+        self.count = 0
+
+
+def failing_operation(c: Counter) -> str:
+    if c.count < 10:
+        c.tick()
+        raise ValueError("nooooope")
+    return "yeees"
+
+
+def test_wait_until_success():
+    counter = Counter()
+
+    with pytest.raises(TimeoutError):
+        wait_until_success(
+            lambda: failing_operation(counter), interval=0.1, timeout_sec=0.5
+        )
+
+    counter.reset()
+
+    wait_until_success(
+        lambda: failing_operation(counter), interval=0.1, timeout_sec=1.1
+    )