feature(backups): intermittent commit for binds, to be replaced

selfprivacy_api/graphql/mutations/services_mutations.py

@@ -7,6 +7,8 @@ from selfprivacy_api.graphql.common_types.jobs import job_to_api_job
 from selfprivacy_api.jobs import JobStatus
 from selfprivacy_api.utils.block_devices import BlockDevices
 
+from traceback import format_tb as format_traceback
+
 from selfprivacy_api.graphql.mutations.mutation_interface import (
     GenericJobMutationReturn,
     GenericMutationReturn,
@@ -171,6 +173,7 @@ class ServicesMutations:
 
         try:
             job = move_service(input.service_id, input.location)
+
         except (ServiceNotFoundError, VolumeNotFoundError) as e:
             return ServiceJobMutationReturn(
                 success=False,
@@ -212,4 +215,5 @@ class ServicesMutations:
 
 
 def pretty_error(e: Exception) -> str:
-    return type(e).__name__ + ": " + str(e)
+    traceback = "/r".join(format_traceback(e.__traceback__))
+    return type(e).__name__ + ": " + str(e) + ": " + traceback

selfprivacy_api/services/moving.py

@@ -1,26 +1,16 @@
 """Generic handler for moving services"""
 
 from __future__ import annotations
-import subprocess
-import pathlib
 import shutil
 from typing import List
 
 from selfprivacy_api.jobs import Job, report_progress
 from selfprivacy_api.utils.block_devices import BlockDevice
-from selfprivacy_api.services.owned_path import OwnedPath
+from selfprivacy_api.services.owned_path import Bind
 
 
 class MoveError(Exception):
-    """Move failed"""
+    """Move of the data has failed"""
-
-
-def get_foldername(p: OwnedPath) -> str:
-    return p.path.split("/")[-1]
-
-
-def location_at_volume(binding_path: OwnedPath, volume_name: str):
-    return f"/volumes/{volume_name}/{get_foldername(binding_path)}"
 
 
 def check_volume(volume: BlockDevice, space_needed: int) -> None:
@@ -33,84 +23,50 @@ def check_volume(volume: BlockDevice, space_needed: int) -> None:
         raise MoveError("Volume is not mounted.")
 
 
-def check_folders(volume_name: str, folders: List[OwnedPath]) -> None:
+def check_binds(volume_name: str, binds: List[Bind]) -> None:
     # Make sure current actual directory exists and if its user and group are correct
-    for folder in folders:
+    for bind in binds:
-        path = pathlib.Path(location_at_volume(folder, volume_name))
+        bind.validate()
-
-        if not path.exists():
-            raise MoveError(f"directory {path} is not found.")
-        if not path.is_dir():
-            raise MoveError(f"{path} is not a directory.")
-        if path.owner() != folder.owner:
-            raise MoveError(f"{path} is not owned by {folder.owner}.")
 
 
-def unbind_folders(owned_folders: List[OwnedPath]) -> None:
+def unbind_folders(owned_folders: List[Bind]) -> None:
     for folder in owned_folders:
-        try:
+        folder.unbind()
-            subprocess.run(
-                ["umount", folder.path],
-                check=True,
-            )
-        except subprocess.CalledProcessError:
-            raise MoveError(f"Unable to unmount folder {folder.path}.")
 
 
-def move_folders_to_volume(
+# May be moved into Bind
-    folders: List[OwnedPath],
+def move_data_to_volume(
-    old_volume_name: str,  # TODO: pass an actual validated block device
+    binds: List[Bind],
     new_volume: BlockDevice,
     job: Job,
-) -> None:
+) -> List[Bind]:
     current_progress = job.progress
     if current_progress is None:
         current_progress = 0
 
-    progress_per_folder = 50 // len(folders)
+    progress_per_folder = 50 // len(binds)
-    for folder in folders:
+    for bind in binds:
-        shutil.move(
+        old_location = bind.location_at_volume()
-            location_at_volume(folder, old_volume_name),
+        bind.drive = new_volume
-            location_at_volume(folder, new_volume.name),
+        new_location = bind.location_at_volume()
-        )
+
+        try:
+            shutil.move(old_location, new_location)
+        except Exception as error:
+            raise MoveError(
+                f"could not move {old_location} to {new_location} : {str(error)}"
+            ) from error
+
         progress = current_progress + progress_per_folder
         report_progress(progress, job, "Moving data to new volume...")
+    return binds
 
 
-def ensure_folder_ownership(folders: List[OwnedPath], volume: BlockDevice) -> None:
+def ensure_folder_ownership(folders: List[Bind]) -> None:
     for folder in folders:
-        true_location = location_at_volume(folder, volume.name)
+        folder.ensure_ownership()
-        try:
-            subprocess.run(
-                [
-                    "chown",
-                    "-R",
-                    f"{folder.owner}:{folder.group}",
-                    # Could we just chown the binded location instead?
-                    true_location,
-                ],
-                check=True,
-            )
-        except subprocess.CalledProcessError as error:
-            print(error.output)
-            error_message = (
-                f"Unable to set ownership of {true_location} :{error.output}"
-            )
-            raise MoveError(error_message)
 
 
-def bind_folders(folders: List[OwnedPath], volume: BlockDevice) -> None:
+def bind_folders(folders: List[Bind], volume: BlockDevice):
     for folder in folders:
-        try:
+        folder.bind()
-            subprocess.run(
-                [
-                    "mount",
-                    "--bind",
-                    location_at_volume(folder, volume.name),
-                    folder.path,
-                ],
-                check=True,
-            )
-        except subprocess.CalledProcessError as error:
-            print(error.output)
-            raise MoveError(f"Unable to mount new volume:{error.output}")

selfprivacy_api/services/owned_path.py

@@ -1,7 +1,103 @@
+from __future__ import annotations
+import subprocess
+import pathlib
 from pydantic import BaseModel
 
+from selfprivacy_api.utils.block_devices import BlockDevice, BlockDevices
 
+
+class BindError(Exception):
+    pass
+
+
+# May be deprecated because of Binds
 class OwnedPath(BaseModel):
     path: str
     owner: str
     group: str
+
+
+class Bind:
+    """
+    A directory that resides on some volume but we mount it into fs
+    where we need it.
+    Used for service data.
+    """
+
+    def __init__(self, binding_path: str, owner: str, group: str, drive: BlockDevice):
+        self.binding_path = binding_path
+        self.owner = owner
+        self.group = group
+        self.drive = drive
+
+    # TODO: make Service return a list of binds instead of owned paths
+    @staticmethod
+    def from_owned_path(path: OwnedPath, drive_name: str) -> Bind:
+        drive = BlockDevices().get_block_device(drive_name)
+        if drive is None:
+            raise BindError(f"No such drive: {drive_name}")
+
+        return Bind(
+            binding_path=path.path, owner=path.owner, group=path.group, drive=drive
+        )
+
+    def bind_foldername(self) -> str:
+        return self.binding_path.split("/")[-1]
+
+    def location_at_volume(self) -> str:
+        return f"/volumes/{self.drive.name}/{self.bind_foldername()}"
+
+    def validate(self) -> str:
+        path = pathlib.Path(self.location_at_volume())
+
+        if not path.exists():
+            raise BindError(f"directory {path} is not found.")
+        if not path.is_dir():
+            raise BindError(f"{path} is not a directory.")
+        if path.owner() != self.owner:
+            raise BindError(f"{path} is not owned by {self.owner}.")
+
+    def bind(self) -> None:
+        try:
+            subprocess.run(
+                [
+                    "mount",
+                    "--bind",
+                    self.location_at_volume(),
+                    self.binding_path,
+                ],
+                check=True,
+            )
+        except subprocess.CalledProcessError as error:
+            print(error.output)
+            raise BindError(f"Unable to mount new volume:{error.output}")
+
+    def unbind(self) -> None:
+        try:
+            subprocess.run(
+                ["umount", self.binding_path],
+                check=True,
+            )
+        except subprocess.CalledProcessError:
+            raise BindError(f"Unable to unmount folder {self.binding_path}.")
+        pass
+
+    def ensure_ownership(self) -> None:
+        true_location = self.location_at_volume()
+        try:
+            subprocess.run(
+                [
+                    "chown",
+                    "-R",
+                    f"{self.owner}:{self.group}",
+                    # Could we just chown the binded location instead?
+                    true_location,
+                ],
+                check=True,
+            )
+        except subprocess.CalledProcessError as error:
+            print(error.output)
+            error_message = (
+                f"Unable to set ownership of {true_location} :{error.output}"
+            )
+            raise BindError(error_message)

selfprivacy_api/services/service.py

@@ -9,15 +9,15 @@ from selfprivacy_api.jobs import Job, Jobs, JobStatus, report_progress
 from selfprivacy_api.utils.block_devices import BlockDevice, BlockDevices
 
 from selfprivacy_api.services.generic_size_counter import get_storage_usage
-from selfprivacy_api.services.owned_path import OwnedPath
+from selfprivacy_api.services.owned_path import OwnedPath, Bind
 from selfprivacy_api.services.moving import (
-    check_folders,
+    check_binds,
     check_volume,
     unbind_folders,
     bind_folders,
     ensure_folder_ownership,
     MoveError,
-    move_folders_to_volume,
+    move_data_to_volume,
 )
 
 from selfprivacy_api import utils
@@ -319,6 +319,13 @@ class Service(ABC):
                 user_data["modules"][service_id] = {}
             user_data["modules"][service_id]["location"] = volume.name
 
+    def binds(self) -> typing.List[Bind]:
+        owned_folders = self.get_owned_folders()
+
+        return [
+            Bind.from_owned_path(folder, self.get_drive()) for folder in owned_folders
+        ]
+
     def assert_can_move(self, new_volume):
         """
         Checks if the service can be moved to new volume
@@ -338,11 +345,10 @@ class Service(ABC):
 
         check_volume(new_volume, space_needed=self.get_storage_usage())
 
-        owned_folders = self.get_owned_folders()
+        binds = self.binds()
-        if owned_folders == []:
+        if binds == []:
             raise MoveError("nothing to move")
-
+        check_binds(current_volume_name, binds)
-        check_folders(current_volume_name, owned_folders)
 
     def do_move_to_volume(
         self,
@@ -351,21 +357,20 @@ class Service(ABC):
     ):
         """
         Move a service to another volume.
+        Note: It may be much simpler to write it per bind, but a bit less safe?
         """
         service_name = self.get_display_name()
-        # TODO : Make sure device exists
+        binds = self.binds()
-        old_volume_name = self.get_drive()
-        owned_folders = self.get_owned_folders()
 
         report_progress(10, job, "Unmounting folders from old volume...")
-        unbind_folders(owned_folders)
+        unbind_folders(binds)
 
         report_progress(20, job, "Moving data to new volume...")
-        move_folders_to_volume(owned_folders, old_volume_name, new_volume, job)
+        binds = move_data_to_volume(binds, new_volume, job)
 
         report_progress(70, job, f"Making sure {service_name} owns its files...")
         try:
-            ensure_folder_ownership(owned_folders, new_volume)
+            ensure_folder_ownership(binds)
         except Exception as error:
             # We have logged it via print and we additionally log it here in the error field
             # We are continuing anyway but Job has no warning field
@@ -377,7 +382,7 @@ class Service(ABC):
             )
 
         report_progress(90, job, f"Mounting {service_name} data...")
-        bind_folders(owned_folders, new_volume)
+        bind_folders(binds)
 
         report_progress(95, job, f"Finishing moving {service_name}...")
         self.set_location(new_volume)

selfprivacy_api/utils/block_devices.py

@@ -4,6 +4,8 @@ import subprocess
 import json
 import typing
 
+from pydantic import BaseModel
+
 from selfprivacy_api.utils import WriteUserData
 from selfprivacy_api.utils.singleton_metaclass import SingletonMetaclass
 

tests/test_graphql/test_services.py

@@ -32,9 +32,9 @@ MOVER_MOCK_PROCESS = CompletedProcess(["ls"], returncode=0)
 
 
 @pytest.fixture()
-def mock_check_service_mover_folders(mocker):
+def mock_check_service_mover_binds(mocker):
     mock = mocker.patch(
-        "selfprivacy_api.services.service.check_folders",
+        "selfprivacy_api.services.service.check_binds",
         autospec=True,
         return_value=None,
     )
@@ -569,7 +569,7 @@ def test_graphql_move_service_without_folders_on_old_volume(
 def test_graphql_move_service(
     authorized_client,
     generic_userdata,
-    mock_check_service_mover_folders,
+    mock_check_service_mover_binds,
     lsblk_singular_mock,
     dummy_service: DummyService,
     mock_subprocess_run,