refactor(services): make moving a part of generic service functionality

selfprivacy_api/jobs/__init__.py

@@ -268,6 +268,18 @@ class Jobs:
         return False
 
 
+# A terse way to call a common operation, for readability
+# job.report_progress() would be even better
+# but it would go against how this file is written
+def report_progress(progress: int, job: Job, status_text: str) -> None:
+    Jobs.update(
+        job=job,
+        status=JobStatus.RUNNING,
+        status_text=status_text,
+        progress=progress,
+    )
+
+
 def _redis_key_from_uuid(uuid_string) -> str:
     return "jobs:" + str(uuid_string)
 

selfprivacy_api/services/generic_service_mover.py

@@ -1,227 +0,0 @@
-"""Generic handler for moving services"""
-
-from __future__ import annotations
-import subprocess
-import pathlib
-import shutil
-from typing import List
-
-from pydantic import BaseModel
-from selfprivacy_api.jobs import Job, JobStatus, Jobs
-from selfprivacy_api.utils.huey import huey
-from selfprivacy_api.utils.block_devices import BlockDevice
-from selfprivacy_api.utils import ReadUserData, WriteUserData
-from selfprivacy_api.services.service import Service
-from selfprivacy_api.services.owned_path import OwnedPath
-
-from selfprivacy_api.services.service import StoppedService
-
-
-class MoveError(Exception):
-    """Move failed"""
-
-
-class FolderMoveNames(BaseModel):
-    name: str
-    bind_location: str
-    owner: str
-    group: str
-
-    @staticmethod
-    def from_owned_path(path: OwnedPath) -> FolderMoveNames:
-        return FolderMoveNames(
-            name=FolderMoveNames.get_foldername(path.path),
-            bind_location=path.path,
-            owner=path.owner,
-            group=path.group,
-        )
-
-    @staticmethod
-    def get_foldername(path: str) -> str:
-        return path.split("/")[-1]
-
-    @staticmethod
-    def default_foldermoves(service: Service) -> list[FolderMoveNames]:
-        return [
-            FolderMoveNames.from_owned_path(folder)
-            for folder in service.get_owned_folders()
-        ]
-
-
-@huey.task()
-def move_service(
-    service: Service,
-    new_volume: BlockDevice,
-    job: Job,
-    folder_names: List[FolderMoveNames],
-    userdata_location: str = None,  # deprecated, not used
-):
-    """
-    Move a service to another volume.
-    Is not allowed to raise errors because it is a task.
-    """
-    service_name = service.get_display_name()
-    old_volume = service.get_drive()
-    report_progress(0, job, "Performing pre-move checks...")
-
-    try:
-        with ReadUserData() as user_data:
-            if not user_data.get("useBinds", False):
-                raise MoveError("Server is not using binds.")
-
-        check_volume(new_volume, service)
-        check_folders(old_volume, folder_names)
-
-        report_progress(5, job, f"Stopping {service_name}...")
-
-        with StoppedService(service):
-            report_progress(10, job, "Unmounting folders from old volume...")
-            unmount_old_volume(folder_names)
-
-            report_progress(20, job, "Moving data to new volume...")
-            move_folders_to_volume(folder_names, old_volume, new_volume, job)
-
-            report_progress(70, job, f"Making sure {service_name} owns its files...")
-            chown_folders(folder_names, new_volume, job, service)
-
-            report_progress(90, job, f"Mounting {service_name} data...")
-            mount_folders(folder_names, new_volume)
-
-            report_progress(95, job, f"Finishing moving {service_name}...")
-            update_volume_in_userdata(service, new_volume)
-
-            Jobs.update(
-                job=job,
-                status=JobStatus.FINISHED,
-                result=f"{service_name} moved successfully.",
-                status_text=f"Starting {service_name}...",
-                progress=100,
-            )
-    except Exception as e:
-        Jobs.update(
-            job=job,
-            status=JobStatus.ERROR,
-            error=type(e).__name__ + " " + str(e),
-        )
-
-
-def check_volume(new_volume: BlockDevice, service: Service) -> bool:
-    service_name = service.get_display_name()
-    old_volume_name: str = service.get_drive()
-
-    # Check if we are on the same volume
-    if old_volume_name == new_volume.name:
-        raise MoveError(f"{service_name} is already on volume {new_volume}")
-
-    # Check if there is enough space on the new volume
-    if int(new_volume.fsavail) < service.get_storage_usage():
-        raise MoveError("Not enough space on the new volume.")
-
-    # Make sure the volume is mounted
-    if (
-        not new_volume.is_root()
-        and f"/volumes/{new_volume.name}" not in new_volume.mountpoints
-    ):
-        raise MoveError("Volume is not mounted.")
-
-
-def check_folders(old_volume: BlockDevice, folder_names: List[FolderMoveNames]) -> None:
-    # Make sure current actual directory exists and if its user and group are correct
-    for folder in folder_names:
-        path = pathlib.Path(f"/volumes/{old_volume}/{folder.name}")
-
-        if not path.exists():
-            raise MoveError(f"{path} is not found.")
-        if not path.is_dir():
-            raise MoveError(f"{path} is not a directory.")
-        if path.owner() != folder.owner:
-            raise MoveError(f"{path} owner is not {folder.owner}.")
-
-
-def unmount_old_volume(folder_names: List[FolderMoveNames]) -> None:
-    for folder in folder_names:
-        try:
-            subprocess.run(
-                ["umount", folder.bind_location],
-                check=True,
-            )
-        except subprocess.CalledProcessError:
-            raise MoveError("Unable to unmount old volume.")
-
-
-def move_folders_to_volume(
-    folder_names: List[FolderMoveNames],
-    old_volume: BlockDevice,
-    new_volume: BlockDevice,
-    job: Job,
-) -> None:
-    # Move data to new volume and set correct permissions
-    current_progress = job.progress
-    folder_percentage = 50 // len(folder_names)
-    for folder in folder_names:
-        shutil.move(
-            f"/volumes/{old_volume}/{folder.name}",
-            f"/volumes/{new_volume.name}/{folder.name}",
-        )
-        progress = current_progress + folder_percentage
-        report_progress(progress, job, "Moving data to new volume...")
-
-
-def chown_folders(
-    folder_names: List[FolderMoveNames], volume: BlockDevice, job: Job, service: Service
-) -> None:
-    service_name = service.get_display_name()
-    for folder in folder_names:
-        try:
-            subprocess.run(
-                [
-                    "chown",
-                    "-R",
-                    f"{folder.owner}:{folder.group}",
-                    f"/volumes/{volume.name}/{folder.name}",
-                ],
-                check=True,
-            )
-        except subprocess.CalledProcessError as error:
-            print(error.output)
-            Jobs.update(
-                job=job,
-                status=JobStatus.RUNNING,
-                error=f"Unable to set ownership of new volume. {service_name} may not be able to access its files. Continuing anyway.",
-            )
-
-
-def mount_folders(folder_names: List[FolderMoveNames], volume: BlockDevice) -> None:
-    for folder in folder_names:
-        try:
-            subprocess.run(
-                [
-                    "mount",
-                    "--bind",
-                    f"/volumes/{volume.name}/{folder.name}",
-                    folder.bind_location,
-                ],
-                check=True,
-            )
-        except subprocess.CalledProcessError as error:
-            print(error.output)
-            raise MoveError(f"Unable to mount new volume:{error.output}")
-
-
-def update_volume_in_userdata(service: Service, volume: BlockDevice):
-    with WriteUserData() as user_data:
-        service_id = service.get_id()
-        if "modules" not in user_data:
-            user_data["modules"] = {}
-        if service_id not in user_data["modules"]:
-            user_data["modules"][service_id] = {}
-        user_data["modules"][service_id]["location"] = volume.name
-
-
-def report_progress(progress: int, job: Job, status_text: str) -> None:
-    Jobs.update(
-        job=job,
-        status=JobStatus.RUNNING,
-        status_text=status_text,
-        progress=progress,
-    )

selfprivacy_api/services/moving.py

@@ -0,0 +1,114 @@
+"""Generic handler for moving services"""
+
+from __future__ import annotations
+import subprocess
+import pathlib
+import shutil
+from typing import List
+
+from selfprivacy_api.jobs import Job, report_progress
+from selfprivacy_api.utils.block_devices import BlockDevice
+from selfprivacy_api.services.owned_path import OwnedPath
+
+
+class MoveError(Exception):
+    """Move failed"""
+
+def get_foldername(path: str) -> str:
+    return path.split("/")[-1]
+
+
+
+
+def check_volume(volume: BlockDevice, space_needed: int) -> bool:
+    # Check if there is enough space on the new volume
+    if int(volume.fsavail) < space_needed:
+        raise MoveError("Not enough space on the new volume.")
+
+    # Make sure the volume is mounted
+    if (
+        not volume.is_root()
+        and f"/volumes/{volume.name}" not in volume.mountpoints
+    ):
+        raise MoveError("Volume is not mounted.")
+
+
+def check_folders(current_volume: BlockDevice, folders: List[OwnedPath]) -> None:
+    # Make sure current actual directory exists and if its user and group are correct
+    for folder in folders:
+        path = pathlib.Path(f"/volumes/{current_volume}/{get_foldername(folder)}")
+
+        if not path.exists():
+            raise MoveError(f"{path} is not found.")
+        if not path.is_dir():
+            raise MoveError(f"{path} is not a directory.")
+        if path.owner() != folder.owner:
+            raise MoveError(f"{path} owner is not {folder.owner}.")
+
+
+def unbind_folders(owned_folders: List[OwnedPath]) -> None:
+    for folder in owned_folders:
+        try:
+            subprocess.run(
+                ["umount", folder.path],
+                check=True,
+            )
+        except subprocess.CalledProcessError:
+            raise MoveError(f"Unable to unmount folder {folder.path}.")
+
+
+def move_folders_to_volume(
+    folders: List[OwnedPath],
+    old_volume: BlockDevice,
+    new_volume: BlockDevice,
+    job: Job,
+) -> None:
+    current_progress = job.progress
+    folder_percentage = 50 // len(folders)
+    for folder in folders:
+        folder_name = get_foldername(folder.path)
+        shutil.move(
+            f"/volumes/{old_volume}/{folder_name}",
+            f"/volumes/{new_volume.name}/{folder_name}",
+        )
+        progress = current_progress + folder_percentage
+        report_progress(progress, job, "Moving data to new volume...")
+
+
+def ensure_folder_ownership(
+    folders: List[OwnedPath], volume: BlockDevice
+) -> None:
+    for folder in folders:
+        true_location = f"/volumes/{volume.name}/{get_foldername(folder.path)}"
+        try:
+            subprocess.run(
+                [
+                    "chown",
+                    "-R",
+                    f"{folder.owner}:{folder.group}",
+                    # Could we just chown the binded location instead?
+                    true_location
+                ],
+                check=True,
+            )
+        except subprocess.CalledProcessError as error:
+            error_message = f"Unable to set ownership of {true_location} :{error.output}"
+            print(error.output)
+            raise MoveError(error_message)
+
+
+def bind_folders(folders: List[OwnedPath], volume: BlockDevice) -> None:
+    for folder in folders:
+        try:
+            subprocess.run(
+                [
+                    "mount",
+                    "--bind",
+                    f"/volumes/{volume.name}/{get_foldername(folder.path)}",
+                    folder.path,
+                ],
+                check=True,
+            )
+        except subprocess.CalledProcessError as error:
+            print(error.output)
+            raise MoveError(f"Unable to mount new volume:{error.output}")

selfprivacy_api/services/service.py

@@ -4,12 +4,14 @@ from enum import Enum
 from typing import List, Optional
 
 from pydantic import BaseModel
-from selfprivacy_api.jobs import Job
+from selfprivacy_api.jobs import Job, Jobs, JobStatus, report_progress
 
 from selfprivacy_api.utils.block_devices import BlockDevice, BlockDevices
 
 from selfprivacy_api.services.generic_size_counter import get_storage_usage
 from selfprivacy_api.services.owned_path import OwnedPath
+from selfprivacy_api.services.moving import check_folders, check_volume, unbind_folders, bind_folders, ensure_folder_ownership, MoveError, move_folders_to_volume
+
 from selfprivacy_api import utils
 from selfprivacy_api.utils.waitloop import wait_until_true
 from selfprivacy_api.utils import ReadUserData, WriteUserData
@@ -294,6 +296,99 @@ class Service(ABC):
     def get_foldername(path: str) -> str:
         return path.split("/")[-1]
 
+    # TODO: with better json utils, it can be one line, and not a separate function
+    @classmethod
+    def set_location(cls, volume: BlockDevice):
+        """
+            Only changes userdata
+        """
+
+        with WriteUserData() as user_data:
+            service_id = cls.get_id()
+            if "modules" not in user_data:
+                user_data["modules"] = {}
+            if service_id not in user_data["modules"]:
+                user_data["modules"][service_id] = {}
+            user_data["modules"][service_id]["location"] = volume.name
+
+    def assert_can_move(self, new_volume):
+        """
+            Checks if the service can be moved to new volume
+            Raises errors if it cannot
+        """
+        with ReadUserData() as user_data:
+            if not user_data.get("useBinds", False):
+                raise MoveError("Server is not using binds.")
+
+        current_volume_name = self.get_drive()
+        service_name = self.get_display_name()
+        if current_volume_name == new_volume.name:
+            raise MoveError(f"{service_name} is already on volume {new_volume}")
+
+        check_volume(new_volume, space_needed=self.get_storage_usage())
+
+        owned_folders = self.get_owned_folders()
+        if owned_folders == []:
+            raise MoveError("nothing to move")
+
+        check_folders(current_volume_name, owned_folders)
+
+
+    def do_move_to_volume(
+        self,
+        new_volume: BlockDevice,
+        job: Job,
+    ):
+        """
+        Move a service to another volume.
+        Is not allowed to raise errors because it is a task.
+        """
+        service_name = self.get_display_name()
+        old_volume_name = self.get_drive()
+        owned_folders = self.get_owned_folders()
+
+        report_progress(0, job, "Performing pre-move checks...")
+
+        # TODO: move trying to the task
+        try:
+            report_progress(5, job, f"Stopping {service_name}...")
+
+            with StoppedService(self):
+                report_progress(10, job, "Unmounting folders from old volume...")
+                unbind_folders(owned_folders)
+
+                report_progress(20, job, "Moving data to new volume...")
+                move_folders_to_volume(owned_folders, old_volume_name, new_volume, job)
+
+                report_progress(70, job, f"Making sure {service_name} owns its files...")
+                try:
+                    ensure_folder_ownership(owned_folders, new_volume, job, self)
+                except Exception as error:
+                    # We have logged it via print and we additionally log it here in the error field
+                    # We are continuing anyway but Job has no warning field
+                    Jobs.update(job, JobStatus.RUNNING, error=f"Service {service_name} will not be able to write files: " + str(error))
+
+                report_progress(90, job, f"Mounting {service_name} data...")
+                bind_folders(owned_folders, new_volume)
+
+                report_progress(95, job, f"Finishing moving {service_name}...")
+                self.set_location(self, new_volume)
+
+                Jobs.update(
+                    job=job,
+                    status=JobStatus.FINISHED,
+                    result=f"{service_name} moved successfully.",
+                    status_text=f"Starting {service_name}...",
+                    progress=100,
+                )
+        except Exception as e:
+            Jobs.update(
+                job=job,
+                status=JobStatus.ERROR,
+                error=type(e).__name__ + " " + str(e),
+            )
+
+
     @abstractmethod
     def move_to_volume(self, volume: BlockDevice) -> Job:
         """Cannot raise errors.

selfprivacy_api/services/tasks.py

@@ -0,0 +1,11 @@
+from selfprivacy_api.services import Service
+from selfprivacy_api.utils.block_devices import BlockDevice
+from selfprivacy_api.utils.huey import huey
+
+
+@huey.task()
+def move_service(
+    service: Service,
+    new_volume: BlockDevice,
+):
+    service.move_to_volume(new_volume)