SelfPrivacy/selfprivacy-rest-api
1
0
Fork 0
mirror of https://git.selfprivacy.org/SelfPrivacy/selfprivacy-rest-api.git synced 2024-11-25 13:31:27 +00:00
Code Issues Projects Releases Wiki Activity

feature(backups): intermittent commit for binds, to be replaced

Browse source
This commit is contained in:
Houkime 2024-02-26 15:01:07 +00:00
parent 18934a53e6
commit 0068272382
6 changed files with 154 additions and 91 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

6
selfprivacy_api/graphql/mutations/services_mutations.py
View file

@ -7,6 +7,8 @@ from selfprivacy_api.graphql.common_types.jobs import job_to_api_job
from selfprivacy_api.jobs import JobStatus
from selfprivacy_api.utils.block_devices import BlockDevices
from traceback import format_tb as format_traceback
from selfprivacy_api.graphql.mutations.mutation_interface import (
GenericJobMutationReturn,
GenericMutationReturn,
@ -171,6 +173,7 @@ class ServicesMutations:
try:
job = move_service(input.service_id, input.location)
except (ServiceNotFoundError, VolumeNotFoundError) as e:
return ServiceJobMutationReturn(
success=False,
@ -212,4 +215,5 @@ class ServicesMutations:
def pretty_error(e: Exception) -> str:
return type(e).__name__ + ": " + str(e)
traceback = "/r".join(format_traceback(e.__traceback__))
return type(e).__name__ + ": " + str(e) + ": " + traceback

102
selfprivacy_api/services/moving.py
View file

@ -1,26 +1,16 @@
"""Generic handler for moving services"""
from __future__ import annotations
import subprocess
import pathlib
import shutil
from typing import List
from selfprivacy_api.jobs import Job, report_progress
from selfprivacy_api.utils.block_devices import BlockDevice
from selfprivacy_api.services.owned_path import OwnedPath
from selfprivacy_api.services.owned_path import Bind
class MoveError(Exception):
"""Move failed"""
def get_foldername(p: OwnedPath) -> str:
return p.path.split("/")[-1]
def location_at_volume(binding_path: OwnedPath, volume_name: str):
return f"/volumes/{volume_name}/{get_foldername(binding_path)}"
"""Move of the data has failed"""
def check_volume(volume: BlockDevice, space_needed: int) -> None:
@ -33,84 +23,50 @@ def check_volume(volume: BlockDevice, space_needed: int) -> None:
raise MoveError("Volume is not mounted.")
def check_folders(volume_name: str, folders: List[OwnedPath]) -> None:
def check_binds(volume_name: str, binds: List[Bind]) -> None:
# Make sure current actual directory exists and if its user and group are correct
for folder in folders:
path = pathlib.Path(location_at_volume(folder, volume_name))
if not path.exists():
raise MoveError(f"directory {path} is not found.")
if not path.is_dir():
raise MoveError(f"{path} is not a directory.")
if path.owner() != folder.owner:
raise MoveError(f"{path} is not owned by {folder.owner}.")
for bind in binds:
bind.validate()
def unbind_folders(owned_folders: List[OwnedPath]) -> None:
def unbind_folders(owned_folders: List[Bind]) -> None:
for folder in owned_folders:
try:
subprocess.run(
["umount", folder.path],
check=True,
)
except subprocess.CalledProcessError:
raise MoveError(f"Unable to unmount folder {folder.path}.")
folder.unbind()
def move_folders_to_volume(
folders: List[OwnedPath],
old_volume_name: str, # TODO: pass an actual validated block device
# May be moved into Bind
def move_data_to_volume(
binds: List[Bind],
new_volume: BlockDevice,
job: Job,
) -> None:
) -> List[Bind]:
current_progress = job.progress
if current_progress is None:
current_progress = 0
progress_per_folder = 50 // len(folders)
for folder in folders:
shutil.move(
location_at_volume(folder, old_volume_name),
location_at_volume(folder, new_volume.name),
)
progress_per_folder = 50 // len(binds)
for bind in binds:
old_location = bind.location_at_volume()
bind.drive = new_volume
new_location = bind.location_at_volume()
try:
shutil.move(old_location, new_location)
except Exception as error:
raise MoveError(
f"could not move {old_location} to {new_location} : {str(error)}"
) from error
progress = current_progress + progress_per_folder
report_progress(progress, job, "Moving data to new volume...")
return binds
def ensure_folder_ownership(folders: List[OwnedPath], volume: BlockDevice) -> None:
def ensure_folder_ownership(folders: List[Bind]) -> None:
for folder in folders:
true_location = location_at_volume(folder, volume.name)
try:
subprocess.run(
[
"chown",
"-R",
f"{folder.owner}:{folder.group}",
# Could we just chown the binded location instead?
true_location,
],
check=True,
)
except subprocess.CalledProcessError as error:
print(error.output)
error_message = (
f"Unable to set ownership of {true_location} :{error.output}"
)
raise MoveError(error_message)
folder.ensure_ownership()
def bind_folders(folders: List[OwnedPath], volume: BlockDevice) -> None:
def bind_folders(folders: List[Bind], volume: BlockDevice):
for folder in folders:
try:
subprocess.run(
[
"mount",
"--bind",
location_at_volume(folder, volume.name),
folder.path,
],
check=True,
)
except subprocess.CalledProcessError as error:
print(error.output)
raise MoveError(f"Unable to mount new volume:{error.output}")
folder.bind()

96
selfprivacy_api/services/owned_path.py
View file

@ -1,7 +1,103 @@
from __future__ import annotations
import subprocess
import pathlib
from pydantic import BaseModel
from selfprivacy_api.utils.block_devices import BlockDevice, BlockDevices
class BindError(Exception):
pass
# May be deprecated because of Binds
class OwnedPath(BaseModel):
path: str
owner: str
group: str
class Bind:
"""
A directory that resides on some volume but we mount it into fs
where we need it.
Used for service data.
"""
def __init__(self, binding_path: str, owner: str, group: str, drive: BlockDevice):
self.binding_path = binding_path
self.owner = owner
self.group = group
self.drive = drive
# TODO: make Service return a list of binds instead of owned paths
@staticmethod
def from_owned_path(path: OwnedPath, drive_name: str) -> Bind:
drive = BlockDevices().get_block_device(drive_name)
if drive is None:
raise BindError(f"No such drive: {drive_name}")
return Bind(
binding_path=path.path, owner=path.owner, group=path.group, drive=drive
)
def bind_foldername(self) -> str:
return self.binding_path.split("/")[-1]
def location_at_volume(self) -> str:
return f"/volumes/{self.drive.name}/{self.bind_foldername()}"
def validate(self) -> str:
path = pathlib.Path(self.location_at_volume())
if not path.exists():
raise BindError(f"directory {path} is not found.")
if not path.is_dir():
raise BindError(f"{path} is not a directory.")
if path.owner() != self.owner:
raise BindError(f"{path} is not owned by {self.owner}.")
def bind(self) -> None:
try:
subprocess.run(
[
"mount",
"--bind",
self.location_at_volume(),
self.binding_path,
],
check=True,
)
except subprocess.CalledProcessError as error:
print(error.output)
raise BindError(f"Unable to mount new volume:{error.output}")
def unbind(self) -> None:
try:
subprocess.run(
["umount", self.binding_path],
check=True,
)
except subprocess.CalledProcessError:
raise BindError(f"Unable to unmount folder {self.binding_path}.")
pass
def ensure_ownership(self) -> None:
true_location = self.location_at_volume()
try:
subprocess.run(
[
"chown",
"-R",
f"{self.owner}:{self.group}",
# Could we just chown the binded location instead?
true_location,
],
check=True,
)
except subprocess.CalledProcessError as error:
print(error.output)
error_message = (
f"Unable to set ownership of {true_location} :{error.output}"
)
raise BindError(error_message)

33
selfprivacy_api/services/service.py
View file

@ -9,15 +9,15 @@ from selfprivacy_api.jobs import Job, Jobs, JobStatus, report_progress
from selfprivacy_api.utils.block_devices import BlockDevice, BlockDevices
from selfprivacy_api.services.generic_size_counter import get_storage_usage
from selfprivacy_api.services.owned_path import OwnedPath
from selfprivacy_api.services.owned_path import OwnedPath, Bind
from selfprivacy_api.services.moving import (
check_folders,
check_binds,
check_volume,
unbind_folders,
bind_folders,
ensure_folder_ownership,
MoveError,
move_folders_to_volume,
move_data_to_volume,
)
from selfprivacy_api import utils
@ -319,6 +319,13 @@ class Service(ABC):
user_data["modules"][service_id] = {}
user_data["modules"][service_id]["location"] = volume.name
def binds(self) -> typing.List[Bind]:
owned_folders = self.get_owned_folders()
return [
Bind.from_owned_path(folder, self.get_drive()) for folder in owned_folders
]
def assert_can_move(self, new_volume):
"""
Checks if the service can be moved to new volume
@ -338,11 +345,10 @@ class Service(ABC):
check_volume(new_volume, space_needed=self.get_storage_usage())
owned_folders = self.get_owned_folders()
if owned_folders == []:
binds = self.binds()
if binds == []:
raise MoveError("nothing to move")
check_folders(current_volume_name, owned_folders)
check_binds(current_volume_name, binds)
def do_move_to_volume(
self,
@ -351,21 +357,20 @@ class Service(ABC):
):
"""
Move a service to another volume.
Note: It may be much simpler to write it per bind, but a bit less safe?
"""
service_name = self.get_display_name()
# TODO : Make sure device exists
old_volume_name = self.get_drive()
owned_folders = self.get_owned_folders()
binds = self.binds()
report_progress(10, job, "Unmounting folders from old volume...")
unbind_folders(owned_folders)
unbind_folders(binds)
report_progress(20, job, "Moving data to new volume...")
move_folders_to_volume(owned_folders, old_volume_name, new_volume, job)
binds = move_data_to_volume(binds, new_volume, job)
report_progress(70, job, f"Making sure {service_name} owns its files...")
try:
ensure_folder_ownership(owned_folders, new_volume)
ensure_folder_ownership(binds)
except Exception as error:
# We have logged it via print and we additionally log it here in the error field
# We are continuing anyway but Job has no warning field
@ -377,7 +382,7 @@ class Service(ABC):
)
report_progress(90, job, f"Mounting {service_name} data...")
bind_folders(owned_folders, new_volume)
bind_folders(binds)
report_progress(95, job, f"Finishing moving {service_name}...")
self.set_location(new_volume)

2
selfprivacy_api/utils/block_devices.py
View file

@ -4,6 +4,8 @@ import subprocess
import json
import typing
from pydantic import BaseModel
from selfprivacy_api.utils import WriteUserData
from selfprivacy_api.utils.singleton_metaclass import SingletonMetaclass

6
tests/test_graphql/test_services.py
View file

@ -32,9 +32,9 @@ MOVER_MOCK_PROCESS = CompletedProcess(["ls"], returncode=0)
@pytest.fixture()
def mock_check_service_mover_folders(mocker):
def mock_check_service_mover_binds(mocker):
mock = mocker.patch(
"selfprivacy_api.services.service.check_folders",
"selfprivacy_api.services.service.check_binds",
autospec=True,
return_value=None,
)
@ -569,7 +569,7 @@ def test_graphql_move_service_without_folders_on_old_volume(
def test_graphql_move_service(
authorized_client,
generic_userdata,
mock_check_service_mover_folders,
mock_check_service_mover_binds,
lsblk_singular_mock,
dummy_service: DummyService,
mock_subprocess_run,