diff --git a/nixos/module.nix b/nixos/module.nix index 35bea4e..aa0a761 100644 --- a/nixos/module.nix +++ b/nixos/module.nix @@ -5,6 +5,19 @@ let config-id = "default"; nixos-rebuild = "${config.system.build.nixos-rebuild}/bin/nixos-rebuild"; nix = "${config.nix.package.out}/bin/nix"; + sp-fetch-remote-module = pkgs.writeShellApplication { + name = "sp-fetch-remote-module"; + runtimeInputs = [ config.nix.package.out ]; + text = '' + if [ "$#" -ne 1 ]; then + echo "Usage: $0 " + exit 1 + fi + + URL="$1" + nix eval --file /etc/sp-fetch-remote-module.nix --raw --apply "f: f { flakeURL = \"$URL\"; }" + ''; + }; in { options.services.selfprivacy-api = { @@ -46,12 +59,12 @@ in pkgs.util-linux pkgs.e2fsprogs pkgs.iproute2 - pkgs.kanidm ]; after = [ "network-online.target" ]; wants = [ "network-online.target" ]; wantedBy = [ "multi-user.target" ]; serviceConfig = { + # Do not forget to edit Postgres identMap if you change the user! User = "root"; ExecStart = "${selfprivacy-graphql-api}/bin/app.py"; Restart = "always"; @@ -82,12 +95,15 @@ in pkgs.util-linux pkgs.e2fsprogs pkgs.iproute2 + pkgs.postgresql_16.out + sp-fetch-remote-module pkgs.kanidm ]; after = [ "network-online.target" ]; wants = [ "network-online.target" ]; wantedBy = [ "multi-user.target" ]; serviceConfig = { + # Do not forget to edit Postgres identMap if you change the user! User = "root"; ExecStart = "${pkgs.python312Packages.huey}/bin/huey_consumer.py selfprivacy_api.task_registry.huey"; Restart = "always"; diff --git a/selfprivacy_api/backup/__init__.py b/selfprivacy_api/backup/__init__.py index 4af7c19..ef07171 100644 --- a/selfprivacy_api/backup/__init__.py +++ b/selfprivacy_api/backup/__init__.py @@ -246,9 +246,10 @@ class Backups: try: if service.can_be_backed_up() is False: raise ValueError("cannot backup a non-backuppable service") - folders = service.get_folders() + folders = service.get_folders_to_back_up() service_name = service.get_id() - service.pre_backup() + service.pre_backup(job=job) + Jobs.update(job, status=JobStatus.RUNNING, status_text="Uploading backup") snapshot = Backups.provider().backupper.start_backup( folders, service_name, @@ -258,12 +259,12 @@ class Backups: Backups._on_new_snapshot_created(service_name, snapshot) if reason == BackupReason.AUTO: Backups._prune_auto_snaps(service) - service.post_backup() + service.post_backup(job=job) except Exception as error: Jobs.update(job, status=JobStatus.ERROR, error=str(error)) raise error - Jobs.update(job, status=JobStatus.FINISHED) + Jobs.update(job, status=JobStatus.FINISHED, result="Backup finished") if reason in [BackupReason.AUTO, BackupReason.PRE_RESTORE]: Jobs.set_expiration(job, AUTOBACKUP_JOB_EXPIRATION_SECONDS) return Backups.sync_date_from_cache(snapshot) @@ -452,6 +453,7 @@ class Backups: with StoppedService(service): if not service.is_always_active(): Backups.assert_dead(service) + service.pre_restore(job=job) if strategy == RestoreStrategy.INPLACE: Backups._inplace_restore(service, snapshot, job) else: # verify_before_download is our default @@ -464,7 +466,7 @@ class Backups: service, snapshot.id, verify=True ) - service.post_restore() + service.post_restore(job=job) Jobs.update( job, status=JobStatus.RUNNING, @@ -515,7 +517,7 @@ class Backups: snapshot_id: str, verify=True, ) -> None: - folders = service.get_folders() + folders = service.get_folders_to_back_up() Backups.provider().backupper.restore_from_backup( snapshot_id, @@ -715,7 +717,7 @@ class Backups: Returns the amount of space available on the volume the given service is located on. """ - folders = service.get_folders() + folders = service.get_folders_to_back_up() if folders == []: raise ValueError("unallocated service", service.get_id()) diff --git a/selfprivacy_api/backup/backuppers/restic_backupper.py b/selfprivacy_api/backup/backuppers/restic_backupper.py index f037dcc..12df3a6 100644 --- a/selfprivacy_api/backup/backuppers/restic_backupper.py +++ b/selfprivacy_api/backup/backuppers/restic_backupper.py @@ -4,12 +4,13 @@ import subprocess import json import datetime import tempfile +import logging +import os from typing import List, Optional, TypeVar, Callable from collections.abc import Iterable from json.decoder import JSONDecodeError -from os.path import exists, join -from os import mkdir +from os.path import exists, join, isfile, islink, isdir from shutil import rmtree from selfprivacy_api.utils.waitloop import wait_until_success @@ -28,6 +29,8 @@ FILESYSTEM_TIMEOUT_SEC = 60 T = TypeVar("T", bound=Callable) +logger = logging.getLogger(__name__) + def unlocked_repo(func: T) -> T: """unlock repo and retry if it appears to be locked""" @@ -363,7 +366,27 @@ class ResticBackupper(AbstractBackupper): parsed_output = ResticBackupper.parse_json_output(output) return parsed_output["total_size"] except ValueError as error: - raise ValueError("cannot restore a snapshot: " + output) from error + raise ValueError("Cannot restore a snapshot: " + output) from error + + def _rm_all_folder_contents(self, folder: str) -> None: + """ + Remove all contents of a folder, including subfolders. + + Raises: + ValueError: If it encounters an error while removing contents. + """ + try: + for filename in os.listdir(folder): + path = join(folder, filename) + try: + if isfile(path) or islink(path): + os.unlink(path) + elif isdir(path): + rmtree(path) + except Exception as error: + raise ValueError("Cannot remove folder contents: ", path) from error + except OSError as error: + raise ValueError("Cannot access folder: ", folder) from error @unlocked_repo def restore_from_backup( @@ -376,7 +399,7 @@ class ResticBackupper(AbstractBackupper): Restore from backup with restic """ if folders is None or folders == []: - raise ValueError("cannot restore without knowing where to!") + raise ValueError("Cannot restore without knowing where to!") with tempfile.TemporaryDirectory() as temp_dir: if verify: @@ -394,9 +417,9 @@ class ResticBackupper(AbstractBackupper): else: # attempting inplace restore for folder in folders: wait_until_success( - lambda: rmtree(folder), timeout_sec=FILESYSTEM_TIMEOUT_SEC + lambda: self._rm_all_folder_contents(folder), + timeout_sec=FILESYSTEM_TIMEOUT_SEC, ) - mkdir(folder) self._raw_verified_restore(snapshot_id, target="/") return diff --git a/selfprivacy_api/dependencies.py b/selfprivacy_api/dependencies.py index b26635a..1efbc4d 100644 --- a/selfprivacy_api/dependencies.py +++ b/selfprivacy_api/dependencies.py @@ -27,4 +27,4 @@ async def get_token_header( def get_api_version() -> str: """Get API version""" - return "3.4.0" + return "3.5.0" diff --git a/selfprivacy_api/graphql/common_types/service.py b/selfprivacy_api/graphql/common_types/service.py index ec81945..cd2329f 100644 --- a/selfprivacy_api/graphql/common_types/service.py +++ b/selfprivacy_api/graphql/common_types/service.py @@ -6,6 +6,7 @@ import strawberry from selfprivacy_api.graphql.common_types.backup import BackupReason from selfprivacy_api.graphql.common_types.dns import DnsRecord +from selfprivacy_api.models.services import License from selfprivacy_api.services import ServiceManager from selfprivacy_api.services import Service as ServiceInterface from selfprivacy_api.services import ServiceDnsRecord @@ -71,6 +72,28 @@ class ServiceStatusEnum(Enum): OFF = "OFF" +@strawberry.enum +class SupportLevelEnum(Enum): + """Enum representing the support level of a service.""" + + NORMAL = "normal" + EXPERIMENTAL = "experimental" + DEPRECATED = "deprecated" + COMMUNITY = "community" + UNKNOWN = "unknown" + + +@strawberry.experimental.pydantic.type(model=License) +class LicenseType: + free: strawberry.auto + full_name: strawberry.auto + redistributable: strawberry.auto + short_name: strawberry.auto + spdx_id: strawberry.auto + url: strawberry.auto + deprecated: strawberry.auto + + def get_storage_usage(root: "Service") -> ServiceStorageUsage: """Get storage usage for a service""" service = ServiceManager.get_service_by_id(root.id) @@ -176,10 +199,15 @@ class Service: is_required: bool is_enabled: bool is_installed: bool + is_system_service: bool can_be_backed_up: bool backup_description: str status: ServiceStatusEnum url: Optional[str] + license: List[LicenseType] + homepage: Optional[str] + source_page: Optional[str] + support_level: SupportLevelEnum @strawberry.field def dns_records(self) -> Optional[List[DnsRecord]]: @@ -207,7 +235,10 @@ class Service: if not config_items: return None # By the "type" field convert every dict into a ConfigItem. In the future there will be more types. - return [config_item_to_graphql(config_items[item]) for item in config_items] + unsorted_config_items = [config_items[item] for item in config_items] + # Sort the items by their weight. If there is no weight, implicitly set it to 50. + config_items = sorted(unsorted_config_items, key=lambda x: x.get("weight", 50)) + return [config_item_to_graphql(item) for item in config_items] # TODO: fill this @strawberry.field @@ -238,6 +269,13 @@ def service_to_graphql_service(service: ServiceInterface) -> Service: backup_description=service.get_backup_description(), status=ServiceStatusEnum(service.get_status().value), url=service.get_url(), + is_system_service=service.is_system_service(), + license=[ + LicenseType.from_pydantic(license) for license in service.get_license() + ], + homepage=service.get_homepage(), + source_page=service.get_source_page(), + support_level=SupportLevelEnum(service.get_support_level().value), ) diff --git a/selfprivacy_api/graphql/queries/backup.py b/selfprivacy_api/graphql/queries/backup.py index 7eba72d..d018f88 100644 --- a/selfprivacy_api/graphql/queries/backup.py +++ b/selfprivacy_api/graphql/queries/backup.py @@ -13,6 +13,7 @@ from selfprivacy_api.graphql.common_types.service import ( Service, ServiceStatusEnum, SnapshotInfo, + SupportLevelEnum, service_to_graphql_service, ) from selfprivacy_api.graphql.common_types.backup import AutobackupQuotas @@ -53,6 +54,11 @@ def tombstone_service(service_id: str) -> Service: can_be_backed_up=False, backup_description="", is_installed=False, + homepage=None, + source_page=None, + license=[], + is_system_service=False, + support_level=SupportLevelEnum.UNKNOWN, ) diff --git a/selfprivacy_api/graphql/queries/services.py b/selfprivacy_api/graphql/queries/services.py index 2f95544..420ade0 100644 --- a/selfprivacy_api/graphql/queries/services.py +++ b/selfprivacy_api/graphql/queries/services.py @@ -16,5 +16,8 @@ from selfprivacy_api.services import ServiceManager class Services: @strawberry.field def all_services(self) -> typing.List[Service]: - services = ServiceManager.get_all_services() - return [service_to_graphql_service(service) for service in services] + services = [ + service_to_graphql_service(service) + for service in ServiceManager.get_all_services() + ] + return sorted(services, key=lambda service: service.display_name) diff --git a/selfprivacy_api/jobs/migrate_to_binds.py b/selfprivacy_api/jobs/migrate_to_binds.py index bde60fb..a41dffa 100644 --- a/selfprivacy_api/jobs/migrate_to_binds.py +++ b/selfprivacy_api/jobs/migrate_to_binds.py @@ -7,11 +7,8 @@ import logging from pydantic import BaseModel from selfprivacy_api.jobs import Job, JobStatus, Jobs -from selfprivacy_api.services.bitwarden import Bitwarden -from selfprivacy_api.services.forgejo import Forgejo +from selfprivacy_api.services import ServiceManager from selfprivacy_api.services.mailserver import MailServer -from selfprivacy_api.services.nextcloud import Nextcloud -from selfprivacy_api.services.pleroma import Pleroma from selfprivacy_api.utils import ReadUserData, WriteUserData from selfprivacy_api.utils.huey import huey from selfprivacy_api.utils.block_devices import BlockDevices @@ -105,6 +102,50 @@ def migrate_to_binds(config: BindMigrationConfig, job: Job): ) return + Jobs.update( + job=job, + status=JobStatus.RUNNING, + progress=0, + status_text="Checking if services are present.", + ) + + nextcloud_service = ServiceManager.get_service_by_id("nextcloud") + bitwarden_service = ServiceManager.get_service_by_id("bitwarden") + gitea_service = ServiceManager.get_service_by_id("gitea") + pleroma_service = ServiceManager.get_service_by_id("pleroma") + + if not nextcloud_service: + Jobs.update( + job=job, + status=JobStatus.ERROR, + error="Nextcloud service not found.", + ) + return + + if not bitwarden_service: + Jobs.update( + job=job, + status=JobStatus.ERROR, + error="Bitwarden service not found.", + ) + return + + if not gitea_service: + Jobs.update( + job=job, + status=JobStatus.ERROR, + error="Gitea service not found.", + ) + return + + if not pleroma_service: + Jobs.update( + job=job, + status=JobStatus.ERROR, + error="Pleroma service not found.", + ) + return + Jobs.update( job=job, status=JobStatus.RUNNING, @@ -172,7 +213,7 @@ def migrate_to_binds(config: BindMigrationConfig, job: Job): status_text="Migrating Nextcloud.", ) - Nextcloud().stop() + nextcloud_service.stop() # If /volumes/sda1/nextcloud or /volumes/sdb/nextcloud exists, skip it. if not pathlib.Path("/volumes/sda1/nextcloud").exists(): @@ -187,7 +228,7 @@ def migrate_to_binds(config: BindMigrationConfig, job: Job): ) # Start Nextcloud - Nextcloud().start() + nextcloud_service.start() # Perform migration of Bitwarden @@ -198,7 +239,7 @@ def migrate_to_binds(config: BindMigrationConfig, job: Job): status_text="Migrating Bitwarden.", ) - Bitwarden().stop() + bitwarden_service.stop() if not pathlib.Path("/volumes/sda1/bitwarden").exists(): if not pathlib.Path("/volumes/sdb/bitwarden").exists(): @@ -223,7 +264,7 @@ def migrate_to_binds(config: BindMigrationConfig, job: Job): ) # Start Bitwarden - Bitwarden().start() + bitwarden_service.start() # Perform migration of Gitea @@ -234,7 +275,7 @@ def migrate_to_binds(config: BindMigrationConfig, job: Job): status_text="Migrating Gitea.", ) - Forgejo().stop() + gitea_service.stop() if not pathlib.Path("/volumes/sda1/gitea").exists(): if not pathlib.Path("/volumes/sdb/gitea").exists(): @@ -245,7 +286,7 @@ def migrate_to_binds(config: BindMigrationConfig, job: Job): group="gitea", ) - Forgejo().start() + gitea_service.start() # Perform migration of Mail server @@ -287,7 +328,7 @@ def migrate_to_binds(config: BindMigrationConfig, job: Job): status_text="Migrating Pleroma.", ) - Pleroma().stop() + pleroma_service.stop() if not pathlib.Path("/volumes/sda1/pleroma").exists(): if not pathlib.Path("/volumes/sdb/pleroma").exists(): @@ -311,7 +352,7 @@ def migrate_to_binds(config: BindMigrationConfig, job: Job): group="postgres", ) - Pleroma().start() + pleroma_service.start() Jobs.update( job=job, diff --git a/selfprivacy_api/models/services.py b/selfprivacy_api/models/services.py index ea9b9dd..1e18bdc 100644 --- a/selfprivacy_api/models/services.py +++ b/selfprivacy_api/models/services.py @@ -1,6 +1,16 @@ from enum import Enum -from typing import Optional -from pydantic import BaseModel +from typing import Optional, List +from pydantic import BaseModel, ConfigDict +from pydantic.alias_generators import to_camel +from selfprivacy_api.services.owned_path import OwnedPath + + +class BaseSchema(BaseModel): + model_config = ConfigDict( + alias_generator=to_camel, + populate_by_name=True, + from_attributes=True, + ) class ServiceStatus(Enum): @@ -15,6 +25,29 @@ class ServiceStatus(Enum): OFF = "OFF" +class SupportLevel(Enum): + """Enum representing the support level of a service.""" + + NORMAL = "normal" + EXPERIMENTAL = "experimental" + DEPRECATED = "deprecated" + COMMUNITY = "community" + UNKNOWN = "unknown" + + @classmethod + def from_str(cls, support_level: str) -> "SupportLevel": + """Return the SupportLevel from a string.""" + if support_level == "normal": + return cls.NORMAL + if support_level == "experimental": + return cls.EXPERIMENTAL + if support_level == "deprecated": + return cls.DEPRECATED + if support_level == "community": + return cls.COMMUNITY + return cls.UNKNOWN + + class ServiceDnsRecord(BaseModel): type: str name: str @@ -23,3 +56,40 @@ class ServiceDnsRecord(BaseModel): ttl: int display_name: str priority: Optional[int] = None + + +class License(BaseSchema): + """Model representing a license.""" + + deprecated: bool + free: bool + full_name: str + redistributable: bool + short_name: str + spdx_id: str + url: str + + +class ServiceMetaData(BaseSchema): + """Model representing the meta data of a service.""" + + id: str + name: str + description: str = "No description found!" + svg_icon: str = "" + showUrl: bool = True + primary_subdomain: Optional[str] = None + is_movable: bool = False + is_required: bool = False + can_be_backed_up: bool = True + backup_description: str = "No backup description found!" + systemd_services: List[str] + user: Optional[str] = None + group: Optional[str] = None + folders: List[str] = [] + owned_folders: List[OwnedPath] = [] + postgre_databases: List[str] = [] + license: List[License] = [] + homepage: Optional[str] = None + source_page: Optional[str] = None + support_level: SupportLevel = SupportLevel.UNKNOWN diff --git a/selfprivacy_api/services/__init__.py b/selfprivacy_api/services/__init__.py index 72d9fe3..11a41f2 100644 --- a/selfprivacy_api/services/__init__.py +++ b/selfprivacy_api/services/__init__.py @@ -3,30 +3,34 @@ import logging import base64 import typing +import subprocess +import json from typing import List from os import path from os import makedirs from os.path import join +from functools import lru_cache + from shutil import copyfile, copytree, rmtree -from selfprivacy_api.services.bitwarden import Bitwarden -from selfprivacy_api.services.forgejo import Forgejo -from selfprivacy_api.services.jitsimeet import JitsiMeet +from selfprivacy_api.jobs import Job, JobStatus, Jobs from selfprivacy_api.services.prometheus import Prometheus -from selfprivacy_api.services.roundcube import Roundcube from selfprivacy_api.services.mailserver import MailServer -from selfprivacy_api.services.nextcloud import Nextcloud -from selfprivacy_api.services.pleroma import Pleroma -from selfprivacy_api.services.ocserv import Ocserv from selfprivacy_api.services.service import Service, ServiceDnsRecord from selfprivacy_api.services.service import ServiceStatus +from selfprivacy_api.utils.cached_call import get_ttl_hash import selfprivacy_api.utils.network as network_utils from selfprivacy_api.services.api_icon import API_ICON -from selfprivacy_api.utils import USERDATA_FILE, DKIM_DIR, SECRETS_FILE, get_domain +from selfprivacy_api.utils import USERDATA_FILE, DKIM_DIR, SECRETS_FILE from selfprivacy_api.utils.block_devices import BlockDevices from selfprivacy_api.utils import read_account_uri +from selfprivacy_api.services.templated_service import ( + SP_MODULES_DEFENITIONS_PATH, + SP_SUGGESTED_MODULES_PATH, + TemplatedService, +) CONFIG_STASH_DIR = "/etc/selfprivacy/dump" KANIDM_A_RECORD = "auth" @@ -39,27 +43,33 @@ class ServiceManager(Service): @staticmethod def get_all_services() -> list[Service]: - return services + return get_services() @staticmethod def get_service_by_id(service_id: str) -> typing.Optional[Service]: - for service in services: + for service in get_services(): if service.get_id() == service_id: return service return None @staticmethod def get_enabled_services() -> list[Service]: - return [service for service in services if service.is_enabled()] + return [service for service in get_services() if service.is_enabled()] # This one is not currently used by any code. @staticmethod def get_disabled_services() -> list[Service]: - return [service for service in services if not service.is_enabled()] + return [service for service in get_services() if not service.is_enabled()] @staticmethod def get_services_by_location(location: str) -> list[Service]: - return [service for service in services if service.get_drive() == location] + return [ + service + for service in get_services( + exclude_remote=True, + ) + if service.get_drive() == location + ] @staticmethod def get_all_required_dns_records() -> list[ServiceDnsRecord]: @@ -76,7 +86,7 @@ class ServiceManager(Service): ), ] - # TODO: Reenable with 3.5.0 release when clients are ready. + # TODO: Reenable with 3.6.0 release when clients are ready. # Do not forget about tests! # try: # dns_records.append( @@ -141,6 +151,10 @@ class ServiceManager(Service): def is_enabled() -> bool: return True + @staticmethod + def is_system_service() -> bool: + return True + @staticmethod def get_backup_description() -> str: return "General server settings." @@ -159,7 +173,9 @@ class ServiceManager(Service): # For now we will just copy settings EXCEPT the locations of services # Stash locations as they are set by user right now locations = {} - for service in services: + for service in get_services( + exclude_remote=True, + ): if service.is_movable(): locations[service.get_id()] = service.get_drive() @@ -167,8 +183,10 @@ class ServiceManager(Service): for p in [USERDATA_FILE, SECRETS_FILE, DKIM_DIR]: cls.retrieve_stashed_path(p) - # Pop locations - for service in services: + # Pop location + for service in get_services( + exclude_remote=True, + ): if service.is_movable(): device = BlockDevices().get_block_device(locations[service.get_id()]) if device is not None: @@ -195,11 +213,6 @@ class ServiceManager(Service): """ pass - @staticmethod - def get_logs(): - # TODO: maybe return the logs for api itself - return "" - @classmethod def get_drive(cls) -> str: return BlockDevices().get_root_block_device().name @@ -234,7 +247,12 @@ class ServiceManager(Service): copyfile(cls.stash_for(p), p) @classmethod - def pre_backup(cls): + def pre_backup(cls, job: Job): + Jobs.update( + job, + status_text="Stashing settings", + status=JobStatus.RUNNING, + ) tempdir = cls.dump_dir() rmtree(join(tempdir), ignore_errors=True) makedirs(tempdir) @@ -243,7 +261,7 @@ class ServiceManager(Service): cls.stash_a_path(p) @classmethod - def post_backup(cls): + def post_backup(cls, job: Job): rmtree(cls.dump_dir(), ignore_errors=True) @classmethod @@ -254,20 +272,80 @@ class ServiceManager(Service): return cls.folders[0] @classmethod - def post_restore(cls): + def post_restore(cls, job: Job): cls.merge_settings() rmtree(cls.dump_dir(), ignore_errors=True) -services: list[Service] = [ - Bitwarden(), - Forgejo(), - MailServer(), - Nextcloud(), - Pleroma(), - Ocserv(), - JitsiMeet(), - Roundcube(), - ServiceManager(), - Prometheus(), -] +# @redis_cached_call(ttl=30) +@lru_cache() +def get_templated_service(service_id: str, ttl_hash=None) -> TemplatedService: + del ttl_hash + return TemplatedService(service_id) + + +# @redis_cached_call(ttl=3600) +@lru_cache() +def get_remote_service(id: str, url: str, ttl_hash=None) -> TemplatedService: + del ttl_hash + response = subprocess.run( + ["sp-fetch-remote-module", url], + capture_output=True, + text=True, + check=True, + ) + return TemplatedService(id, response.stdout) + + +DUMMY_SERVICES = [] +TEST_FLAGS: list[str] = [] + + +def get_services(exclude_remote=False) -> List[Service]: + if "ONLY_DUMMY_SERVICE" in TEST_FLAGS: + return DUMMY_SERVICES + if "DUMMY_SERVICE_AND_API" in TEST_FLAGS: + return DUMMY_SERVICES + [ServiceManager()] + + hardcoded_services: list[Service] = [ + MailServer(), + ServiceManager(), + Prometheus(), + ] + if DUMMY_SERVICES: + hardcoded_services += DUMMY_SERVICES + service_ids = [service.get_id() for service in hardcoded_services] + + templated_services: List[Service] = [] + if path.exists(SP_MODULES_DEFENITIONS_PATH): + for module in listdir(SP_MODULES_DEFENITIONS_PATH): + if module in service_ids: + continue + try: + templated_services.append( + get_templated_service(module, ttl_hash=get_ttl_hash(30)) + ) + service_ids.append(module) + except Exception as e: + logger.error(f"Failed to load service {module}: {e}") + + if not exclude_remote and path.exists(SP_SUGGESTED_MODULES_PATH): + # It is a file with a JSON array + with open(SP_SUGGESTED_MODULES_PATH) as f: + suggested_modules = json.load(f) + for module in suggested_modules: + if module in service_ids: + continue + try: + templated_services.append( + get_remote_service( + module, + f"git+https://git.selfprivacy.org/SelfPrivacy/selfprivacy-nixos-config.git?ref=flakes&dir=sp-modules/{module}", + ttl_hash=get_ttl_hash(3600), + ) + ) + service_ids.append(module) + except Exception as e: + logger.error(f"Failed to load service {module}: {e}") + + return hardcoded_services + templated_services diff --git a/selfprivacy_api/services/bitwarden/__init__.py b/selfprivacy_api/services/bitwarden/__init__.py deleted file mode 100644 index 358cb7e..0000000 --- a/selfprivacy_api/services/bitwarden/__init__.py +++ /dev/null @@ -1,113 +0,0 @@ -"""Class representing Bitwarden service""" - -import base64 -import subprocess -from typing import List - -from selfprivacy_api.utils.systemd import get_service_status -from selfprivacy_api.services.service import Service, ServiceStatus -from selfprivacy_api.services.bitwarden.icon import BITWARDEN_ICON -from selfprivacy_api.services.config_item import ( - StringServiceConfigItem, - BoolServiceConfigItem, - ServiceConfigItem, -) -from selfprivacy_api.utils.regex_strings import SUBDOMAIN_REGEX - - -class Bitwarden(Service): - """Class representing Bitwarden service.""" - - config_items: dict[str, ServiceConfigItem] = { - "subdomain": StringServiceConfigItem( - id="subdomain", - default_value="password", - description="Subdomain", - regex=SUBDOMAIN_REGEX, - widget="subdomain", - ), - "signupsAllowed": BoolServiceConfigItem( - id="signupsAllowed", - default_value=True, - description="Allow new user signups", - ), - "sendsAllowed": BoolServiceConfigItem( - id="sendsAllowed", - default_value=True, - description="Allow users to use Bitwarden Send", - ), - "emergencyAccessAllowed": BoolServiceConfigItem( - id="emergencyAccessAllowed", - default_value=True, - description="Allow users to enable Emergency Access", - ), - } - - @staticmethod - def get_id() -> str: - """Return service id.""" - return "bitwarden" - - @staticmethod - def get_display_name() -> str: - """Return service display name.""" - return "Bitwarden" - - @staticmethod - def get_description() -> str: - """Return service description.""" - return "Bitwarden is a password manager." - - @staticmethod - def get_svg_icon() -> str: - """Read SVG icon from file and return it as base64 encoded string.""" - return base64.b64encode(BITWARDEN_ICON.encode("utf-8")).decode("utf-8") - - @staticmethod - def get_user() -> str: - return "vaultwarden" - - @staticmethod - def is_movable() -> bool: - return True - - @staticmethod - def is_required() -> bool: - return False - - @staticmethod - def get_backup_description() -> str: - return "Password database, encryption certificate and attachments." - - @staticmethod - def get_status() -> ServiceStatus: - """ - Return Bitwarden status from systemd. - Use command return code to determine status. - - Return code 0 means service is running. - Return code 1 or 2 means service is in error stat. - Return code 3 means service is stopped. - Return code 4 means service is off. - """ - return get_service_status("vaultwarden.service") - - @staticmethod - def stop(): - subprocess.run(["systemctl", "stop", "vaultwarden.service"]) - - @staticmethod - def start(): - subprocess.run(["systemctl", "start", "vaultwarden.service"]) - - @staticmethod - def restart(): - subprocess.run(["systemctl", "restart", "vaultwarden.service"]) - - @staticmethod - def get_logs(): - return "" - - @staticmethod - def get_folders() -> List[str]: - return ["/var/lib/bitwarden", "/var/lib/bitwarden_rs"] diff --git a/selfprivacy_api/services/bitwarden/bitwarden.svg b/selfprivacy_api/services/bitwarden/bitwarden.svg deleted file mode 100644 index ced270c..0000000 --- a/selfprivacy_api/services/bitwarden/bitwarden.svg +++ /dev/null @@ -1,3 +0,0 @@ - - - diff --git a/selfprivacy_api/services/bitwarden/icon.py b/selfprivacy_api/services/bitwarden/icon.py deleted file mode 100644 index f9280e0..0000000 --- a/selfprivacy_api/services/bitwarden/icon.py +++ /dev/null @@ -1,5 +0,0 @@ -BITWARDEN_ICON = """ - - - -""" diff --git a/selfprivacy_api/services/config_item.py b/selfprivacy_api/services/config_item.py index eba557a..bb22c57 100644 --- a/selfprivacy_api/services/config_item.py +++ b/selfprivacy_api/services/config_item.py @@ -14,6 +14,7 @@ class ServiceConfigItem(ABC): description: str widget: str type: str + weight: int @abstractmethod def get_value(self, service_id): @@ -27,13 +28,14 @@ class ServiceConfigItem(ABC): def validate_value(self, value): return True - def as_dict(self, service_options): + def as_dict(self, service_id: str): return { "id": self.id, "type": self.type, "description": self.description, "widget": self.widget, - "value": self.get_value(service_options), + "value": self.get_value(service_id), + "weight": self.weight, } @@ -46,6 +48,7 @@ class StringServiceConfigItem(ServiceConfigItem): regex: Optional[str] = None, widget: Optional[str] = None, allow_empty: bool = False, + weight: int = 50, ): if widget == "subdomain" and not regex: raise ValueError("Subdomain widget requires regex") @@ -56,6 +59,7 @@ class StringServiceConfigItem(ServiceConfigItem): self.regex = re.compile(regex) if regex else None self.widget = widget if widget else "text" self.allow_empty = allow_empty + self.weight = weight def get_value(self, service_id): with ReadUserData() as user_data: @@ -73,15 +77,16 @@ class StringServiceConfigItem(ServiceConfigItem): user_data["modules"][service_id] = {} user_data["modules"][service_id][self.id] = value - def as_dict(self, service_options): + def as_dict(self, service_id): return { "id": self.id, "type": self.type, "description": self.description, "widget": self.widget, - "value": self.get_value(service_options), + "value": self.get_value(service_id), "default_value": self.default_value, "regex": self.regex.pattern if self.regex else None, + "weight": self.weight, } def validate_value(self, value): @@ -104,12 +109,14 @@ class BoolServiceConfigItem(ServiceConfigItem): default_value: bool, description: str, widget: Optional[str] = None, + weight: int = 50, ): self.id = id self.type = "bool" self.default_value = default_value self.description = description self.widget = widget if widget else "switch" + self.weight = weight def get_value(self, service_id): with ReadUserData() as user_data: @@ -127,14 +134,15 @@ class BoolServiceConfigItem(ServiceConfigItem): user_data["modules"][service_id] = {} user_data["modules"][service_id][self.id] = value - def as_dict(self, service_options): + def as_dict(self, service_id): return { "id": self.id, "type": self.type, "description": self.description, "widget": self.widget, - "value": self.get_value(service_options), + "value": self.get_value(service_id), "default_value": self.default_value, + "weight": self.weight, } def validate_value(self, value): @@ -149,6 +157,7 @@ class EnumServiceConfigItem(ServiceConfigItem): description: str, options: list[str], widget: Optional[str] = None, + weight: int = 50, ): self.id = id self.type = "enum" @@ -156,6 +165,7 @@ class EnumServiceConfigItem(ServiceConfigItem): self.description = description self.options = options self.widget = widget if widget else "select" + self.weight = weight def get_value(self, service_id): with ReadUserData() as user_data: @@ -173,15 +183,16 @@ class EnumServiceConfigItem(ServiceConfigItem): user_data["modules"][service_id] = {} user_data["modules"][service_id][self.id] = value - def as_dict(self, service_options): + def as_dict(self, service_id): return { "id": self.id, "type": self.type, "description": self.description, "widget": self.widget, - "value": self.get_value(service_options), + "value": self.get_value(service_id), "default_value": self.default_value, "options": self.options, + "weight": self.weight, } def validate_value(self, value): @@ -200,6 +211,7 @@ class IntServiceConfigItem(ServiceConfigItem): widget: Optional[str] = None, min_value: Optional[int] = None, max_value: Optional[int] = None, + weight: int = 50, ) -> None: self.id = id self.type = "int" @@ -208,6 +220,7 @@ class IntServiceConfigItem(ServiceConfigItem): self.widget = widget if widget else "number" self.min_value = min_value self.max_value = max_value + self.weight = weight def get_value(self, service_id): with ReadUserData() as user_data: @@ -225,16 +238,17 @@ class IntServiceConfigItem(ServiceConfigItem): user_data["modules"][service_id] = {} user_data["modules"][service_id][self.id] = value - def as_dict(self, service_options): + def as_dict(self, service_id): return { "id": self.id, "type": self.type, "description": self.description, "widget": self.widget, - "value": self.get_value(service_options), + "value": self.get_value(service_id), "default_value": self.default_value, "min_value": self.min_value, "max_value": self.max_value, + "weight": self.weight, } def validate_value(self, value): diff --git a/selfprivacy_api/services/forgejo/__init__.py b/selfprivacy_api/services/forgejo/__init__.py deleted file mode 100644 index 350c004..0000000 --- a/selfprivacy_api/services/forgejo/__init__.py +++ /dev/null @@ -1,138 +0,0 @@ -"""Class representing Bitwarden service""" - -import base64 -import subprocess -from typing import List - -from selfprivacy_api.utils import ReadUserData, WriteUserData - -from selfprivacy_api.utils.systemd import get_service_status -from selfprivacy_api.services.service import Service, ServiceStatus -from selfprivacy_api.services.forgejo.icon import FORGEJO_ICON -from selfprivacy_api.services.config_item import ( - StringServiceConfigItem, - BoolServiceConfigItem, - EnumServiceConfigItem, - ServiceConfigItem, -) -from selfprivacy_api.utils.regex_strings import SUBDOMAIN_REGEX - - -class Forgejo(Service): - """Class representing Forgejo service. - - Previously was Gitea, so some IDs are still called gitea for compatibility. - """ - - config_items: dict[str, ServiceConfigItem] = { - "subdomain": StringServiceConfigItem( - id="subdomain", - default_value="git", - description="Subdomain", - regex=SUBDOMAIN_REGEX, - widget="subdomain", - ), - "appName": StringServiceConfigItem( - id="appName", - default_value="SelfPrivacy git Service", - description="The name displayed in the web interface", - ), - "enableLfs": BoolServiceConfigItem( - id="enableLfs", - default_value=True, - description="Enable Git LFS", - ), - "forcePrivate": BoolServiceConfigItem( - id="forcePrivate", - default_value=False, - description="Force all new repositories to be private", - ), - "disableRegistration": BoolServiceConfigItem( - id="disableRegistration", - default_value=False, - description="Disable registration of new users", - ), - "requireSigninView": BoolServiceConfigItem( - id="requireSigninView", - default_value=False, - description="Force users to log in to view any page", - ), - "defaultTheme": EnumServiceConfigItem( - id="defaultTheme", - default_value="forgejo-auto", - description="Default theme", - options=[ - "forgejo-auto", - "forgejo-light", - "forgejo-dark", - "gitea-auto", - "gitea-light", - "gitea-dark", - ], - ), - } - - @staticmethod - def get_id() -> str: - """Return service id. For compatibility keep in gitea.""" - return "gitea" - - @staticmethod - def get_display_name() -> str: - """Return service display name.""" - return "Forgejo" - - @staticmethod - def get_description() -> str: - """Return service description.""" - return "Forgejo is a Git forge." - - @staticmethod - def get_svg_icon() -> str: - """Read SVG icon from file and return it as base64 encoded string.""" - return base64.b64encode(FORGEJO_ICON.encode("utf-8")).decode("utf-8") - - @staticmethod - def is_movable() -> bool: - return True - - @staticmethod - def is_required() -> bool: - return False - - @staticmethod - def get_backup_description() -> str: - return "Git repositories, database and user data." - - @staticmethod - def get_status() -> ServiceStatus: - """ - Return Gitea status from systemd. - Use command return code to determine status. - Return code 0 means service is running. - Return code 1 or 2 means service is in error stat. - Return code 3 means service is stopped. - Return code 4 means service is off. - """ - return get_service_status("forgejo.service") - - @staticmethod - def stop(): - subprocess.run(["systemctl", "stop", "forgejo.service"]) - - @staticmethod - def start(): - subprocess.run(["systemctl", "start", "forgejo.service"]) - - @staticmethod - def restart(): - subprocess.run(["systemctl", "restart", "forgejo.service"]) - - @staticmethod - def get_logs(): - return "" - - @staticmethod - def get_folders() -> List[str]: - """The data folder is still called gitea for compatibility.""" - return ["/var/lib/gitea"] diff --git a/selfprivacy_api/services/forgejo/gitea.svg b/selfprivacy_api/services/forgejo/gitea.svg deleted file mode 100644 index 9ba8a76..0000000 --- a/selfprivacy_api/services/forgejo/gitea.svg +++ /dev/null @@ -1,3 +0,0 @@ - - - diff --git a/selfprivacy_api/services/forgejo/icon.py b/selfprivacy_api/services/forgejo/icon.py deleted file mode 100644 index 5e600cf..0000000 --- a/selfprivacy_api/services/forgejo/icon.py +++ /dev/null @@ -1,5 +0,0 @@ -FORGEJO_ICON = """ - - - -""" diff --git a/selfprivacy_api/services/jitsimeet/__init__.py b/selfprivacy_api/services/jitsimeet/__init__.py deleted file mode 100644 index ee7bc85..0000000 --- a/selfprivacy_api/services/jitsimeet/__init__.py +++ /dev/null @@ -1,113 +0,0 @@ -"""Class representing Jitsi Meet service""" - -import base64 -import subprocess -from typing import List - -from selfprivacy_api.jobs import Job -from selfprivacy_api.utils.systemd import ( - get_service_status_from_several_units, -) -from selfprivacy_api.services.service import Service, ServiceStatus -from selfprivacy_api.utils.block_devices import BlockDevice -from selfprivacy_api.services.jitsimeet.icon import JITSI_ICON -from selfprivacy_api.services.config_item import ( - StringServiceConfigItem, - ServiceConfigItem, -) -from selfprivacy_api.utils.regex_strings import SUBDOMAIN_REGEX - - -class JitsiMeet(Service): - """Class representing Jitsi service""" - - config_items: dict[str, ServiceConfigItem] = { - "subdomain": StringServiceConfigItem( - id="subdomain", - default_value="meet", - description="Subdomain", - regex=SUBDOMAIN_REGEX, - widget="subdomain", - ), - "appName": StringServiceConfigItem( - id="appName", - default_value="Jitsi Meet", - description="The name displayed in the web interface", - ), - } - - @staticmethod - def get_id() -> str: - """Return service id.""" - return "jitsi-meet" - - @staticmethod - def get_display_name() -> str: - """Return service display name.""" - return "JitsiMeet" - - @staticmethod - def get_description() -> str: - """Return service description.""" - return "Jitsi Meet is a free and open-source video conferencing solution." - - @staticmethod - def get_svg_icon() -> str: - """Read SVG icon from file and return it as base64 encoded string.""" - return base64.b64encode(JITSI_ICON.encode("utf-8")).decode("utf-8") - - @staticmethod - def is_movable() -> bool: - return False - - @staticmethod - def is_required() -> bool: - return False - - @staticmethod - def get_backup_description() -> str: - return "Secrets that are used to encrypt the communication." - - @staticmethod - def get_status() -> ServiceStatus: - return get_service_status_from_several_units( - ["prosody.service", "jitsi-videobridge2.service", "jicofo.service"] - ) - - @staticmethod - def stop(): - subprocess.run( - ["systemctl", "stop", "jitsi-videobridge2.service"], - check=False, - ) - subprocess.run(["systemctl", "stop", "jicofo.service"], check=False) - subprocess.run(["systemctl", "stop", "prosody.service"], check=False) - - @staticmethod - def start(): - subprocess.run(["systemctl", "start", "prosody.service"], check=False) - subprocess.run( - ["systemctl", "start", "jitsi-videobridge2.service"], - check=False, - ) - subprocess.run(["systemctl", "start", "jicofo.service"], check=False) - - @staticmethod - def restart(): - subprocess.run(["systemctl", "restart", "prosody.service"], check=False) - subprocess.run( - ["systemctl", "restart", "jitsi-videobridge2.service"], - check=False, - ) - subprocess.run(["systemctl", "restart", "jicofo.service"], check=False) - - @staticmethod - def get_logs(): - return "" - - @staticmethod - def get_folders() -> List[str]: - return ["/var/lib/jitsi-meet"] - - def move_to_volume(self, volume: BlockDevice) -> Job: - raise NotImplementedError("jitsi-meet service is not movable") diff --git a/selfprivacy_api/services/jitsimeet/icon.py b/selfprivacy_api/services/jitsimeet/icon.py deleted file mode 100644 index 08bcbb1..0000000 --- a/selfprivacy_api/services/jitsimeet/icon.py +++ /dev/null @@ -1,5 +0,0 @@ -JITSI_ICON = """ - - - -""" diff --git a/selfprivacy_api/services/mailserver/__init__.py b/selfprivacy_api/services/mailserver/__init__.py index d5dd481..05ce56a 100644 --- a/selfprivacy_api/services/mailserver/__init__.py +++ b/selfprivacy_api/services/mailserver/__init__.py @@ -89,10 +89,6 @@ class MailServer(Service): subprocess.run(["systemctl", "restart", "dovecot2.service"], check=False) subprocess.run(["systemctl", "restart", "postfix.service"], check=False) - @staticmethod - def get_logs(): - return "" - @staticmethod def get_folders() -> List[str]: return ["/var/vmail", "/var/sieve"] diff --git a/selfprivacy_api/services/nextcloud/__init__.py b/selfprivacy_api/services/nextcloud/__init__.py deleted file mode 100644 index 866ad03..0000000 --- a/selfprivacy_api/services/nextcloud/__init__.py +++ /dev/null @@ -1,104 +0,0 @@ -"""Class representing Nextcloud service.""" - -import base64 -import subprocess -from typing import List - -from selfprivacy_api.utils.systemd import get_service_status -from selfprivacy_api.services.service import Service, ServiceStatus - -from selfprivacy_api.services.nextcloud.icon import NEXTCLOUD_ICON -from selfprivacy_api.services.config_item import ( - StringServiceConfigItem, - BoolServiceConfigItem, - ServiceConfigItem, -) -from selfprivacy_api.utils.regex_strings import SUBDOMAIN_REGEX - - -class Nextcloud(Service): - """Class representing Nextcloud service.""" - - config_items: dict[str, ServiceConfigItem] = { - "subdomain": StringServiceConfigItem( - id="subdomain", - default_value="cloud", - description="Subdomain", - regex=SUBDOMAIN_REGEX, - widget="subdomain", - ), - "enableImagemagick": BoolServiceConfigItem( - id="enableImagemagick", - default_value=True, - description="Enable ImageMagick", - ), - } - - @staticmethod - def get_id() -> str: - """Return service id.""" - return "nextcloud" - - @staticmethod - def get_display_name() -> str: - """Return service display name.""" - return "Nextcloud" - - @staticmethod - def get_description() -> str: - """Return service description.""" - return "Nextcloud is a cloud storage service that offers a web interface and a desktop client." - - @staticmethod - def get_svg_icon() -> str: - """Read SVG icon from file and return it as base64 encoded string.""" - return base64.b64encode(NEXTCLOUD_ICON.encode("utf-8")).decode("utf-8") - - @staticmethod - def is_movable() -> bool: - return True - - @staticmethod - def is_required() -> bool: - return False - - @staticmethod - def get_backup_description() -> str: - return "All the files and other data stored in Nextcloud." - - @staticmethod - def get_status() -> ServiceStatus: - """ - Return Nextcloud status from systemd. - Use command return code to determine status. - - Return code 0 means service is running. - Return code 1 or 2 means service is in error stat. - Return code 3 means service is stopped. - Return code 4 means service is off. - """ - return get_service_status("phpfpm-nextcloud.service") - - @staticmethod - def stop(): - """Stop Nextcloud service.""" - subprocess.Popen(["systemctl", "stop", "phpfpm-nextcloud.service"]) - - @staticmethod - def start(): - """Start Nextcloud service.""" - subprocess.Popen(["systemctl", "start", "phpfpm-nextcloud.service"]) - - @staticmethod - def restart(): - """Restart Nextcloud service.""" - subprocess.Popen(["systemctl", "restart", "phpfpm-nextcloud.service"]) - - @staticmethod - def get_logs(): - """Return Nextcloud logs.""" - return "" - - @staticmethod - def get_folders() -> List[str]: - return ["/var/lib/nextcloud"] diff --git a/selfprivacy_api/services/nextcloud/icon.py b/selfprivacy_api/services/nextcloud/icon.py deleted file mode 100644 index d178640..0000000 --- a/selfprivacy_api/services/nextcloud/icon.py +++ /dev/null @@ -1,12 +0,0 @@ -NEXTCLOUD_ICON = """ - - - - - - - - - - -""" diff --git a/selfprivacy_api/services/nextcloud/nextcloud.svg b/selfprivacy_api/services/nextcloud/nextcloud.svg deleted file mode 100644 index d7dbcb5..0000000 --- a/selfprivacy_api/services/nextcloud/nextcloud.svg +++ /dev/null @@ -1,10 +0,0 @@ - - - - - - - - - - diff --git a/selfprivacy_api/services/ocserv/__init__.py b/selfprivacy_api/services/ocserv/__init__.py deleted file mode 100644 index c180f62..0000000 --- a/selfprivacy_api/services/ocserv/__init__.py +++ /dev/null @@ -1,86 +0,0 @@ -"""Class representing ocserv service.""" - -import base64 -import subprocess -import typing -from selfprivacy_api.jobs import Job -from selfprivacy_api.utils.systemd import get_service_status -from selfprivacy_api.services.service import Service, ServiceStatus -from selfprivacy_api.utils.block_devices import BlockDevice -from selfprivacy_api.services.ocserv.icon import OCSERV_ICON - - -class Ocserv(Service): - """Class representing ocserv service.""" - - @staticmethod - def get_id() -> str: - return "ocserv" - - @staticmethod - def get_display_name() -> str: - return "OpenConnect VPN" - - @staticmethod - def get_description() -> str: - return "OpenConnect VPN to connect your devices and access the internet." - - @staticmethod - def get_svg_icon() -> str: - return base64.b64encode(OCSERV_ICON.encode("utf-8")).decode("utf-8") - - @classmethod - def get_url(cls) -> typing.Optional[str]: - """Return service url.""" - return None - - @staticmethod - def is_movable() -> bool: - return False - - @staticmethod - def is_required() -> bool: - return False - - @staticmethod - def can_be_backed_up() -> bool: - return False - - @staticmethod - def get_backup_description() -> str: - return "Nothing to backup." - - @staticmethod - def get_status() -> ServiceStatus: - return get_service_status("ocserv.service") - - @staticmethod - def stop(): - subprocess.run(["systemctl", "stop", "ocserv.service"], check=False) - - @staticmethod - def start(): - subprocess.run(["systemctl", "start", "ocserv.service"], check=False) - - @staticmethod - def restart(): - subprocess.run(["systemctl", "restart", "ocserv.service"], check=False) - - @classmethod - def get_configuration(cls): - return {} - - @classmethod - def set_configuration(cls, config_items): - return super().set_configuration(config_items) - - @staticmethod - def get_logs(): - return "" - - @staticmethod - def get_folders() -> typing.List[str]: - return [] - - def move_to_volume(self, volume: BlockDevice) -> Job: - raise NotImplementedError("ocserv service is not movable") diff --git a/selfprivacy_api/services/ocserv/icon.py b/selfprivacy_api/services/ocserv/icon.py deleted file mode 100644 index 6585c5e..0000000 --- a/selfprivacy_api/services/ocserv/icon.py +++ /dev/null @@ -1,5 +0,0 @@ -OCSERV_ICON = """ - - - -""" diff --git a/selfprivacy_api/services/ocserv/ocserv.svg b/selfprivacy_api/services/ocserv/ocserv.svg deleted file mode 100644 index 288f743..0000000 --- a/selfprivacy_api/services/ocserv/ocserv.svg +++ /dev/null @@ -1,3 +0,0 @@ - - - diff --git a/selfprivacy_api/services/owned_path.py b/selfprivacy_api/services/owned_path.py index 1bbe4df..c1d97c2 100644 --- a/selfprivacy_api/services/owned_path.py +++ b/selfprivacy_api/services/owned_path.py @@ -105,8 +105,8 @@ class Bind: ["umount", self.binding_path], check=True, ) - except subprocess.CalledProcessError: - raise BindError(f"Unable to unmount folder {self.binding_path}.") + except subprocess.CalledProcessError as error: + raise BindError(f"Unable to unmount folder {self.binding_path}. {error}") pass def ensure_ownership(self) -> None: diff --git a/selfprivacy_api/services/pleroma/__init__.py b/selfprivacy_api/services/pleroma/__init__.py deleted file mode 100644 index 0c260a8..0000000 --- a/selfprivacy_api/services/pleroma/__init__.py +++ /dev/null @@ -1,93 +0,0 @@ -"""Class representing Nextcloud service.""" - -import base64 -import subprocess -from typing import List - -from selfprivacy_api.services.owned_path import OwnedPath -from selfprivacy_api.utils.systemd import get_service_status -from selfprivacy_api.services.service import Service, ServiceStatus - -from selfprivacy_api.services.pleroma.icon import PLEROMA_ICON - - -class Pleroma(Service): - """Class representing Pleroma service.""" - - @staticmethod - def get_id() -> str: - return "pleroma" - - @staticmethod - def get_display_name() -> str: - return "Pleroma" - - @staticmethod - def get_description() -> str: - return "Pleroma is a microblogging service that offers a web interface and a desktop client." - - @staticmethod - def get_svg_icon() -> str: - return base64.b64encode(PLEROMA_ICON.encode("utf-8")).decode("utf-8") - - @staticmethod - def is_movable() -> bool: - return True - - @staticmethod - def is_required() -> bool: - return False - - @staticmethod - def get_backup_description() -> str: - return "Your Pleroma accounts, posts and media." - - @staticmethod - def get_status() -> ServiceStatus: - return get_service_status("pleroma.service") - - @staticmethod - def stop(): - subprocess.run(["systemctl", "stop", "pleroma.service"]) - subprocess.run(["systemctl", "stop", "postgresql.service"]) - - @staticmethod - def start(): - subprocess.run(["systemctl", "start", "pleroma.service"]) - subprocess.run(["systemctl", "start", "postgresql.service"]) - - @staticmethod - def restart(): - subprocess.run(["systemctl", "restart", "pleroma.service"]) - subprocess.run(["systemctl", "restart", "postgresql.service"]) - - @classmethod - def get_configuration(cls): - return {} - - @classmethod - def set_configuration(cls, config_items): - return super().set_configuration(config_items) - - @staticmethod - def get_logs(): - return "" - - @staticmethod - def get_owned_folders() -> List[OwnedPath]: - """ - Get a list of occupied directories with ownership info - Pleroma has folders that are owned by different users - """ - return [ - OwnedPath( - path="/var/lib/pleroma", - owner="pleroma", - group="pleroma", - ), - OwnedPath( - path="/var/lib/postgresql", - owner="postgres", - group="postgres", - ), - ] diff --git a/selfprivacy_api/services/pleroma/icon.py b/selfprivacy_api/services/pleroma/icon.py deleted file mode 100644 index c0c4d2b..0000000 --- a/selfprivacy_api/services/pleroma/icon.py +++ /dev/null @@ -1,12 +0,0 @@ -PLEROMA_ICON = """ - - - - - - - - - - -""" diff --git a/selfprivacy_api/services/pleroma/pleroma.svg b/selfprivacy_api/services/pleroma/pleroma.svg deleted file mode 100644 index f87c438..0000000 --- a/selfprivacy_api/services/pleroma/pleroma.svg +++ /dev/null @@ -1,10 +0,0 @@ - - - - - - - - - - diff --git a/selfprivacy_api/services/prometheus/__init__.py b/selfprivacy_api/services/prometheus/__init__.py index c18a117..8b4bf4c 100644 --- a/selfprivacy_api/services/prometheus/__init__.py +++ b/selfprivacy_api/services/prometheus/__init__.py @@ -47,6 +47,10 @@ class Prometheus(Service): def is_required() -> bool: return True + @staticmethod + def is_system_service() -> bool: + return True + @staticmethod def can_be_backed_up() -> bool: return False @@ -71,10 +75,6 @@ class Prometheus(Service): def restart(): subprocess.run(["systemctl", "restart", "prometheus.service"]) - @staticmethod - def get_logs(): - return "" - @staticmethod def get_owned_folders() -> List[OwnedPath]: return [ diff --git a/selfprivacy_api/services/roundcube/__init__.py b/selfprivacy_api/services/roundcube/__init__.py deleted file mode 100644 index 8349ccd..0000000 --- a/selfprivacy_api/services/roundcube/__init__.py +++ /dev/null @@ -1,104 +0,0 @@ -"""Class representing Roundcube service""" - -import base64 -import subprocess -from typing import List - -from selfprivacy_api.jobs import Job -from selfprivacy_api.utils.systemd import ( - get_service_status_from_several_units, -) -from selfprivacy_api.services.service import Service, ServiceStatus -from selfprivacy_api.utils.block_devices import BlockDevice -from selfprivacy_api.services.roundcube.icon import ROUNDCUBE_ICON -from selfprivacy_api.services.config_item import ( - StringServiceConfigItem, - ServiceConfigItem, -) -from selfprivacy_api.utils.regex_strings import SUBDOMAIN_REGEX - - -class Roundcube(Service): - """Class representing roundcube service""" - - config_items: dict[str, ServiceConfigItem] = { - "subdomain": StringServiceConfigItem( - id="subdomain", - default_value="roundcube", - description="Subdomain", - regex=SUBDOMAIN_REGEX, - widget="subdomain", - ), - } - - @staticmethod - def get_id() -> str: - """Return service id.""" - return "roundcube" - - @staticmethod - def get_display_name() -> str: - """Return service display name.""" - return "Roundcube" - - @staticmethod - def get_description() -> str: - """Return service description.""" - return "Roundcube is an open source webmail software." - - @staticmethod - def get_svg_icon() -> str: - """Read SVG icon from file and return it as base64 encoded string.""" - return base64.b64encode(ROUNDCUBE_ICON.encode("utf-8")).decode("utf-8") - - @staticmethod - def is_movable() -> bool: - return False - - @staticmethod - def is_required() -> bool: - return False - - @staticmethod - def can_be_backed_up() -> bool: - return False - - @staticmethod - def get_backup_description() -> str: - return "Nothing to backup." - - @staticmethod - def get_status() -> ServiceStatus: - return get_service_status_from_several_units(["phpfpm-roundcube.service"]) - - @staticmethod - def stop(): - subprocess.run( - ["systemctl", "stop", "phpfpm-roundcube.service"], - check=False, - ) - - @staticmethod - def start(): - subprocess.run( - ["systemctl", "start", "phpfpm-roundcube.service"], - check=False, - ) - - @staticmethod - def restart(): - subprocess.run( - ["systemctl", "restart", "phpfpm-roundcube.service"], - check=False, - ) - - @staticmethod - def get_logs(): - return "" - - @staticmethod - def get_folders() -> List[str]: - return [] - - def move_to_volume(self, volume: BlockDevice) -> Job: - raise NotImplementedError("roundcube service is not movable") diff --git a/selfprivacy_api/services/roundcube/icon.py b/selfprivacy_api/services/roundcube/icon.py deleted file mode 100644 index 4a08207..0000000 --- a/selfprivacy_api/services/roundcube/icon.py +++ /dev/null @@ -1,7 +0,0 @@ -ROUNDCUBE_ICON = """ - - - - - -""" diff --git a/selfprivacy_api/services/service.py b/selfprivacy_api/services/service.py index b71ee00..ca180f3 100644 --- a/selfprivacy_api/services/service.py +++ b/selfprivacy_api/services/service.py @@ -1,6 +1,7 @@ """Abstract class for a service running on a server""" from abc import ABC, abstractmethod +import logging from typing import List, Optional from os.path import exists @@ -14,7 +15,12 @@ from selfprivacy_api.utils.block_devices import BlockDevice, BlockDevices from selfprivacy_api.jobs import Job, Jobs, JobStatus, report_progress from selfprivacy_api.jobs.upgrade_system import rebuild_system -from selfprivacy_api.models.services import ServiceStatus, ServiceDnsRecord +from selfprivacy_api.models.services import ( + License, + ServiceStatus, + ServiceDnsRecord, + SupportLevel, +) from selfprivacy_api.services.generic_size_counter import get_storage_usage from selfprivacy_api.services.owned_path import OwnedPath, Bind from selfprivacy_api.services.moving import ( @@ -30,6 +36,8 @@ from selfprivacy_api.services.moving import ( DEFAULT_START_STOP_TIMEOUT = 5 * 60 +logger = logging.getLogger(__name__) + class Service(ABC): """ @@ -163,6 +171,37 @@ class Service(ABC): with ReadUserData() as user_data: return user_data.get("modules", {}).get(name, {}) != {} + def is_system_service(self) -> bool: + """ + `True` if the service is a system service and should be hidden from the user. + `False` if it is not a system service. + """ + return False + + def get_license(self) -> List[License]: + """ + The licenses of the service. + """ + return [] + + def get_homepage(self) -> Optional[str]: + """ + The homepage of the service. + """ + return None + + def get_source_page(self) -> Optional[str]: + """ + The source page of the service. + """ + return None + + def get_support_level(self) -> SupportLevel: + """ + The support level of the service. + """ + return SupportLevel.NORMAL + @staticmethod @abstractmethod def get_status() -> ServiceStatus: @@ -226,11 +265,6 @@ class Service(ABC): cls.get_id(), ) - @staticmethod - @abstractmethod - def get_logs(): - pass - @classmethod def get_storage_usage(cls) -> int: """ @@ -314,6 +348,10 @@ class Service(ABC): ) return [owned_folder.path for owned_folder in cls.get_owned_folders()] + @classmethod + def get_folders_to_back_up(cls) -> List[str]: + return cls.get_folders() + @classmethod def get_owned_folders(cls) -> List[OwnedPath]: """ @@ -330,6 +368,9 @@ class Service(ABC): def get_foldername(path: str) -> str: return path.split("/")[-1] + def get_postgresql_databases(self) -> List[str]: + return [] + # TODO: with better json utils, it can be one line, and not a separate function @classmethod def set_location(cls, volume: BlockDevice): @@ -477,13 +518,16 @@ class Service(ABC): group=group, ) - def pre_backup(self): + def pre_backup(self, job: Job): pass - def post_backup(self): + def post_backup(self, job: Job): pass - def post_restore(self): + def pre_restore(self, job: Job): + pass + + def post_restore(self, job: Job): pass diff --git a/selfprivacy_api/services/templated_service.py b/selfprivacy_api/services/templated_service.py new file mode 100644 index 0000000..aeffb5f --- /dev/null +++ b/selfprivacy_api/services/templated_service.py @@ -0,0 +1,514 @@ +"""A Service implementation that loads all needed data from a JSON file""" + +import base64 +import logging +import json +import subprocess +from typing import List, Optional +from os.path import join, exists +from os import mkdir, remove + +from selfprivacy_api.utils.postgres import PostgresDumper +from selfprivacy_api.jobs import Job, JobStatus, Jobs +from selfprivacy_api.models.services import ( + License, + ServiceDnsRecord, + ServiceMetaData, + ServiceStatus, + SupportLevel, +) +from selfprivacy_api.services.flake_service_manager import FlakeServiceManager +from selfprivacy_api.services.generic_size_counter import get_storage_usage +from selfprivacy_api.services.owned_path import OwnedPath +from selfprivacy_api.services.service import Service +from selfprivacy_api.utils import ReadUserData, WriteUserData, get_domain +from selfprivacy_api.services.config_item import ( + ServiceConfigItem, + StringServiceConfigItem, + BoolServiceConfigItem, + EnumServiceConfigItem, + IntServiceConfigItem, +) +from selfprivacy_api.utils.block_devices import BlockDevice, BlockDevices +from selfprivacy_api.utils.systemd import get_service_status_from_several_units + +SP_MODULES_DEFENITIONS_PATH = "/etc/sp-modules" +SP_SUGGESTED_MODULES_PATH = "/etc/suggested-sp-modules" + +logger = logging.getLogger(__name__) + + +def config_item_from_json(json_data: dict) -> Optional[ServiceConfigItem]: + """Create a ServiceConfigItem from JSON data.""" + weight = json_data.get("meta", {}).get("weight", 50) + if json_data["meta"]["type"] == "enable": + return None + if json_data["meta"]["type"] == "location": + return None + if json_data["meta"]["type"] == "string": + return StringServiceConfigItem( + id=json_data["name"], + default_value=json_data["default"], + description=json_data["description"], + regex=json_data["meta"].get("regex"), + widget=json_data["meta"].get("widget"), + allow_empty=json_data["meta"].get("allowEmpty", False), + weight=weight, + ) + if json_data["meta"]["type"] == "bool": + return BoolServiceConfigItem( + id=json_data["name"], + default_value=json_data["default"], + description=json_data["description"], + widget=json_data["meta"].get("widget"), + weight=weight, + ) + if json_data["meta"]["type"] == "enum": + return EnumServiceConfigItem( + id=json_data["name"], + default_value=json_data["default"], + description=json_data["description"], + options=json_data["meta"]["options"], + widget=json_data["meta"].get("widget"), + weight=weight, + ) + if json_data["meta"]["type"] == "int": + return IntServiceConfigItem( + id=json_data["name"], + default_value=json_data["default"], + description=json_data["description"], + widget=json_data["meta"].get("widget"), + min_value=json_data["meta"].get("minValue"), + max_value=json_data["meta"].get("maxValue"), + weight=weight, + ) + raise ValueError("Unknown config item type") + + +class TemplatedService(Service): + """Class representing a dynamically loaded service.""" + + def __init__(self, service_id: str, source_data: Optional[str] = None) -> None: + if source_data: + self.definition_data = json.loads(source_data) + else: + # Check if the service exists + if not exists(join(SP_MODULES_DEFENITIONS_PATH, service_id)): + raise FileNotFoundError(f"Service {service_id} not found") + # Load the service + with open(join(SP_MODULES_DEFENITIONS_PATH, service_id)) as file: + self.definition_data = json.load(file) + # Check if required fields are present + if "meta" not in self.definition_data: + raise ValueError("meta not found in service definition") + if "options" not in self.definition_data: + raise ValueError("options not found in service definition") + # Load the meta data + self.meta = ServiceMetaData(**self.definition_data["meta"]) + # Load the options + self.options = self.definition_data["options"] + # Load the config items + self.config_items = {} + for option in self.options.values(): + config_item = config_item_from_json(option) + if config_item: + self.config_items[config_item.id] = config_item + # If it is movable, check for the location option + if self.meta.is_movable and "location" not in self.options: + raise ValueError("Service is movable but does not have a location option") + # Load all subdomains via options with "subdomain" widget + self.subdomain_options: List[str] = [] + for option in self.options.values(): + if option.get("meta", {}).get("widget") == "subdomain": + self.subdomain_options.append(option["name"]) + + def get_id(self) -> str: + # Check if ID contains elements that might be a part of the path + if "/" in self.meta.id or "\\" in self.meta.id: + raise ValueError("Invalid ID") + return self.meta.id + + def get_display_name(self) -> str: + return self.meta.name + + def get_description(self) -> str: + return self.meta.description + + def get_svg_icon(self) -> str: + return base64.b64encode(self.meta.svg_icon.encode("utf-8")).decode("utf-8") + + def get_subdomain(self) -> Optional[str]: + # If there are no subdomain options, return None + if not self.subdomain_options: + return None + # If primary_subdomain is set, try to find it in the options + if ( + self.meta.primary_subdomain + and self.meta.primary_subdomain in self.subdomain_options + ): + option_name = self.meta.primary_subdomain + # Otherwise, use the first subdomain option + else: + option_name = self.subdomain_options[0] + + # Now, read the value from the userdata + name = self.get_id() + with ReadUserData() as user_data: + if "modules" in user_data: + if name in user_data["modules"]: + if option_name in user_data["modules"][name]: + return user_data["modules"][name][option_name] + # Otherwise, return default value for the option + return self.options[option_name].get("default") + + def get_subdomains(self) -> List[str]: + # Return a current subdomain for every subdomain option + subdomains = [] + with ReadUserData() as user_data: + for option in self.subdomain_options: + if "modules" in user_data: + if self.get_id() in user_data["modules"]: + if option in user_data["modules"][self.get_id()]: + subdomains.append( + user_data["modules"][self.get_id()][option] + ) + continue + subdomains.append(self.options[option]["default"]) + return subdomains + + def get_url(self) -> Optional[str]: + if not self.meta.showUrl: + return None + subdomain = self.get_subdomain() + if not subdomain: + return None + return f"https://{subdomain}.{get_domain()}" + + def get_user(self) -> Optional[str]: + if not self.meta.user: + return self.get_id() + return self.meta.user + + def get_group(self) -> Optional[str]: + if not self.meta.group: + return self.get_user() + return self.meta.group + + def is_movable(self) -> bool: + return self.meta.is_movable + + def is_required(self) -> bool: + return self.meta.is_required + + def can_be_backed_up(self) -> bool: + return self.meta.can_be_backed_up + + def get_backup_description(self) -> str: + return self.meta.backup_description + + def is_enabled(self) -> bool: + name = self.get_id() + with ReadUserData() as user_data: + return user_data.get("modules", {}).get(name, {}).get("enable", False) + + def is_installed(self) -> bool: + name = self.get_id() + with FlakeServiceManager() as service_manager: + return name in service_manager.services + + def get_license(self) -> List[License]: + return self.meta.license + + def get_homepage(self) -> Optional[str]: + return self.meta.homepage + + def get_source_page(self) -> Optional[str]: + return self.meta.source_page + + def get_support_level(self) -> SupportLevel: + return self.meta.support_level + + def get_status(self) -> ServiceStatus: + if not self.meta.systemd_services: + return ServiceStatus.INACTIVE + return get_service_status_from_several_units(self.meta.systemd_services) + + def _set_enable(self, enable: bool): + name = self.get_id() + with WriteUserData() as user_data: + if "modules" not in user_data: + user_data["modules"] = {} + if name not in user_data["modules"]: + user_data["modules"][name] = {} + user_data["modules"][name]["enable"] = enable + + def enable(self): + """Enable the service. Usually this means enabling systemd unit.""" + name = self.get_id() + if not self.is_installed(): + # First, double-check that it is a suggested module + if exists(SP_SUGGESTED_MODULES_PATH): + with open(SP_SUGGESTED_MODULES_PATH) as file: + suggested_modules = json.load(file) + if name not in suggested_modules: + raise ValueError("Service is not a suggested module") + else: + raise FileNotFoundError("Suggested modules file not found") + with FlakeServiceManager() as service_manager: + service_manager.services[name] = ( + f"git+https://git.selfprivacy.org/SelfPrivacy/selfprivacy-nixos-config.git?ref=flakes&dir=sp-modules/{name}" + ) + if "location" in self.options: + with WriteUserData() as user_data: + if "modules" not in user_data: + user_data["modules"] = {} + if name not in user_data["modules"]: + user_data["modules"][name] = {} + if "location" not in user_data["modules"][name]: + user_data["modules"][name]["location"] = ( + BlockDevices().get_root_block_device().name + ) + + self._set_enable(True) + + def disable(self): + """Disable the service. Usually this means disabling systemd unit.""" + self._set_enable(False) + + def start(self): + """Start the systemd units""" + for unit in self.meta.systemd_services: + subprocess.run(["systemctl", "start", unit], check=False) + + def stop(self): + """Stop the systemd units""" + for unit in self.meta.systemd_services: + subprocess.run(["systemctl", "stop", unit], check=False) + + def restart(self): + """Restart the systemd units""" + for unit in self.meta.systemd_services: + subprocess.run(["systemctl", "restart", unit], check=False) + + def get_configuration(self) -> dict: + # If there are no options, return an empty dict + if not self.config_items: + return {} + return { + key: self.config_items[key].as_dict(self.get_id()) + for key in self.config_items + } + + def set_configuration(self, config_items): + for key, value in config_items.items(): + if key not in self.config_items: + raise ValueError(f"Key {key} is not valid for {self.get_id()}") + if self.config_items[key].validate_value(value) is False: + raise ValueError(f"Value {value} is not valid for {key}") + for key, value in config_items.items(): + self.config_items[key].set_value( + value, + self.get_id(), + ) + + def get_storage_usage(self) -> int: + """ + Calculate the real storage usage of folders occupied by service + Calculate using pathlib. + Do not follow symlinks. + """ + storage_used = 0 + for folder in self.get_folders(): + storage_used += get_storage_usage(folder) + return storage_used + + def has_folders(self) -> int: + """ + If there are no folders on disk, moving is noop + """ + for folder in self.get_folders(): + if exists(folder): + return True + return False + + def get_dns_records(self, ip4: str, ip6: Optional[str]) -> List[ServiceDnsRecord]: + display_name = self.get_display_name() + subdomains = self.get_subdomains() + # Generate records for every subdomain + records: List[ServiceDnsRecord] = [] + for subdomain in subdomains: + if not subdomain: + continue + records.append( + ServiceDnsRecord( + type="A", + name=subdomain, + content=ip4, + ttl=3600, + display_name=display_name, + ) + ) + if ip6: + records.append( + ServiceDnsRecord( + type="AAAA", + name=subdomain, + content=ip6, + ttl=3600, + display_name=display_name, + ) + ) + return records + + def get_drive(self) -> str: + """ + Get the name of the drive/volume where the service is located. + Example values are `sda1`, `vda`, `sdb`. + """ + root_device: str = BlockDevices().get_root_block_device().name + if not self.is_movable(): + return root_device + with ReadUserData() as userdata: + if userdata.get("useBinds", False): + return ( + userdata.get("modules", {}) + .get(self.get_id(), {}) + .get( + "location", + root_device, + ) + ) + else: + return root_device + + def _get_db_dumps_folder(self) -> str: + # Get the drive where the service is located and append the folder name + return join("/var/lib/postgresql-dumps", self.get_id()) + + def get_folders(self) -> List[str]: + folders = self.meta.folders + owned_folders = self.meta.owned_folders + # Include the contents of folders list + resulting_folders = folders.copy() + for folder in owned_folders: + resulting_folders.append(folder.path) + return resulting_folders + + def get_owned_folders(self) -> List[OwnedPath]: + folders = self.meta.folders + owned_folders = self.meta.owned_folders + resulting_folders = owned_folders.copy() + for folder in folders: + resulting_folders.append(self.owned_path(folder)) + return resulting_folders + + def get_folders_to_back_up(self) -> List[str]: + resulting_folders = self.meta.folders.copy() + if self.get_postgresql_databases(): + resulting_folders.append(self._get_db_dumps_folder()) + return resulting_folders + + def set_location(self, volume: BlockDevice): + """ + Only changes userdata + """ + + service_id = self.get_id() + with WriteUserData() as user_data: + if "modules" not in user_data: + user_data["modules"] = {} + if service_id not in user_data["modules"]: + user_data["modules"][service_id] = {} + user_data["modules"][service_id]["location"] = volume.name + + def get_postgresql_databases(self) -> List[str]: + return self.meta.postgre_databases + + def owned_path(self, path: str): + """Default folder ownership""" + service_name = self.get_display_name() + + try: + owner = self.get_user() + if owner is None: + # TODO: assume root? + # (if we do not want to do assumptions, maybe not declare user optional?) + raise LookupError(f"no user for service: {service_name}") + group = self.get_group() + if group is None: + raise LookupError(f"no group for service: {service_name}") + except Exception as error: + raise LookupError( + f"when deciding a bind for folder {path} of service {service_name}, error: {str(error)}" + ) + + return OwnedPath( + path=path, + owner=owner, + group=group, + ) + + def pre_backup(self, job: Job): + if self.get_postgresql_databases(): + db_dumps_folder = self._get_db_dumps_folder() + # Create folder for the dumps if it does not exist + if not exists(db_dumps_folder): + mkdir(db_dumps_folder) + # Dump the databases + for db_name in self.get_postgresql_databases(): + Jobs.update( + job, + status_text=f"Creating a dump of database {db_name}", + status=JobStatus.RUNNING, + ) + db_dumper = PostgresDumper(db_name) + backup_file = join(db_dumps_folder, f"{db_name}.dump") + db_dumper.backup_database(backup_file) + + def _clear_db_dumps(self): + db_dumps_folder = self._get_db_dumps_folder() + for db_name in self.get_postgresql_databases(): + backup_file = join(db_dumps_folder, f"{db_name}.dump") + if exists(backup_file): + remove(backup_file) + unpacked_file = backup_file.replace(".gz", "") + if exists(unpacked_file): + remove(unpacked_file) + + def post_backup(self, job: Job): + if self.get_postgresql_databases(): + db_dumps_folder = self._get_db_dumps_folder() + # Remove the backup files + for db_name in self.get_postgresql_databases(): + backup_file = join(db_dumps_folder, f"{db_name}.dump") + if exists(backup_file): + remove(backup_file) + + def pre_restore(self, job: Job): + if self.get_postgresql_databases(): + # Create folder for the dumps if it does not exist + db_dumps_folder = self._get_db_dumps_folder() + if not exists(db_dumps_folder): + mkdir(db_dumps_folder) + # Remove existing dumps if they exist + self._clear_db_dumps() + + def post_restore(self, job: Job): + if self.get_postgresql_databases(): + # Recover the databases + db_dumps_folder = self._get_db_dumps_folder() + for db_name in self.get_postgresql_databases(): + if exists(join(db_dumps_folder, f"{db_name}.dump")): + Jobs.update( + job, + status_text=f"Restoring database {db_name}", + status=JobStatus.RUNNING, + ) + db_dumper = PostgresDumper(db_name) + backup_file = join(db_dumps_folder, f"{db_name}.dump") + db_dumper.restore_database(backup_file) + else: + logger.error(f"Database dump for {db_name} not found") + raise FileNotFoundError(f"Database dump for {db_name} not found") + # Remove the dumps + self._clear_db_dumps() diff --git a/selfprivacy_api/services/test_service/__init__.py b/selfprivacy_api/services/test_service/__init__.py index d5044b3..8f0e50d 100644 --- a/selfprivacy_api/services/test_service/__init__.py +++ b/selfprivacy_api/services/test_service/__init__.py @@ -95,7 +95,7 @@ class DummyService(Service): def get_status(cls) -> ServiceStatus: filepath = cls.status_file() if filepath in [None, ""]: - raise ValueError(f"We do not have a path for our test dummy status file!") + raise ValueError("We do not have a path for our test dummy status file!") if not path.exists(filepath): raise FileNotFoundError(filepath) @@ -195,10 +195,6 @@ class DummyService(Service): def set_configuration(cls, config_items): return super().set_configuration(config_items) - @staticmethod - def get_logs(): - return "" - @staticmethod def get_storage_usage() -> int: storage_usage = 0 diff --git a/selfprivacy_api/utils/cached_call.py b/selfprivacy_api/utils/cached_call.py new file mode 100644 index 0000000..14195fb --- /dev/null +++ b/selfprivacy_api/utils/cached_call.py @@ -0,0 +1,6 @@ +import time + + +def get_ttl_hash(seconds=3600): + """Return the same value withing `seconds` time period""" + return round(time.time() / seconds) diff --git a/selfprivacy_api/utils/postgres.py b/selfprivacy_api/utils/postgres.py new file mode 100644 index 0000000..3dd99c5 --- /dev/null +++ b/selfprivacy_api/utils/postgres.py @@ -0,0 +1,34 @@ +import subprocess + + +class PostgresDumper: + """--dbname=postgresql://postgres@%2Frun%2Fpostgresql/pleroma""" + + def __init__(self, db_name: str): + self.db_name = db_name + self.user = "postgres" + self.socket_dir = r"%2Frun%2Fpostgresql" + + def backup_database(self, backup_file: str): + # Create the database dump in custom format + dump_command = [ + "pg_dump", + f"--dbname=postgresql://{self.user}@{self.socket_dir}/{self.db_name}", + "--format=custom", + f"--file={backup_file}", + ] + + subprocess.run(dump_command, check=True) + + return backup_file + + def restore_database(self, backup_file: str): + restore_command = [ + "pg_restore", + f"--dbname=postgresql://{self.user}@{self.socket_dir}", + "--clean", + "--create", + "--exit-on-error", + backup_file, + ] + subprocess.run(restore_command, check=True) diff --git a/selfprivacy_api/utils/redis_pool.py b/selfprivacy_api/utils/redis_pool.py index 64f5758..bdad60e 100644 --- a/selfprivacy_api/utils/redis_pool.py +++ b/selfprivacy_api/utils/redis_pool.py @@ -29,6 +29,7 @@ class RedisPool: url, decode_responses=True, ) + self._raw_pool = redis.ConnectionPool.from_url(url) @staticmethod def connection_url(dbnumber: int) -> str: @@ -44,6 +45,12 @@ class RedisPool: """ return redis.Redis(connection_pool=self._pool) + def get_raw_connection(self): + """ + Get a raw connection from the pool. + """ + return redis.Redis(connection_pool=self._raw_pool) + def get_connection_async(self) -> redis_async.Redis: """ Get an async connection from the pool. diff --git a/setup.py b/setup.py index ecdb02c..8415ee3 100644 --- a/setup.py +++ b/setup.py @@ -2,7 +2,7 @@ from setuptools import setup, find_packages setup( name="selfprivacy_api", - version="3.4.0", + version="3.5.0", packages=find_packages(), scripts=[ "selfprivacy_api/app.py", diff --git a/tests/conftest.py b/tests/conftest.py index 275389c..a9766ed 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -258,7 +258,7 @@ def dummy_service( ensure_user_exists(user) # register our service - services.services.append(service) + services.DUMMY_SERVICES.append(service) huey.immediate = True assert huey.immediate is True @@ -269,8 +269,8 @@ def dummy_service( # Cleanup because apparently it matters wrt tasks # Some tests may remove it from the list intentionally, this is fine - if service in services.services: - services.services.remove(service) + if service in services.DUMMY_SERVICES: + services.DUMMY_SERVICES.remove(service) def prepare_nixos_rebuild_calls(fp, unit_name): diff --git a/tests/test_autobackup.py b/tests/test_autobackup.py index 08d61ca..98aca85 100644 --- a/tests/test_autobackup.py +++ b/tests/test_autobackup.py @@ -177,7 +177,9 @@ def test_services_to_autobackup(backups, dummy_service): Backups.set_autobackup_period_minutes(backup_period) services = Backups.services_to_back_up(now) - assert set(services) == set(backuppable_services()) + assert set(service.get_id() for service in services) == set( + service.get_id() for service in backuppable_services() + ) assert dummy_service.get_id() in [ service.get_id() for service in backuppable_services() ] diff --git a/tests/test_backup.py b/tests/test_backup.py index 16589ec..3e23109 100644 --- a/tests/test_backup.py +++ b/tests/test_backup.py @@ -250,6 +250,7 @@ def test_error_censoring_encryptionkey(dummy_service, backups): Backups.back_up(dummy_service) job = get_backup_fail(dummy_service) + assert job is not None assert_job_errored(job) job_text = all_job_text(job) @@ -287,6 +288,7 @@ def test_error_censoring_loginkey(dummy_service, backups, fp): Backups.back_up(dummy_service) job = get_backup_fail(dummy_service) + assert job is not None assert_job_errored(job) job_text = all_job_text(job) @@ -705,6 +707,7 @@ def test_provider_storage(backups): Storage.store_provider(test_provider) restored_provider_model = Storage.load_provider() + assert restored_provider_model is not None assert restored_provider_model.kind == "BACKBLAZE" assert restored_provider_model.login == test_login assert restored_provider_model.key == test_key diff --git a/tests/test_graphql/test_api_backup.py b/tests/test_graphql/test_api_backup.py index dbb76c2..1e1424e 100644 --- a/tests/test_graphql/test_api_backup.py +++ b/tests/test_graphql/test_api_backup.py @@ -434,7 +434,7 @@ def test_snapshots_orphaned_service(authorized_client, dummy_service, backups): snaps = api_snapshots(authorized_client) assert len(snaps) == 1 - all_services.services.remove(dummy_service) + all_services.DUMMY_SERVICES.remove(dummy_service) assert ServiceManager.get_service_by_id(dummy_service.get_id()) is None snaps = api_snapshots(authorized_client) diff --git a/tests/test_graphql/test_services.py b/tests/test_graphql/test_services.py index 8d0f0a4..e2754af 100644 --- a/tests/test_graphql/test_services.py +++ b/tests/test_graphql/test_services.py @@ -88,20 +88,26 @@ def dummy_service_with_binds(dummy_service, mock_lsblk_devices, volume_folders): @pytest.fixture() def only_dummy_service(dummy_service) -> Generator[DummyService, None, None]: # because queries to services that are not really there error out - back_copy = service_module.services.copy() - service_module.services.clear() - service_module.services.append(dummy_service) + service_module.TEST_FLAGS.clear() + service_module.TEST_FLAGS.append("ONLY_DUMMY_SERVICE") + service_module.DUMMY_SERVICES.clear() + service_module.DUMMY_SERVICES.append(dummy_service) yield dummy_service - service_module.services.clear() - service_module.services.extend(back_copy) + service_module.TEST_FLAGS.clear() + service_module.DUMMY_SERVICES.clear() @pytest.fixture def only_dummy_service_and_api( - only_dummy_service, generic_userdata, dkim_file + generic_userdata, dkim_file, dummy_service ) -> Generator[DummyService, None, None]: - service_module.services.append(ServiceManager()) - return only_dummy_service + service_module.TEST_FLAGS.clear() + service_module.TEST_FLAGS.append("DUMMY_SERVICE_AND_API") + service_module.DUMMY_SERVICES.clear() + service_module.DUMMY_SERVICES.append(dummy_service) + yield dummy_service + service_module.TEST_FLAGS.clear() + service_module.DUMMY_SERVICES.clear() @pytest.fixture() @@ -727,6 +733,7 @@ def test_graphql_move_service( def test_mailservice_cannot_enable_disable(authorized_client): mailservice = ServiceManager.get_service_by_id("simple-nixos-mailserver") + assert mailservice is not None mutation_response = api_enable(authorized_client, mailservice) data = get_data(mutation_response)["services"]["enableService"] diff --git a/tests/test_graphql/test_system.py b/tests/test_graphql/test_system.py index 3897033..dd5a286 100644 --- a/tests/test_graphql/test_system.py +++ b/tests/test_graphql/test_system.py @@ -337,30 +337,30 @@ def test_graphql_get_domain( assert is_dns_record_in_array( dns_records, dns_record(name="api", record_type="AAAA") ) - assert is_dns_record_in_array(dns_records, dns_record(name="cloud")) - assert is_dns_record_in_array( - dns_records, dns_record(name="cloud", record_type="AAAA") - ) - assert is_dns_record_in_array(dns_records, dns_record(name="git")) - assert is_dns_record_in_array( - dns_records, dns_record(name="git", record_type="AAAA") - ) - assert is_dns_record_in_array(dns_records, dns_record(name="meet")) - assert is_dns_record_in_array( - dns_records, dns_record(name="meet", record_type="AAAA") - ) - assert is_dns_record_in_array(dns_records, dns_record(name="password")) - assert is_dns_record_in_array( - dns_records, dns_record(name="password", record_type="AAAA") - ) - assert is_dns_record_in_array(dns_records, dns_record(name="social")) - assert is_dns_record_in_array( - dns_records, dns_record(name="social", record_type="AAAA") - ) - assert is_dns_record_in_array(dns_records, dns_record(name="vpn")) - assert is_dns_record_in_array( - dns_records, dns_record(name="vpn", record_type="AAAA") - ) + # assert is_dns_record_in_array(dns_records, dns_record(name="cloud")) + # assert is_dns_record_in_array( + # dns_records, dns_record(name="cloud", record_type="AAAA") + # ) + # assert is_dns_record_in_array(dns_records, dns_record(name="git")) + # assert is_dns_record_in_array( + # dns_records, dns_record(name="git", record_type="AAAA") + # ) + # assert is_dns_record_in_array(dns_records, dns_record(name="meet")) + # assert is_dns_record_in_array( + # dns_records, dns_record(name="meet", record_type="AAAA") + # ) + # assert is_dns_record_in_array(dns_records, dns_record(name="password")) + # assert is_dns_record_in_array( + # dns_records, dns_record(name="password", record_type="AAAA") + # ) + # assert is_dns_record_in_array(dns_records, dns_record(name="social")) + # assert is_dns_record_in_array( + # dns_records, dns_record(name="social", record_type="AAAA") + # ) + # assert is_dns_record_in_array(dns_records, dns_record(name="vpn")) + # assert is_dns_record_in_array( + # dns_records, dns_record(name="vpn", record_type="AAAA") + # ) assert is_dns_record_in_array( dns_records, dns_record( @@ -490,30 +490,30 @@ def test_graphql_get_domain_no_uri_account_file( assert is_dns_record_in_array( dns_records, dns_record(name="api", record_type="AAAA") ) - assert is_dns_record_in_array(dns_records, dns_record(name="cloud")) - assert is_dns_record_in_array( - dns_records, dns_record(name="cloud", record_type="AAAA") - ) - assert is_dns_record_in_array(dns_records, dns_record(name="git")) - assert is_dns_record_in_array( - dns_records, dns_record(name="git", record_type="AAAA") - ) - assert is_dns_record_in_array(dns_records, dns_record(name="meet")) - assert is_dns_record_in_array( - dns_records, dns_record(name="meet", record_type="AAAA") - ) - assert is_dns_record_in_array(dns_records, dns_record(name="password")) - assert is_dns_record_in_array( - dns_records, dns_record(name="password", record_type="AAAA") - ) - assert is_dns_record_in_array(dns_records, dns_record(name="social")) - assert is_dns_record_in_array( - dns_records, dns_record(name="social", record_type="AAAA") - ) - assert is_dns_record_in_array(dns_records, dns_record(name="vpn")) - assert is_dns_record_in_array( - dns_records, dns_record(name="vpn", record_type="AAAA") - ) + # assert is_dns_record_in_array(dns_records, dns_record(name="cloud")) + # assert is_dns_record_in_array( + # dns_records, dns_record(name="cloud", record_type="AAAA") + # ) + # assert is_dns_record_in_array(dns_records, dns_record(name="git")) + # assert is_dns_record_in_array( + # dns_records, dns_record(name="git", record_type="AAAA") + # ) + # assert is_dns_record_in_array(dns_records, dns_record(name="meet")) + # assert is_dns_record_in_array( + # dns_records, dns_record(name="meet", record_type="AAAA") + # ) + # assert is_dns_record_in_array(dns_records, dns_record(name="password")) + # assert is_dns_record_in_array( + # dns_records, dns_record(name="password", record_type="AAAA") + # ) + # assert is_dns_record_in_array(dns_records, dns_record(name="social")) + # assert is_dns_record_in_array( + # dns_records, dns_record(name="social", record_type="AAAA") + # ) + # assert is_dns_record_in_array(dns_records, dns_record(name="vpn")) + # assert is_dns_record_in_array( + # dns_records, dns_record(name="vpn", record_type="AAAA") + # ) assert is_dns_record_in_array( dns_records, dns_record( @@ -590,30 +590,30 @@ def test_graphql_get_domain_not_found_account_file( assert is_dns_record_in_array( dns_records, dns_record(name="api", record_type="AAAA") ) - assert is_dns_record_in_array(dns_records, dns_record(name="cloud")) - assert is_dns_record_in_array( - dns_records, dns_record(name="cloud", record_type="AAAA") - ) - assert is_dns_record_in_array(dns_records, dns_record(name="git")) - assert is_dns_record_in_array( - dns_records, dns_record(name="git", record_type="AAAA") - ) - assert is_dns_record_in_array(dns_records, dns_record(name="meet")) - assert is_dns_record_in_array( - dns_records, dns_record(name="meet", record_type="AAAA") - ) - assert is_dns_record_in_array(dns_records, dns_record(name="password")) - assert is_dns_record_in_array( - dns_records, dns_record(name="password", record_type="AAAA") - ) - assert is_dns_record_in_array(dns_records, dns_record(name="social")) - assert is_dns_record_in_array( - dns_records, dns_record(name="social", record_type="AAAA") - ) - assert is_dns_record_in_array(dns_records, dns_record(name="vpn")) - assert is_dns_record_in_array( - dns_records, dns_record(name="vpn", record_type="AAAA") - ) + # assert is_dns_record_in_array(dns_records, dns_record(name="cloud")) + # assert is_dns_record_in_array( + # dns_records, dns_record(name="cloud", record_type="AAAA") + # ) + # assert is_dns_record_in_array(dns_records, dns_record(name="git")) + # assert is_dns_record_in_array( + # dns_records, dns_record(name="git", record_type="AAAA") + # ) + # assert is_dns_record_in_array(dns_records, dns_record(name="meet")) + # assert is_dns_record_in_array( + # dns_records, dns_record(name="meet", record_type="AAAA") + # ) + # assert is_dns_record_in_array(dns_records, dns_record(name="password")) + # assert is_dns_record_in_array( + # dns_records, dns_record(name="password", record_type="AAAA") + # ) + # assert is_dns_record_in_array(dns_records, dns_record(name="social")) + # assert is_dns_record_in_array( + # dns_records, dns_record(name="social", record_type="AAAA") + # ) + # assert is_dns_record_in_array(dns_records, dns_record(name="vpn")) + # assert is_dns_record_in_array( + # dns_records, dns_record(name="vpn", record_type="AAAA") + # ) assert is_dns_record_in_array( dns_records, dns_record( @@ -690,30 +690,30 @@ def test_graphql_get_domain_black_account_file( assert is_dns_record_in_array( dns_records, dns_record(name="api", record_type="AAAA") ) - assert is_dns_record_in_array(dns_records, dns_record(name="cloud")) - assert is_dns_record_in_array( - dns_records, dns_record(name="cloud", record_type="AAAA") - ) - assert is_dns_record_in_array(dns_records, dns_record(name="git")) - assert is_dns_record_in_array( - dns_records, dns_record(name="git", record_type="AAAA") - ) - assert is_dns_record_in_array(dns_records, dns_record(name="meet")) - assert is_dns_record_in_array( - dns_records, dns_record(name="meet", record_type="AAAA") - ) - assert is_dns_record_in_array(dns_records, dns_record(name="password")) - assert is_dns_record_in_array( - dns_records, dns_record(name="password", record_type="AAAA") - ) - assert is_dns_record_in_array(dns_records, dns_record(name="social")) - assert is_dns_record_in_array( - dns_records, dns_record(name="social", record_type="AAAA") - ) - assert is_dns_record_in_array(dns_records, dns_record(name="vpn")) - assert is_dns_record_in_array( - dns_records, dns_record(name="vpn", record_type="AAAA") - ) + # assert is_dns_record_in_array(dns_records, dns_record(name="cloud")) + # assert is_dns_record_in_array( + # dns_records, dns_record(name="cloud", record_type="AAAA") + # ) + # assert is_dns_record_in_array(dns_records, dns_record(name="git")) + # assert is_dns_record_in_array( + # dns_records, dns_record(name="git", record_type="AAAA") + # ) + # assert is_dns_record_in_array(dns_records, dns_record(name="meet")) + # assert is_dns_record_in_array( + # dns_records, dns_record(name="meet", record_type="AAAA") + # ) + # assert is_dns_record_in_array(dns_records, dns_record(name="password")) + # assert is_dns_record_in_array( + # dns_records, dns_record(name="password", record_type="AAAA") + # ) + # assert is_dns_record_in_array(dns_records, dns_record(name="social")) + # assert is_dns_record_in_array( + # dns_records, dns_record(name="social", record_type="AAAA") + # ) + # assert is_dns_record_in_array(dns_records, dns_record(name="vpn")) + # assert is_dns_record_in_array( + # dns_records, dns_record(name="vpn", record_type="AAAA") + # ) assert is_dns_record_in_array( dns_records, dns_record( diff --git a/tests/test_services.py b/tests/test_services.py index 4c31743..6920ea0 100644 --- a/tests/test_services.py +++ b/tests/test_services.py @@ -10,8 +10,6 @@ from selfprivacy_api.utils.waitloop import wait_until_true import selfprivacy_api.services as services_module -from selfprivacy_api.services.bitwarden import Bitwarden -from selfprivacy_api.services.pleroma import Pleroma from selfprivacy_api.services.mailserver import MailServer from selfprivacy_api.services.owned_path import OwnedPath @@ -63,22 +61,22 @@ def test_delayed_start_stop(raw_dummy_service): assert dummy.get_status() == ServiceStatus.ACTIVE -def test_owned_folders_from_not_owned(): - assert Bitwarden.get_owned_folders() == [ - OwnedPath( - path=folder, - group="vaultwarden", - owner="vaultwarden", - ) - for folder in Bitwarden.get_folders() - ] +# def test_owned_folders_from_not_owned(): +# assert Bitwarden.get_owned_folders() == [ +# OwnedPath( +# path=folder, +# group="vaultwarden", +# owner="vaultwarden", +# ) +# for folder in Bitwarden.get_folders() +# ] -def test_paths_from_owned_paths(): - assert len(Pleroma.get_folders()) == 2 - assert Pleroma.get_folders() == [ - ownedpath.path for ownedpath in Pleroma.get_owned_folders() - ] +# def test_paths_from_owned_paths(): +# assert len(Pleroma.get_folders()) == 2 +# assert Pleroma.get_folders() == [ +# ownedpath.path for ownedpath in Pleroma.get_owned_folders() +# ] def test_enabling_disabling_reads_json(dummy_service: DummyService): @@ -163,5 +161,5 @@ def test_mailserver_with_no_dkim_returns_no_dns(no_dkim_file): assert MailServer().get_dns_records("157.90.247.192", "2a01:4f8:c17:7e3d::2") == [] -def test_services_enabled_by_default(generic_userdata): - assert set(ServiceManager.get_enabled_services()) == set(services_module.services) +# def test_services_enabled_by_default(generic_userdata): +# assert set(ServiceManager.get_enabled_services()) == set(services_module.services) diff --git a/tests/test_services_systemctl.py b/tests/test_services_systemctl.py index 43805e8..4dbb080 100644 --- a/tests/test_services_systemctl.py +++ b/tests/test_services_systemctl.py @@ -1,12 +1,7 @@ import pytest from selfprivacy_api.services.service import ServiceStatus -from selfprivacy_api.services.bitwarden import Bitwarden -from selfprivacy_api.services.forgejo import Forgejo from selfprivacy_api.services.mailserver import MailServer -from selfprivacy_api.services.nextcloud import Nextcloud -from selfprivacy_api.services.ocserv import Ocserv -from selfprivacy_api.services.pleroma import Pleroma def expected_status_call(service_name: str): @@ -74,21 +69,11 @@ def mock_popen_systemctl_service_not_ok(mocker): ############################################################################### -def test_systemctl_ok(mock_popen_systemctl_service_ok): - assert MailServer.get_status() == ServiceStatus.ACTIVE - assert Bitwarden.get_status() == ServiceStatus.ACTIVE - assert Forgejo.get_status() == ServiceStatus.ACTIVE - assert Nextcloud.get_status() == ServiceStatus.ACTIVE - assert Ocserv.get_status() == ServiceStatus.ACTIVE - assert Pleroma.get_status() == ServiceStatus.ACTIVE - call_args_asserts(mock_popen_systemctl_service_ok) +# def test_systemctl_ok(mock_popen_systemctl_service_ok): +# assert MailServer.get_status() == ServiceStatus.ACTIVE +# call_args_asserts(mock_popen_systemctl_service_ok) -def test_systemctl_failed_service(mock_popen_systemctl_service_not_ok): - assert MailServer.get_status() == ServiceStatus.FAILED - assert Bitwarden.get_status() == ServiceStatus.FAILED - assert Forgejo.get_status() == ServiceStatus.FAILED - assert Nextcloud.get_status() == ServiceStatus.FAILED - assert Ocserv.get_status() == ServiceStatus.FAILED - assert Pleroma.get_status() == ServiceStatus.FAILED - call_args_asserts(mock_popen_systemctl_service_not_ok) +# def test_systemctl_failed_service(mock_popen_systemctl_service_not_ok): +# assert MailServer.get_status() == ServiceStatus.FAILED +# call_args_asserts(mock_popen_systemctl_service_not_ok)