diff --git a/.drone.yml b/.drone.yml index a1bd384..fff99ae 100644 --- a/.drone.yml +++ b/.drone.yml @@ -7,6 +7,9 @@ steps: commands: - kill $(ps aux | grep 'redis-server 127.0.0.1:6389' | awk '{print $2}') || true - redis-server --bind 127.0.0.1 --port 6389 >/dev/null & + # We do not care about persistance on CI + - sleep 10 + - redis-cli -h 127.0.0.1 -p 6389 config set stop-writes-on-bgsave-error no - coverage run -m pytest -q - coverage xml - sonar-scanner -Dsonar.projectKey=SelfPrivacy-REST-API -Dsonar.sources=. -Dsonar.host.url=http://analyzer.lan:9000 -Dsonar.login="$SONARQUBE_TOKEN" diff --git a/.gitignore b/.gitignore index 7941396..7f93e02 100755 --- a/.gitignore +++ b/.gitignore @@ -147,3 +147,4 @@ cython_debug/ # End of https://www.toptal.com/developers/gitignore/api/flask *.db +*.rdb diff --git a/api.nix b/api.nix deleted file mode 100644 index 83bc695..0000000 --- a/api.nix +++ /dev/null @@ -1,64 +0,0 @@ -{ lib, python39Packages }: -with python39Packages; -buildPythonApplication { - pname = "selfprivacy-api"; - version = "2.0.0"; - - propagatedBuildInputs = [ - setuptools - portalocker - pytz - pytest - pytest-mock - pytest-datadir - huey - gevent - mnemonic - pydantic - typing-extensions - psutil - fastapi - uvicorn - (buildPythonPackage rec { - pname = "strawberry-graphql"; - version = "0.123.0"; - format = "pyproject"; - patches = [ - ./strawberry-graphql.patch - ]; - propagatedBuildInputs = [ - typing-extensions - python-multipart - python-dateutil - # flask - pydantic - pygments - poetry - # flask-cors - (buildPythonPackage rec { - pname = "graphql-core"; - version = "3.2.0"; - format = "setuptools"; - src = fetchPypi { - inherit pname version; - sha256 = "sha256-huKgvgCL/eGe94OI3opyWh2UKpGQykMcJKYIN5c4A84="; - }; - checkInputs = [ - pytest-asyncio - pytest-benchmark - pytestCheckHook - ]; - pythonImportsCheck = [ - "graphql" - ]; - }) - ]; - src = fetchPypi { - inherit pname version; - sha256 = "KsmZ5Xv8tUg6yBxieAEtvoKoRG60VS+iVGV0X6oCExo="; - }; - }) - ]; - - src = ./.; -} diff --git a/default.nix b/default.nix deleted file mode 100644 index 740c7ce..0000000 --- a/default.nix +++ /dev/null @@ -1,2 +0,0 @@ -{ pkgs ? import {} }: -pkgs.callPackage ./api.nix {} diff --git a/selfprivacy_api/backup/__init__.py b/selfprivacy_api/backup/__init__.py index 56150db..bbcebb7 100644 --- a/selfprivacy_api/backup/__init__.py +++ b/selfprivacy_api/backup/__init__.py @@ -1,12 +1,16 @@ from datetime import datetime, timedelta from operator import add -from os import statvfs, path, walk +from os import statvfs from typing import List, Optional from selfprivacy_api.utils import ReadUserData, WriteUserData from selfprivacy_api.services import get_service_by_id -from selfprivacy_api.services.service import Service, ServiceStatus, StoppedService +from selfprivacy_api.services.service import ( + Service, + ServiceStatus, + StoppedService, +) from selfprivacy_api.jobs import Jobs, JobStatus, Job @@ -41,16 +45,17 @@ class NotDeadError(AssertionError): def __str__(self): return f""" - Service {self.service_name} should be either stopped or dead from an error before we back up. - Normally, this error is unreachable because we do try ensure this. - Apparently, not this time. - """ + Service {self.service_name} should be either stopped or dead from + an error before we back up. + Normally, this error is unreachable because we do try ensure this. + Apparently, not this time. + """ class Backups: """A stateless controller class for backups""" - ### Providers + # Providers @staticmethod def provider(): @@ -172,7 +177,7 @@ class Backups: user_data["backup"] = DEFAULT_JSON_PROVIDER - ### Init + # Init @staticmethod def init_repo(): @@ -191,7 +196,7 @@ class Backups: return False - ### Backup + # Backup @staticmethod def back_up(service: Service): @@ -221,7 +226,8 @@ class Backups: Jobs.update(job, status=JobStatus.FINISHED) return snapshot - ### Restoring + # Restoring + @staticmethod def _ensure_queued_restore_job(service, snapshot) -> Job: job = get_restore_job(service) @@ -237,12 +243,17 @@ class Backups: Jobs.update(job, status=JobStatus.RUNNING) try: - Backups._restore_service_from_snapshot(service, snapshot.id, verify=False) + Backups._restore_service_from_snapshot( + service, + snapshot.id, + verify=False, + ) except Exception as e: Backups._restore_service_from_snapshot( service, failsafe_snapshot.id, verify=False ) raise e + # TODO: Do we really have to forget this snapshot? — Inex Backups.forget_snapshot(failsafe_snapshot) @staticmethod @@ -295,8 +306,9 @@ class Backups: else: raise NotImplementedError( """ - We do not know if there is enough space for restoration because there is some novel restore strategy used! - This is a developer's fault, open a issue please + We do not know if there is enough space for restoration because + there is some novel restore strategy used! + This is a developer's fault, open an issue please """ ) available_space = Backups.space_usable_for_service(service) @@ -307,15 +319,20 @@ class Backups: ) @staticmethod - def _restore_service_from_snapshot(service: Service, snapshot_id: str, verify=True): + def _restore_service_from_snapshot( + service: Service, + snapshot_id: str, + verify=True, + ): folders = service.get_folders() Backups.provider().backupper.restore_from_backup( snapshot_id, folders, + verify=verify, ) - ### Snapshots + # Snapshots @staticmethod def get_snapshots(service: Service) -> List[Snapshot]: @@ -377,7 +394,7 @@ class Backups: # expiring cache entry Storage.cache_snapshot(snapshot) - ### Autobackup + # Autobackup @staticmethod def is_autobackup_enabled(service: Service) -> bool: @@ -472,7 +489,7 @@ class Backups: ) ] - ### Helpers + # Helpers @staticmethod def space_usable_for_service(service: Service) -> int: @@ -500,6 +517,9 @@ class Backups: def assert_dead(service: Service): # if we backup the service that is failing to restore it to the # previous snapshot, its status can be FAILED - # And obviously restoring a failed service is the moun route - if service.get_status() not in [ServiceStatus.INACTIVE, ServiceStatus.FAILED]: + # And obviously restoring a failed service is the main route + if service.get_status() not in [ + ServiceStatus.INACTIVE, + ServiceStatus.FAILED, + ]: raise NotDeadError(service) diff --git a/selfprivacy_api/backup/backuppers/__init__.py b/selfprivacy_api/backup/backuppers/__init__.py index 24eb108..05adede 100644 --- a/selfprivacy_api/backup/backuppers/__init__.py +++ b/selfprivacy_api/backup/backuppers/__init__.py @@ -30,7 +30,12 @@ class AbstractBackupper(ABC): raise NotImplementedError @abstractmethod - def restore_from_backup(self, snapshot_id: str, folders: List[str], verify=True): + def restore_from_backup( + self, + snapshot_id: str, + folders: List[str], + verify=True, + ): """Restore a target folder using a snapshot""" raise NotImplementedError diff --git a/selfprivacy_api/backup/backuppers/restic_backupper.py b/selfprivacy_api/backup/backuppers/restic_backupper.py index 565a084..e04eaaf 100644 --- a/selfprivacy_api/backup/backuppers/restic_backupper.py +++ b/selfprivacy_api/backup/backuppers/restic_backupper.py @@ -81,7 +81,7 @@ class ResticBackupper(AbstractBackupper): mount_command.insert(0, "nohup") handle = subprocess.Popen(mount_command, stdout=subprocess.DEVNULL, shell=False) sleep(2) - if not "ids" in listdir(dir): + if "ids" not in listdir(dir): raise IOError("failed to mount dir ", dir) return handle @@ -211,7 +211,12 @@ class ResticBackupper(AbstractBackupper): except ValueError as e: raise ValueError("cannot restore a snapshot: " + output) from e - def restore_from_backup(self, snapshot_id, folders: List[str], verify=True): + def restore_from_backup( + self, + snapshot_id, + folders: List[str], + verify=True, + ): """ Restore from backup with restic """ @@ -236,6 +241,9 @@ class ResticBackupper(AbstractBackupper): dst = folder sync(src, dst) + if not verify: + self.unmount_repo(dir) + def do_restore(self, snapshot_id, target="/", verify=False): """barebones restic restore""" restore_command = self.restic_command( diff --git a/selfprivacy_api/dependencies.py b/selfprivacy_api/dependencies.py index 3284fd8..d7b12fe 100644 --- a/selfprivacy_api/dependencies.py +++ b/selfprivacy_api/dependencies.py @@ -27,4 +27,4 @@ async def get_token_header( def get_api_version() -> str: """Get API version""" - return "2.1.2" + return "2.1.3" diff --git a/selfprivacy_api/migrations/__init__.py b/selfprivacy_api/migrations/__init__.py index adb7d24..33472b9 100644 --- a/selfprivacy_api/migrations/__init__.py +++ b/selfprivacy_api/migrations/__init__.py @@ -22,6 +22,9 @@ from selfprivacy_api.migrations.providers import CreateProviderFields from selfprivacy_api.migrations.prepare_for_nixos_2211 import ( MigrateToSelfprivacyChannelFrom2205, ) +from selfprivacy_api.migrations.prepare_for_nixos_2305 import ( + MigrateToSelfprivacyChannelFrom2211, +) migrations = [ FixNixosConfigBranch(), @@ -31,6 +34,7 @@ migrations = [ CheckForFailedBindsMigration(), CreateProviderFields(), MigrateToSelfprivacyChannelFrom2205(), + MigrateToSelfprivacyChannelFrom2211(), ] diff --git a/selfprivacy_api/migrations/prepare_for_nixos_2305.py b/selfprivacy_api/migrations/prepare_for_nixos_2305.py new file mode 100644 index 0000000..d9fed28 --- /dev/null +++ b/selfprivacy_api/migrations/prepare_for_nixos_2305.py @@ -0,0 +1,58 @@ +import os +import subprocess + +from selfprivacy_api.migrations.migration import Migration + + +class MigrateToSelfprivacyChannelFrom2211(Migration): + """Migrate to selfprivacy Nix channel. + For some reason NixOS 22.11 servers initialized with the nixos channel instead of selfprivacy. + This stops us from upgrading to NixOS 23.05 + """ + + def get_migration_name(self): + return "migrate_to_selfprivacy_channel_from_2211" + + def get_migration_description(self): + return "Migrate to selfprivacy Nix channel from NixOS 22.11." + + def is_migration_needed(self): + try: + output = subprocess.check_output( + ["nix-channel", "--list"], start_new_session=True + ) + output = output.decode("utf-8") + first_line = output.split("\n", maxsplit=1)[0] + return first_line.startswith("nixos") and ( + first_line.endswith("nixos-22.11") + ) + except subprocess.CalledProcessError: + return False + + def migrate(self): + # Change the channel and update them. + # Also, go to /etc/nixos directory and make a git pull + current_working_directory = os.getcwd() + try: + print("Changing channel") + os.chdir("/etc/nixos") + subprocess.check_output( + [ + "nix-channel", + "--add", + "https://channel.selfprivacy.org/nixos-selfpricacy", + "nixos", + ] + ) + subprocess.check_output(["nix-channel", "--update"]) + nixos_config_branch = subprocess.check_output( + ["git", "rev-parse", "--abbrev-ref", "HEAD"], start_new_session=True + ) + if nixos_config_branch.decode("utf-8").strip() == "api-redis": + print("Also changing nixos-config branch from api-redis to master") + subprocess.check_output(["git", "checkout", "master"]) + subprocess.check_output(["git", "pull"]) + os.chdir(current_working_directory) + except subprocess.CalledProcessError: + os.chdir(current_working_directory) + print("Error") diff --git a/setup.py b/setup.py index 51606b6..d20bf9a 100755 --- a/setup.py +++ b/setup.py @@ -2,7 +2,7 @@ from setuptools import setup, find_packages setup( name="selfprivacy_api", - version="2.1.2", + version="2.1.3", packages=find_packages(), scripts=[ "selfprivacy_api/app.py",