diff --git a/.gitignore b/.gitignore index 7f93e02..bd62fff 100755 --- a/.gitignore +++ b/.gitignore @@ -148,3 +148,5 @@ cython_debug/ *.db *.rdb + +/result diff --git a/README.md b/README.md new file mode 100644 index 0000000..01ffd88 --- /dev/null +++ b/README.md @@ -0,0 +1,67 @@ +# SelfPrivacy GraphQL API which allows app to control your server + +## build + +```console +$ nix build +``` + +As a result, you should get the `./result` symlink to a folder (in `/nix/store`) with build contents. + +## develop & test + +```console +$ nix develop +$ [SP devshell] pytest . +=================================== test session starts ===================================== +platform linux -- Python 3.10.11, pytest-7.1.3, pluggy-1.0.0 +rootdir: /data/selfprivacy/selfprivacy-rest-api +plugins: anyio-3.5.0, datadir-1.4.1, mock-3.8.2 +collected 692 items + +tests/test_block_device_utils.py ................. [ 2%] +tests/test_common.py ..... [ 3%] +tests/test_jobs.py ........ [ 4%] +tests/test_model_storage.py .. [ 4%] +tests/test_models.py .. [ 4%] +tests/test_network_utils.py ...... [ 5%] +tests/test_services.py ...... [ 6%] +tests/test_graphql/test_api.py . [ 6%] +tests/test_graphql/test_api_backup.py ............... [ 8%] +tests/test_graphql/test_api_devices.py ................. [ 11%] +tests/test_graphql/test_api_recovery.py ......... [ 12%] +tests/test_graphql/test_api_version.py .. [ 13%] +tests/test_graphql/test_backup.py ............................... [ 21%] +tests/test_graphql/test_localsecret.py ... [ 22%] +tests/test_graphql/test_ssh.py ............ [ 23%] +tests/test_graphql/test_system.py ............................. [ 28%] +tests/test_graphql/test_system_nixos_tasks.py ........ [ 29%] +tests/test_graphql/test_users.py .................................. [ 42%] +tests/test_graphql/test_repository/test_json_tokens_repository.py [ 44%] +tests/test_graphql/test_repository/test_tokens_repository.py .... [ 53%] +tests/test_rest_endpoints/test_auth.py .......................... [ 58%] +tests/test_rest_endpoints/test_system.py ........................ [ 63%] +tests/test_rest_endpoints/test_users.py ................................ [ 76%] +tests/test_rest_endpoints/services/test_bitwarden.py ............ [ 78%] +tests/test_rest_endpoints/services/test_gitea.py .............. [ 80%] +tests/test_rest_endpoints/services/test_mailserver.py ..... [ 81%] +tests/test_rest_endpoints/services/test_nextcloud.py ............ [ 83%] +tests/test_rest_endpoints/services/test_ocserv.py .............. [ 85%] +tests/test_rest_endpoints/services/test_pleroma.py .............. [ 87%] +tests/test_rest_endpoints/services/test_services.py .... [ 88%] +tests/test_rest_endpoints/services/test_ssh.py ..................... [100%] + +============================== 692 passed in 352.76s (0:05:52) =============================== +``` + +## dependencies and dependant modules + +Current flake inherits nixpkgs from NixOS configuration flake. So there is no need to refer to extra nixpkgs dependency if you want to be aligned with exact NixOS configuration. + +![diagram](http://www.plantuml.com/plantuml/proxy?src=https://git.selfprivacy.org/SelfPrivacy/selfprivacy-rest-api/raw/branch/master/nix-dependencies-diagram.puml) + +Nix code for NixOS service module for API is located in NixOS configuration repository. + +## current issues + +- It's not clear how to store in this repository information about several compatible NixOS configuration commits, where API application tests pass. Currently, here is only a single `flake.lock`. diff --git a/default.nix b/default.nix new file mode 100644 index 0000000..1c779d9 --- /dev/null +++ b/default.nix @@ -0,0 +1,33 @@ +{ pythonPackages, rev ? "local" }: + +pythonPackages.buildPythonPackage rec { + pname = "selfprivacy-graphql-api"; + version = rev; + src = builtins.filterSource (p: t: p != ".git" && t != "symlink") ./.; + nativeCheckInputs = [ pythonPackages.pytestCheckHook ]; + propagatedBuildInputs = with pythonPackages; [ + fastapi + gevent + huey + mnemonic + portalocker + psutil + pydantic + pytest + pytest-datadir + pytest-mock + pytz + redis + setuptools + strawberry-graphql + typing-extensions + uvicorn + ]; + pythonImportsCheck = [ "selfprivacy_api" ]; + doCheck = false; + meta = { + description = '' + SelfPrivacy Server Management API + ''; + }; +} diff --git a/flake.lock b/flake.lock new file mode 100644 index 0000000..3dd8a15 --- /dev/null +++ b/flake.lock @@ -0,0 +1,26 @@ +{ + "nodes": { + "nixpkgs": { + "locked": { + "lastModified": 1702780907, + "narHash": "sha256-blbrBBXjjZt6OKTcYX1jpe9SRof2P9ZYWPzq22tzXAA=", + "owner": "nixos", + "repo": "nixpkgs", + "rev": "1e2e384c5b7c50dbf8e9c441a9e58d85f408b01f", + "type": "github" + }, + "original": { + "owner": "nixos", + "repo": "nixpkgs", + "type": "github" + } + }, + "root": { + "inputs": { + "nixpkgs": "nixpkgs" + } + } + }, + "root": "root", + "version": 7 +} diff --git a/flake.nix b/flake.nix new file mode 100644 index 0000000..c133604 --- /dev/null +++ b/flake.nix @@ -0,0 +1,50 @@ +{ + description = "SelfPrivacy API flake"; + + inputs.nixpkgs.url = "github:nixos/nixpkgs"; + + outputs = { self, nixpkgs, ... }: + let + system = "x86_64-linux"; + pkgs = nixpkgs.legacyPackages.${system}; + selfprivacy-graphql-api = pkgs.callPackage ./default.nix { + pythonPackages = pkgs.python310Packages; + rev = self.shortRev or self.dirtyShortRev or "dirty"; + }; + in + { + packages.${system}.default = selfprivacy-graphql-api; + nixosModules.default = + import ./nixos/module.nix self.packages.${system}.default; + devShells.${system}.default = pkgs.mkShell { + packages = + let + # TODO is there a better way to get environment for VS Code? + python3 = + nixpkgs.lib.findFirst (p: p.pname == "python3") (abort "wtf") + self.packages.${system}.default.propagatedBuildInputs; + python-env = + python3.withPackages + (_: self.packages.${system}.default.propagatedBuildInputs); + in + with pkgs; [ + python-env + black + rclone + redis + restic + ]; + shellHook = '' + # envs set with export and as attributes are treated differently. + # for example. printenv will not fetch the value of an attribute. + export USE_REDIS_PORT=6379 + export TEST_MODE=true + pkill redis-server + sleep 2 + setsid redis-server --bind 127.0.0.1 --port $USE_REDIS_PORT >/dev/null 2>/dev/null & + # maybe set more env-vars + ''; + }; + }; + nixConfig.bash-prompt = ''\n\[\e[1;32m\][\[\e[0m\]\[\e[1;34m\]SP devshell\[\e[0m\]\[\e[1;32m\]:\w]\$\[\[\e[0m\] ''; +} diff --git a/nix-dependencies-diagram.puml b/nix-dependencies-diagram.puml new file mode 100644 index 0000000..de98bf7 --- /dev/null +++ b/nix-dependencies-diagram.puml @@ -0,0 +1,22 @@ +@startuml + +left to right direction + +title repositories and flake inputs relations diagram + +cloud nixpkgs as nixpkgs_transit +control "nixos-rebuild" as nixos_rebuild +component "SelfPrivacy\nAPI app" as selfprivacy_app +component "SelfPrivacy\nNixOS configuration" as nixos_configuration + +note top of nixos_configuration : SelfPrivacy\nAPI service module + +nixos_configuration ).. nixpkgs_transit +nixpkgs_transit ..> selfprivacy_app +selfprivacy_app --> nixos_configuration +[nixpkgs] --> nixos_configuration +nixos_configuration -> nixos_rebuild + +footer %date("yyyy-MM-dd'T'HH:mmZ") + +@enduml diff --git a/nixos/module.nix b/nixos/module.nix new file mode 100644 index 0000000..7790e18 --- /dev/null +++ b/nixos/module.nix @@ -0,0 +1,166 @@ +selfprivacy-graphql-api: { config, lib, pkgs, ... }: + +let + cfg = config.services.selfprivacy-api; + config-id = "default"; + nixos-rebuild = "${config.system.build.nixos-rebuild}/bin/nixos-rebuild"; + nix = "${config.nix.package.out}/bin/nix"; +in +{ + options.services.selfprivacy-api = { + enable = lib.mkOption { + default = true; + type = lib.types.bool; + description = '' + Enable SelfPrivacy API service + ''; + }; + }; + config = lib.mkIf cfg.enable { + users.users."selfprivacy-api" = { + isNormalUser = false; + isSystemUser = true; + extraGroups = [ "opendkim" ]; + group = "selfprivacy-api"; + }; + users.groups."selfprivacy-api".members = [ "selfprivacy-api" ]; + + systemd.services.selfprivacy-api = { + description = "API Server used to control system from the mobile application"; + environment = config.nix.envVars // { + HOME = "/root"; + PYTHONUNBUFFERED = "1"; + } // config.networking.proxy.envVars; + path = [ + "/var/" + "/var/dkim/" + pkgs.coreutils + pkgs.gnutar + pkgs.xz.bin + pkgs.gzip + pkgs.gitMinimal + config.nix.package.out + pkgs.restic + pkgs.mkpasswd + pkgs.util-linux + pkgs.e2fsprogs + pkgs.iproute2 + ]; + after = [ "network-online.target" ]; + wantedBy = [ "network-online.target" ]; + serviceConfig = { + User = "root"; + ExecStart = "${selfprivacy-graphql-api}/bin/app.py"; + Restart = "always"; + RestartSec = "5"; + }; + }; + systemd.services.selfprivacy-api-worker = { + description = "Task worker for SelfPrivacy API"; + environment = config.nix.envVars // { + HOME = "/root"; + PYTHONUNBUFFERED = "1"; + PYTHONPATH = + pkgs.python310Packages.makePythonPath [ selfprivacy-graphql-api ]; + } // config.networking.proxy.envVars; + path = [ + "/var/" + "/var/dkim/" + pkgs.coreutils + pkgs.gnutar + pkgs.xz.bin + pkgs.gzip + pkgs.gitMinimal + config.nix.package.out + pkgs.restic + pkgs.mkpasswd + pkgs.util-linux + pkgs.e2fsprogs + pkgs.iproute2 + ]; + after = [ "network-online.target" ]; + wantedBy = [ "network-online.target" ]; + serviceConfig = { + User = "root"; + ExecStart = "${pkgs.python310Packages.huey}/bin/huey_consumer.py selfprivacy_api.task_registry.huey"; + Restart = "always"; + RestartSec = "5"; + }; + }; + # One shot systemd service to rebuild NixOS using nixos-rebuild + systemd.services.sp-nixos-rebuild = { + description = "nixos-rebuild switch"; + environment = config.nix.envVars // { + HOME = "/root"; + } // config.networking.proxy.envVars; + # TODO figure out how to get dependencies list reliably + path = [ pkgs.coreutils pkgs.gnutar pkgs.xz.bin pkgs.gzip pkgs.gitMinimal config.nix.package.out ]; + # TODO set proper timeout for reboot instead of service restart + serviceConfig = { + User = "root"; + WorkingDirectory = "/etc/nixos"; + # sync top-level flake with sp-modules sub-flake + # (https://github.com/NixOS/nix/issues/9339) + ExecStartPre = '' + ${nix} flake lock --override-input sp-modules path:./sp-modules + ''; + ExecStart = '' + ${nixos-rebuild} switch --flake .#${config-id} + ''; + KillMode = "none"; + SendSIGKILL = "no"; + }; + restartIfChanged = false; + unitConfig.X-StopOnRemoval = false; + }; + # One shot systemd service to upgrade NixOS using nixos-rebuild + systemd.services.sp-nixos-upgrade = { + # protection against simultaneous runs + after = [ "sp-nixos-rebuild.service" ]; + description = "Upgrade NixOS and SP modules to latest versions"; + environment = config.nix.envVars // { + HOME = "/root"; + } // config.networking.proxy.envVars; + # TODO figure out how to get dependencies list reliably + path = [ pkgs.coreutils pkgs.gnutar pkgs.xz.bin pkgs.gzip pkgs.gitMinimal config.nix.package.out ]; + serviceConfig = { + User = "root"; + WorkingDirectory = "/etc/nixos"; + # TODO get URL from systemd template parameter? + ExecStartPre = '' + ${nix} flake update \ + --override-input selfprivacy-nixos-config git+https://git.selfprivacy.org/SelfPrivacy/selfprivacy-nixos-config.git?ref=flakes + ''; + ExecStart = '' + ${nixos-rebuild} switch --flake .#${config-id} + ''; + KillMode = "none"; + SendSIGKILL = "no"; + }; + restartIfChanged = false; + unitConfig.X-StopOnRemoval = false; + }; + # One shot systemd service to rollback NixOS using nixos-rebuild + systemd.services.sp-nixos-rollback = { + # protection against simultaneous runs + after = [ "sp-nixos-rebuild.service" "sp-nixos-upgrade.service" ]; + description = "Rollback NixOS using nixos-rebuild"; + environment = config.nix.envVars // { + HOME = "/root"; + } // config.networking.proxy.envVars; + # TODO figure out how to get dependencies list reliably + path = [ pkgs.coreutils pkgs.gnutar pkgs.xz.bin pkgs.gzip pkgs.gitMinimal config.nix.package.out ]; + serviceConfig = { + User = "root"; + WorkingDirectory = "/etc/nixos"; + ExecStart = '' + ${nixos-rebuild} switch --rollback --flake .#${config-id} + ''; + KillMode = "none"; + SendSIGKILL = "no"; + }; + restartIfChanged = false; + unitConfig.X-StopOnRemoval = false; + }; + }; +} diff --git a/shell.nix b/shell.nix deleted file mode 100644 index bce16bd..0000000 --- a/shell.nix +++ /dev/null @@ -1,48 +0,0 @@ -{ pkgs ? import { } }: -let - sp-python = pkgs.python310.withPackages (p: with p; [ - setuptools - portalocker - pytz - pytest - pytest-mock - pytest-datadir - huey - gevent - mnemonic - coverage - pylint - rope - mypy - pylsp-mypy - pydantic - typing-extensions - psutil - black - fastapi - uvicorn - redis - strawberry-graphql - flake8-bugbear - flake8 - ]); -in -pkgs.mkShell { - buildInputs = [ - sp-python - pkgs.black - pkgs.redis - pkgs.restic - pkgs.rclone - ]; - shellHook = '' - PYTHONPATH=${sp-python}/${sp-python.sitePackages} - # envs set with export and as attributes are treated differently. - # for example. printenv will not fetch the value of an attribute. - export USE_REDIS_PORT=6379 - pkill redis-server - sleep 2 - setsid redis-server --bind 127.0.0.1 --port $USE_REDIS_PORT >/dev/null 2>/dev/null & - # maybe set more env-vars - ''; -}