refactor: Move from nix-shell to nix flake

2024-11-17 08:02:36 +00:00 · 2024-01-09 20:20:58 +03:00 · 2024-01-09 20:20:58 +03:00 · 1e9744227b
parent 6b4920a0e7
commit 1e9744227b
8 changed files with 366 additions and 48 deletions
--- a/.gitignore
+++ b/.gitignore
@ -148,3 +148,5 @@ cython_debug/

 *.db
 *.rdb
+
+/result
--- a/README.md
+++ b/README.md
@ -0,0 +1,67 @@
+# SelfPrivacy GraphQL API which allows app to control your server
+
+## build
+
+```console
+$ nix build
+```
+
+As a result, you should get the `./result` symlink to a folder (in `/nix/store`) with build contents.
+
+## develop & test
+
+```console
+$ nix develop
+$ [SP devshell] pytest .
+=================================== test session starts =====================================
+platform linux -- Python 3.10.11, pytest-7.1.3, pluggy-1.0.0
+rootdir: /data/selfprivacy/selfprivacy-rest-api
+plugins: anyio-3.5.0, datadir-1.4.1, mock-3.8.2
+collected 692 items
+
+tests/test_block_device_utils.py .................                                    [  2%]
+tests/test_common.py .....                                                            [  3%]
+tests/test_jobs.py ........                                                           [  4%]
+tests/test_model_storage.py ..                                                        [  4%]
+tests/test_models.py ..                                                               [  4%]
+tests/test_network_utils.py ......                                                    [  5%]
+tests/test_services.py ......                                                         [  6%]
+tests/test_graphql/test_api.py .                                                      [  6%]
+tests/test_graphql/test_api_backup.py ...............                                 [  8%]
+tests/test_graphql/test_api_devices.py .................                              [ 11%]
+tests/test_graphql/test_api_recovery.py .........                                     [ 12%]
+tests/test_graphql/test_api_version.py ..                                             [ 13%]
+tests/test_graphql/test_backup.py ...............................                     [ 21%]
+tests/test_graphql/test_localsecret.py ...                                            [ 22%]
+tests/test_graphql/test_ssh.py ............                                           [ 23%]
+tests/test_graphql/test_system.py .............................                       [ 28%]
+tests/test_graphql/test_system_nixos_tasks.py ........                                [ 29%]
+tests/test_graphql/test_users.py ..................................                   [ 42%]
+tests/test_graphql/test_repository/test_json_tokens_repository.py                     [ 44%]
+tests/test_graphql/test_repository/test_tokens_repository.py ....                     [ 53%]
+tests/test_rest_endpoints/test_auth.py ..........................                     [ 58%]
+tests/test_rest_endpoints/test_system.py ........................                     [ 63%]
+tests/test_rest_endpoints/test_users.py ................................              [ 76%]
+tests/test_rest_endpoints/services/test_bitwarden.py ............                     [ 78%]
+tests/test_rest_endpoints/services/test_gitea.py ..............                       [ 80%]
+tests/test_rest_endpoints/services/test_mailserver.py .....                           [ 81%]
+tests/test_rest_endpoints/services/test_nextcloud.py ............                     [ 83%]
+tests/test_rest_endpoints/services/test_ocserv.py ..............                      [ 85%]
+tests/test_rest_endpoints/services/test_pleroma.py ..............                     [ 87%]
+tests/test_rest_endpoints/services/test_services.py ....                              [ 88%]
+tests/test_rest_endpoints/services/test_ssh.py .....................                  [100%]
+
+============================== 692 passed in 352.76s (0:05:52) ===============================
+```
+
+## dependencies and dependant modules
+
+Current flake inherits nixpkgs from NixOS configuration flake. So there is no need to refer to extra nixpkgs dependency if you want to be aligned with exact NixOS configuration.
+
+![diagram](http://www.plantuml.com/plantuml/proxy?src=https://git.selfprivacy.org/SelfPrivacy/selfprivacy-rest-api/raw/branch/master/nix-dependencies-diagram.puml)
+
+Nix code for NixOS service module for API is located in NixOS configuration repository.
+
+## current issues
+
+- It's not clear how to store in this repository information about several compatible NixOS configuration commits, where API application tests pass. Currently, here is only a single `flake.lock`.
--- a/default.nix
+++ b/default.nix
@ -0,0 +1,33 @@
+{ pythonPackages, rev ? "local" }:
+
+pythonPackages.buildPythonPackage rec {
+  pname = "selfprivacy-graphql-api";
+  version = rev;
+  src = builtins.filterSource (p: t: p != ".git" && t != "symlink") ./.;
+  nativeCheckInputs = [ pythonPackages.pytestCheckHook ];
+  propagatedBuildInputs = with pythonPackages; [
+    fastapi
+    gevent
+    huey
+    mnemonic
+    portalocker
+    psutil
+    pydantic
+    pytest
+    pytest-datadir
+    pytest-mock
+    pytz
+    redis
+    setuptools
+    strawberry-graphql
+    typing-extensions
+    uvicorn
+  ];
+  pythonImportsCheck = [ "selfprivacy_api" ];
+  doCheck = false;
+  meta = {
+    description = ''
+      SelfPrivacy Server Management API
+    '';
+  };
+}
--- a/flake.lock
+++ b/flake.lock
@ -0,0 +1,26 @@
+{
+  "nodes": {
+    "nixpkgs": {
+      "locked": {
+        "lastModified": 1702780907,
+        "narHash": "sha256-blbrBBXjjZt6OKTcYX1jpe9SRof2P9ZYWPzq22tzXAA=",
+        "owner": "nixos",
+        "repo": "nixpkgs",
+        "rev": "1e2e384c5b7c50dbf8e9c441a9e58d85f408b01f",
+        "type": "github"
+      },
+      "original": {
+        "owner": "nixos",
+        "repo": "nixpkgs",
+        "type": "github"
+      }
+    },
+    "root": {
+      "inputs": {
+        "nixpkgs": "nixpkgs"
+      }
+    }
+  },
+  "root": "root",
+  "version": 7
+}
--- a/flake.nix
+++ b/flake.nix
@ -0,0 +1,50 @@
+{
+  description = "SelfPrivacy API flake";
+
+  inputs.nixpkgs.url = "github:nixos/nixpkgs";
+
+  outputs = { self, nixpkgs, ... }:
+    let
+      system = "x86_64-linux";
+      pkgs = nixpkgs.legacyPackages.${system};
+      selfprivacy-graphql-api = pkgs.callPackage ./default.nix {
+        pythonPackages = pkgs.python310Packages;
+        rev = self.shortRev or self.dirtyShortRev or "dirty";
+      };
+    in
+    {
+      packages.${system}.default = selfprivacy-graphql-api;
+      nixosModules.default =
+        import ./nixos/module.nix self.packages.${system}.default;
+      devShells.${system}.default = pkgs.mkShell {
+        packages =
+          let
+            # TODO is there a better way to get environment for VS Code?
+            python3 =
+              nixpkgs.lib.findFirst (p: p.pname == "python3") (abort "wtf")
+                self.packages.${system}.default.propagatedBuildInputs;
+            python-env =
+              python3.withPackages
+                (_: self.packages.${system}.default.propagatedBuildInputs);
+          in
+          with pkgs; [
+            python-env
+            black
+            rclone
+            redis
+            restic
+          ];
+        shellHook = ''
+          # envs set with export and as attributes are treated differently.
+          # for example. printenv <Name> will not fetch the value of an attribute.
+          export USE_REDIS_PORT=6379
+          export TEST_MODE=true
+          pkill redis-server
+          sleep 2
+          setsid redis-server --bind 127.0.0.1 --port $USE_REDIS_PORT >/dev/null 2>/dev/null &
+          # maybe set more env-vars
+        '';
+      };
+    };
+  nixConfig.bash-prompt = ''\n\[\e[1;32m\][\[\e[0m\]\[\e[1;34m\]SP devshell\[\e[0m\]\[\e[1;32m\]:\w]\$\[\[\e[0m\] '';
+}
--- a/nix-dependencies-diagram.puml
+++ b/nix-dependencies-diagram.puml
@ -0,0 +1,22 @@
+@startuml
+
+left to right direction
+
+title repositories and flake inputs relations diagram
+
+cloud nixpkgs as nixpkgs_transit
+control "<font:monospaced><size:15>nixos-rebuild" as nixos_rebuild
+component "SelfPrivacy\nAPI app" as selfprivacy_app
+component "SelfPrivacy\nNixOS configuration" as nixos_configuration
+
+note top of nixos_configuration : SelfPrivacy\nAPI service module
+
+nixos_configuration ).. nixpkgs_transit
+nixpkgs_transit ..> selfprivacy_app
+selfprivacy_app --> nixos_configuration
+[nixpkgs] --> nixos_configuration
+nixos_configuration -> nixos_rebuild
+
+footer %date("yyyy-MM-dd'T'HH:mmZ")
+
+@enduml
--- a/nixos/module.nix
+++ b/nixos/module.nix
@ -0,0 +1,166 @@
+selfprivacy-graphql-api: { config, lib, pkgs, ... }:
+
+let
+  cfg = config.services.selfprivacy-api;
+  config-id = "default";
+  nixos-rebuild = "${config.system.build.nixos-rebuild}/bin/nixos-rebuild";
+  nix = "${config.nix.package.out}/bin/nix";
+in
+{
+  options.services.selfprivacy-api = {
+    enable = lib.mkOption {
+      default = true;
+      type = lib.types.bool;
+      description = ''
+        Enable SelfPrivacy API service
+      '';
+    };
+  };
+  config = lib.mkIf cfg.enable {
+    users.users."selfprivacy-api" = {
+      isNormalUser = false;
+      isSystemUser = true;
+      extraGroups = [ "opendkim" ];
+      group = "selfprivacy-api";
+    };
+    users.groups."selfprivacy-api".members = [ "selfprivacy-api" ];
+
+    systemd.services.selfprivacy-api = {
+      description = "API Server used to control system from the mobile application";
+      environment = config.nix.envVars // {
+        HOME = "/root";
+        PYTHONUNBUFFERED = "1";
+      } // config.networking.proxy.envVars;
+      path = [
+        "/var/"
+        "/var/dkim/"
+        pkgs.coreutils
+        pkgs.gnutar
+        pkgs.xz.bin
+        pkgs.gzip
+        pkgs.gitMinimal
+        config.nix.package.out
+        pkgs.restic
+        pkgs.mkpasswd
+        pkgs.util-linux
+        pkgs.e2fsprogs
+        pkgs.iproute2
+      ];
+      after = [ "network-online.target" ];
+      wantedBy = [ "network-online.target" ];
+      serviceConfig = {
+        User = "root";
+        ExecStart = "${selfprivacy-graphql-api}/bin/app.py";
+        Restart = "always";
+        RestartSec = "5";
+      };
+    };
+    systemd.services.selfprivacy-api-worker = {
+      description = "Task worker for SelfPrivacy API";
+      environment = config.nix.envVars // {
+        HOME = "/root";
+        PYTHONUNBUFFERED = "1";
+        PYTHONPATH =
+          pkgs.python310Packages.makePythonPath [ selfprivacy-graphql-api ];
+      } // config.networking.proxy.envVars;
+      path = [
+        "/var/"
+        "/var/dkim/"
+        pkgs.coreutils
+        pkgs.gnutar
+        pkgs.xz.bin
+        pkgs.gzip
+        pkgs.gitMinimal
+        config.nix.package.out
+        pkgs.restic
+        pkgs.mkpasswd
+        pkgs.util-linux
+        pkgs.e2fsprogs
+        pkgs.iproute2
+      ];
+      after = [ "network-online.target" ];
+      wantedBy = [ "network-online.target" ];
+      serviceConfig = {
+        User = "root";
+        ExecStart = "${pkgs.python310Packages.huey}/bin/huey_consumer.py selfprivacy_api.task_registry.huey";
+        Restart = "always";
+        RestartSec = "5";
+      };
+    };
+    # One shot systemd service to rebuild NixOS using nixos-rebuild
+    systemd.services.sp-nixos-rebuild = {
+      description = "nixos-rebuild switch";
+      environment = config.nix.envVars // {
+        HOME = "/root";
+      } // config.networking.proxy.envVars;
+      # TODO figure out how to get dependencies list reliably
+      path = [ pkgs.coreutils pkgs.gnutar pkgs.xz.bin pkgs.gzip pkgs.gitMinimal config.nix.package.out ];
+      # TODO set proper timeout for reboot instead of service restart
+      serviceConfig = {
+        User = "root";
+        WorkingDirectory = "/etc/nixos";
+        # sync top-level flake with sp-modules sub-flake
+        # (https://github.com/NixOS/nix/issues/9339)
+        ExecStartPre = ''
+          ${nix} flake lock --override-input sp-modules path:./sp-modules
+        '';
+        ExecStart = ''
+          ${nixos-rebuild} switch --flake .#${config-id}
+        '';
+        KillMode = "none";
+        SendSIGKILL = "no";
+      };
+      restartIfChanged = false;
+      unitConfig.X-StopOnRemoval = false;
+    };
+    # One shot systemd service to upgrade NixOS using nixos-rebuild
+    systemd.services.sp-nixos-upgrade = {
+      # protection against simultaneous runs
+      after = [ "sp-nixos-rebuild.service" ];
+      description = "Upgrade NixOS and SP modules to latest versions";
+      environment = config.nix.envVars // {
+        HOME = "/root";
+      } // config.networking.proxy.envVars;
+      # TODO figure out how to get dependencies list reliably
+      path = [ pkgs.coreutils pkgs.gnutar pkgs.xz.bin pkgs.gzip pkgs.gitMinimal config.nix.package.out ];
+      serviceConfig = {
+        User = "root";
+        WorkingDirectory = "/etc/nixos";
+        # TODO get URL from systemd template parameter?
+        ExecStartPre = ''
+          ${nix} flake update \
+          --override-input selfprivacy-nixos-config git+https://git.selfprivacy.org/SelfPrivacy/selfprivacy-nixos-config.git?ref=flakes
+        '';
+        ExecStart = ''
+          ${nixos-rebuild} switch --flake .#${config-id}
+        '';
+        KillMode = "none";
+        SendSIGKILL = "no";
+      };
+      restartIfChanged = false;
+      unitConfig.X-StopOnRemoval = false;
+    };
+    # One shot systemd service to rollback NixOS using nixos-rebuild
+    systemd.services.sp-nixos-rollback = {
+      # protection against simultaneous runs
+      after = [ "sp-nixos-rebuild.service" "sp-nixos-upgrade.service" ];
+      description = "Rollback NixOS using nixos-rebuild";
+      environment = config.nix.envVars // {
+        HOME = "/root";
+      } // config.networking.proxy.envVars;
+      # TODO figure out how to get dependencies list reliably
+      path = [ pkgs.coreutils pkgs.gnutar pkgs.xz.bin pkgs.gzip pkgs.gitMinimal config.nix.package.out ];
+      serviceConfig = {
+        User = "root";
+        WorkingDirectory = "/etc/nixos";
+        ExecStart = ''
+          ${nixos-rebuild} switch --rollback --flake .#${config-id}
+        '';
+        KillMode = "none";
+        SendSIGKILL = "no";
+      };
+      restartIfChanged = false;
+      unitConfig.X-StopOnRemoval = false;
+    };
+  };
+}
--- a/shell.nix
+++ b/shell.nix
@ -1,48 +0,0 @@
-{ pkgs ? import <nixos-22.11> { } }:
-let
-  sp-python = pkgs.python310.withPackages (p: with p; [
-    setuptools
-    portalocker
-    pytz
-    pytest
-    pytest-mock
-    pytest-datadir
-    huey
-    gevent
-    mnemonic
-    coverage
-    pylint
-    rope
-    mypy
-    pylsp-mypy
-    pydantic
-    typing-extensions
-    psutil
-    black
-    fastapi
-    uvicorn
-    redis
-    strawberry-graphql
-    flake8-bugbear
-    flake8
-  ]);
-in
-pkgs.mkShell {
-  buildInputs = [
-    sp-python
-    pkgs.black
-    pkgs.redis
-    pkgs.restic
-    pkgs.rclone
-  ];
-  shellHook = ''
-    PYTHONPATH=${sp-python}/${sp-python.sitePackages}
-    # envs set with export and as attributes are treated differently.
-    # for example. printenv <Name> will not fetch the value of an attribute.
-    export USE_REDIS_PORT=6379
-    pkill redis-server
-    sleep 2
-    setsid redis-server --bind 127.0.0.1 --port $USE_REDIS_PORT >/dev/null 2>/dev/null &
-    # maybe set more env-vars
-  '';
-}