Merge pull request 'run pytest inside an ephemeral NixOS VM; update readme' (#85) from flake-check-and-readme into master

Reviewed-on: https://git.selfprivacy.org/SelfPrivacy/selfprivacy-rest-api/pulls/85
Reviewed-by: Inex Code <inex.code@selfprivacy.org>

.gitignore

@@ -150,3 +150,4 @@ cython_debug/
 *.rdb
 
 /result
+/.nixos-test-history

README.md

@@ -6,52 +6,16 @@
 $ nix build
 ```
 
-As a result, you should get the `./result` symlink to a folder (in `/nix/store`) with build contents.
+In case of successful build, you should get the `./result` symlink to a folder (in `/nix/store`) with build contents.
 
-## develop & test
+## develop
 
 ```console
 $ nix develop
-$ [SP devshell] pytest .
+[SP devshell:/dir/selfprivacy-rest-api]$ python
-=================================== test session starts =====================================
+Python 3.10.13 (main, Aug 24 2023, 12:59:26) [GCC 12.3.0] on linux
-platform linux -- Python 3.10.11, pytest-7.1.3, pluggy-1.0.0
+Type "help", "copyright", "credits" or "license" for more information.
-rootdir: /data/selfprivacy/selfprivacy-rest-api
+(ins)>>>
-plugins: anyio-3.5.0, datadir-1.4.1, mock-3.8.2
-collected 692 items
-
-tests/test_block_device_utils.py .................                                    [  2%]
-tests/test_common.py .....                                                            [  3%]
-tests/test_jobs.py ........                                                           [  4%]
-tests/test_model_storage.py ..                                                        [  4%]
-tests/test_models.py ..                                                               [  4%]
-tests/test_network_utils.py ......                                                    [  5%]
-tests/test_services.py ......                                                         [  6%]
-tests/test_graphql/test_api.py .                                                      [  6%]
-tests/test_graphql/test_api_backup.py ...............                                 [  8%]
-tests/test_graphql/test_api_devices.py .................                              [ 11%]
-tests/test_graphql/test_api_recovery.py .........                                     [ 12%]
-tests/test_graphql/test_api_version.py ..                                             [ 13%]
-tests/test_graphql/test_backup.py ...............................                     [ 21%]
-tests/test_graphql/test_localsecret.py ...                                            [ 22%]
-tests/test_graphql/test_ssh.py ............                                           [ 23%]
-tests/test_graphql/test_system.py .............................                       [ 28%]
-tests/test_graphql/test_system_nixos_tasks.py ........                                [ 29%]
-tests/test_graphql/test_users.py ..................................                   [ 42%]
-tests/test_graphql/test_repository/test_json_tokens_repository.py                     [ 44%]
-tests/test_graphql/test_repository/test_tokens_repository.py ....                     [ 53%]
-tests/test_rest_endpoints/test_auth.py ..........................                     [ 58%]
-tests/test_rest_endpoints/test_system.py ........................                     [ 63%]
-tests/test_rest_endpoints/test_users.py ................................              [ 76%]
-tests/test_rest_endpoints/services/test_bitwarden.py ............                     [ 78%]
-tests/test_rest_endpoints/services/test_gitea.py ..............                       [ 80%]
-tests/test_rest_endpoints/services/test_mailserver.py .....                           [ 81%]
-tests/test_rest_endpoints/services/test_nextcloud.py ............                     [ 83%]
-tests/test_rest_endpoints/services/test_ocserv.py ..............                      [ 85%]
-tests/test_rest_endpoints/services/test_pleroma.py ..............                     [ 87%]
-tests/test_rest_endpoints/services/test_services.py ....                              [ 88%]
-tests/test_rest_endpoints/services/test_ssh.py .....................                  [100%]
-
-============================== 692 passed in 352.76s (0:05:52) ===============================
 ```
 
 If you don't have experimental flakes enabled, you can use the following command:
@@ -60,14 +24,67 @@ If you don't have experimental flakes enabled, you can use the following command
 nix --extra-experimental-features nix-command --extra-experimental-features flakes develop
 ```
 
+## testing
+
+Run the test suite by running coverage with pytest inside an ephemeral NixOS VM with redis service enabled:
+```console
+$ nix flake check -L
+```
+
+Run the same test suite, but additionally create `./result/coverage.xml` in the current directory:
+```console
+$ nix build .#checks.x86_64-linux.default -L
+```
+
+Alternatively, just print the path to `/nix/store/...coverage.xml` without creating any files in the current directory:
+```console
+$ nix build .#checks.x86_64-linux.default -L --print-out-paths --no-link
+```
+
+Run the same test suite with arbitrary pytest options:
+```console
+$ pytest-vm.sh # specify pytest options here, e.g. `--last-failed`
+```
+When running using the script, pytest cache is preserved between runs in `.pytest_cache` folder.
+NixOS VM state temporary resides in `${TMPDIR:=/tmp}/nixos-vm-tmp-dir/vm-state-machine` during the test.
+Git workdir directory is shared read-write with VM via `.nixos-vm-tmp-dir/shared-xchg` symlink. VM accesses workdir contents via `/tmp/shared` mount point and `/root/source` symlink.
+
+Launch VM and execute commands manually either in Linux console (user `root`) or using python NixOS tests driver API (refer to [NixOS documentation](https://nixos.org/manual/nixos/stable/#ssec-machine-objects)):
+```console
+$ nix run .#checks.x86_64-linux.default.driverInteractive
+```
+
+You can add `--keep-vm-state` in order to keep VM state between runs:
+```console
+$ TMPDIR=".nixos-vm-tmp-dir" nix run .#checks.x86_64-linux.default.driverInteractive --keep-vm-state
+```
+
+Option `-L`/`--print-build-logs` is optional for all nix commands. It tells nix to print each log line one after another instead of overwriting a single one.
+
 ## dependencies and dependant modules
 
-Current flake inherits nixpkgs from NixOS configuration flake. So there is no need to refer to extra nixpkgs dependency if you want to be aligned with exact NixOS configuration.
+This flake depends on a single Nix flake input - nixpkgs repository. nixpkgs repository is used for all software packages used to build, run API service, tests, etc.
 
-![diagram](http://www.plantuml.com/plantuml/proxy?src=https://git.selfprivacy.org/SelfPrivacy/selfprivacy-rest-api/raw/branch/master/nix-dependencies-diagram.puml)
+In order to synchronize nixpkgs input with the same from selfprivacy-nixos-config repository, use this command:
+
+```console
+$ nix flake lock --override-input nixpkgs nixpkgs --inputs-from git+https://git.selfprivacy.org/SelfPrivacy/selfprivacy-nixos-config.git?ref=BRANCH
+```
+
+Replace BRANCH with the branch name of selfprivacy-nixos-config repository you want to sync with. During development nixpkgs input update might be required in both selfprivacy-rest-api and selfprivacy-nixos-config repositories simultaneously. So, a new feature branch might be temporarily used until selfprivacy-nixos-config gets the feature branch merged.
+
+Show current flake inputs (e.g. nixpkgs):
+```console
+$ nix flake metadata
+```
+
+Show selfprivacy-nixos-config Nix flake inputs (including nixpkgs):
+```console
+$ nix flake metadata git+https://git.selfprivacy.org/SelfPrivacy/selfprivacy-nixos-config.git?ref=BRANCH
+```
 
 Nix code for NixOS service module for API is located in NixOS configuration repository.
 
-## current issues
+## troubleshooting
 
-- It's not clear how to store in this repository information about several compatible NixOS configuration commits, where API application tests pass. Currently, here is only a single `flake.lock`.
+Sometimes commands inside `nix develop` refuse to work properly if the calling shell lacks `LANG` environment variable. Try to set it before entering `nix develop`.

default.nix

@@ -4,7 +4,6 @@ pythonPackages.buildPythonPackage rec {
   pname = "selfprivacy-graphql-api";
   version = rev;
   src = builtins.filterSource (p: t: p != ".git" && t != "symlink") ./.;
-  nativeCheckInputs = [ pythonPackages.pytestCheckHook ];
   propagatedBuildInputs = with pythonPackages; [
     fastapi
     gevent
@@ -13,9 +12,6 @@ pythonPackages.buildPythonPackage rec {
     portalocker
     psutil
     pydantic
-    pytest
-    pytest-datadir
-    pytest-mock
     pytz
     redis
     setuptools

flake.nix

@@ -11,40 +11,131 @@
         pythonPackages = pkgs.python310Packages;
         rev = self.shortRev or self.dirtyShortRev or "dirty";
       };
+      python = self.packages.${system}.default.pythonModule;
+      python-env =
+        python.withPackages (ps:
+          self.packages.${system}.default.propagatedBuildInputs ++ (with ps; [
+            coverage
+            pytest
+            pytest-datadir
+            pytest-mock
+          ]));
+      vmtest-src-dir = "/root/source";
+      shellMOTD = ''
+        Welcome to SP API development shell!
+
+        [formatters]
+
+          black
+          nixpkgs-fmt
+
+        [testing in NixOS VM]
+
+          nixos-test-driver - run an interactive NixOS VM with with all dependencies
+          pytest-vm         - run pytest in an ephemeral NixOS VM with Redis, accepting pytest arguments
+      '';
     in
     {
-      packages.${system}.default = selfprivacy-graphql-api;
+      # see https://github.com/NixOS/nixpkgs/blob/66a9817cec77098cfdcbb9ad82dbb92651987a84/nixos/lib/test-driver/test_driver/machine.py#L359
+      packages.${system} = {
+        default = selfprivacy-graphql-api;
+        pytest-vm = pkgs.writeShellScriptBin "pytest-vm" ''
+          set -o errexit
+          set -o nounset
+          set -o xtrace
+
+          # see https://github.com/NixOS/nixpkgs/blob/66a9817cec77098cfdcbb9ad82dbb92651987a84/nixos/lib/test-driver/test_driver/machine.py#L359
+          export TMPDIR=''${TMPDIR:=/tmp}/nixos-vm-tmp-dir
+          readonly NIXOS_VM_SHARED_DIR_HOST="$TMPDIR/shared-xchg"
+          readonly NIXOS_VM_SHARED_DIR_GUEST="/tmp/shared"
+
+          mkdir -p "$TMPDIR"
+          ln -sfv "$PWD" -T "$NIXOS_VM_SHARED_DIR_HOST"
+
+          SCRIPT=$(cat <<EOF
+          start_all()
+          machine.succeed("ln -sf $NIXOS_VM_SHARED_DIR_GUEST -T ${vmtest-src-dir} >&2")
+          machine.succeed("cd ${vmtest-src-dir} && coverage run -m pytest -v $@ >&2")
+          machine.succeed("cd ${vmtest-src-dir} && coverage report >&2")
+          EOF
+          )
+
+          if [ -f "/etc/arch-release" ]; then
+              ${self.checks.${system}.default.driverInteractive}/bin/nixos-test-driver --no-interactive <(printf "%s" "$SCRIPT")
+          else
+              ${self.checks.${system}.default.driver}/bin/nixos-test-driver -- <(printf "%s" "$SCRIPT")
+          fi
+        '';
+      };
       nixosModules.default =
         import ./nixos/module.nix self.packages.${system}.default;
       devShells.${system}.default = pkgs.mkShell {
-        packages =
+        name = "SP API dev shell";
-          let
+        packages = with pkgs; [
-            # TODO is there a better way to get environment for VS Code?
-            python3 =
-              nixpkgs.lib.findFirst (p: p.pname == "python3") (abort "wtf")
-                self.packages.${system}.default.propagatedBuildInputs;
-            python-env =
-              python3.withPackages
-                (_: self.packages.${system}.default.propagatedBuildInputs);
-          in
-          with pkgs; [
           python-env
-            black
           rclone
           redis
           restic
+          self.packages.${system}.pytest-vm
+          # FIXME consider loading this explicitly only after ArchLinux issue is solved
+          self.checks.x86_64-linux.default.driverInteractive
         ];
         shellHook = ''
           # envs set with export and as attributes are treated differently.
           # for example. printenv <Name> will not fetch the value of an attribute.
-          export USE_REDIS_PORT=6379
+          export TEST_MODE="true"
-          export TEST_MODE=true
+
-          pkill redis-server
+          # more tips for bash-completion to work on non-NixOS:
-          sleep 2
+          # https://discourse.nixos.org/t/whats-the-nix-way-of-bash-completion-for-packages/20209/16?u=alexoundos
-          setsid redis-server --bind 127.0.0.1 --port $USE_REDIS_PORT >/dev/null 2>/dev/null &
+          # Load installed profiles
-          # maybe set more env-vars
+          for file in "/etc/profile.d/"*.sh; do
+            # If that folder doesn't exist, bash loves to return the whole glob
+            [[ -f "$file" ]] && source "$file"
+          done
+
+          printf "%s" "${shellMOTD}"
         '';
       };
+      checks.${system} = {
+        fmt-check = pkgs.runCommandLocal "sp-api-fmt-check"
+          { nativeBuildInputs = [ pkgs.black ]; }
+          "black --check ${self.outPath} > $out";
+        default =
+          pkgs.testers.runNixOSTest {
+            imports = [{
+              name = "default";
+              nodes.machine = { lib, pkgs, ... }: {
+                imports = [{
+                  boot.consoleLogLevel = lib.mkForce 3;
+                  documentation.enable = false;
+                  services.journald.extraConfig = lib.mkForce "";
+                  services.redis.servers.sp-api = {
+                    enable = true;
+                    save = [ ];
+                    port = 6379; # FIXME
+                    settings.notify-keyspace-events = "KEA";
+                  };
+                  environment.systemPackages = with pkgs; [
+                    python-env
+                    # TODO: these can be passed via wrapper script around app
+                    rclone
+                    restic
+                  ];
+                  environment.variables.TEST_MODE = "true";
+                  systemd.tmpfiles.settings.src.${vmtest-src-dir}.L.argument =
+                    self.outPath;
+                }];
+              };
+              testScript = ''
+                start_all()
+                machine.succeed("cd ${vmtest-src-dir} && coverage run --data-file=/tmp/.coverage -m pytest -p no:cacheprovider -v >&2")
+                machine.succeed("coverage xml --rcfile=${vmtest-src-dir}/.coveragerc --data-file=/tmp/.coverage >&2")
+                machine.copy_from_vm("coverage.xml", ".")
+                machine.succeed("coverage report >&2")
+              '';
+            }];
+          };
+      };
     };
   nixConfig.bash-prompt = ''\n\[\e[1;32m\][\[\e[0m\]\[\e[1;34m\]SP devshell\[\e[0m\]\[\e[1;32m\]:\w]\$\[\[\e[0m\] '';
 }