feat: Migration to 23.11

selfprivacy_api/migrations/__init__.py

@@ -22,8 +22,8 @@ from selfprivacy_api.migrations.providers import CreateProviderFields
 from selfprivacy_api.migrations.prepare_for_nixos_2211 import (
     MigrateToSelfprivacyChannelFrom2205,
 )
-from selfprivacy_api.migrations.prepare_for_nixos_2305 import (
+from selfprivacy_api.migrations.migrate_to_flakes import (
-    MigrateToSelfprivacyChannelFrom2211,
+    MigrateToFlakes,
 )
 from selfprivacy_api.migrations.redis_tokens import LoadTokensToRedis
 
@@ -35,8 +35,8 @@ migrations = [
     CheckForFailedBindsMigration(),
     CreateProviderFields(),
     MigrateToSelfprivacyChannelFrom2205(),
-    MigrateToSelfprivacyChannelFrom2211(),
     LoadTokensToRedis(),
+    MigrateToFlakes(),
 ]
 
 

selfprivacy_api/migrations/migrate_to_flakes.py

@@ -0,0 +1,240 @@
+import json
+import os
+import subprocess
+from selfprivacy_api.jobs import JobStatus, Jobs
+
+from selfprivacy_api.migrations.migration import Migration
+
+
+class MigrateToFlakes(Migration):
+    """Migrate to selfprivacy Nix channel.
+    For some reason NixOS 22.11 servers initialized with the nixos channel instead of selfprivacy.
+    This stops us from upgrading to NixOS 23.05
+    """
+
+    def get_migration_name(self):
+        return "migrate_to_flakes"
+
+    def get_migration_description(self):
+        return "Migrate to selfprivacy Nix flakes."
+
+    def is_migration_needed(self):
+        return True
+
+    def migrate(self):
+        # Change the channel and update them.
+        current_working_directory = os.getcwd()
+        try:
+            print("===========================")
+            print("STARTING MIGRATION TO 23.11")
+            print("===========================")
+            os.chdir("/")
+
+            print("Disabling automatic upgrades to prevent conflicts.")
+            subprocess.check_output(
+                [
+                    "systemctl",
+                    "stop",
+                    "nixos-upgrade.service",
+                    "nixos-upgrade.timer",
+                ]
+            )
+            print("Disabled automatic upgrades.")
+
+            print("Reading the userdata file.")
+            userdata_file = open(
+                "/etc/nixos/userdata/userdata.json", "r", encoding="utf-8"
+            )
+            userdata: dict = json.load(userdata_file)
+            userdata_file.close()
+            print("Read file. Validating contents...")
+            assert userdata["dns"]["provider"] != None
+            assert userdata["dns"]["apiKey"] != None
+            assert userdata["useBinds"] == True
+            assert userdata["username"] != None
+            assert userdata["username"] != ""
+            print("Userdata file is probably fine.")
+
+            print(
+                "Moving old NixOS configuration from /etc/nixos to /etc/nixos.pre-flakes"
+            )
+            subprocess.check_output(
+                [
+                    "mv",
+                    "-v",
+                    "/etc/nixos",
+                    "/etc/nixos.pre-flakes",
+                ]
+            )
+            assert os.path.exists("/etc/nixos.pre-flakes/userdata/userdata.json")
+            print("Moved")
+
+            print("Downloading the template of the new NixOS configuration")
+            subprocess.check_output(
+                [
+                    "mkdir",
+                    "/etc/nixos",
+                ]
+            )
+            archive = subprocess.Popen(
+                (
+                    "curl",
+                    "--fail",
+                    "https://git.selfprivacy.org/SelfPrivacy/selfprivacy-nixos-template/archive/master.tar.gz",
+                ),
+                stdout=subprocess.PIPE,
+            )
+            extract = subprocess.check_output(
+                (
+                    "tar",
+                    "-xz",
+                    "-C",
+                    "/etc/nixos",
+                    "--strip-components=1",
+                    '--exclude=".*"',
+                ),
+                stdin=archive.stdout,
+            )
+            archive.wait()
+            print("Downloaded")
+
+            print("Copying hardware-configuration.nix")
+            subprocess.check_output(
+                [
+                    "cp",
+                    "-v",
+                    "/etc/nixos.pre-flakes/hardware-configuration.nix",
+                    "/etc/nixos/hardware-configuration.nix",
+                ]
+            )
+            print("Copied")
+
+            print("Checking if /etc/nixos.pre-flakes/networking.nix exists")
+            if os.path.exists("/etc/nixos.pre-flakes/networking.nix"):
+                print("Transforming networking.nix to /etc/nixos/deployment.nix")
+                raise NotImplementedError
+            else:
+                print("Generating /etc/nixos/deployment.nix")
+                deployment_contents = '{ lib, ... }: {\n  system.stateVersion = lib.mkDefault "23.11";\n  nixpkgs.hostPlatform = lib.mkDefault "x86_64-linux";\n}'
+                with open("/etc/nixos/deployment.nix", "w") as file:
+                    file.write(deployment_contents)
+
+            print("Generated.")
+
+            print("Generating the new userdata file.")
+            new_userdata = {
+                "dns": {
+                    "provider": userdata.get("dns", {}).get("provider", "CLOUDFLARE"),
+                    "useStagingACME": True,
+                },
+                "server": {
+                    "provider": userdata.get("server", {}).get("provider", "HETZNER")
+                },
+                "domain": userdata.get("domain"),
+                "hashedMasterPassword": userdata.get("hashedMasterPassword"),
+                "hostname": userdata.get("hostname"),
+                "timezone": userdata.get("timezone", "Etc/UTC"),
+                "username": userdata.get("username"),
+                "useBinds": True,
+                "sshKeys": userdata.get("sshKeys", []),
+                "users": userdata.get("users", []),
+                "autoUpgrade": {
+                    "enable": userdata.get("autoUpgrade", {}).get("enable", True),
+                    "allowReboot": userdata.get("autoUpgrade", {}).get(
+                        "allowReboot", False
+                    ),
+                },
+                "modules": {
+                    "bitwarden": {
+                        "enable": userdata.get("bitwarden", {}).get("enable", False),
+                        "location": userdata.get("bitwarden", {}).get(
+                            "location", "sda1"
+                        ),
+                    },
+                    "gitea": {
+                        "enable": userdata.get("gitea", {}).get("enable", False),
+                        "location": userdata.get("gitea", {}).get("location", "sda1"),
+                    },
+                    "jitsi-meet": {
+                        "enable": userdata.get("jitsi", {}).get("enable", False),
+                    },
+                    "nextcloud": {
+                        "enable": userdata.get("nextcloud", {}).get("enable", True),
+                        "location": userdata.get("nextcloud", {}).get(
+                            "location", "sda1"
+                        ),
+                    },
+                    "ocserv": {
+                        "enable": userdata.get("ocserv", {}).get("enable", False),
+                    },
+                    "pleroma": {
+                        "enable": userdata.get("pleroma", {}).get("enable", False),
+                        "location": userdata.get("pleroma", {}).get("location", "sda1"),
+                    },
+                    "simple-nixos-mailserver": {
+                        "enable": True,
+                        "location": userdata.get("email", {}).get("location", "sda1"),
+                    },
+                },
+                "volumes": userdata.get("volumes", []),
+                "ssh": {"rootKeys": userdata.get("ssh", {}).get("rootKeys", [])},
+                "stateVersion": "23.11",
+            }
+            with open("/etc/nixos/userdata.json", "w") as file:
+                json.dump(new_userdata, file, indent=4)
+            print("New userdata.json generated.")
+
+            print("Generating /etc/selfprivacy/secrets.json")
+            secrets_contents = {
+                "databasePassword": userdata.get("databasePassword"),
+                "dns": {"apiKey": userdata.get("dns", {}).get("apiKey", "INVALID")},
+                "modules": {
+                    "nextcloud": {
+                        "adminPassword": userdata.get("nextcloud", {}).get(
+                            "adminPassword", "INVALID"
+                        ),
+                        "databasePassword": userdata.get("nextcloud", {}).get(
+                            "databasePassword", "INVALID"
+                        ),
+                    }
+                },
+            }
+            with open("/etc/selfprivacy/secrets.json", "w") as file:
+                json.dump(secrets_contents, file, indent=4)
+            os.chmod("/etc/selfprivacy/secrets.json", 0o600)
+            print("secrets.json generated.")
+
+            print("Building NixOS")
+            subprocess.check_output(
+                ["nix", "flake", "lock", "/etc/nixos", "--update-input", "sp-modules"]
+            )
+            subprocess.check_output(
+                [
+                    "nixos-rebuild",
+                    "boot",
+                    "--flake",
+                    "/etc/nixos#default",
+                ]
+            )
+            print(
+                "NixOS built. You may reboot now! Creating a notification for the app"
+            )
+
+            Jobs.add(
+                name="NixOS upgrade to 23.11",
+                type_id="migrations.migrate_to_flakes",
+                status=JobStatus.FINISHED,
+                status_text="New system built. Reboot your server to apply.",
+                progress=100,
+            )
+
+        except Exception as error:
+            os.chdir(current_working_directory)
+            print("Error")
+            print(error)
+            Jobs.add(
+                name="NixOS upgrade to 23.11",
+                type_id="migrations.migrate_to_flakes",
+                status=JobStatus.ERROR,
+                status_text=str(error),
+            )

selfprivacy_api/migrations/prepare_for_nixos_2305.py

@@ -1,58 +0,0 @@
-import os
-import subprocess
-
-from selfprivacy_api.migrations.migration import Migration
-
-
-class MigrateToSelfprivacyChannelFrom2211(Migration):
-    """Migrate to selfprivacy Nix channel.
-    For some reason NixOS 22.11 servers initialized with the nixos channel instead of selfprivacy.
-    This stops us from upgrading to NixOS 23.05
-    """
-
-    def get_migration_name(self):
-        return "migrate_to_selfprivacy_channel_from_2211"
-
-    def get_migration_description(self):
-        return "Migrate to selfprivacy Nix channel from NixOS 22.11."
-
-    def is_migration_needed(self):
-        try:
-            output = subprocess.check_output(
-                ["nix-channel", "--list"], start_new_session=True
-            )
-            output = output.decode("utf-8")
-            first_line = output.split("\n", maxsplit=1)[0]
-            return first_line.startswith("nixos") and (
-                first_line.endswith("nixos-22.11")
-            )
-        except subprocess.CalledProcessError:
-            return False
-
-    def migrate(self):
-        # Change the channel and update them.
-        # Also, go to /etc/nixos directory and make a git pull
-        current_working_directory = os.getcwd()
-        try:
-            print("Changing channel")
-            os.chdir("/etc/nixos")
-            subprocess.check_output(
-                [
-                    "nix-channel",
-                    "--add",
-                    "https://channel.selfprivacy.org/nixos-selfpricacy",
-                    "nixos",
-                ]
-            )
-            subprocess.check_output(["nix-channel", "--update"])
-            nixos_config_branch = subprocess.check_output(
-                ["git", "rev-parse", "--abbrev-ref", "HEAD"], start_new_session=True
-            )
-            if nixos_config_branch.decode("utf-8").strip() == "api-redis":
-                print("Also changing nixos-config branch from api-redis to master")
-                subprocess.check_output(["git", "checkout", "master"])
-            subprocess.check_output(["git", "pull"])
-            os.chdir(current_working_directory)
-        except subprocess.CalledProcessError:
-            os.chdir(current_working_directory)
-            print("Error")