[OPS-1467] Introduce mkActivateScript for user-level systemd services

Problem: We have many projects where we deploy similar
approaches with user-level systemd services. We need to reduce duplication.

Solution: create a flexible enough lib to cover all the best practices

lib/systemd/default.nix

@@ -4,4 +4,5 @@
 
   hardenServices = import ./harden-services.nix;
 
+  userLevelServices = import ./user-level-services.nix;
 }

lib/systemd/user-level-services.nix

@@ -0,0 +1,107 @@
+{ nixpkgs
+, system ? "x86_64-linux"
+, pkgs ? nixpkgs.legacyPackages.${system}, ... }:
+let
+  mkVM = moduleConfig: serviceConfig: nixpkgs.lib.nixosSystem {
+    inherit system;
+    modules = [ ({ ... }: {
+
+      imports = [ moduleConfig.module ];
+
+      services.${moduleConfig.nixOsServiceName} = {
+        enable = true;
+      } // serviceConfig;
+
+    }) ];
+  };
+
+in {
+  mkActivateScript = activateScriptName:
+      { module, nixOsServiceName, primaryService, auxiliaryServices}@moduleConfig:
+      { backupAction ? "", checkAction ? null, restoreAction ? ""}:
+      serviceConfig:
+    let
+      vm = mkVM moduleConfig serviceConfig;
+      allServices = [primaryService] ++ auxiliaryServices;
+    in
+    pkgs.writeShellScriptBin activateScriptName ''
+      set -euo pipefail
+      export XDG_RUNTIME_DIR="/run/user/$UID"
+      mkdir -v -p "$HOME/.config/systemd/user"
+      ${if backupAction != ""
+        then
+          ''
+            # Deploy may change DB schema via migration and we should be able
+            # to restore its state to a point before an update to successfully rollback
+            # failed deployment
+            echo Backing up the existing state
+            ${backupAction}
+          ''
+        else
+          ""
+      }
+
+      ${pkgs.lib.concatStringsSep "\n" (builtins.map (service:
+        let
+          serviceUnit = vm.config.systemd.units.${service};
+        in
+          ''
+            rm -v -f -- "$HOME/.config/systemd/user/${service}"
+            ln -v -s ${serviceUnit.unit}/${service} "$HOME/.config/systemd/user/${service}"
+          ''
+          +
+          (pkgs.lib.concatStringsSep "\n" (builtins.map (wantedByTarget:
+              ''
+                rm -v -f -- "$HOME/.config/systemd/user/${wantedByTarget}.wants/${service}"
+                mkdir -v -p "$HOME/.config/systemd/user/${wantedByTarget}.wants"
+                ln -v -s "$HOME/.config/systemd/user/${service}" "$HOME/.config/systemd/user/${wantedByTarget}.wants"
+              ''
+          ) serviceUnit.wantedBy))
+        ) allServices)}
+      systemctl --user daemon-reload
+
+      ${if restoreAction != ""
+        then
+          ''
+          rollback() {
+            exit_code="$?"
+            if [[ $exit_code -ne 0 ]]; then
+              echo Activation failed, restoring the previous working state
+              ${restoreAction}
+            fi
+            exit "$exit_code"
+          }
+          trap rollback EXIT
+          ''
+        else ""
+      }
+
+      ${pkgs.lib.concatStringsSep "\n" (builtins.map (service:
+        if builtins.elem "default.target" vm.config.systemd.units.${service}.wantedBy
+        then
+          ''
+            echo Restarting ${service}
+            systemctl --user restart ${service}
+          ''
+        else ""
+      ) (auxiliaryServices ++ [primaryService]))}
+
+      ${if checkAction != null
+        then
+          ''
+            retry_count=0
+            while [[ "$retry_count" -lt 10 ]]; do
+              echo "Checking if the server is up, round $((retry_count+1))"
+              set +e
+              ${checkAction vm.config.services.${moduleConfig.nixOsServiceName}}
+              set -e
+              retry_count=$((retry_count+1))
+              sleep 5
+            done
+            # If we didn't manage to get 200, fail and cause a rollback
+            exit 1
+          ''
+        else ""
+      }
+    '';
+}