[OPS-1467] Introduce mkActivateScript for user-level systemd services

Problem: We have many projects where we deploy similar
approaches with user-level systemd services. We need to reduce duplication.

Solution: create a flexible enough lib to cover all the best practices

lib/systemd/default.nix

@@ -4,4 +4,5 @@
 
   hardenServices = import ./harden-services.nix;
 
+  userLevelServices = import ./user-level-services.nix;
 }

lib/systemd/user-level-services.nix

@@ -0,0 +1,88 @@
+{ nixpkgs
+, system ? "x86_64-linux"
+, pkgs ? nixpkgs.legacyPackages.${system}, ... }:
+let
+  mkVM = moduleConfig: serviceConfig: nixpkgs.lib.nixosSystem {
+    inherit system;
+    modules = [ ({ ... }: {
+
+      imports = [ moduleConfig.module ];
+
+      services.${moduleConfig.nixOsServiceName} = {
+        enable = true;
+      } // serviceConfig;
+
+    }) ];
+  };
+
+in {
+  mkActivateScript = activateScriptName:
+      { module, nixOsServiceName, primaryService, primaryPortOptionName, auxiliaryServices, oneShotServices}@moduleConfig:
+      { backupAction ? "", restoreAction ? ""}:
+      serviceConfig:
+    let
+      vm = mkVM moduleConfig serviceConfig;
+      wantedServices = [primaryService] ++ auxiliaryServices;
+      allServices =  wantedServices ++ oneShotServices;
+    in
+    pkgs.writeShellScriptBin activateScriptName ''
+      set -euo pipefail
+      export XDG_RUNTIME_DIR="/run/user/$UID"
+      mkdir -v -p "$HOME/.config/systemd/user/default.target.wants"
+
+      # Deploy may change DB schema via migration and we should be able
+      # to restore its state to a point before an update to successfully rollback
+      # failed deployment
+      echo Stopping ${primaryService}
+      systemctl --user stop ${primaryService} || true
+      echo Backing up the existing state
+      ${backupAction}
+
+      ${pkgs.lib.concatStringsSep "\n" (builtins.map (service:
+        ''
+          rm -v -f -- "$HOME/.config/systemd/user/${service}"
+          ln -v -s ${vm.config.systemd.units.${service}.unit}/${service} "$HOME/.config/systemd/user/${service}"
+        ''
+        ) allServices)}
+      ${pkgs.lib.concatStringsSep "\n" (builtins.map (service:
+        ''
+          rm -v -f -- "$HOME/.config/systemd/user/default.target.wants/${service}"
+          ln -v -s "$HOME/.config/systemd/user/${service}" "$HOME/.config/systemd/user/default.target.wants"
+        ''
+        ) wantedServices)}
+      systemctl --user daemon-reload
+
+      rollback() {
+        exit_code="$?"
+        if [[ $exit_code -ne 0 ]]; then
+          echo Activation failed, restoring the previous working state
+          ${restoreAction}
+        fi
+        exit "$exit_code"
+      }
+      trap rollback EXIT
+
+      ${pkgs.lib.concatStringsSep "\n" (builtins.map (service:
+        ''
+          echo Restarting ${service}
+          systemctl --user restart ${service}
+        ''
+      ) auxiliaryServices)}
+
+      echo Restarting ${primaryService}
+      systemctl --user restart ${primaryService}
+
+      # Attempt to get a 200 code for main page
+      retry_count=0
+      while [[ "$retry_count" -lt 10 ]]; do
+        echo "Checking if the server is up, round $((retry_count+1))"
+        set +e
+        curl --silent --show-error --fail http://127.0.0.1:${builtins.toString vm.config.services.${nixOsServiceName}.${primaryPortOptionName}} > /dev/null && exit 0
+        set -e
+        retry_count=$((retry_count+1))
+        sleep 5
+      done
+      # If we didn't manage to get 200, fail and cause a rollback
+      exit 1
+    '';
+}