From bc12c21efbaf63e273903a93a5965680c7739f41 Mon Sep 17 00:00:00 2001 From: Sergey Gulin Date: Tue, 5 Dec 2023 15:15:24 +0300 Subject: [PATCH] [OPS-1463] Docker networks and volumes Problem: Our nixpkgs fork has a commit that makes it possible to define named docker networks declaratively. However, it seems like it doesn't work as expected. We want to fix it and move to serokell.nix repo instead. Solution. Fix problems, add the ability to recreate networks, add warnings, extract everything as a separate module. --- .github/workflows/check.yml | 2 +- flake.lock | 6 +- flake.nix | 5 +- modules/virtualization/docker.nix | 197 ++++++++++++++++++++++++++++++ tests/docker.nix | 44 +++++++ 5 files changed, 249 insertions(+), 5 deletions(-) create mode 100644 modules/virtualization/docker.nix create mode 100644 tests/docker.nix diff --git a/.github/workflows/check.yml b/.github/workflows/check.yml index 485e3d8..73de4fa 100644 --- a/.github/workflows/check.yml +++ b/.github/workflows/check.yml @@ -5,7 +5,7 @@ jobs: check: runs-on: self-hosted steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: check flake run: nix flake check -L diff --git a/flake.lock b/flake.lock index a11ee6e..66dce92 100644 --- a/flake.lock +++ b/flake.lock @@ -900,11 +900,11 @@ }, "nixpkgs_7": { "locked": { - "lastModified": 1701769088, - "narHash": "sha256-YSsVHZmNq/9zqHpLoL8bGhuYQo2s465NzQWJf5gBdKo=", + "lastModified": 1702386253, + "narHash": "sha256-gWyY0ZnlyugHRthZQBmFfxeKNDq2o6g7kaSU1lwyj74=", "owner": "serokell", "repo": "nixpkgs", - "rev": "bc9197946f30ffe4265f7e3c7825f8bf70f7416e", + "rev": "4a0f28c92f803406ca2eed0cce08230447ad9d01", "type": "github" }, "original": { diff --git a/flake.nix b/flake.nix index 9e52d51..78f0baa 100644 --- a/flake.nix +++ b/flake.nix @@ -12,7 +12,6 @@ url = "github:hercules-ci/gitignore.nix"; flake = false; }; - flake-compat = { flake = false; }; @@ -62,6 +61,7 @@ upload-daemon = import ./modules/services/upload-daemon.nix; hetzner-cloud = import ./modules/virtualization/hetzner-cloud.nix; ec2 = import ./modules/virtualization/ec2.nix; + docker = import ./modules/virtualization/docker.nix; wireguard-monitoring = import ./modules/wireguard-monitoring/default.nix; postgresql-migration = import ./modules/postgresql-migration.nix; }; @@ -80,6 +80,9 @@ packages = pkgs.lib.optionalAttrs (! lib.hasInfix "darwin" system) { inherit (pkgs) benchwrapper; }; + checks = { + docker = import ./tests/docker.nix (inputs // { inherit pkgs; }); + }; } )); } diff --git a/modules/virtualization/docker.nix b/modules/virtualization/docker.nix new file mode 100644 index 0000000..3c1dd2f --- /dev/null +++ b/modules/virtualization/docker.nix @@ -0,0 +1,197 @@ +# SPDX-FileCopyrightText: 2023 Serokell +# +# SPDX-License-Identifier: MPL-2.0 + +{ config, lib, pkgs, ... }: +with lib; let + cfg = config.virtualisation.docker; + inherit (builtins) attrNames; + + mkUncreateMaybe = networks: volumes: '' + set -euo pipefail + nexisting=$(${pkgs.coreutils}/bin/mktemp) + nwanted=$(${pkgs.coreutils}/bin/mktemp) + vexisting=$(${pkgs.coreutils}/bin/mktemp) + vwanted=$(${pkgs.coreutils}/bin/mktemp) + cleanup() { + rm -f "$nexisting" "$nwanted" "$vexisting" "$vwanted" + } + trap cleanup EXIT + ${pkgs.docker}/bin/docker network ls --format '{{.Name}}' > "$nexisting" + echo -e "bridge\nhost\nnone\n${concatStringsSep "\n" networks}" > "$nwanted" + ${pkgs.docker}/bin/docker volume ls --format '{{.Name}}' > "$vexisting" + echo -e "${concatStringsSep "\n" volumes}" > "$vwanted" + nsuperfluous="$(${pkgs.gnugrep}/bin/grep -vxF -f $nwanted $nexisting || true)" + vsuperfluous="$(${pkgs.gnugrep}/bin/grep -vxF -f $vwanted $vexisting || true)" + while read -r net; do + if [[ ! -z "$net" ]]; then + if [[ -f /etc/docker/network-opts/$net ]]; then + echo -n "Removed superfluous Docker network: " + ${pkgs.docker}/bin/docker network rm "$net" || true + rm -f /etc/docker/network-opts/$net + else + echo "Skipped deleting Docker network $net as it was manually created (/etc/docker/network-opts/$net is missing)." + fi + fi + done <<< "$nsuperfluous" + while read -r vol; do + if [[ ! -z "$vol" ]]; then + if [[ -f /etc/docker/volumes/$vol ]]; then + echo -n "Removed superfluous Docker volume: " + ${pkgs.docker}/bin/docker volume rm "$vol" || true + rm -f /etc/docker/volumes/$vol + else + echo "Skipped deleting Docker volume $vol as it was manually created (/etc/docker/volumes/$vol is missing)." + fi + fi + done <<< "$vsuperfluous" + ''; + + mkNetworkOpts = opts: concatStringsSep " " + ([ "--driver=${opts.driver}" ] + ++ optional (opts ? subnet && opts.subnet != null) "--subnet=${opts.subnet}" + ++ optional (opts ? ip-range && opts.ip-range != null) "--ip-range=${opts.ip-range}" + ++ optional (opts ? gateway && opts.gateway != null) "--gateway=${opts.gateway}" + ++ optional (opts ? ipv6 && opts.ipv6) "--ipv6" + ++ optional (opts ? internal && opts.internal) "--internal"); + + mkNetwork = recreate: name: opts: let + create = '' + ln -s ${pkgs.writeText name (mkNetworkOpts opts)} "/etc/docker/network-opts/${name}" + echo "*** docker network create ${mkNetworkOpts opts} ${name}" + ${pkgs.docker}/bin/docker network create ${mkNetworkOpts opts} ${name} + ''; + in '' + mkdir -p /etc/docker/network-opts/ + if [[ $(${pkgs.docker}/bin/docker network ls --quiet --filter name=^${name}$ | wc -c) -eq 0 ]]; then + rm -f /etc/docker/network-opts/${name} + ${create} + elif [[ "${toString recreate}" ]]; then + oldOpts="$(cat /etc/docker/network-opts/${name} || true)" + if [ "$oldOpts" != "${mkNetworkOpts opts}" ]; then + # If oldOpts is different from new ones, disconnect all containers and recreate the network + for i in `${pkgs.docker}/bin/docker network inspect -f '{{range .Containers}}{{.Name}} {{end}}' ${name}`; do + echo "*** disconnect container $i from network ${name}" + ${pkgs.docker}/bin/docker network disconnect -f ${name} $i + done + ${pkgs.docker}/bin/docker network rm ${name} + rm -f /etc/docker/network-opts/${name} + ${create} + fi + fi + ''; + + mkVolume = name: '' + mkdir -p /etc/docker/volumes/ + if [[ $(${pkgs.docker}/bin/docker volume ls --quiet --filter name=^${name}$ | wc -c) -eq 0 ]]; then + echo "*** docker volume create ${name}" + ${pkgs.docker}/bin/docker volume create ${name} + touch /etc/docker/volumes/${name} + fi + ''; +in { + options.virtualisation.docker = { + volumes = mkOption { + default = []; + type = types.listOf types.str; + example = [ "volume_1" "volume_2" ]; + description = '' + A list of named volumes that should be created. + ''; + }; + + networks = mkOption { + default = {}; + type = types.attrsOf (types.submodule { + options = { + driver = mkOption { + default = "bridge"; + type = types.str; + example = "overlay"; + description = '' + Driver to manage the network. One of bridge, or overlay. + ''; + }; + + subnet = mkOption { + default = null; + type = types.nullOr types.str; + example = "172.28.0.0/16"; + description = '' + Subnet in CIDR format that represents a network segment. + ''; + }; + + ip-range = mkOption { + default = null; + type = types.nullOr types.str; + example = "172.28.5.0/24"; + description = '' + Allocate container ip from a sub-range. + ''; + }; + + gateway = mkOption { + default = null; + type = types.nullOr types.str; + example = "172.28.5.254"; + description = '' + IPv4 or IPv6 Gateway for the master subnet. + ''; + }; + + ipv6 = mkOption { + default = false; + type = types.bool; + example = true; + description = '' + Enable IPv6 networking. + ''; + }; + + internal = mkOption { + default = false; + type = types.bool; + example = true; + description = '' + Restrict external access to the network. + ''; + }; + }; + }); + + example = { + my-network = { + driver = "bridge"; + subnet = "172.28.0.0/16"; + ip-range = "172.28.5.0/24"; + gateway = "172.28.5.254"; + }; + }; + + description = '' + A list of named networks to be created. + ''; + }; + unsafeRecreateNetworks = mkEnableOption '' + When enabled, docker will disconnect all containers + connected to the modified network and recreate it. + Unmodified networks will not be affected. + ''; + }; + + config = { + systemd.services.docker.postStart = + mkUncreateMaybe (attrNames cfg.networks) cfg.volumes + + concatStrings (mapAttrsToList (mkNetwork cfg.unsafeRecreateNetworks) cfg.networks) + + concatStrings (map mkVolume cfg.volumes); + + virtualisation.docker.daemon.settings.log-level = lib.mkDefault "info"; + + warnings = + optional cfg.unsafeRecreateNetworks + "The cfg.unsafeRecreateNetworks option is enabled, all containers connected to the modified networks will be disabled." ++ + optional (!cfg.unsafeRecreateNetworks) + "The cfg.unsafeRecreateNetworks option is disabled, no modification to existing networks will be applied."; + }; +} diff --git a/tests/docker.nix b/tests/docker.nix new file mode 100644 index 0000000..0a8e143 --- /dev/null +++ b/tests/docker.nix @@ -0,0 +1,44 @@ +# SPDX-FileCopyrightText: 2023 Serokell +# +# SPDX-License-Identifier: MPL-2.0 + +{ self, nixpkgs, pkgs, ... }: +import "${nixpkgs}/nixos/tests/make-test-python.nix" ({...} : { + name = "docker"; + nodes = { + docker = {...}: { + imports = [ self.nixosModules.docker ]; + virtualisation.docker = { + enable = true; + volumes = [ "thevolume" ]; + networks.thenetwork = { + driver = "bridge"; + subnet = "172.28.0.0/16"; + ip-range = "172.28.5.0/24"; + gateway = "172.28.5.254"; + }; + }; + }; + }; + + testScript = '' + start_all() + + docker.wait_for_unit("sockets.target") + docker.wait_for_unit("docker.service") + + docker.succeed("docker volume ls | grep thevolume") + docker.succeed("docker network ls | grep thenetwork") + + docker.succeed("docker network inspect thenetwork --format {{.IPAM.Config}} | grep '172.28.0.0/16 172.28.5.0/24 172.28.5.254'") + + docker.succeed("docker volume create newvolume"); + docker.succeed("docker network create newnetwork") + docker.systemctl("restart docker") + docker.wait_for_unit("docker.service") + + # don't remove manually created networks and volumes + docker.succeed("docker volume ls | grep newvolume") + docker.succeed("docker network ls | grep newnetwork") + ''; +}) { inherit pkgs; }