Setup deploy_test for provisioner (#305)

* Update renovatebot for docker compose

* create deploy_test test

* extract 1password item helper

* added additional tests

* test teleport health

* fix quoting issue with regression test

.bazelrc

@@ -17,6 +17,10 @@ test --test_verbose_timeout_warnings
 # Output test logs to the console when there are errors
 test --test_output=errors
 
+# Secrets necessary also while testing
+test --test_env=ONEPASSWORD_SERVICE_ACCOUNT_TOKEN_PROD
+test --test_env=ONEPASSWORD_SERVICE_ACCOUNT_TOKEN_DEV
+
 common --enable_bzlmod=true
 
 # Currently a bug in rules_docker where the wrong platform is selected when building docker images

buildbuddy.yaml

@@ -12,16 +12,6 @@ actions:
     bazel_commands:
       - "test //... @rules_task//... --config buildbuddy --config buildbuddy_rbe"
 
-  - name: "Test Teleport connection"
-    user: buildbuddy
-    container_image: "ubuntu-20.04"
-    triggers:
-      pull_request:
-        branches:
-          - "*"
-    bazel_commands:
-      - "run //tools/teleport:connection_test --config buildbuddy"
-
   - name: "Deploy Provisioner"
     user: buildbuddy
     container_image: "ubuntu-20.04"

provisioner/BUILD.bazel

@@ -2,6 +2,7 @@ load("//tools/pyinfra:defs.bzl", "pyinfra_run")
 load("@rules_task//:defs.bzl", "cmd", "task", "task_test")
 load("//tools/docker:docker.bzl", "docker_load")
 load("@pip-setup//:requirements.bzl", "requirement")
+load("//tools/onepassword:defs.bzl", "secrets")
 
 pyinfra_run(
     name = "provision",
@@ -38,22 +39,40 @@ pyinfra_run(
         "OP_BINARY": cmd.executable("//tools/onepassword:op"),
     },
     inventory = "inventory.py",
+    deps = [
+        "//tools/onepassword:lib",
+    ],
 )
 
 task(
     name = "deploy",
     cmds = [
+        secrets({
+            "TELEPORT_BUILDBUDDY_IDENTITY": "teleport_buildbuddy_identity.notesPlain",
+        }),
         "export TELEPORT_IDENTITY=$(mktemp)",
         {"defer": "rm -f $TELEPORT_IDENTITY"},
         'echo "$TELEPORT_BUILDBUDDY_IDENTITY" > $TELEPORT_IDENTITY',
         cmd.executable(":provision"),
     ],
     env = {
         "SETUP_ENV": "prod",
+        "OP_BINARY": cmd.executable("//tools/onepassword:op"),
     },
     exec_properties = {
         "include-secrets": "true",
     },
+    deps = ["//tools/onepassword:lib"],
+)
+
+task_test(
+    name = "deploy_test",
+    cmds = [
+        cmd.executable(":deploy"),
+    ],
+    env = {
+        "PYINFRA_RUN_ARGS": "--dry",
+    },
 )
 
 docker_load(

provisioner/deploys/docker/tasks/install_docker.py

@@ -3,6 +3,7 @@
 from pyinfra.facts.server import LsbRelease
 from pyinfra.api.deploy import deploy
 from pyinfra.facts.deb import DebArch
+from pyinfra.facts.server import Users
 
 
 @deploy("Install Docker")
@@ -56,6 +57,17 @@ def install_docker():
         cache_time=0 if add_apt_repo.changed else 24 * 60 * 60,
     )
 
+    existing_groups = host.get_fact(Users)["ubuntu"]["groups"]
+
+    if "docker" not in existing_groups:
+        server.shell(
+            name="Add ubuntu to docker group",
+            commands=[
+                "usermod -a -G docker ubuntu",
+            ],
+            _sudo=True,
+        )
+
     systemd.service(
         name="Enable the docker service",
         service="docker.service",

provisioner/deploys/microk8s/tasks/install_microk8s.py

@@ -61,15 +61,15 @@ def install_microk8s():
         )
 
     existing_groups = host.get_fact(Users)["ubuntu"]["groups"]
-    new_groups = existing_groups + ["microk8s"]
 
-    server.user(
-        name="Add ubuntu to microk8s group",
-        user="ubuntu",
-        groups=new_groups,
-        present=True,
-        _sudo=True,
-    )
+    if "microk8s" not in existing_groups:
+        server.shell(
+            name="Add ubuntu to microk8s group",
+            commands=[
+                "usermod -a -G microk8s ubuntu",
+            ],
+            _sudo=True,
+        )
 
     files.directory(
         name="Create and own .kube directory",

provisioner/deploys/monitoring/files/docker-compose.yml.j2

@@ -3,6 +3,7 @@ version: '3.8'
 services:
   # https://promhippie.github.io/github_exporter/#getting-started
   github_exporter:
+    container_name: github_exporter
     image: promhippie/github-exporter:latest@sha256:ad5cfc76d534d4c67ded2042b3ad343a8bcee9fd02f06638b39f74fbee17796e
     restart: always
     environment:
@@ -16,6 +17,7 @@ services:
       - GITHUB_EXPORTER_COLLECTOR_ADMIN=false
 
   nri-prometheus:
+    container_name: nri-prometheus
     image: newrelic/nri-prometheus:2.18.0@sha256:6b32ce98a098625b980342aae25634c7025d2dac996ac60642ffe0fc47e92bb9
     restart: always
     environment:

provisioner/deploys/monitoring/tasks/install_monitoring.py

@@ -1,7 +1,7 @@
 from pyinfra.api.deploy import deploy
 from pyinfra.operations import files, server, apt, systemd
 from pyinfra import host
-from provisioner.utils import one_password_item
+from tools.onepassword.lib import get_item_path
 from pyinfra.facts.server import LsbRelease
 from pyinfra.facts.deb import DebArch
 
@@ -17,8 +17,8 @@ def install_monitoring():
         _sudo=True,
     )
 
-    new_relic_license_key = one_password_item("new_relic_license_key")["password"]
-    github_exporter_token = one_password_item("github_exporter_token")["password"]
+    new_relic_license_key = get_item_path("new_relic_license_key.password")
+    github_exporter_token = get_item_path("github_exporter_token.password")
 
     docker_compose = files.template(
         name="Copy the docker-compose file",

provisioner/group_data/dev.py

@@ -4,5 +4,4 @@
 teleport_public_addr = "127.0.0.1:10443"
 teleport_acme_email = ""
 teleport_acme_enabled = "no"
-onepassword_vault_id = "vgijssel-dev"
 new_relic_display_name = "provisioner_dev"

provisioner/group_data/prod.py

@@ -4,5 +4,4 @@
 teleport_public_addr = "tele.vgijssel.nl:443"
 teleport_acme_email = "[email protected]"
 teleport_acme_enabled = "yes"
-onepassword_vault_id = "vgijssel-prod"
 new_relic_display_name = "provisioner"

provisioner/group_data/test.py

@@ -4,5 +4,4 @@
 teleport_public_addr = "127.0.0.1:10443"
 teleport_acme_email = ""
 teleport_acme_enabled = "no"
-onepassword_vault_id = "vgijssel-dev"
 new_relic_display_name = "provisioner_test"

provisioner/inventory.py

@@ -1,6 +1,5 @@
 import os
 import pkg_resources
-from pathlib import Path
 
 import pyinfra.api.connectors
 import provisioner.connectors.teleport
@@ -22,23 +21,6 @@ def patched_get_all_connectors():
 setup_env = os.environ.get("SETUP_ENV", "dev")
 
 
-def _get_onepassword_service_account_token(env_key, tmp_file):
-    if env_key in os.environ:
-        return os.environ[env_key]
-
-    file = os.path.join(
-        os.environ.get("BUILD_WORKSPACE_DIRECTORY", ""),
-        "tmp",
-        tmp_file,
-    )
-
-    if os.path.exists(file):
-        return Path(file).read_text()
-
-    else:
-        raise ValueError(f"Either set env variable '{env_key}' or create file '{file}'")
-
-
 if setup_env == "prod":
     if (
         "TELEPORT_IDENTITY" in os.environ
@@ -59,38 +41,18 @@ def _get_onepassword_service_account_token(env_key, tmp_file):
                 "teleport_proxy": "tele.vgijssel.nl",
                 "teleport_user": teleport_user,
                 "teleport_identity": teleport_identity,
-                "onepassword_service_account_token": _get_onepassword_service_account_token(
-                    "ONEPASSWORD_SERVICE_ACCOUNT_TOKEN_PROD",
-                    "1password-service-account-token-prod",
-                ),
             },
         ),
     ]
 
 elif setup_env == "test":
     container_id = os.environ["CONTAINER_ID"]
     test = [
-        (
-            f"@docker/{container_id}",
-            {
-                "onepassword_service_account_token": _get_onepassword_service_account_token(
-                    "ONEPASSWORD_SERVICE_ACCOUNT_TOKEN_DEV",
-                    "1password-service-account-token-dev",
-                ),
-            },
-        ),
+        (f"@docker/{container_id}", {}),
     ]
 
 else:
     container_id = "provisioner_dev"
     dev = [
-        (
-            f"@docker/{container_id}",
-            {
-                "onepassword_service_account_token": _get_onepassword_service_account_token(
-                    "ONEPASSWORD_SERVICE_ACCOUNT_TOKEN_DEV",
-                    "1password-service-account-token-dev",
-                ),
-            },
-        ),
+        (f"@docker/{container_id}", {}),
     ]

provisioner/test_provisioner.py

@@ -42,6 +42,70 @@ def test_ubuntu_focal(host):
     assert host.system_info.codename == "jammy"
 
 
+def test_user_added_to_sudo_group(host):
+    assert "sudo" in host.user("ubuntu").groups
+
+
+def test_docker_installed(host):
+    docker = host.package("docker-ce")
+    assert docker.is_installed
+
+
+def test_docker_compose_installed(host):
+    docker_compose = host.package("docker-compose-plugin")
+    assert docker_compose.is_installed
+
+
+def test_docker_service(host):
+    docker = host.service("docker")
+    assert docker.is_running
+    assert docker.is_enabled
+
+
+def test_user_added_to_docker_group(host):
+    assert "docker" in host.user("ubuntu").groups
+
+
+def test_newrelic_infra_installed(host):
+    newrelic_infra = host.package("newrelic-infra")
+    assert newrelic_infra.is_installed
+
+    server_version = semver.VersionInfo.parse(newrelic_infra.version)
+    wanted_version = semver.VersionInfo.parse("1.42.2")
+    assert server_version >= wanted_version
+
+
+def test_newrelic_infra_service(host):
+    newrelic_infra = host.service("newrelic-infra")
+    assert newrelic_infra.is_running
+    assert newrelic_infra.is_enabled
+
+
+def test_newrelic_infra_config(host):
+    config = host.file("/etc/newrelic-infra.yml")
+    assert config.exists
+    assert config.contains("license_key:")
+    assert "config validation finished without errors" in host.check_output(
+        "newrelic-infra -validate"
+    )
+
+
+def test_github_exporter_service(host):
+    github_exporter = host.docker("github_exporter")
+    assert github_exporter.is_running
+
+
+def test_nri_prometheus_service(host):
+    nri_prometheus = host.docker("nri-prometheus")
+    assert nri_prometheus.is_running
+
+
+def test_nri_prometheus_config(host):
+    config = host.file("/opt/monitoring/nri-prometheus-config.yaml")
+    assert config.exists
+    assert config.contains("http://github_exporter:9504/metrics")
+
+
 def test_microk8s_installed(host):
     assert "microk8s" in host.check_output("snap list")
 
@@ -90,6 +154,12 @@ def test_teleport_service(host):
     assert teleport.is_enabled
 
 
+def test_teleport_health(host):
+    assert '"status":"ok"' in host.check_output(
+        "curl --fail -L http://localhost:3000/healthz"
+    )
+
+
 def test_https_port_is_open(host):
     assert host.socket("tcp://0.0.0.0:443").is_listening
 

provisioner/utils.py

@@ -35,39 +35,3 @@ def wait_for_reconnect(name):
         name=name,
         function=_wait_for_reconnect,
     )
-
-
-def one_password_item(
-    item_title,
-    onepassword_vault_id=None,
-    onepassword_service_account_token=None,
-):
-    if onepassword_vault_id is None:
-        onepassword_vault_id = host.data.onepassword_vault_id
-
-    if onepassword_service_account_token is None:
-        onepassword_service_account_token = host.data.onepassword_service_account_token
-
-    command = "{op_binary} item get '{item_title}' --vault='{onepassword_vault_id}' --format=json".format(
-        op_binary=os.environ["OP_BINARY"],
-        item_title=item_title,
-        onepassword_vault_id=onepassword_vault_id,
-        onepassword_service_account_token=onepassword_service_account_token,
-    )
-
-    try:
-        # NOTE: this environ hackery is so that if the command fail the secret is not printed to stdout/stderr.
-        os.environ["OP_SERVICE_ACCOUNT_TOKEN"] = onepassword_service_account_token
-        json_string = local.shell(command, print_input=False)
-    finally:
-        del os.environ["OP_SERVICE_ACCOUNT_TOKEN"]
-        print("An exception occurred")
-
-    raw_data = json.loads(json_string)
-
-    data = {}
-
-    for field in raw_data["fields"]:
-        data[field["id"]] = field.get("value", None)
-
-    return data

renovate.json

@@ -41,5 +41,8 @@
   },
   "bazel-module": {
     "enabled": true
+  },
+  "docker-compose": {
+    "fileMatch": ["(^|/)docker-compose\\.yml\\.j2$"]
   }
 }

requirements.in

@@ -10,4 +10,5 @@ pytest-homeassistant-custom-component==0.13.32;
 tzdata==2023.3;
 pyinfra==2.7
 sqlalchemy==2.0.15;
-semver==2.13.0;
+semver==2.13.0;
+bazel-runfiles==0.22.0;