Add rancher-monitoring for E2E

Signed-off-by: Paulo Gomes <[email protected]>

.github/workflows/tests.yml

@@ -24,13 +24,14 @@ jobs:
   e2e:
     strategy:
       matrix:
-        distro: [rocky-8, rocky-9, leap]
+        distro: [centos8, centos9]
 
     runs-on: ubuntu-latest
     steps:
     - name: Checkout code
       uses: actions/checkout@v4
 
+    # https://lima-vm.io/docs/examples/gha/
     - name: "Install QEMU"
       run: |
         set -eux

Makefile

@@ -1,6 +1,7 @@
 RUNNER ?= docker
 POLICIES = $(shell find policy -mindepth 1 -maxdepth 1 -type d | sort -u | cut -f 2 -d'/')
 DISTROS = $(shell find hack/e2e -type f | grep .yaml | sort -u | cut -f3 -d'/' | cut -f1 -d.)
+LIMA_DEBUG :=
 
 # GPG Signing
 DRY_RUN ?= false
@@ -99,18 +100,22 @@ endif
 	@echo RPM_CHANNEL: $(RPM_CHANNEL)
 	@echo VERSION: $(VERSION)
 
-LIMA_DEBUG :=
-LIMA_DEBUG = --debug
-
 e2e:
 	$(MAKE) $(addprefix push-tool-, $(DISTROS))
 
-# TODO: push the selinux into the VM
-# TODO: Apply the selinux rancher on helm install
 e2e-%:
+	make $(subst :,/,$*)-build-image
+	make $(subst :,/,$*)-build-artefacts
+
 	limactl start $(LIMA_DEBUG) --tty=false --cpus 6 --memory 8 --plain --name=$(subst :,/,$*) hack/e2e/$(subst :,/,$*).yaml
+	limactl cp build/$(subst :,/,$*)/noarch/rancher-*.rpm $(subst :,/,$*):/tmp/rancher-selinux.rpm
 	limactl cp hack/e2e/setup-vm.sh $(subst :,/,$*):/tmp/setup-vm.sh
 	limactl shell $(subst :,/,$*) sudo /tmp/setup-vm.sh
+
+	limactl stop $(subst :,/,$*)
+	limactl delete $(subst :,/,$*)
+
+e2e-%-clean:
 	limactl stop $(subst :,/,$*)
 	limactl delete $(subst :,/,$*)
 

hack/e2e/rocky-8.yaml → hack/e2e/centos8.yaml

@@ -19,11 +19,15 @@ images:
 - location: "https://dl.rockylinux.org/pub/rocky/8/images/aarch64/Rocky-8-GenericCloud.latest.aarch64.qcow2"
   arch: "aarch64"
 mountTypesUnsupported: ["9p"]
-mounts:
-- location: "~"
-- location: "/tmp/lima"
-  writable: true
+mounts: []
 cpuType:
   # Workaround for "vmx_write_mem: mmu_gva_to_gpa XXXXXXXXXXXXXXXX failed" on Intel Mac
   # https://bugs.launchpad.net/qemu/+bug/1838390
   x86_64: "Haswell-v4"
+
+provision:
+- mode: system
+  script: |
+    #!/bin/sh
+
+    yum in -y git container-selinux

hack/e2e/rocky-9.yaml → hack/e2e/centos9.yaml

@@ -17,7 +17,11 @@ images:
 - location: "https://dl.rockylinux.org/pub/rocky/9/images/aarch64/Rocky-9-GenericCloud.latest.aarch64.qcow2"
   arch: "aarch64"
 mountTypesUnsupported: ["9p"]
-mounts:
-- location: "~"
-- location: "/tmp/lima"
-  writable: true
+mounts: []
+
+provision:
+- mode: system
+  script: |
+    #!/bin/sh
+
+    yum in -y git container-selinux

hack/e2e/setup-vm.sh

@@ -1,19 +1,21 @@
 #!/bin/bash
 
-set -xo pipefail
+set -euxo pipefail
 
 function enforceSELinux(){
     echo "> Check SELinux status"
     # Short circuit if SELinux is not being enforced.
     getenforce | grep -q Enforcing
+
+    sudo dnf install -y /tmp/rancher-selinux.rpm
 }
 
 function installDependencies(){
     echo 'echo "export PATH=$PATH:/usr/local/bin"' >> ~/.bashrc
     echo 'echo "export TERM=xterm"' >> ~/.bashrc
 
     # Git is required by helm
-    yum in -y git
+    # yum in -y git
 
     echo "> Installing Helm 3"
     curl https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3 | bash
@@ -62,6 +64,38 @@ function installRancher(){
 
     kubectl wait --for=condition=ready -n cattle-system pod -l app=rancher --timeout=60s
     kubectl wait --for=condition=ready -n cattle-system pod -l app=rancher-webhook --timeout=60s
+
+    # Background processes, such as Fleet deployment need to take place, which
+    # may result in intermittent errors. Add some additional waiting time to
+    # accommodate such processes.
+    sleep 60
+}
+
+
+function installRancherMonitoring(){
+    helm repo add rancher-charts https://charts.rancher.io/
+
+    helm upgrade --install=true \
+        --labels=catalog.cattle.io/cluster-repo-name=rancher-charts \
+        --namespace=cattle-monitoring-system --timeout=10m0s --wait=true \
+        --create-namespace \
+        rancher-monitoring-crd rancher-charts/rancher-monitoring-crd
+
+    helm upgrade --install=true \
+        --labels=catalog.cattle.io/cluster-repo-name=rancher-charts \
+        --namespace=cattle-monitoring-system --timeout=10m0s --wait=true \
+        --create-namespace \
+        rancher-monitoring rancher-charts/rancher-monitoring
+
+    # Ensure exporter is working before SELinux policy is applied
+    kubectl wait --for=condition=ready -n cattle-monitoring-system pod -l app.kubernetes.io/name=prometheus-node-exporter --timeout=60s
+
+    # TODO: Move this to a helm chart value
+    kubectl patch daemonset rancher-monitoring-prometheus-node-exporter  -n cattle-monitoring-system -p '{"spec": {"template": {"spec": 
+    { "securityContext": {"seLinuxOptions": {"type": "prom_node_exporter_t"}}}}}}'
+
+    # Ensure exporter comes back after SELinux policy is applied
+    kubectl wait --for=condition=ready -n cattle-monitoring-system pod -l app.kubernetes.io/name=prometheus-node-exporter --timeout=60s
 }
 
 function E2E(){
@@ -73,6 +107,7 @@ function main(){
     installDependencies
     installRKE2
     installRancher
+    installRancherMonitoring
 
     E2E
 }