WIP

Signed-off-by: Pete Wall <[email protected]>
grafana · Feb 12, 2025 · 9c27fc0 · 9c27fc0
1 parent 9abacd7
commit 9c27fc0
Show file tree

Hide file tree

Showing 8 changed files with 195 additions and 7 deletions.
diff --git a/charts/k8s-monitoring/tests/platform/openshift/.envrc b/charts/k8s-monitoring/tests/platform/openshift/.envrc
@@ -1,7 +1,8 @@
-export GCP_SERVICE_ACCOUNT=$(op --account grafana.1password.com read "op://Kubernetes Monitoring/GCP k8s-monitoring-helm-cluster-creator/username")
-op --account grafana.1password.com read --out-file sak.json "op://Kubernetes Monitoring/GCP k8s-monitoring-helm-cluster-creator/credential"
-gcloud auth activate-service-account "${GCP_SERVICE_ACCOUNT}" --key-file=sak.json
-rm sak.json
+mkdir -p cluster-installer-files
+rm -f cluster-installer-files/gcp_service_account_key.json
+op --account grafana.1password.com read --out-file cluster-installer-files/gcp_service_account_key.json "op://Kubernetes Monitoring/GCP k8s-monitoring-helm-cluster-creator/credential"
+export GOOGLE_CLOUD_KEYFILE_JSON=$(pwd)/cluster-installer-files/gcp_service_account_key.json
+export KUBECONFIG=$(pwd)/cluster-installer-files/auth/kubeconfig
 
 export GRAFANA_CLOUD_METRICS_USERNAME=$(op --account grafana.1password.com read "op://Kubernetes Monitoring/helmchart Prometheus/username")
 export GRAFANA_CLOUD_LOGS_USERNAME=$(op --account grafana.1password.com read "op://Kubernetes Monitoring/helmchart Loki/username")

diff --git a/charts/k8s-monitoring/tests/platform/openshift/.gitignore b/charts/k8s-monitoring/tests/platform/openshift/.gitignore
@@ -0,0 +1,3 @@
+cluster-installer-files
+deployments/grafana-cloud-credentials.yaml
+deployments/test-variables.yaml
diff --git a/charts/k8s-monitoring/tests/platform/openshift/Makefile b/charts/k8s-monitoring/tests/platform/openshift/Makefile
@@ -0,0 +1,36 @@
+all: deployments/test-variables.yaml deployments/grafana-cloud-credentials.yaml
+clean:
+	rm -f deployments/test-variables.yaml deployments/grafana-cloud-credentials.yaml
+
+deployments/test-variables.yaml:
+	echo "---" > $@
+	kubectl create configmap test-variables \
+		--from-literal=CLUSTER="$(shell yq eval '.cluster.name' values.yaml)-$$RANDOM_NUMBER" \
+		--from-literal=RANDOM_NUMBER="$$RANDOM_NUMBER" \
+		--from-literal=ANNOTATION_POD_LOG_SERVICE_NAME="hello-kubernetes-annotation" \
+		--from-literal=LABEL_POD_LOG_SERVICE_NAME="hello-kubernetes-label" \
+		-o yaml --dry-run=client >> $@
+
+deployments/grafana-cloud-credentials.yaml:
+	echo "---" > $@
+	echo "# yamllint disable rule:line-length" >> $@
+	kubectl create secret generic grafana-cloud-credentials \
+  		--from-literal=PROMETHEUS_USER="$$GRAFANA_CLOUD_METRICS_USERNAME" \
+  		--from-literal=PROMETHEUS_PASS="$$GRAFANA_CLOUD_RW_POLICY_TOKEN" \
+  		--from-literal=LOKI_USER="$$GRAFANA_CLOUD_LOGS_USERNAME" \
+  		--from-literal=LOKI_PASS="$$GRAFANA_CLOUD_RW_POLICY_TOKEN" \
+		-o yaml --dry-run=client >> $@
+
+flux-manifests/flux.yaml:
+	flux install \
+		--components source-controller,helm-controller \
+		--export > $@
+
+flux-manifests/scc.yaml:
+	wget https://raw.githubusercontent.com/fluxcd/flux2/main/manifests/openshift/scc.yaml -O $@
+
+.PHONY: flux-manifests
+flux-manifests: flux-manifests/flux.yaml flux-manifests/scc.yaml
+
+run-test:
+	../../../../../scripts/run-cluster-test.sh .
diff --git a/charts/k8s-monitoring/tests/platform/openshift/deployments/query-test.yaml b/charts/k8s-monitoring/tests/platform/openshift/deployments/query-test.yaml
@@ -0,0 +1,45 @@
+---
+apiVersion: source.toolkit.fluxcd.io/v1
+kind: GitRepository
+metadata:
+  name: k8s-monitoring-test
+spec:
+  interval: 1m
+  url: https://github.com/grafana/k8s-monitoring-helm
+  ref:
+    branch: main
+  ignore: |
+    /*
+    !/charts/k8s-monitoring-test
+---
+apiVersion: helm.toolkit.fluxcd.io/v2
+kind: HelmRelease
+metadata:
+  name: k8s-monitoring-test
+spec:
+  interval: 1m
+  chart:
+    spec:
+      chart: charts/k8s-monitoring-test
+      sourceRef:
+        kind: GitRepository
+        name: k8s-monitoring-test
+      interval: 1m
+  values:
+    tests:
+      - env:
+          PROMETHEUS_URL: https://prometheus-prod-13-prod-us-east-0.grafana.net/api/prom/api/v1/query
+          LOKI_URL: https://logs-prod-006.grafana.net/loki/api/v1/query
+        envFrom:
+          - secretRef: {name: grafana-cloud-credentials}
+          - configMapRef: {name: test-variables}
+        queries:
+          # Self reporting metrics
+          - query: grafana_kubernetes_monitoring_build_info{cluster="$CLUSTER"}
+            type: promql
+          - query: grafana_kubernetes_monitoring_feature_info{cluster="$CLUSTER", feature="clusterMetrics"}
+            type: promql
+          - query: grafana_kubernetes_monitoring_feature_info{cluster="$CLUSTER", feature="clusterEvents"}
+            type: promql
+          - query: grafana_kubernetes_monitoring_feature_info{cluster="$CLUSTER", feature="podLogs", method="volumes"}
+            type: promql
diff --git a/charts/k8s-monitoring/tests/platform/openshift/openshift-cluster-config.yaml b/charts/k8s-monitoring/tests/platform/openshift/openshift-cluster-config.yaml
@@ -0,0 +1,36 @@
+additionalTrustBundlePolicy: Proxyonly
+apiVersion: v1
+#metadata:
+#  name: WILL BE REPLACED BY scripts/include/cluster/openshift.sh
+baseDomain: okd.grafana.petewall.net
+controlPlane:
+  name: master
+  replicas: 1
+  platform:
+    gcp:
+      type: e2-standard-8
+  architecture: amd64
+  hyperthreading: Enabled
+compute:
+- name: worker
+  replicas: 0
+  platform:
+    gcp:
+      type: custom-6-20480
+  architecture: amd64
+  hyperthreading: Enabled
+networking:
+  clusterNetwork:
+  - cidr: 10.128.0.0/14
+    hostPrefix: 23
+  machineNetwork:
+  - cidr: 10.0.0.0/16
+  networkType: OVNKubernetes
+  serviceNetwork:
+  - 172.30.0.0/16
+platform:
+  gcp:
+    projectID: grafana-k8s-monitoring
+    region: us-central1
+publish: External
+pullSecret: '{"auths":{"fake":{"auth":"aWQ6cGFzcwo="}}}'
diff --git a/charts/k8s-monitoring/tests/platform/openshift/values.yaml b/charts/k8s-monitoring/tests/platform/openshift/values.yaml
@@ -0,0 +1,57 @@
+cluster:
+  name: openshift-test
+
+global:
+  platform: openshift
+
+destinations:
+  - name: grafanaCloudMetrics
+    type: prometheus
+    url: https://prometheus-prod-13-prod-us-east-0.grafana.net/api/prom/push
+    auth:
+      type: basic
+      usernameKey: PROMETHEUS_USER
+      passwordKey: PROMETHEUS_PASS
+    secret:
+      create: false
+      name: grafana-cloud-credentials
+  - name: grafanaCloudLogs
+    type: loki
+    url: https://logs-prod-006.grafana.net/loki/api/v1/push
+    auth:
+      type: basic
+      usernameKey: LOKI_USER
+      passwordKey: LOKI_PASS
+    secret:
+      create: false
+      name: grafana-cloud-credentials
+
+clusterMetrics:
+  enabled: true
+  opencost:
+    enabled: true
+    metricsSource: grafanaCloudMetrics
+    opencost:
+      exporter:
+        defaultClusterId: k8s-monitoring-gc-feature-test
+      prometheus:
+        external:
+          url: https://prometheus-prod-13-prod-us-east-0.grafana.net/api/prom
+        existingSecretName: grafana-cloud-credentials
+        username_key: PROMETHEUS_USER
+        password_key: PROMETHEUS_PASS
+  kepler:
+    enabled: true
+
+clusterEvents:
+  enabled: true
+
+podLogs:
+  enabled: true
+
+integrations:
+  alloy:
+    instances:
+      - name: alloy
+        labelSelectors:
+          app.kubernetes.io/name: [alloy-metrics, alloy-singleton, alloy-logs]
diff --git a/scripts/includes/cluster/openshift.sh b/scripts/includes/cluster/openshift.sh
@@ -4,12 +4,16 @@ createOpenShiftCluster() {
   local clusterName=$1
   local clusterConfig=$2
 
-  ssh-keygen -t ed25519 -N '' -f "${clusterName}-key"
-
+  clusterInstallerFilesDir=$(dirname "${clusterConfig}")/cluster-installer-files
+  mkdir -p "${clusterInstallerFilesDir}"
+  yq ".metadata.name=\"${clusterName}\"" "${clusterConfig}" > "${clusterInstallerFilesDir}/install-config.yaml"
+#  openshift-install create cluster --dir "${clusterInstallerFilesDir}"
 }
 
 deleteOpenShiftCluster() {
   local clusterName=$1
   local clusterConfig=$2
 
-}
+  clusterInstallerFilesDir=$(dirname "${clusterConfig}")/cluster-installer-files
+  openshift-install destroy cluster --dir "${clusterInstallerFilesDir}"
+}
diff --git a/scripts/run-cluster-test.sh b/scripts/run-cluster-test.sh
@@ -6,6 +6,7 @@ source "${PARENT_DIR}/scripts/includes/logging.sh"
 source "${PARENT_DIR}/scripts/includes/cluster/eks.sh"
 source "${PARENT_DIR}/scripts/includes/cluster/gke.sh"
 source "${PARENT_DIR}/scripts/includes/cluster/kind.sh"
+source "${PARENT_DIR}/scripts/includes/cluster/openshift.sh"
 
 heading "Kubernetes Monitoring Helm" "Cluster Test Runner"
 
@@ -22,6 +23,7 @@ usage() {
   echo "    eks-cluster-config.yaml           - Config file for creating an EKS cluster."
   echo "    gke-cluster-config.yaml           - Config file for creating a GKE cluster."
   echo "    gke-autopilot-cluster-config.yaml - Config file for creating a GKE Autopilot cluster."
+  echo "    openshift-cluster-config.yaml     - Config file for creating an OpenShift cluster."
 }
 
 CREATE_CLUSTER=${CREATE_CLUSTER:-true}
@@ -55,6 +57,8 @@ if [ "${CREATE_CLUSTER}" == "true" ]; then
     createGKECluster "${clusterName}" "${TEST_DIRECTORY}/gke-cluster-config.yaml"
   elif [ -f "${TEST_DIRECTORY}/gke-autopilot-cluster-config.yaml" ]; then
     createGKEAutopilotCluster "${clusterName}" "${TEST_DIRECTORY}/gke-autopilot-cluster-config.yaml"
+  elif [ -f "${TEST_DIRECTORY}/openshift-cluster-config.yaml" ]; then
+    createOpenShiftCluster "${clusterName}" "${TEST_DIRECTORY}/openshift-cluster-config.yaml"
   else
     createKindCluster "${clusterName}"
   fi
@@ -69,6 +73,8 @@ deleteCluster() {
     deleteGKECluster "${clusterName}" "${TEST_DIRECTORY}/gke-cluster-config.yaml"
   elif [ -f "${TEST_DIRECTORY}/gke-autopilot-cluster-config.yaml" ]; then
     deleteGKECluster "${clusterName}" "${TEST_DIRECTORY}/gke-autopilot-cluster-config.yaml"
+  elif [ -f "${TEST_DIRECTORY}/openshift-cluster-config.yaml" ]; then
+    deleteOpenShiftCluster "${clusterName}" "${TEST_DIRECTORY}/openshift-cluster-config.yaml"
   else
     deleteKindCluster "${clusterName}"
   fi