From e5bd8b1eeaf9517e1e400e9d70d11df7981904e5 Mon Sep 17 00:00:00 2001 From: Chris Sibbitt Date: Tue, 12 Dec 2023 15:23:08 -0500 Subject: [PATCH] update smoketest for prometheus auth --- .../tasks/component_grafana.yml | 10 ++-- .../tasks/component_prometheus.yml | 40 ------------- .../tasks/component_prometheus_reader.yml | 58 +++++++++++++++++++ roles/servicetelemetry/tasks/main.yml | 2 + .../templates/manifest_grafana_ds.j2 | 5 +- .../templates/manifest_prometheus.j2 | 9 +-- tests/smoketest/smoketest.sh | 6 +- .../smoketest_ceilometer_entrypoint.sh | 6 +- .../smoketest_collectd_entrypoint.sh | 8 +-- tests/smoketest/smoketest_job.yaml.template | 8 +-- 10 files changed, 84 insertions(+), 68 deletions(-) create mode 100644 roles/servicetelemetry/tasks/component_prometheus_reader.yml diff --git a/roles/servicetelemetry/tasks/component_grafana.yml b/roles/servicetelemetry/tasks/component_grafana.yml index 068507610..2a839b5d8 100644 --- a/roles/servicetelemetry/tasks/component_grafana.yml +++ b/roles/servicetelemetry/tasks/component_grafana.yml @@ -82,18 +82,18 @@ namespace: '{{ ansible_operator_meta.namespace }}' register: serving_certs_ca - - name: Retrieve prometheus secret + - name: Retrieve prometheus reader token k8s_info: api_version: v1 kind: Secret namespace: '{{ ansible_operator_meta.namespace }}' - name: '{{ ansible_operator_meta.name }}-prometheus-htpasswd' - register: prometheus_secret + name: stf-prometheus-reader-token + register: prometheus_reader_secret - - name: Decode prometheus password + - name: Decode prometheus reader token no_log: true set_fact: - prom_basicauth_passwd: '{{ prometheus_secret.resources[0].data.password | b64decode }}' + prometheus_reader_token: '{{ prometheus_reader_secret.resources[0].data.token | b64decode }}' # Lookup existing datasources - name: Remove legacy datasources diff --git a/roles/servicetelemetry/tasks/component_prometheus.yml b/roles/servicetelemetry/tasks/component_prometheus.yml index 2e865abd2..ac65044d5 100644 --- a/roles/servicetelemetry/tasks/component_prometheus.yml +++ b/roles/servicetelemetry/tasks/component_prometheus.yml @@ -173,46 +173,6 @@ name: prometheus-k8s-{{ ansible_operator_meta.namespace }} namespace: '{{ ansible_operator_meta.namespace }}' -- name: Check for existing prometheus htpasswd user secret - k8s_info: - api_version: v1 - kind: Secret - namespace: '{{ ansible_operator_meta.namespace }}' - name: '{{ ansible_operator_meta.name }}-prometheus-htpasswd' - register: prometheus_htpasswd - -- name: Create a new prometheus password if it doesn't exist yet - when: prometheus_htpasswd.resources|length == 0 - block: - - name: Set prometheus htpasswd - no_log: true - set_fact: - prom_basicauth_passwd: "{{ lookup('password', '/dev/null') }}" - - - name: Create htpasswd secret # Contains both the htpasswd version and plaintext for lookup - no_log: true - k8s: - definition: - api_version: v1 - kind: Secret - metadata: - name: '{{ ansible_operator_meta.name }}-prometheus-htpasswd' - namespace: '{{ ansible_operator_meta.namespace }}' - type: Opaque - stringData: - auth: 'internal:{{ prom_basicauth_passwd | password_hash("bcrypt") | replace("$2b$","$2y$", 1)}}' - password: '{{ prom_basicauth_passwd }}' - tags: - - skip_ansible_lint - - - name: Re-register new object for use in the annotation - k8s_info: - api_version: v1 - kind: Secret - namespace: '{{ ansible_operator_meta.namespace }}' - name: '{{ ansible_operator_meta.name }}-prometheus-htpasswd' - register: prometheus_htpasswd - - name: Lookup template debug: msg: "{{ lookup('template', './manifest_prometheus.j2') | from_yaml }}" diff --git a/roles/servicetelemetry/tasks/component_prometheus_reader.yml b/roles/servicetelemetry/tasks/component_prometheus_reader.yml new file mode 100644 index 000000000..6cbee8b42 --- /dev/null +++ b/roles/servicetelemetry/tasks/component_prometheus_reader.yml @@ -0,0 +1,58 @@ +- name: Create ServiceAccount/stf-prometheus-reader + k8s: + state: '{{ "present" if servicetelemetry_vars.backends.metrics.prometheus.enabled else "absent" }}' + definition: + apiVersion: v1 + kind: ServiceAccount + metadata: + name: stf-prometheus-reader + namespace: '{{ ansible_operator_meta.namespace }}' + +- name: Create prometheus-reader Role + k8s: + state: '{{ "present" if servicetelemetry_vars.backends.metrics.prometheus.enabled else "absent" }}' + definition: + apiVersion: rbac.authorization.k8s.io/v1 + kind: Role + metadata: + name: prometheus-reader + namespace: '{{ ansible_operator_meta.namespace }}' + rules: + - apiGroups: + - '{{ prometheus_operator_api_string | replace("/v1","") }}' + resources: + - prometheus + verbs: + - get + namespaces: + - '{{ ansible_operator_meta.namespace }}' + +- name: Create prometheus-reader RoleBinding for stf-prometheus-reader + k8s: + state: '{{ "present" if servicetelemetry_vars.backends.metrics.prometheus.enabled else "absent" }}' + definition: + apiVersion: rbac.authorization.k8s.io/v1 + kind: RoleBinding + metadata: + name: stf-prometheus-reader + namespace: '{{ ansible_operator_meta.namespace }}' + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: prometheus-reader + subjects: + - kind: ServiceAccount + name: stf-prometheus-reader + +- name: Create an access token for stf-prometheus-reader + k8s: + state: '{{ "present" if servicetelemetry_vars.backends.metrics.prometheus.enabled else "absent" }}' + definition: + apiVersion: v1 + kind: Secret + metadata: + name: stf-prometheus-reader-token + namespace: '{{ ansible_operator_meta.namespace }}' + annotations: + kubernetes.io/service-account.name: stf-prometheus-reader + type: kubernetes.io/service-account-token diff --git a/roles/servicetelemetry/tasks/main.yml b/roles/servicetelemetry/tasks/main.yml index bc33df647..dc3e881c7 100644 --- a/roles/servicetelemetry/tasks/main.yml +++ b/roles/servicetelemetry/tasks/main.yml @@ -47,6 +47,8 @@ - block: - name: Create Prometheus instance include_tasks: component_prometheus.yml + - name: Create Prometheus read-only user + include_tasks: component_prometheus_reader.yml # --> alerting - name: Create Alertmanager instance diff --git a/roles/servicetelemetry/templates/manifest_grafana_ds.j2 b/roles/servicetelemetry/templates/manifest_grafana_ds.j2 index d0f0478d1..a453b311a 100644 --- a/roles/servicetelemetry/templates/manifest_grafana_ds.j2 +++ b/roles/servicetelemetry/templates/manifest_grafana_ds.j2 @@ -12,14 +12,13 @@ spec: jsonData: timeInterval: 5s tlsAuthWithCACert: true + httpHeaderName1: 'Authorization' name: STFPrometheus type: prometheus url: 'https://{{ ansible_operator_meta.name }}-prometheus-proxy.{{ ansible_operator_meta.namespace }}.svc:9092' version: 1 - basicAuth: true - basicAuthUser: internal secureJsonData: - basicAuthPassword: '{{ prom_basicauth_passwd }}' + httpHeaderValue1: 'Bearer {{prometheus_reader_token}}' tlsCACert: | {{ serving_certs_ca.resources[0].data['service-ca.crt'] | indent(10) }} {% endif %} diff --git a/roles/servicetelemetry/templates/manifest_prometheus.j2 b/roles/servicetelemetry/templates/manifest_prometheus.j2 index 2bdf408b9..e9eb63786 100644 --- a/roles/servicetelemetry/templates/manifest_prometheus.j2 +++ b/roles/servicetelemetry/templates/manifest_prometheus.j2 @@ -20,7 +20,6 @@ spec: labels: prometheus: '{{ ansible_operator_meta.name }}' annotations: - hash-of-entire-htpasswd-secret-to-force-restart-if-changed: {{ prometheus_htpasswd | sha1 }} {% if servicetelemetry_vars.alerting.enabled %} alerting: alertmanagers: @@ -42,10 +41,11 @@ spec: - -tls-cert=/etc/tls/private/tls.crt - -tls-key=/etc/tls/private/tls.key - -upstream=http://localhost:9090/ - - -htpasswd-file=/etc/proxy/htpasswd/auth - -cookie-secret-file=/etc/proxy/secrets/session_secret - -openshift-service-account=prometheus-stf - - '-openshift-sar={"resource": "namespaces", "verb": "get"}' + - '-openshift-sar={"namespace":"{{ ansible_operator_meta.namespace }}","resource": "prometheus", "group":"{{ prometheus_operator_api_string | replace("/v1","") }}", "verb":"get"}' + - '-openshift-delegate-urls={"/":{"namespace":"{{ ansible_operator_meta.namespace }}","resource": "prometheus", "group":"{{ prometheus_operator_api_string | replace("/v1","") }}", "verb":"get"}}' + ports: - containerPort: 9092 name: https @@ -55,14 +55,11 @@ spec: name: secret-{{ ansible_operator_meta.name }}-prometheus-proxy-tls - mountPath: /etc/proxy/secrets name: secret-{{ ansible_operator_meta.name }}-session-secret - - mountPath: /etc/proxy/htpasswd - name: secret-{{ ansible_operator_meta.name }}-prometheus-htpasswd configMaps: - serving-certs-ca-bundle secrets: - '{{ ansible_operator_meta.name }}-prometheus-proxy-tls' - '{{ ansible_operator_meta.name }}-session-secret' - - '{{ ansible_operator_meta.name }}-prometheus-htpasswd' {% if servicetelemetry_vars.backends.metrics.prometheus.storage.strategy == "persistent" %} storage: volumeClaimTemplate: diff --git a/tests/smoketest/smoketest.sh b/tests/smoketest/smoketest.sh index caaeb4e88..177157fe3 100755 --- a/tests/smoketest/smoketest.sh +++ b/tests/smoketest/smoketest.sh @@ -47,8 +47,8 @@ echo "*** [INFO] Working in project ${OCP_PROJECT}" echo "*** [INFO] Getting ElasticSearch authentication password" ELASTICSEARCH_AUTH_PASS=$(oc get secret elasticsearch-es-elastic-user -ogo-template='{{ .data.elastic | base64decode }}') -echo "*** [INFO] Getting Prometheus authentication password" -PROMETHEUS_AUTH_PASS=$(oc get secret default-prometheus-htpasswd -ogo-template='{{ .data.password | base64decode }}') +echo "*** [INFO] Getting Prometheus authentication token" +PROMETHEUS_AUTH_TOKEN=$(oc create token stf-prometheus-reader) echo "*** [INFO] Creating configmaps..." oc delete configmap/stf-smoketest-healthcheck-log configmap/stf-smoketest-collectd-config configmap/stf-smoketest-sensubility-config configmap/stf-smoketest-collectd-entrypoint-script configmap/stf-smoketest-ceilometer-publisher configmap/stf-smoketest-ceilometer-entrypoint-script job/stf-smoketest || true @@ -76,7 +76,7 @@ oc wait --for=jsonpath='{.status.phase}'=Running pod/qdr-test echo "*** [INFO] Creating smoketest jobs..." oc delete job -l app=stf-smoketest for NAME in "${CLOUDNAMES[@]}"; do - oc create -f <(sed -e "s/<>/${NAME}/;s/<>/${ELASTICSEARCH_AUTH_PASS}/;s/<>/${PROMETHEUS_AUTH_PASS}/" ${REL}/smoketest_job.yaml.template) + oc create -f <(sed -e "s/<>/${NAME}/;s/<>/${ELASTICSEARCH_AUTH_PASS}/;s/<>/${PROMETHEUS_AUTH_TOKEN}/" ${REL}/smoketest_job.yaml.template) done echo "*** [INFO] Triggering an alertmanager notification..." diff --git a/tests/smoketest/smoketest_ceilometer_entrypoint.sh b/tests/smoketest/smoketest_ceilometer_entrypoint.sh index adf3a9046..0fc6f232f 100644 --- a/tests/smoketest/smoketest_ceilometer_entrypoint.sh +++ b/tests/smoketest/smoketest_ceilometer_entrypoint.sh @@ -5,7 +5,7 @@ set +e PROMETHEUS=${PROMETHEUS:-"https://default-prometheus-proxy:9092"} ELASTICSEARCH=${ELASTICSEARCH:-"https://elasticsearch-es-http:9200"} ELASTICSEARCH_AUTH_PASS=${ELASTICSEARCH_AUTH_PASS:-""} -PROMETHEUS_AUTH_PASS=${PROMETHEUS_AUTH_PASS:-""} +PROMETHEUS_AUTH_TOKEN=${PROMETHEUS_AUTH_TOKEN:-""} CLOUDNAME=${CLOUDNAME:-"smoke1"} POD=$(hostname) @@ -20,14 +20,14 @@ echo "*** [INFO] Sleeping for 30 seconds to produce all metrics and events" sleep 30 echo "*** [INFO] List of metric names for debugging..." -curl -sk -u "internal:${PROMETHEUS_AUTH_PASS}" -g "${PROMETHEUS}/api/v1/label/__name__/values" 2>&2 | tee /tmp/label_names +curl -sk -H "Authorization: Bearer ${PROMETHEUS_AUTH_TOKEN}" -g "${PROMETHEUS}/api/v1/label/__name__/values" 2>&2 | tee /tmp/label_names echo; echo # Checks that the metrics actually appear in prometheus echo "*** [INFO] Checking for recent image metrics..." echo "[DEBUG] Running the curl command to return a query" -curl -k -u "internal:${PROMETHEUS_AUTH_PASS}" -g "${PROMETHEUS}/api/v1/query?" --data-urlencode 'query=ceilometer_image_size' 2>&1 | grep '"result":\[{"metric":{"__name__":"ceilometer_image_size"' +curl -k -H "Authorization: Bearer ${PROMETHEUS_AUTH_TOKEN}" -g "${PROMETHEUS}/api/v1/query?" --data-urlencode 'query=ceilometer_image_size' 2>&1 | grep '"result":\[{"metric":{"__name__":"ceilometer_image_size"' metrics_result=$? echo "[DEBUG] Set metrics_result to $metrics_result" diff --git a/tests/smoketest/smoketest_collectd_entrypoint.sh b/tests/smoketest/smoketest_collectd_entrypoint.sh index d7f5132e8..d0dd800c3 100755 --- a/tests/smoketest/smoketest_collectd_entrypoint.sh +++ b/tests/smoketest/smoketest_collectd_entrypoint.sh @@ -5,7 +5,7 @@ set +e PROMETHEUS=${PROMETHEUS:-"https://default-prometheus-proxy:9092"} ELASTICSEARCH=${ELASTICSEARCH:-"https://elasticsearch-es-http:9200"} ELASTICSEARCH_AUTH_PASS=${ELASTICSEARCH_AUTH_PASS:-""} -PROMETHEUS_AUTH_PASS=${PROMETHEUS_AUTH_PASS:-""} +PROMETHEUS_AUTH_TOKEN=${PROMETHEUS_AUTH_TOKEN:-""} CLOUDNAME=${CLOUDNAME:-"smoke1"} POD=$(hostname) @@ -37,12 +37,12 @@ sleep 30 echo "*** [INFO] List of metric names for debugging..." -curl -k -u "internal:${PROMETHEUS_AUTH_PASS}" -g "${PROMETHEUS}/api/v1/label/__name__/values" 2>&2 | tee /tmp/label_names +curl -k -H "Authorization: Bearer ${PROMETHEUS_AUTH_TOKEN}" -g "${PROMETHEUS}/api/v1/label/__name__/values" 2>&2 | tee /tmp/label_names echo; echo # Checks that the metrics actually appear in prometheus echo "*** [INFO] Checking for recent CPU metrics..." -curl -k -u "internal:${PROMETHEUS_AUTH_PASS}" -g "${PROMETHEUS}/api/v1/query?" --data-urlencode 'query=collectd_cpu_total{container="sg-core",plugin_instance="0",type_instance="user",service="default-cloud1-coll-meter",host="'"${POD}"'"}[1m]' 2>&2 | tee /tmp/query_output +curl -k -H "Authorization: Bearer ${PROMETHEUS_AUTH_TOKEN}" -g "${PROMETHEUS}/api/v1/query?" --data-urlencode 'query=collectd_cpu_total{container="sg-core",plugin_instance="0",type_instance="user",service="default-cloud1-coll-meter",host="'"${POD}"'"}[1m]' 2>&2 | tee /tmp/query_output echo; echo # The egrep exit code is the result of the test and becomes the container/pod/job exit code @@ -53,7 +53,7 @@ echo; echo # Checks that the metrics actually appear in prometheus echo "*** [INFO] Checking for recent healthcheck metrics..." -curl -k -u "internal:${PROMETHEUS_AUTH_PASS}" -g "${PROMETHEUS}/api/v1/query?" --data-urlencode 'query=sensubility_container_health_status{container="sg-core",service="default-cloud1-sens-meter",host="'"${POD}"'"}[1m]' 2>&2 | tee /tmp/query_output +curl -k -H "Authorization: Bearer ${PROMETHEUS_AUTH_TOKEN}" -g "${PROMETHEUS}/api/v1/query?" --data-urlencode 'query=sensubility_container_health_status{container="sg-core",service="default-cloud1-sens-meter",host="'"${POD}"'"}[1m]' 2>&2 | tee /tmp/query_output echo; echo # The egrep exit code is the result of the test and becomes the container/pod/job exit code diff --git a/tests/smoketest/smoketest_job.yaml.template b/tests/smoketest/smoketest_job.yaml.template index 4a9c20cc9..12626f3b2 100644 --- a/tests/smoketest/smoketest_job.yaml.template +++ b/tests/smoketest/smoketest_job.yaml.template @@ -22,8 +22,8 @@ spec: value: <> - name: ELASTICSEARCH_AUTH_PASS value: "<>" - - name: PROMETHEUS_AUTH_PASS - value: "<>" + - name: PROMETHEUS_AUTH_TOKEN + value: "<>" - name: OBSERVABILITY_STRATEGY value: "<>" volumeMounts: @@ -51,8 +51,8 @@ spec: value: <> - name: ELASTICSEARCH_AUTH_PASS value: "<>" - - name: PROMETHEUS_AUTH_PASS - value: "<>" + - name: PROMETHEUS_AUTH_TOKEN + value: "<>" - name: OBSERVABILITY_STRATEGY value: "<>" volumeMounts: