From af438370829488eb35bf71b3c4eb8f71d47b0b05 Mon Sep 17 00:00:00 2001 From: Robert Lankford Date: Thu, 6 Feb 2025 12:04:30 -0800 Subject: [PATCH] add application-observability platform test (#1206) * add application-observability platform test --------- Signed-off-by: Robbie Lankford --- .../grafana-cloud/app-observability/.envrc | 5 + .../app-observability}/.gitignore | 0 .../app-observability/.rendered/output.yaml | 1068 +++++++++++++++++ .../grafana-cloud/app-observability/Makefile | 30 + .../grafana-cloud/app-observability/README.md | 0 .../deployments/otel-demo-manifest.yaml | 69 ++ .../deployments/query-test.yaml | 39 + .../app-observability/values.yaml | 72 ++ .../k8s-monitoring/.envrc | 0 .../grafana-cloud/k8s-monitoring/.gitignore | 2 + .../k8s-monitoring/.rendered/output.yaml | 0 .../k8s-monitoring/Makefile | 0 .../deployments/query-test.yaml | 0 .../k8s-monitoring/values.yaml | 0 14 files changed, 1285 insertions(+) create mode 100644 charts/k8s-monitoring/tests/platform/grafana-cloud/app-observability/.envrc rename charts/k8s-monitoring/tests/platform/{grafana-cloud-features/k8s-monitoring => grafana-cloud/app-observability}/.gitignore (100%) create mode 100644 charts/k8s-monitoring/tests/platform/grafana-cloud/app-observability/.rendered/output.yaml create mode 100644 charts/k8s-monitoring/tests/platform/grafana-cloud/app-observability/Makefile create mode 100644 charts/k8s-monitoring/tests/platform/grafana-cloud/app-observability/README.md create mode 100644 charts/k8s-monitoring/tests/platform/grafana-cloud/app-observability/deployments/otel-demo-manifest.yaml create mode 100644 charts/k8s-monitoring/tests/platform/grafana-cloud/app-observability/deployments/query-test.yaml create mode 100644 charts/k8s-monitoring/tests/platform/grafana-cloud/app-observability/values.yaml rename charts/k8s-monitoring/tests/platform/{grafana-cloud-features => grafana-cloud}/k8s-monitoring/.envrc (100%) create mode 100644 charts/k8s-monitoring/tests/platform/grafana-cloud/k8s-monitoring/.gitignore rename charts/k8s-monitoring/tests/platform/{grafana-cloud-features => grafana-cloud}/k8s-monitoring/.rendered/output.yaml (100%) rename charts/k8s-monitoring/tests/platform/{grafana-cloud-features => grafana-cloud}/k8s-monitoring/Makefile (100%) rename charts/k8s-monitoring/tests/platform/{grafana-cloud-features => grafana-cloud}/k8s-monitoring/deployments/query-test.yaml (100%) rename charts/k8s-monitoring/tests/platform/{grafana-cloud-features => grafana-cloud}/k8s-monitoring/values.yaml (100%) diff --git a/charts/k8s-monitoring/tests/platform/grafana-cloud/app-observability/.envrc b/charts/k8s-monitoring/tests/platform/grafana-cloud/app-observability/.envrc new file mode 100644 index 000000000..285ec8878 --- /dev/null +++ b/charts/k8s-monitoring/tests/platform/grafana-cloud/app-observability/.envrc @@ -0,0 +1,5 @@ +export GRAFANA_CLOUD_OTLP_USERNAME=$(op --account grafana.1password.com read "op://Kubernetes Monitoring/helmchart OTLP/username") +export GRAFANA_CLOUD_METRICS_USERNAME=$(op --account grafana.1password.com read "op://Kubernetes Monitoring/helmchart Prometheus/username") +export GRAFANA_CLOUD_LOGS_USERNAME=$(op --account grafana.1password.com read "op://Kubernetes Monitoring/helmchart Loki/username") +export GRAFANA_CLOUD_RW_POLICY_TOKEN=$(op --account grafana.1password.com read "op://Kubernetes Monitoring/helmchart Loki/password") +export RANDOM_NUMBER=$(shuf -i 100000-999999 -n 1) diff --git a/charts/k8s-monitoring/tests/platform/grafana-cloud-features/k8s-monitoring/.gitignore b/charts/k8s-monitoring/tests/platform/grafana-cloud/app-observability/.gitignore similarity index 100% rename from charts/k8s-monitoring/tests/platform/grafana-cloud-features/k8s-monitoring/.gitignore rename to charts/k8s-monitoring/tests/platform/grafana-cloud/app-observability/.gitignore diff --git a/charts/k8s-monitoring/tests/platform/grafana-cloud/app-observability/.rendered/output.yaml b/charts/k8s-monitoring/tests/platform/grafana-cloud/app-observability/.rendered/output.yaml new file mode 100644 index 000000000..88630eccf --- /dev/null +++ b/charts/k8s-monitoring/tests/platform/grafana-cloud/app-observability/.rendered/output.yaml @@ -0,0 +1,1068 @@ +--- +# Source: k8s-monitoring/charts/alloy-logs/templates/serviceaccount.yaml +apiVersion: v1 +kind: ServiceAccount +metadata: + name: k8smon-alloy-logs + namespace: default + labels: + helm.sh/chart: alloy-logs-0.11.0 + app.kubernetes.io/name: alloy-logs + app.kubernetes.io/instance: k8smon + + app.kubernetes.io/version: "v1.6.1" + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/part-of: alloy + app.kubernetes.io/component: rbac +--- +# Source: k8s-monitoring/charts/alloy-receiver/templates/serviceaccount.yaml +apiVersion: v1 +kind: ServiceAccount +metadata: + name: k8smon-alloy-receiver + namespace: default + labels: + helm.sh/chart: alloy-receiver-0.11.0 + app.kubernetes.io/name: alloy-receiver + app.kubernetes.io/instance: k8smon + + app.kubernetes.io/version: "v1.6.1" + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/part-of: alloy + app.kubernetes.io/component: rbac +--- +# Source: k8s-monitoring/templates/alloy-config.yaml +apiVersion: v1 +kind: ConfigMap +metadata: + name: k8smon-alloy-logs + namespace: default +data: + config.alloy: |- + // Destination: grafanaCloudLogs (loki) + otelcol.exporter.loki "grafanacloudlogs" { + forward_to = [loki.write.grafanacloudlogs.receiver] + } + + loki.write "grafanacloudlogs" { + endpoint { + url = "https://logs-prod-006.grafana.net/loki/api/v1/push" + tenant_id = nonsensitive(remote.kubernetes.secret.grafanacloudlogs.data["tenantId"]) + basic_auth { + username = nonsensitive(remote.kubernetes.secret.grafanacloudlogs.data["LOKI_USER"]) + password = remote.kubernetes.secret.grafanacloudlogs.data["LOKI_PASS"] + } + tls_config { + insecure_skip_verify = false + ca_pem = nonsensitive(remote.kubernetes.secret.grafanacloudlogs.data["ca"]) + cert_pem = nonsensitive(remote.kubernetes.secret.grafanacloudlogs.data["cert"]) + key_pem = remote.kubernetes.secret.grafanacloudlogs.data["key"] + } + } + external_labels = { + cluster = "application-observability-gc-feature-test", + "k8s_cluster_name" = "application-observability-gc-feature-test", + } + } + + remote.kubernetes.secret "grafanacloudlogs" { + name = "grafana-cloud-credentials" + namespace = "default" + } + // Feature: Pod Logs + declare "pod_logs" { + argument "logs_destinations" { + comment = "Must be a list of log destinations where collected logs should be forwarded to" + } + + discovery.relabel "filtered_pods" { + targets = discovery.kubernetes.pods.targets + rule { + source_labels = ["__meta_kubernetes_namespace"] + action = "replace" + target_label = "namespace" + } + rule { + source_labels = ["__meta_kubernetes_pod_name"] + action = "replace" + target_label = "pod" + } + rule { + source_labels = ["__meta_kubernetes_pod_container_name"] + action = "replace" + target_label = "container" + } + rule { + source_labels = ["__meta_kubernetes_namespace", "__meta_kubernetes_pod_container_name"] + separator = "/" + action = "replace" + replacement = "$1" + target_label = "job" + } + + // set the container runtime as a label + rule { + action = "replace" + source_labels = ["__meta_kubernetes_pod_container_id"] + regex = "^(\\S+):\\/\\/.+$" + replacement = "$1" + target_label = "tmp_container_runtime" + } + + // make all labels on the pod available to the pipeline as labels, + // they are omitted before write to loki via stage.label_keep unless explicitly set + rule { + action = "labelmap" + regex = "__meta_kubernetes_pod_label_(.+)" + } + + // make all annotations on the pod available to the pipeline as labels, + // they are omitted before write to loki via stage.label_keep unless explicitly set + rule { + action = "labelmap" + regex = "__meta_kubernetes_pod_annotation_(.+)" + } + + // explicitly set service_name. if not set, loki will automatically try to populate a default. + // see https://grafana.com/docs/loki/latest/get-started/labels/#default-labels-for-all-users + // + // choose the first value found from the following ordered list: + // - pod.annotation[resource.opentelemetry.io/service.name] + // - pod.label[app.kubernetes.io/name] + // - k8s.pod.name + // - k8s.container.name + rule { + action = "replace" + source_labels = [ + "__meta_kubernetes_pod_annotation_resource_opentelemetry_io_service_name", + "__meta_kubernetes_pod_label_app_kubernetes_io_name", + "__meta_kubernetes_pod_name", + "__meta_kubernetes_pod_container_name", + ] + separator = ";" + regex = "^(?:;*)?([^;]+).*$" + replacement = "$1" + target_label = "service_name" + } + + // set service_namespace + rule { + action = "replace" + source_labels = ["__meta_kubernetes_pod_annotation_resource_opentelemetry_io_service_namespace"] + target_label = "service_namespace" + } + + // set deployment_environment and deployment_environment_name + rule { + action = "replace" + source_labels = ["__meta_kubernetes_pod_annotation_resource_opentelemetry_io_deployment_environment_name"] + target_label = "deployment_environment_name" + } + rule { + action = "replace" + source_labels = ["__meta_kubernetes_pod_annotation_resource_opentelemetry_io_deployment_environment"] + target_label = "deployment_environment" + } + rule { + source_labels = ["__meta_kubernetes_pod_annotation_k8s_grafana_com_logs_job"] + target_label = "job" + } + rule { + source_labels = ["__meta_kubernetes_pod_label_app_kubernetes_io_name"] + target_label = "app_kubernetes_io_name" + } + } + + discovery.kubernetes "pods" { + role = "pod" + selectors { + role = "pod" + field = "spec.nodeName=" + sys.env("HOSTNAME") + } + } + + discovery.relabel "filtered_pods_with_paths" { + targets = discovery.relabel.filtered_pods.output + + rule { + source_labels = ["__meta_kubernetes_pod_uid", "__meta_kubernetes_pod_container_name"] + separator = "/" + action = "replace" + replacement = "/var/log/pods/*$1/*.log" + target_label = "__path__" + } + } + + local.file_match "pod_logs" { + path_targets = discovery.relabel.filtered_pods_with_paths.output + } + + loki.source.file "pod_logs" { + targets = local.file_match.pod_logs.targets + forward_to = [loki.process.pod_logs.receiver] + } + + loki.process "pod_logs" { + stage.match { + selector = "{tmp_container_runtime=~\"containerd|cri-o\"}" + // the cri processing stage extracts the following k/v pairs: log, stream, time, flags + stage.cri {} + + // Set the extract flags and stream values as labels + stage.labels { + values = { + flags = "", + stream = "", + } + } + } + + stage.match { + selector = "{tmp_container_runtime=\"docker\"}" + // the docker processing stage extracts the following k/v pairs: log, stream, time + stage.docker {} + + // Set the extract stream value as a label + stage.labels { + values = { + stream = "", + } + } + } + + // Drop the filename label, since it's not really useful in the context of Kubernetes, where we already have cluster, + // namespace, pod, and container labels. Drop any structured metadata. Also drop the temporary + // container runtime label as it is no longer needed. + stage.label_drop { + values = [ + "filename", + "tmp_container_runtime", + ] + } + + // Only keep the labels that are defined in the `keepLabels` list. + stage.label_keep { + values = ["app_kubernetes_io_name","container","instance","job","level","namespace","pod","service_name","service_namespace","deployment_environment","deployment_environment_name","integration"] + } + + forward_to = argument.logs_destinations.value + } + } + pod_logs "feature" { + logs_destinations = [ + loki.write.grafanacloudlogs.receiver, + ] + } +--- +# Source: k8s-monitoring/templates/alloy-config.yaml +apiVersion: v1 +kind: ConfigMap +metadata: + name: k8smon-alloy-receiver + namespace: default +data: + config.alloy: |- + // Destination: grafanaCloudLogs (loki) + otelcol.exporter.loki "grafanacloudlogs" { + forward_to = [loki.write.grafanacloudlogs.receiver] + } + + loki.write "grafanacloudlogs" { + endpoint { + url = "https://logs-prod-006.grafana.net/loki/api/v1/push" + tenant_id = nonsensitive(remote.kubernetes.secret.grafanacloudlogs.data["tenantId"]) + basic_auth { + username = nonsensitive(remote.kubernetes.secret.grafanacloudlogs.data["LOKI_USER"]) + password = remote.kubernetes.secret.grafanacloudlogs.data["LOKI_PASS"] + } + tls_config { + insecure_skip_verify = false + ca_pem = nonsensitive(remote.kubernetes.secret.grafanacloudlogs.data["ca"]) + cert_pem = nonsensitive(remote.kubernetes.secret.grafanacloudlogs.data["cert"]) + key_pem = remote.kubernetes.secret.grafanacloudlogs.data["key"] + } + } + external_labels = { + cluster = "application-observability-gc-feature-test", + "k8s_cluster_name" = "application-observability-gc-feature-test", + } + } + + remote.kubernetes.secret "grafanacloudlogs" { + name = "grafana-cloud-credentials" + namespace = "default" + } + + // Destination: otlp-gateway (otlp) + otelcol.receiver.prometheus "otlp_gateway" { + output { + metrics = [otelcol.processor.attributes.otlp_gateway.input] + } + } + otelcol.auth.basic "otlp_gateway" { + username = nonsensitive(remote.kubernetes.secret.otlp_gateway.data["OTLP_GATEWAY_USER"]) + password = remote.kubernetes.secret.otlp_gateway.data["OTLP_GATEWAY_PASS"] + } + + otelcol.processor.attributes "otlp_gateway" { + action { + key = "cluster" + action = "upsert" + value = "application-observability-gc-feature-test" + } + action { + key = "k8s.cluster.name" + action = "upsert" + value = "application-observability-gc-feature-test" + } + output { + metrics = [otelcol.processor.transform.otlp_gateway.input] + logs = [otelcol.processor.transform.otlp_gateway.input] + traces = [otelcol.processor.transform.otlp_gateway.input] + } + } + + otelcol.processor.transform "otlp_gateway" { + error_mode = "ignore" + + output { + metrics = [otelcol.processor.batch.otlp_gateway.input] + traces = [otelcol.processor.batch.otlp_gateway.input] + } + } + + otelcol.processor.batch "otlp_gateway" { + timeout = "2s" + send_batch_size = 4096 + send_batch_max_size = 4096 + + output { + metrics = [otelcol.exporter.otlphttp.otlp_gateway.input] + traces = [otelcol.exporter.otlphttp.otlp_gateway.input] + } + } + otelcol.exporter.otlphttp "otlp_gateway" { + client { + endpoint = "https://otlp-gateway-prod-us-east-0.grafana.net/otlp" + auth = otelcol.auth.basic.otlp_gateway.handler + headers = { + "X-Scope-OrgID" = nonsensitive(remote.kubernetes.secret.otlp_gateway.data["tenantId"]), + } + tls { + insecure = false + insecure_skip_verify = false + ca_pem = nonsensitive(remote.kubernetes.secret.otlp_gateway.data["ca"]) + cert_pem = nonsensitive(remote.kubernetes.secret.otlp_gateway.data["cert"]) + key_pem = remote.kubernetes.secret.otlp_gateway.data["key"] + } + } + } + + remote.kubernetes.secret "otlp_gateway" { + name = "grafana-cloud-credentials" + namespace = "default" + } + // Feature: Application Observability + declare "application_observability" { + argument "metrics_destinations" { + comment = "Must be a list of metrics destinations where collected metrics should be forwarded to" + } + + argument "logs_destinations" { + comment = "Must be a list of log destinations where collected logs should be forwarded to" + } + + argument "traces_destinations" { + comment = "Must be a list of trace destinations where collected trace should be forwarded to" + } + + // OTLP Receiver + otelcol.receiver.otlp "receiver" { + grpc { + endpoint = "0.0.0.0:4317" + } + http { + endpoint = "0.0.0.0:4318" + } + debug_metrics { + disable_high_cardinality_metrics = true + } + output { + metrics = [otelcol.processor.resourcedetection.default.input] + logs = [otelcol.processor.resourcedetection.default.input] + traces = [otelcol.processor.resourcedetection.default.input] + } + } + + // Resource Detection Processor + otelcol.processor.resourcedetection "default" { + detectors = ["env", "system"] + system { + hostname_sources = ["os"] + } + + output { + metrics = [otelcol.processor.k8sattributes.default.input] + logs = [otelcol.processor.k8sattributes.default.input] + traces = [otelcol.processor.k8sattributes.default.input] + } + } + + // K8s Attributes Processor + otelcol.processor.k8sattributes "default" { + extract { + metadata = ["k8s.namespace.name","k8s.pod.name","k8s.deployment.name","k8s.statefulset.name","k8s.daemonset.name","k8s.cronjob.name","k8s.job.name","k8s.node.name","k8s.pod.uid","k8s.pod.start_time"] + } + pod_association { + source { + from = "connection" + } + } + + output { + metrics = [otelcol.processor.transform.default.input] + logs = [otelcol.processor.transform.default.input] + traces = [otelcol.processor.transform.default.input, otelcol.connector.host_info.default.input] + } + } + + // Host Info Connector + otelcol.connector.host_info "default" { + host_identifiers = [ "k8s.node.name" ] + + output { + metrics = [otelcol.processor.batch.default.input] + } + } + + // Transform Processor + otelcol.processor.transform "default" { + error_mode = "ignore" + log_statements { + context = "resource" + statements = [ + "set(attributes[\"pod\"], attributes[\"k8s.pod.name\"])", + "set(attributes[\"namespace\"], attributes[\"k8s.namespace.name\"])", + "set(attributes[\"loki.resource.labels\"], \"cluster, namespace, job, pod\")", + ] + } + trace_statements { + context = "span" + statements = [ + "replace_pattern(name, \"\\\\?.*\", \"\")", + "replace_match(name, \"GET /api/products/*\", \"GET /api/products/{productId}\")", + ] + } + + output { + metrics = [otelcol.processor.batch.default.input] + logs = [otelcol.processor.batch.default.input] + traces = [otelcol.processor.batch.default.input] + } + } + + // Batch Processor + otelcol.processor.batch "default" { + send_batch_size = 16384 + send_batch_max_size = 0 + timeout = "2s" + + output { + metrics = argument.metrics_destinations.value + logs = argument.logs_destinations.value + traces = argument.traces_destinations.value + } + } + } + application_observability "feature" { + metrics_destinations = [ + otelcol.processor.attributes.otlp_gateway.input, + ] + logs_destinations = [ + otelcol.exporter.loki.grafanacloudlogs.input, + ] + traces_destinations = [ + otelcol.processor.attributes.otlp_gateway.input, + ] + } + // Self Reporting + prometheus.exporter.unix "kubernetes_monitoring_telemetry" { + set_collectors = ["textfile"] + textfile { + directory = "/etc/alloy" + } + } + + discovery.relabel "kubernetes_monitoring_telemetry" { + targets = prometheus.exporter.unix.kubernetes_monitoring_telemetry.targets + rule { + target_label = "instance" + action = "replace" + replacement = "k8smon" + } + rule { + target_label = "job" + action = "replace" + replacement = "integrations/kubernetes/kubernetes_monitoring_telemetry" + } + } + + prometheus.scrape "kubernetes_monitoring_telemetry" { + job_name = "integrations/kubernetes/kubernetes_monitoring_telemetry" + targets = discovery.relabel.kubernetes_monitoring_telemetry.output + scrape_interval = "60s" + clustering { + enabled = true + } + forward_to = [prometheus.relabel.kubernetes_monitoring_telemetry.receiver] + } + + prometheus.relabel "kubernetes_monitoring_telemetry" { + rule { + source_labels = ["__name__"] + regex = "grafana_kubernetes_monitoring_.*" + action = "keep" + } + forward_to = [ + otelcol.receiver.prometheus.otlp_gateway.receiver, + ] + } + + self-reporting-metric.prom: | + + # HELP grafana_kubernetes_monitoring_build_info A metric to report the version of the Kubernetes Monitoring Helm chart + # TYPE grafana_kubernetes_monitoring_build_info gauge + grafana_kubernetes_monitoring_build_info{version="2.0.8", namespace="default"} 1 + # HELP grafana_kubernetes_monitoring_feature_info A metric to report the enabled features of the Kubernetes Monitoring Helm chart + # TYPE grafana_kubernetes_monitoring_feature_info gauge + grafana_kubernetes_monitoring_feature_info{feature="applicationObservability", protocols="otlpgrpc,otlphttp", version="1.0.0"} 1 + grafana_kubernetes_monitoring_feature_info{feature="podLogs", method="volumes", version="1.0.0"} 1 +--- +# Source: k8s-monitoring/charts/alloy-logs/templates/rbac.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: k8smon-alloy-logs + labels: + helm.sh/chart: alloy-logs-0.11.0 + app.kubernetes.io/name: alloy-logs + app.kubernetes.io/instance: k8smon + + app.kubernetes.io/version: "v1.6.1" + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/part-of: alloy + app.kubernetes.io/component: rbac +rules: + # Rules which allow discovery.kubernetes to function. + - apiGroups: + - "" + - "discovery.k8s.io" + - "networking.k8s.io" + resources: + - endpoints + - endpointslices + - ingresses + - nodes + - nodes/proxy + - nodes/metrics + - pods + - services + verbs: + - get + - list + - watch + # Rules which allow loki.source.kubernetes and loki.source.podlogs to work. + - apiGroups: + - "" + resources: + - pods + - pods/log + - namespaces + verbs: + - get + - list + - watch + - apiGroups: + - "monitoring.grafana.com" + resources: + - podlogs + verbs: + - get + - list + - watch + # Rules which allow mimir.rules.kubernetes to work. + - apiGroups: ["monitoring.coreos.com"] + resources: + - prometheusrules + verbs: + - get + - list + - watch + - nonResourceURLs: + - /metrics + verbs: + - get + # Rules for prometheus.kubernetes.* + - apiGroups: ["monitoring.coreos.com"] + resources: + - podmonitors + - servicemonitors + - probes + verbs: + - get + - list + - watch + # Rules which allow eventhandler to work. + - apiGroups: + - "" + resources: + - events + verbs: + - get + - list + - watch + # needed for remote.kubernetes.* + - apiGroups: [""] + resources: + - "configmaps" + - "secrets" + verbs: + - get + - list + - watch + # needed for otelcol.processor.k8sattributes + - apiGroups: ["apps"] + resources: ["replicasets"] + verbs: ["get", "list", "watch"] + - apiGroups: ["extensions"] + resources: ["replicasets"] + verbs: ["get", "list", "watch"] +--- +# Source: k8s-monitoring/charts/alloy-receiver/templates/rbac.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: k8smon-alloy-receiver + labels: + helm.sh/chart: alloy-receiver-0.11.0 + app.kubernetes.io/name: alloy-receiver + app.kubernetes.io/instance: k8smon + + app.kubernetes.io/version: "v1.6.1" + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/part-of: alloy + app.kubernetes.io/component: rbac +rules: + # Rules which allow discovery.kubernetes to function. + - apiGroups: + - "" + - "discovery.k8s.io" + - "networking.k8s.io" + resources: + - endpoints + - endpointslices + - ingresses + - nodes + - nodes/proxy + - nodes/metrics + - pods + - services + verbs: + - get + - list + - watch + # Rules which allow loki.source.kubernetes and loki.source.podlogs to work. + - apiGroups: + - "" + resources: + - pods + - pods/log + - namespaces + verbs: + - get + - list + - watch + - apiGroups: + - "monitoring.grafana.com" + resources: + - podlogs + verbs: + - get + - list + - watch + # Rules which allow mimir.rules.kubernetes to work. + - apiGroups: ["monitoring.coreos.com"] + resources: + - prometheusrules + verbs: + - get + - list + - watch + - nonResourceURLs: + - /metrics + verbs: + - get + # Rules for prometheus.kubernetes.* + - apiGroups: ["monitoring.coreos.com"] + resources: + - podmonitors + - servicemonitors + - probes + verbs: + - get + - list + - watch + # Rules which allow eventhandler to work. + - apiGroups: + - "" + resources: + - events + verbs: + - get + - list + - watch + # needed for remote.kubernetes.* + - apiGroups: [""] + resources: + - "configmaps" + - "secrets" + verbs: + - get + - list + - watch + # needed for otelcol.processor.k8sattributes + - apiGroups: ["apps"] + resources: ["replicasets"] + verbs: ["get", "list", "watch"] + - apiGroups: ["extensions"] + resources: ["replicasets"] + verbs: ["get", "list", "watch"] +--- +# Source: k8s-monitoring/charts/alloy-logs/templates/rbac.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: k8smon-alloy-logs + labels: + helm.sh/chart: alloy-logs-0.11.0 + app.kubernetes.io/name: alloy-logs + app.kubernetes.io/instance: k8smon + + app.kubernetes.io/version: "v1.6.1" + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/part-of: alloy + app.kubernetes.io/component: rbac +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: k8smon-alloy-logs +subjects: + - kind: ServiceAccount + name: k8smon-alloy-logs + namespace: default +--- +# Source: k8s-monitoring/charts/alloy-receiver/templates/rbac.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: k8smon-alloy-receiver + labels: + helm.sh/chart: alloy-receiver-0.11.0 + app.kubernetes.io/name: alloy-receiver + app.kubernetes.io/instance: k8smon + + app.kubernetes.io/version: "v1.6.1" + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/part-of: alloy + app.kubernetes.io/component: rbac +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: k8smon-alloy-receiver +subjects: + - kind: ServiceAccount + name: k8smon-alloy-receiver + namespace: default +--- +# Source: k8s-monitoring/charts/alloy-logs/templates/service.yaml +apiVersion: v1 +kind: Service +metadata: + name: k8smon-alloy-logs + labels: + helm.sh/chart: alloy-logs-0.11.0 + app.kubernetes.io/name: alloy-logs + app.kubernetes.io/instance: k8smon + + app.kubernetes.io/version: "v1.6.1" + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/part-of: alloy + app.kubernetes.io/component: networking +spec: + type: ClusterIP + selector: + app.kubernetes.io/name: alloy-logs + app.kubernetes.io/instance: k8smon + internalTrafficPolicy: Cluster + ports: + - name: http-metrics + port: 12345 + targetPort: 12345 + protocol: "TCP" +--- +# Source: k8s-monitoring/charts/alloy-receiver/templates/service.yaml +apiVersion: v1 +kind: Service +metadata: + name: k8smon-alloy-receiver + labels: + helm.sh/chart: alloy-receiver-0.11.0 + app.kubernetes.io/name: alloy-receiver + app.kubernetes.io/instance: k8smon + + app.kubernetes.io/version: "v1.6.1" + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/part-of: alloy + app.kubernetes.io/component: networking +spec: + type: ClusterIP + selector: + app.kubernetes.io/name: alloy-receiver + app.kubernetes.io/instance: k8smon + internalTrafficPolicy: Cluster + ports: + - name: http-metrics + port: 12345 + targetPort: 12345 + protocol: "TCP" + - name: otlp-http + port: 4318 + targetPort: 4318 + protocol: TCP + - name: otlp-grpc + port: 4317 + targetPort: 4317 + protocol: TCP +--- +# Source: k8s-monitoring/charts/alloy-logs/templates/controllers/daemonset.yaml +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: k8smon-alloy-logs + labels: + helm.sh/chart: alloy-logs-0.11.0 + app.kubernetes.io/name: alloy-logs + app.kubernetes.io/instance: k8smon + + app.kubernetes.io/version: "v1.6.1" + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/part-of: alloy +spec: + minReadySeconds: 10 + selector: + matchLabels: + app.kubernetes.io/name: alloy-logs + app.kubernetes.io/instance: k8smon + template: + metadata: + annotations: + kubectl.kubernetes.io/default-container: alloy + k8s.grafana.com/logs.job: integrations/alloy + labels: + app.kubernetes.io/name: alloy-logs + app.kubernetes.io/instance: k8smon + spec: + serviceAccountName: k8smon-alloy-logs + containers: + - name: alloy + image: docker.io/grafana/alloy:v1.6.1 + imagePullPolicy: IfNotPresent + args: + - run + - /etc/alloy/config.alloy + - --storage.path=/tmp/alloy + - --server.http.listen-addr=0.0.0.0:12345 + - --server.http.ui-path-prefix=/ + - --stability.level=generally-available + env: + - name: ALLOY_DEPLOY_MODE + value: "helm" + - name: HOSTNAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + ports: + - containerPort: 12345 + name: http-metrics + readinessProbe: + httpGet: + path: /-/ready + port: 12345 + scheme: HTTP + initialDelaySeconds: 10 + timeoutSeconds: 1 + securityContext: + allowPrivilegeEscalation: false + capabilities: + add: + - CHOWN + - DAC_OVERRIDE + - FOWNER + - FSETID + - KILL + - SETGID + - SETUID + - SETPCAP + - NET_BIND_SERVICE + - NET_RAW + - SYS_CHROOT + - MKNOD + - AUDIT_WRITE + - SETFCAP + drop: + - ALL + seccompProfile: + type: RuntimeDefault + volumeMounts: + - name: config + mountPath: /etc/alloy + - name: varlog + mountPath: /var/log + readOnly: true + - name: dockercontainers + mountPath: /var/lib/docker/containers + readOnly: true + - name: config-reloader + image: ghcr.io/jimmidyson/configmap-reload:v0.14.0 + args: + - --volume-dir=/etc/alloy + - --webhook-url=http://localhost:12345/-/reload + volumeMounts: + - name: config + mountPath: /etc/alloy + resources: + requests: + cpu: 1m + memory: 5Mi + dnsPolicy: ClusterFirst + nodeSelector: + kubernetes.io/os: linux + volumes: + - name: config + configMap: + name: k8smon-alloy-logs + - name: varlog + hostPath: + path: /var/log + - name: dockercontainers + hostPath: + path: /var/lib/docker/containers +--- +# Source: k8s-monitoring/charts/alloy-receiver/templates/controllers/deployment.yaml +apiVersion: apps/v1 +kind: Deployment +metadata: + name: k8smon-alloy-receiver + labels: + helm.sh/chart: alloy-receiver-0.11.0 + app.kubernetes.io/name: alloy-receiver + app.kubernetes.io/instance: k8smon + + app.kubernetes.io/version: "v1.6.1" + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/part-of: alloy +spec: + replicas: 1 + minReadySeconds: 10 + selector: + matchLabels: + app.kubernetes.io/name: alloy-receiver + app.kubernetes.io/instance: k8smon + template: + metadata: + annotations: + kubectl.kubernetes.io/default-container: alloy + k8s.grafana.com/logs.job: integrations/alloy + labels: + app.kubernetes.io/name: alloy-receiver + app.kubernetes.io/instance: k8smon + spec: + serviceAccountName: k8smon-alloy-receiver + containers: + - name: alloy + image: docker.io/grafana/alloy:v1.6.1 + imagePullPolicy: IfNotPresent + args: + - run + - /etc/alloy/config.alloy + - --storage.path=/tmp/alloy + - --server.http.listen-addr=0.0.0.0:12345 + - --server.http.ui-path-prefix=/ + - --stability.level=generally-available + env: + - name: ALLOY_DEPLOY_MODE + value: "helm" + - name: HOSTNAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + ports: + - containerPort: 12345 + name: http-metrics + - containerPort: 4318 + name: otlp-http + protocol: TCP + - containerPort: 4317 + name: otlp-grpc + protocol: TCP + readinessProbe: + httpGet: + path: /-/ready + port: 12345 + scheme: HTTP + initialDelaySeconds: 10 + timeoutSeconds: 1 + securityContext: + allowPrivilegeEscalation: false + capabilities: + add: + - CHOWN + - DAC_OVERRIDE + - FOWNER + - FSETID + - KILL + - SETGID + - SETUID + - SETPCAP + - NET_BIND_SERVICE + - NET_RAW + - SYS_CHROOT + - MKNOD + - AUDIT_WRITE + - SETFCAP + drop: + - ALL + seccompProfile: + type: RuntimeDefault + volumeMounts: + - name: config + mountPath: /etc/alloy + - name: config-reloader + image: ghcr.io/jimmidyson/configmap-reload:v0.14.0 + args: + - --volume-dir=/etc/alloy + - --webhook-url=http://localhost:12345/-/reload + volumeMounts: + - name: config + mountPath: /etc/alloy + resources: + requests: + cpu: 1m + memory: 5Mi + dnsPolicy: ClusterFirst + nodeSelector: + kubernetes.io/os: linux + volumes: + - name: config + configMap: + name: k8smon-alloy-receiver diff --git a/charts/k8s-monitoring/tests/platform/grafana-cloud/app-observability/Makefile b/charts/k8s-monitoring/tests/platform/grafana-cloud/app-observability/Makefile new file mode 100644 index 000000000..5c92eb517 --- /dev/null +++ b/charts/k8s-monitoring/tests/platform/grafana-cloud/app-observability/Makefile @@ -0,0 +1,30 @@ +all: deployments/test-variables.yaml deployments/grafana-cloud-credentials.yaml +clean: + rm -f deployments/test-variables.yaml deployments/grafana-cloud-credentials.yaml + +deployments/test-variables.yaml: + echo "---" > $@ + kubectl create configmap test-variables \ + --from-literal=CLUSTER="$(shell yq eval '.cluster.name' values.yaml)-$$RANDOM_NUMBER" \ + --from-literal=RANDOM_NUMBER="$$RANDOM_NUMBER" \ + -o yaml --dry-run=client >> $@ + echo "---" >> $@ + kubectl create configmap test-variables --namespace=otel-demo \ + --from-literal=CLUSTER="$(shell yq eval '.cluster.name' values.yaml)-$$RANDOM_NUMBER" \ + --from-literal=RANDOM_NUMBER="$$RANDOM_NUMBER" \ + -o yaml --dry-run=client >> $@ + +deployments/grafana-cloud-credentials.yaml: + echo "---" > $@ + echo "# yamllint disable rule:line-length" >> $@ + kubectl create secret generic grafana-cloud-credentials \ + --from-literal=OTLP_GATEWAY_USER="$$GRAFANA_CLOUD_OTLP_USERNAME" \ + --from-literal=OTLP_GATEWAY_PASS="$$GRAFANA_CLOUD_RW_POLICY_TOKEN" \ + --from-literal=LOKI_USER="$$GRAFANA_CLOUD_LOGS_USERNAME" \ + --from-literal=LOKI_PASS="$$GRAFANA_CLOUD_RW_POLICY_TOKEN" \ + --from-literal=PROMETHEUS_USER="$$GRAFANA_CLOUD_METRICS_USERNAME" \ + --from-literal=PROMETHEUS_PASS="$$GRAFANA_CLOUD_RW_POLICY_TOKEN" \ + -o yaml --dry-run=client >> $@ + +run-test: + ../../../../../../scripts/run-cluster-test.sh . diff --git a/charts/k8s-monitoring/tests/platform/grafana-cloud/app-observability/README.md b/charts/k8s-monitoring/tests/platform/grafana-cloud/app-observability/README.md new file mode 100644 index 000000000..e69de29bb diff --git a/charts/k8s-monitoring/tests/platform/grafana-cloud/app-observability/deployments/otel-demo-manifest.yaml b/charts/k8s-monitoring/tests/platform/grafana-cloud/app-observability/deployments/otel-demo-manifest.yaml new file mode 100644 index 000000000..f9a786c41 --- /dev/null +++ b/charts/k8s-monitoring/tests/platform/grafana-cloud/app-observability/deployments/otel-demo-manifest.yaml @@ -0,0 +1,69 @@ +--- +apiVersion: source.toolkit.fluxcd.io/v1 +kind: HelmRepository +metadata: + name: open-telemetry + namespace: default +spec: + interval: 1h + url: https://open-telemetry.github.io/opentelemetry-helm-charts +--- +apiVersion: v1 +kind: Namespace +metadata: + name: otel-demo +--- +apiVersion: helm.toolkit.fluxcd.io/v2 +kind: HelmRelease +metadata: + name: otel-demo + namespace: otel-demo +spec: + interval: 10m + timeout: 10m + chart: + spec: + chart: opentelemetry-demo + version: 0.33.8 + sourceRef: + kind: HelmRepository + name: open-telemetry + namespace: default + interval: 10m + values: + default: + # List of environment variables applied to all components + env: + - name: RANDOM_NUMBER + valueFrom: + configMapKeyRef: + name: test-variables + key: RANDOM_NUMBER + - name: CLUSTER + valueFrom: + configMapKeyRef: + name: test-variables + key: CLUSTER + - name: OTEL_SERVICE_NAME + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: "metadata.labels['app.kubernetes.io/component']" + - name: OTEL_COLLECTOR_NAME + value: 'k8smon-alloy-receiver.default' + - name: OTEL_EXPORTER_OTLP_METRICS_TEMPORALITY_PREFERENCE + value: cumulative + - name: OTEL_RESOURCE_ATTRIBUTES + value: 'service.name=$(OTEL_SERVICE_NAME),service.namespace=opentelemetry-demo,service.version={{ .Chart.AppVersion }},k8s.cluster.name=$(CLUSTER)' + + opentelemetry-collector: + enabled: false + + jaeger: + enabled: false + + prometheus: + enabled: false + + grafana: + enabled: false diff --git a/charts/k8s-monitoring/tests/platform/grafana-cloud/app-observability/deployments/query-test.yaml b/charts/k8s-monitoring/tests/platform/grafana-cloud/app-observability/deployments/query-test.yaml new file mode 100644 index 000000000..5d8e8212a --- /dev/null +++ b/charts/k8s-monitoring/tests/platform/grafana-cloud/app-observability/deployments/query-test.yaml @@ -0,0 +1,39 @@ +--- +apiVersion: source.toolkit.fluxcd.io/v1 +kind: GitRepository +metadata: + name: k8s-monitoring-test +spec: + interval: 1m + url: https://github.com/grafana/k8s-monitoring-helm + ref: + branch: main + ignore: | + /* + !/charts/k8s-monitoring-test +--- +apiVersion: helm.toolkit.fluxcd.io/v2 +kind: HelmRelease +metadata: + name: k8s-monitoring-test +spec: + interval: 1m + chart: + spec: + chart: charts/k8s-monitoring-test + sourceRef: + kind: GitRepository + name: k8s-monitoring-test + interval: 1m + values: + tests: + - env: + PROMETHEUS_URL: https://prometheus-prod-13-prod-us-east-0.grafana.net/api/prom/api/v1/query + LOKI_URL: https://logs-prod-006.grafana.net/loki/api/v1/query + envFrom: + - secretRef: {name: grafana-cloud-credentials} + - configMapRef: {name: test-variables} + queries: + # otel demo services + - query: group by (telemetry_sdk_language, job) (traces_target_info{deployment_environment=~".*", k8s_cluster_name="$CLUSTER"}) + type: promql diff --git a/charts/k8s-monitoring/tests/platform/grafana-cloud/app-observability/values.yaml b/charts/k8s-monitoring/tests/platform/grafana-cloud/app-observability/values.yaml new file mode 100644 index 000000000..c032e230c --- /dev/null +++ b/charts/k8s-monitoring/tests/platform/grafana-cloud/app-observability/values.yaml @@ -0,0 +1,72 @@ +--- +cluster: + name: application-observability-gc-feature-test + +destinations: + - name: grafanaCloudLogs + type: loki + url: https://logs-prod-006.grafana.net/loki/api/v1/push + auth: + type: basic + usernameKey: LOKI_USER + passwordKey: LOKI_PASS + secret: + create: false + name: grafana-cloud-credentials + - name: otlp-gateway + type: otlp + url: https://otlp-gateway-prod-us-east-0.grafana.net/otlp + protocol: http + auth: + type: basic + usernameKey: OTLP_GATEWAY_USER + passwordKey: OTLP_GATEWAY_PASS + secret: + create: false + name: grafana-cloud-credentials + metrics: + enabled: true + logs: + enabled: false + traces: + enabled: true + processors: + batch: + size: 4096 + maxSize: 4096 + +applicationObservability: + enabled: true + traces: + transforms: + span: + - replace_pattern(name, "\\?.*", "") + - replace_match(name, "GET /api/products/*", "GET /api/products/{productId}") + + receivers: + otlp: + grpc: + enabled: true + http: + enabled: true + +podLogs: + enabled: true + +alloy-logs: + enabled: true + +alloy-receiver: + enabled: true + controller: + type: deployment + alloy: + extraPorts: + - name: otlp-http + port: 4318 + targetPort: 4318 + protocol: TCP + - name: otlp-grpc + port: 4317 + targetPort: 4317 + protocol: TCP diff --git a/charts/k8s-monitoring/tests/platform/grafana-cloud-features/k8s-monitoring/.envrc b/charts/k8s-monitoring/tests/platform/grafana-cloud/k8s-monitoring/.envrc similarity index 100% rename from charts/k8s-monitoring/tests/platform/grafana-cloud-features/k8s-monitoring/.envrc rename to charts/k8s-monitoring/tests/platform/grafana-cloud/k8s-monitoring/.envrc diff --git a/charts/k8s-monitoring/tests/platform/grafana-cloud/k8s-monitoring/.gitignore b/charts/k8s-monitoring/tests/platform/grafana-cloud/k8s-monitoring/.gitignore new file mode 100644 index 000000000..a9f8e80bd --- /dev/null +++ b/charts/k8s-monitoring/tests/platform/grafana-cloud/k8s-monitoring/.gitignore @@ -0,0 +1,2 @@ +deployments/grafana-cloud-credentials.yaml +deployments/test-variables.yaml diff --git a/charts/k8s-monitoring/tests/platform/grafana-cloud-features/k8s-monitoring/.rendered/output.yaml b/charts/k8s-monitoring/tests/platform/grafana-cloud/k8s-monitoring/.rendered/output.yaml similarity index 100% rename from charts/k8s-monitoring/tests/platform/grafana-cloud-features/k8s-monitoring/.rendered/output.yaml rename to charts/k8s-monitoring/tests/platform/grafana-cloud/k8s-monitoring/.rendered/output.yaml diff --git a/charts/k8s-monitoring/tests/platform/grafana-cloud-features/k8s-monitoring/Makefile b/charts/k8s-monitoring/tests/platform/grafana-cloud/k8s-monitoring/Makefile similarity index 100% rename from charts/k8s-monitoring/tests/platform/grafana-cloud-features/k8s-monitoring/Makefile rename to charts/k8s-monitoring/tests/platform/grafana-cloud/k8s-monitoring/Makefile diff --git a/charts/k8s-monitoring/tests/platform/grafana-cloud-features/k8s-monitoring/deployments/query-test.yaml b/charts/k8s-monitoring/tests/platform/grafana-cloud/k8s-monitoring/deployments/query-test.yaml similarity index 100% rename from charts/k8s-monitoring/tests/platform/grafana-cloud-features/k8s-monitoring/deployments/query-test.yaml rename to charts/k8s-monitoring/tests/platform/grafana-cloud/k8s-monitoring/deployments/query-test.yaml diff --git a/charts/k8s-monitoring/tests/platform/grafana-cloud-features/k8s-monitoring/values.yaml b/charts/k8s-monitoring/tests/platform/grafana-cloud/k8s-monitoring/values.yaml similarity index 100% rename from charts/k8s-monitoring/tests/platform/grafana-cloud-features/k8s-monitoring/values.yaml rename to charts/k8s-monitoring/tests/platform/grafana-cloud/k8s-monitoring/values.yaml