From ddb9e13ae4229b86d14b6f40bb663180b6aa399a Mon Sep 17 00:00:00 2001 From: Tim Campbell Date: Fri, 31 Jan 2025 15:54:29 +0100 Subject: [PATCH 01/13] Add environment variables/sidecars for prometheus metrics --- templates/deployment-sidekiq.yaml | 20 ++++++++++++++++++++ templates/deployment-web.yaml | 30 ++++++++++++++++++++++++++++++ values.yaml | 24 ++++++++++++++++++++++++ 3 files changed, 74 insertions(+) diff --git a/templates/deployment-sidekiq.yaml b/templates/deployment-sidekiq.yaml index 1b3042d..d549819 100644 --- a/templates/deployment-sidekiq.yaml +++ b/templates/deployment-sidekiq.yaml @@ -206,6 +206,26 @@ spec: - name: OTEL_SERVICE_NAME_SEPARATOR value: "{{ coalesce $context.Values.mastodon.sidekiq.otel.nameSeparator $context.Values.mastodon.otel.nameSeparator }}" {{- end }} + {{- if $context.Values.mastodon.metrics.prometheus.enabled }} + - name: MASTODON_PROMETHEUS_EXPORTER_ENABLED + value: "true" + - name: PROMETHEUS_EXPORTER_HOST + value: {{ $context.Values.mastodon.metrics.prometheus.host }} + - name: PROMETHEUS_EXPORTER_PORT + value: {{ $context.Values.mastodon.metrics.prometheus.port }} + {{- if $context.Values.mastodon.metrics.prometheus.sidekiq.detailed }} + - name: MASTODON_PROMETHEUS_EXPORTER_SIDEKIQ_DETAILED_METRICS + value: "true" + {{- end }} + {{- if $context.Values.mastodon.metrics.prometheus.sidekiq.local.enabled }} + - name: MASTODON_PROMETHEUS_EXPORTER_LOCAL + value: "true" + - name: MASTODON_PROMETHEUS_EXPORTER_HOST + value: {{ $context.Values.mastodon.metrics.prometheus.sidekiq.local.host }} + - name: MASTODON_PROMETHEUS_EXPORTER_PORT + value: {{ $context.Values.mastodon.metrics.prometheus.sidekiq.local.port }} + {{- end }} + {{- end }} volumeMounts: {{- if (not $context.Values.mastodon.s3.enabled) }} - name: assets diff --git a/templates/deployment-web.yaml b/templates/deployment-web.yaml index d787c3f..3df2733 100644 --- a/templates/deployment-web.yaml +++ b/templates/deployment-web.yaml @@ -196,6 +196,20 @@ spec: - name: OTEL_SERVICE_NAME_SEPARATOR value: "{{ coalesce .Values.mastodon.web.otel.nameSeparator .Values.mastodon.otel.nameSeparator }}" {{- end }} + {{- if .Values.mastodon.metrics.prometheus.enabled }} + - name: MASTODON_PROMETHEUS_EXPORTER_ENABLED + value: "true" + - name: PROMETHEUS_EXPORTER_HOST + value: {{ $.Values.mastodon.metrics.prometheus.host }} + - name: PROMETHEUS_EXPORTER_PORT + value: {{ .Values.mastodon.metrics.prometheus.port }} + {{- if .Values.mastodon.metrics.prometheus.web.detailed }} + - name: MASTODON_PROMETHEUS_EXPORTER_WEB_DETAILED_METRICS + value: "true" + {{- end }} + {{- end }} + - name: TEST_ENV_VALUE + value: {{ .Values.mastodon.metrics.statsd.address }} volumeMounts: {{- if (not .Values.mastodon.s3.enabled) }} - name: assets @@ -233,6 +247,22 @@ spec: resources: {{- toYaml . | nindent 12 }} {{- end }} + {{- if .Values.mastodon.metrics.prometheus.enabled }} + - name: prometheus-exporter + image: "{{ coalesce .Values.mastodon.web.image.repository .Values.image.repository }}:{{ coalesce .Values.mastodon.web.image.tag .Values.image.tag .Chart.AppVersion }}" + command: + - ./bin/prometheus_exporter + resources: + requests: + cpu: "0.1" + memory: "180M" + limits: + cpu: "0.5" + memory: "250M" + ports: + - name: prometheus + containerPort: {{ .Values.mastodon.metrics.prometheus.port }} + {{- end }} {{- include "mastodon.statsdExporterContainer" $ | indent 8 }} {{- with .Values.nodeSelector }} nodeSelector: diff --git a/values.yaml b/values.yaml index 1bf977d..59ebe01 100644 --- a/values.yaml +++ b/values.yaml @@ -431,6 +431,30 @@ mastodon: enabled: false port: 9102 + # Settings for Prometheus metrics. + # For more information, see: + # https://docs.joinmastodon.org/admin/config/#prometheus + prometheus: + enabled: false + # Host & port to send metrics to + host: localhost + port: 9394 + + # Prometheus for web pods + web: + # Collect per-controller/action metrics for every request + detailed: false + + # Prometheus for sidekiq pods + sidekiq: + # Collect per-job metrics for every job + detailed: false + # Use a local in-process exporter instead + local: + enabled: true + host: localhost + port: 9394 + # Open Telemetry configuration for all deployments. Component-specific # configuration will override these values. otel: From 42d0c93ea9e34e16a16a7552dff870772d40301c Mon Sep 17 00:00:00 2001 From: Tim Campbell Date: Fri, 31 Jan 2025 15:58:52 +0100 Subject: [PATCH 02/13] Bugfix --- templates/deployment-sidekiq.yaml | 4 ++-- templates/deployment-web.yaml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/templates/deployment-sidekiq.yaml b/templates/deployment-sidekiq.yaml index d549819..ce012ff 100644 --- a/templates/deployment-sidekiq.yaml +++ b/templates/deployment-sidekiq.yaml @@ -212,7 +212,7 @@ spec: - name: PROMETHEUS_EXPORTER_HOST value: {{ $context.Values.mastodon.metrics.prometheus.host }} - name: PROMETHEUS_EXPORTER_PORT - value: {{ $context.Values.mastodon.metrics.prometheus.port }} + value: "{{ $context.Values.mastodon.metrics.prometheus.port }}" {{- if $context.Values.mastodon.metrics.prometheus.sidekiq.detailed }} - name: MASTODON_PROMETHEUS_EXPORTER_SIDEKIQ_DETAILED_METRICS value: "true" @@ -223,7 +223,7 @@ spec: - name: MASTODON_PROMETHEUS_EXPORTER_HOST value: {{ $context.Values.mastodon.metrics.prometheus.sidekiq.local.host }} - name: MASTODON_PROMETHEUS_EXPORTER_PORT - value: {{ $context.Values.mastodon.metrics.prometheus.sidekiq.local.port }} + value: "{{ $context.Values.mastodon.metrics.prometheus.sidekiq.local.port }}" {{- end }} {{- end }} volumeMounts: diff --git a/templates/deployment-web.yaml b/templates/deployment-web.yaml index 3df2733..fb48b29 100644 --- a/templates/deployment-web.yaml +++ b/templates/deployment-web.yaml @@ -202,7 +202,7 @@ spec: - name: PROMETHEUS_EXPORTER_HOST value: {{ $.Values.mastodon.metrics.prometheus.host }} - name: PROMETHEUS_EXPORTER_PORT - value: {{ .Values.mastodon.metrics.prometheus.port }} + value: "{{ .Values.mastodon.metrics.prometheus.port }}" {{- if .Values.mastodon.metrics.prometheus.web.detailed }} - name: MASTODON_PROMETHEUS_EXPORTER_WEB_DETAILED_METRICS value: "true" From 97b61a4bd2489045e51e4742821cc7d55a6c90fe Mon Sep 17 00:00:00 2001 From: Tim Campbell Date: Tue, 4 Feb 2025 13:55:01 +0100 Subject: [PATCH 03/13] Local exporter is enabled by default for sidekiq --- templates/deployment-sidekiq.yaml | 8 +++----- values.yaml | 5 ----- 2 files changed, 3 insertions(+), 10 deletions(-) diff --git a/templates/deployment-sidekiq.yaml b/templates/deployment-sidekiq.yaml index ce012ff..f1af6cb 100644 --- a/templates/deployment-sidekiq.yaml +++ b/templates/deployment-sidekiq.yaml @@ -213,17 +213,15 @@ spec: value: {{ $context.Values.mastodon.metrics.prometheus.host }} - name: PROMETHEUS_EXPORTER_PORT value: "{{ $context.Values.mastodon.metrics.prometheus.port }}" - {{- if $context.Values.mastodon.metrics.prometheus.sidekiq.detailed }} - - name: MASTODON_PROMETHEUS_EXPORTER_SIDEKIQ_DETAILED_METRICS - value: "true" - {{- end }} - {{- if $context.Values.mastodon.metrics.prometheus.sidekiq.local.enabled }} - name: MASTODON_PROMETHEUS_EXPORTER_LOCAL value: "true" - name: MASTODON_PROMETHEUS_EXPORTER_HOST value: {{ $context.Values.mastodon.metrics.prometheus.sidekiq.local.host }} - name: MASTODON_PROMETHEUS_EXPORTER_PORT value: "{{ $context.Values.mastodon.metrics.prometheus.sidekiq.local.port }}" + {{- if $context.Values.mastodon.metrics.prometheus.sidekiq.detailed }} + - name: MASTODON_PROMETHEUS_EXPORTER_SIDEKIQ_DETAILED_METRICS + value: "true" {{- end }} {{- end }} volumeMounts: diff --git a/values.yaml b/values.yaml index 59ebe01..ec15799 100644 --- a/values.yaml +++ b/values.yaml @@ -449,11 +449,6 @@ mastodon: sidekiq: # Collect per-job metrics for every job detailed: false - # Use a local in-process exporter instead - local: - enabled: true - host: localhost - port: 9394 # Open Telemetry configuration for all deployments. Component-specific # configuration will override these values. From c2f6cefb81c3a4d9e882dfb993961237051bb0d2 Mon Sep 17 00:00:00 2001 From: Tim Campbell Date: Thu, 6 Feb 2025 16:07:12 +0100 Subject: [PATCH 04/13] Minor bug fix --- templates/deployment-sidekiq.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/templates/deployment-sidekiq.yaml b/templates/deployment-sidekiq.yaml index f1af6cb..0edf272 100644 --- a/templates/deployment-sidekiq.yaml +++ b/templates/deployment-sidekiq.yaml @@ -216,9 +216,9 @@ spec: - name: MASTODON_PROMETHEUS_EXPORTER_LOCAL value: "true" - name: MASTODON_PROMETHEUS_EXPORTER_HOST - value: {{ $context.Values.mastodon.metrics.prometheus.sidekiq.local.host }} + value: {{ $context.Values.mastodon.metrics.prometheus.host }} - name: MASTODON_PROMETHEUS_EXPORTER_PORT - value: "{{ $context.Values.mastodon.metrics.prometheus.sidekiq.local.port }}" + value: "{{ $context.Values.mastodon.metrics.prometheus.port }}" {{- if $context.Values.mastodon.metrics.prometheus.sidekiq.detailed }} - name: MASTODON_PROMETHEUS_EXPORTER_SIDEKIQ_DETAILED_METRICS value: "true" From f0dba547ac2e0007d8a16b484348576f465ee80b Mon Sep 17 00:00:00 2001 From: Tim Campbell Date: Tue, 4 Mar 2025 17:49:36 +0100 Subject: [PATCH 05/13] Remove unnecessary vars from sidekiq --- templates/deployment-sidekiq.yaml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/templates/deployment-sidekiq.yaml b/templates/deployment-sidekiq.yaml index 0edf272..de78f46 100644 --- a/templates/deployment-sidekiq.yaml +++ b/templates/deployment-sidekiq.yaml @@ -209,10 +209,6 @@ spec: {{- if $context.Values.mastodon.metrics.prometheus.enabled }} - name: MASTODON_PROMETHEUS_EXPORTER_ENABLED value: "true" - - name: PROMETHEUS_EXPORTER_HOST - value: {{ $context.Values.mastodon.metrics.prometheus.host }} - - name: PROMETHEUS_EXPORTER_PORT - value: "{{ $context.Values.mastodon.metrics.prometheus.port }}" - name: MASTODON_PROMETHEUS_EXPORTER_LOCAL value: "true" - name: MASTODON_PROMETHEUS_EXPORTER_HOST From 0ddba8c4e831328938858d873b31d2b47c129978 Mon Sep 17 00:00:00 2001 From: Tim Campbell Date: Tue, 4 Mar 2025 17:50:49 +0100 Subject: [PATCH 06/13] Fix comment wording --- values.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/values.yaml b/values.yaml index 5ee94a3..6492019 100644 --- a/values.yaml +++ b/values.yaml @@ -439,7 +439,7 @@ mastodon: # https://docs.joinmastodon.org/admin/config/#prometheus prometheus: enabled: false - # Host & port to send metrics to + # Host & port for the exporter to listen on host: localhost port: 9394 From 43b68198010a97620a9317b89a7443f5fdae43b9 Mon Sep 17 00:00:00 2001 From: Tim Campbell Date: Tue, 4 Mar 2025 17:59:27 +0100 Subject: [PATCH 07/13] Add port definition when using prometheus --- templates/deployment-sidekiq.yaml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/templates/deployment-sidekiq.yaml b/templates/deployment-sidekiq.yaml index de78f46..e69d0cd 100644 --- a/templates/deployment-sidekiq.yaml +++ b/templates/deployment-sidekiq.yaml @@ -220,6 +220,11 @@ spec: value: "true" {{- end }} {{- end }} + {{- if or $context.Values.mastodon.sidekiq.otel.enabled (and $context.Values.mastodon.otel.enabled (ne $context.Values.mastodon.sidekiq.otel.enabled false)) }} + ports: + - name: prometheus + containerPort: {{ .Values.mastodon.metrics.prometheus.port }} + {{- end }} volumeMounts: {{- if (not $context.Values.mastodon.s3.enabled) }} - name: assets From 4b52f5f4bde249caf7a7e2be8944376ae5786964 Mon Sep 17 00:00:00 2001 From: Tim Campbell Date: Tue, 4 Mar 2025 18:01:11 +0100 Subject: [PATCH 08/13] Add port for web sidecar command --- templates/deployment-web.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/templates/deployment-web.yaml b/templates/deployment-web.yaml index fb48b29..f5115b2 100644 --- a/templates/deployment-web.yaml +++ b/templates/deployment-web.yaml @@ -252,6 +252,9 @@ spec: image: "{{ coalesce .Values.mastodon.web.image.repository .Values.image.repository }}:{{ coalesce .Values.mastodon.web.image.tag .Values.image.tag .Chart.AppVersion }}" command: - ./bin/prometheus_exporter + args: + - "--port" + - "{{ .Values.mastodon.metrics.prometheus.port }}" resources: requests: cpu: "0.1" From 529843ec098d625749a7d2d1802b70ccb2c04a9b Mon Sep 17 00:00:00 2001 From: Tim Campbell Date: Tue, 4 Mar 2025 18:09:00 +0100 Subject: [PATCH 09/13] Clarification for prometheus metrics --- values.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/values.yaml b/values.yaml index 6492019..8d4209c 100644 --- a/values.yaml +++ b/values.yaml @@ -434,7 +434,7 @@ mastodon: enabled: false port: 9102 - # Settings for Prometheus metrics. + # Settings for Prometheus metrics. NOTE: Only available in Mastodon v4.4. # For more information, see: # https://docs.joinmastodon.org/admin/config/#prometheus prometheus: From 487c77958c51dec24c5df90fb23eece942080951 Mon Sep 17 00:00:00 2001 From: Tim Campbell Date: Tue, 4 Mar 2025 18:15:25 +0100 Subject: [PATCH 10/13] Make sure everything listens on the correct ports (localhost for web pods because of sidecar, 0.0.0.0 for sidekiq --- templates/deployment-sidekiq.yaml | 2 +- templates/deployment-web.yaml | 2 +- values.yaml | 3 +-- 3 files changed, 3 insertions(+), 4 deletions(-) diff --git a/templates/deployment-sidekiq.yaml b/templates/deployment-sidekiq.yaml index e69d0cd..f2e692d 100644 --- a/templates/deployment-sidekiq.yaml +++ b/templates/deployment-sidekiq.yaml @@ -212,7 +212,7 @@ spec: - name: MASTODON_PROMETHEUS_EXPORTER_LOCAL value: "true" - name: MASTODON_PROMETHEUS_EXPORTER_HOST - value: {{ $context.Values.mastodon.metrics.prometheus.host }} + value: "0.0.0.0" - name: MASTODON_PROMETHEUS_EXPORTER_PORT value: "{{ $context.Values.mastodon.metrics.prometheus.port }}" {{- if $context.Values.mastodon.metrics.prometheus.sidekiq.detailed }} diff --git a/templates/deployment-web.yaml b/templates/deployment-web.yaml index f5115b2..024cc6c 100644 --- a/templates/deployment-web.yaml +++ b/templates/deployment-web.yaml @@ -200,7 +200,7 @@ spec: - name: MASTODON_PROMETHEUS_EXPORTER_ENABLED value: "true" - name: PROMETHEUS_EXPORTER_HOST - value: {{ $.Values.mastodon.metrics.prometheus.host }} + value: "127.0.0.1" - name: PROMETHEUS_EXPORTER_PORT value: "{{ .Values.mastodon.metrics.prometheus.port }}" {{- if .Values.mastodon.metrics.prometheus.web.detailed }} diff --git a/values.yaml b/values.yaml index 8d4209c..223a3be 100644 --- a/values.yaml +++ b/values.yaml @@ -439,8 +439,7 @@ mastodon: # https://docs.joinmastodon.org/admin/config/#prometheus prometheus: enabled: false - # Host & port for the exporter to listen on - host: localhost + # Port for the exporter to listen on port: 9394 # Prometheus for web pods From 3b19a5435c35d545c271d823046cd6380d6984a4 Mon Sep 17 00:00:00 2001 From: Tim Campbell Date: Wed, 5 Mar 2025 11:59:11 +0100 Subject: [PATCH 11/13] Fix port check --- templates/deployment-sidekiq.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates/deployment-sidekiq.yaml b/templates/deployment-sidekiq.yaml index f2e692d..b8c9ec7 100644 --- a/templates/deployment-sidekiq.yaml +++ b/templates/deployment-sidekiq.yaml @@ -220,7 +220,7 @@ spec: value: "true" {{- end }} {{- end }} - {{- if or $context.Values.mastodon.sidekiq.otel.enabled (and $context.Values.mastodon.otel.enabled (ne $context.Values.mastodon.sidekiq.otel.enabled false)) }} + {{- if $context.Values.mastodon.metrics.prometheus.enabled }} ports: - name: prometheus containerPort: {{ .Values.mastodon.metrics.prometheus.port }} From 3f6396d84d132d1e3dd35ab23334b68d5aded29a Mon Sep 17 00:00:00 2001 From: Tim Campbell Date: Wed, 5 Mar 2025 16:46:09 +0100 Subject: [PATCH 12/13] Minor bugfix --- templates/deployment-sidekiq.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates/deployment-sidekiq.yaml b/templates/deployment-sidekiq.yaml index b8c9ec7..5951bd6 100644 --- a/templates/deployment-sidekiq.yaml +++ b/templates/deployment-sidekiq.yaml @@ -223,7 +223,7 @@ spec: {{- if $context.Values.mastodon.metrics.prometheus.enabled }} ports: - name: prometheus - containerPort: {{ .Values.mastodon.metrics.prometheus.port }} + containerPort: {{ $context.Values.mastodon.metrics.prometheus.port }} {{- end }} volumeMounts: {{- if (not $context.Values.mastodon.s3.enabled) }} From c3862faa04f26289d64872e2532274c696bf7a51 Mon Sep 17 00:00:00 2001 From: Tim Campbell Date: Thu, 6 Mar 2025 09:48:44 +0100 Subject: [PATCH 13/13] Bugfix for sidecar prometheus exporter --- templates/deployment-web.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/templates/deployment-web.yaml b/templates/deployment-web.yaml index 024cc6c..078bb5f 100644 --- a/templates/deployment-web.yaml +++ b/templates/deployment-web.yaml @@ -253,6 +253,8 @@ spec: command: - ./bin/prometheus_exporter args: + - "--bind" + - "0.0.0.0" - "--port" - "{{ .Values.mastodon.metrics.prometheus.port }}" resources: