From 851d24f470321a78ff2453be91ec7e49a0fcb379 Mon Sep 17 00:00:00 2001 From: Mike Wooskey Date: Thu, 11 Jul 2024 13:21:16 +0000 Subject: [PATCH] Fixed #254. Fixing #253. --- ...onfigure_compose_profiles_default_optional | 27 +++++++++++++++++++ prometheus/.env-dist | 5 ++++ prometheus/Makefile | 19 +++++++++---- prometheus/README.md | 11 +++++--- prometheus/config/Dockerfile | 2 +- prometheus/config/prometheus.yml | 13 ++++++--- prometheus/config/setup.sh | 3 +++ prometheus/docker-compose.instance.yaml | 7 ++++- prometheus/docker-compose.yaml | 7 +++++ 9 files changed, 81 insertions(+), 13 deletions(-) create mode 100644 _scripts/reconfigure_compose_profiles_default_optional diff --git a/_scripts/reconfigure_compose_profiles_default_optional b/_scripts/reconfigure_compose_profiles_default_optional new file mode 100644 index 00000000..3ff1f63f --- /dev/null +++ b/_scripts/reconfigure_compose_profiles_default_optional @@ -0,0 +1,27 @@ +#!/bin/bash + +## reconfigure_compose_profiles ${ENV_FILE} VAR1=profile1 VAR2=profile2 ... + +BIN=$(dirname ${BASH_SOURCE}) +source ${BIN}/funcs.sh +set -e + +ENV_FILE=${1} +shift +## Make new .env if it doesn't exist: +test -f ${ENV_FILE} || cp .env-dist ${ENV_FILE} + +#PROFILES="default" +for arg in "$@"; do + parts=(${arg//=/ }); var=${parts[0]}; profile=${parts[@]:1}; + if [[ $(${BIN}/dotenv -f ${ENV_FILE} get ${var}) == "true" ]]; then + if [ -z "$PROFILES" ]; then + PROFILES="${profile}" + else + PROFILES="${PROFILES},${profile}" + fi + fi +done + +${BIN}/dotenv -f ${ENV_FILE} set DOCKER_COMPOSE_PROFILES="${PROFILES}" +echo "Set DOCKER_COMPOSE_PROFILES=${PROFILES}" diff --git a/prometheus/.env-dist b/prometheus/.env-dist index f15a9ef1..e8800c37 100644 --- a/prometheus/.env-dist +++ b/prometheus/.env-dist @@ -16,6 +16,7 @@ PROMETHEUS_CONFIG_YTT_VERSION=v0.49.0 PROMETHEUS_INSTANCE= +PROMETHEUS_ENABLED=true PROMETHEUS_NODE_EXPORTER_ENABLED=true PROMETHEUS_CADVISOR_ENABLED=true PROMETHEUS_ALERTMANAGER_ENABLED=false @@ -66,3 +67,7 @@ PROMETHEUS_MTLS_AUTH=false # Enter a comma separated list of client domains allowed to connect via mTLS. # Wildcards are allowed and encouraged on a per-app basis: PROMETHEUS_MTLS_AUTHORIZED_CERTS=*.clients.whoami.example.com + +PROMETHEUS_NODE_EXPORTER_TARGETS=node-exporter:9100 +PROMETHEUS_CADVISOR_TARGETS=cadvisor:8080 +PROMETHEUS_ALERTMANAGER_TARGETS=alertmanager:9093 diff --git a/prometheus/Makefile b/prometheus/Makefile index a33eb65e..c5fa0638 100644 --- a/prometheus/Makefile +++ b/prometheus/Makefile @@ -8,19 +8,27 @@ config-hook: @${BIN}/reconfigure ${ENV_FILE} PROMETHEUS_INSTANCE=$${instance:-default} @${BIN}/reconfigure_auth ${ENV_FILE} PROMETHEUS @echo - @${BIN}/confirm $$(test "$$(${BIN}/dotenv -f ${ENV_FILE} get PROMETHEUS_NODE_EXPORTER_ENABLED)" == true && echo yes || echo no) "Do you want to run node-exporter to collect the Host system metrics" "?" && ${BIN}/reconfigure ${ENV_FILE} PROMETHEUS_NODE_EXPORTER_ENABLED=true || ${BIN}/reconfigure ${ENV_FILE} PROMETHEUS_NODE_EXPORTER_ENABLED=false + @${BIN}/confirm $$(test "$$(${BIN}/dotenv -f ${ENV_FILE} get PROMETHEUS_ENABLED)" == true && echo yes || echo no) "Do you want to install Prometheus (the only reason *not* to install it is if you're installing cADvisor, Node Exporter, and/or Alertmanager on a host other than the host where Prometheus itself is installed, as additional targets for that instance of Prometheus)" "?" && ${BIN}/reconfigure ${ENV_FILE} PROMETHEUS_ENABLED=true || ${BIN}/reconfigure ${ENV_FILE} PROMETHEUS_ENABLED=false + @echo + @${BIN}/confirm $$(test "$$(${BIN}/dotenv -f ${ENV_FILE} get PROMETHEUS_NODE_EXPORTER_ENABLED)" == true && echo yes || echo no) "Do you want to run Node Exporter to collect the Host system metrics" "?" && ${BIN}/reconfigure ${ENV_FILE} PROMETHEUS_NODE_EXPORTER_ENABLED=true || ${BIN}/reconfigure ${ENV_FILE} PROMETHEUS_NODE_EXPORTER_ENABLED=false @echo @${BIN}/confirm $$(test "$$(${BIN}/dotenv -f ${ENV_FILE} get PROMETHEUS_CADVISOR_ENABLED)" == true && echo yes || echo no) "Do you want to run cAdvisor to collect container metrics" "?" && ${BIN}/reconfigure ${ENV_FILE} PROMETHEUS_CADVISOR_ENABLED=true || ${BIN}/reconfigure ${ENV_FILE} PROMETHEUS_CADVISOR_ENABLED=false @echo - @${BIN}/confirm $$(test "$$(${BIN}/dotenv -f ${ENV_FILE} get PROMETHEUS_ALERTMANAGER_ENABLED)" == true && echo yes || echo no) "Do you want to run alertmanager to collect container metrics" "?" && ${BIN}/reconfigure ${ENV_FILE} PROMETHEUS_ALERTMANAGER_ENABLED=true || ${BIN}/reconfigure ${ENV_FILE} PROMETHEUS_ALERTMANAGER_ENABLED=false + @${BIN}/confirm $$(test "$$(${BIN}/dotenv -f ${ENV_FILE} get PROMETHEUS_ALERTMANAGER_ENABLED)" == true && echo yes || echo no) "Do you want to run Alertmanager to collect container metrics" "?" && ${BIN}/reconfigure ${ENV_FILE} PROMETHEUS_ALERTMANAGER_ENABLED=true || ${BIN}/reconfigure ${ENV_FILE} PROMETHEUS_ALERTMANAGER_ENABLED=false @echo @test "$$(${BIN}/dotenv -f ${ENV_FILE} get PROMETHEUS_ALERTMANAGER_ENABLED)" == true && (${BIN}/confirm $$(test "$$(${BIN}/dotenv -f ${ENV_FILE} get PROMETHEUS_ALERTMANAGER_SMTP_ENABLED)" == true && echo yes || echo no) "Do you want to send notifications via SMTP (email)" "?" && ${BIN}/reconfigure ${ENV_FILE} PROMETHEUS_ALERTMANAGER_SMTP_ENABLED=true || ${BIN}/reconfigure ${ENV_FILE} PROMETHEUS_ALERTMANAGER_SMTP_ENABLED=false) || ${BIN}/reconfigure ${ENV_FILE} PROMETHEUS_ALERTMANAGER_SMTP_ENABLED=false || true @test "$$(${BIN}/dotenv -f ${ENV_FILE} get PROMETHEUS_ALERTMANAGER_SMTP_ENABLED)" == true && ${BIN}/reconfigure_ask ${ENV_FILE} PROMETHEUS_ALERTMANAGER_SMTP_FROM "Enter the SMTP default sender from address" prometheus@${ROOT_DOMAIN} || true @test "$$(${BIN}/dotenv -f ${ENV_FILE} get PROMETHEUS_ALERTMANAGER_SMTP_ENABLED)" == true && ${BIN}/reconfigure_ask ${ENV_FILE} PROMETHEUS_ALERTMANAGER_SMTP_TO "Enter the email address to send notifications to" || true @test "$$(${BIN}/dotenv -f ${ENV_FILE} get PROMETHEUS_ALERTMANAGER_SMTP_ENABLED)" == true && ${BIN}/reconfigure_ask_host_port ${ENV_FILE} PROMETHEUS_ALERTMANAGER_SMTP_SMARTHOST "Enter the SMTP smarthost domain and port" "smtp.example.com:465" || true @test "$$(${BIN}/dotenv -f ${ENV_FILE} get PROMETHEUS_ALERTMANAGER_SMTP_ENABLED)" == true && ${BIN}/reconfigure_ask_email_address ${ENV_FILE} PROMETHEUS_ALERTMANAGER_SMTP_AUTH_USERNAME "Enter the SMTP authentication username" "user@example.com" || true - @test "$$(${BIN}/dotenv -f ${ENV_FILE} get PROMETHEUS_ALERTMANAGER_SMTP_ENABLED)" == true && ${BIN}/reconfigure_ask ${ENV_FILE} PROMETHEUS_ALERTMANAGER_SMTP_AUTH_PASSWORD "Enter the SMTP authentication password" || true + @test "$$(${BIN}/dotenv -f ${ENV_FILE} get PROMETHEUS_ALERTMANAGER_SMTP_ENABLED)" == true && ${BIN}/reconfigure_ask ${ENV_FILE} PROMETHEUS_ALERTMANAGER_SMTP_AUTH_PASSWORD "Enter the SMTP authentication password" || true + @echo + @test "$$(${BIN}/dotenv -f ${ENV_FILE} get PROMETHEUS_ENABLED)" == true && test "$$(${BIN}/dotenv -f ${ENV_FILE} get PROMETHEUS_NODE_EXPORTER_ENABLED)" == true && ${BIN}/reconfigure_ask ${ENV_FILE} PROMETHEUS_NODE_EXPORTER_TARGETS "Enter a comma-separated list of Node Exporter targets" || true + @echo + @test "$$(${BIN}/dotenv -f ${ENV_FILE} get PROMETHEUS_ENABLED)" == true && test "$$(${BIN}/dotenv -f ${ENV_FILE} get PROMETHEUS_CADVISOR_ENABLED)" == true && ${BIN}/reconfigure_ask ${ENV_FILE} PROMETHEUS_CADVISOR_TARGETS "Enter a comma-separated list of cAdvisor targets" || true @echo + @test "$$(${BIN}/dotenv -f ${ENV_FILE} get PROMETHEUS_ENABLED)" == true && test "$$(${BIN}/dotenv -f ${ENV_FILE} get PROMETHEUS_ALERTMANAGER_ENABLED)" == true && ${BIN}/reconfigure_ask ${ENV_FILE} PROMETHEUS_ALERTMANAGER_TARGETS "Enter a comma-separated list of AlertManager targets" || true + @make --no-print-directory compose-profiles .PHONY: override-hook @@ -35,11 +43,11 @@ override-hook: #### # (this hardcodes the string into docker-compose.override.yaml) #### name=@VARIABLE_NAME # sets the template 'name' field to the literal string '${VARIABLE_NAME}' #### # (used for regular docker-compose expansion of env vars by name.) - @${BIN}/docker_compose_override ${ENV_FILE} project=:prometheus instance=@PROMETHEUS_INSTANCE traefik_host=@PROMETHEUS_METRICS_TRAEFIK_HOST http_auth=PROMETHEUS_HTTP_AUTH http_auth_var=@PROMETHEUS_HTTP_AUTH ip_sourcerange=@PROMETHEUS_METRICS_IP_SOURCERANGE oauth2=PROMETHEUS_OAUTH2 authorized_group=PROMETHEUS_OAUTH2_AUTHORIZED_GROUP enable_mtls_auth=PROMETHEUS_MTLS_AUTH mtls_authorized_certs=PROMETHEUS_MTLS_AUTHORIZED_CERTS + @${BIN}/docker_compose_override ${ENV_FILE} project=:prometheus instance=@PROMETHEUS_INSTANCE traefik_host=@PROMETHEUS_METRICS_TRAEFIK_HOST http_auth=PROMETHEUS_HTTP_AUTH http_auth_var=@PROMETHEUS_HTTP_AUTH ip_sourcerange=@PROMETHEUS_METRICS_IP_SOURCERANGE oauth2=PROMETHEUS_OAUTH2 authorized_group=PROMETHEUS_OAUTH2_AUTHORIZED_GROUP enable_mtls_auth=PROMETHEUS_MTLS_AUTH mtls_authorized_certs=PROMETHEUS_MTLS_AUTHORIZED_CERTS node_exporter_enabled=PROMETHEUS_NODE_EXPORTER_ENABLED node_exporter_targets=PROMETHEUS_NODE_EXPORTER_TARGETS cadvisor_enabled=PROMETHEUS_CADVISOR_ENABLED cadvisor_targets=PROMETHEUS_CADVISOR_TARGETS alertmanager_enabled=PROMETHEUS_ALERTMANAGER_ENABLED alertmanager_targets=PROMETHEUS_ALERTMANAGER_TARGETS prometheus_enabled=PROMETHEUS_ENABLED .PHONY: compose-profiles compose-profiles: - @${BIN}/reconfigure_compose_profiles ${ENV_FILE} PROMETHEUS_NODE_EXPORTER_ENABLED=node-exporter PROMETHEUS_CADVISOR_ENABLED=cadvisor PROMETHEUS_ALERTMANAGER_ENABLED=alertmanager + @${BIN}/reconfigure_compose_profiles_default_optional ${ENV_FILE} PROMETHEUS_ENABLED=default PROMETHEUS_NODE_EXPORTER_ENABLED=node-exporter PROMETHEUS_CADVISOR_ENABLED=cadvisor PROMETHEUS_ALERTMANAGER_ENABLED=alertmanager .PHONY: shell shell: @@ -54,3 +62,4 @@ test-alert: @docker compose --env-file ${ENV_FILE} run --rm -it alert-test @echo @echo "Sent test alert" + diff --git a/prometheus/README.md b/prometheus/README.md index 65aee51e..44be834b 100644 --- a/prometheus/README.md +++ b/prometheus/README.md @@ -24,6 +24,11 @@ make config * Choose whether to run node-exporter or not (For collecting the Host metrics). * Choose whether to run cAdvisor or not (For collecting container metrics). +By default, this installs Node Exporter, cAdvisor, and Alertmanager targets +for the host (of (i.e., your Docker context). When you run `make config` you'll +have the opportunity to add or change targets. You can also add targets from +within the Prometheus UI, but they will not be saved if the container is reinstalled. + ``` make install ``` @@ -48,13 +53,13 @@ This configurations comes with the following dashboards preinstalled: You can find new dashboards on [grafana.com](https://grafana.com/grafana/dashboards/). Add new -dashboards, by downloading the JSON file into the -[grafana/dashboards](grafana/dashboards) directory, and they will be +dashboards by downloading the JSON file into the +[grafana/provisioning/dashboards](grafana/provisioning/dashboards) directory, and they will be made available the next time you run `make install`. ## Alerts -If you enabled alertmanager and SMTP via `make config`, you can test that alerts are working: +If you enabled Alertmanager and SMTP via `make config`, you can test that alerts are working: ``` make test-alert diff --git a/prometheus/config/Dockerfile b/prometheus/config/Dockerfile index 3a965a51..92303e52 100644 --- a/prometheus/config/Dockerfile +++ b/prometheus/config/Dockerfile @@ -1,4 +1,4 @@ -FROM debian:stable-slim as ytt +FROM debian:stable-slim AS ytt ARG PROMETHEUS_CONFIG_YTT_VERSION RUN apt update && apt install -y wget && wget "https://github.com/vmware-tanzu/carvel-ytt/releases/download/${PROMETHEUS_CONFIG_YTT_VERSION}/ytt-linux-$(dpkg --print-architecture)" -O ytt && install ytt /usr/local/bin/ytt diff --git a/prometheus/config/prometheus.yml b/prometheus/config/prometheus.yml index 37d2af84..f6062e6d 100644 --- a/prometheus/config/prometheus.yml +++ b/prometheus/config/prometheus.yml @@ -1,7 +1,13 @@ #@ load("@ytt:data", "data") + +#@ node_exporter_targets = data.values.node_exporter_targets.split(",") +#@ cadvisor_targets = data.values.cadvisor_targets.split(",") +#@ alertmanager_targets = data.values.alertmanager_targets.split(",") + global: scrape_interval: 1m +#@yaml/text-templated-strings scrape_configs: - job_name: "prometheus" scrape_interval: 1m @@ -11,19 +17,20 @@ scrape_configs: #@ if data.values.node_exporter_enabled == "true": - job_name: "node" static_configs: - - targets: ["node-exporter:9100"] + - targets: #@ node_exporter_targets #@ end #@ if data.values.cadvisor_enabled == "true": - job_name: "cadvisor" scrape_interval: 5s static_configs: - - targets: ["cadvisor:8080"] + - targets: #@ cadvisor_targets #@ end +#@yaml/text-templated-strings #@ if data.values.alertmanager_enabled == "true": alerting: alertmanagers: - static_configs: - - targets: ['alertmanager:9093'] + - targets: #@ alertmanager_targets #@ end diff --git a/prometheus/config/setup.sh b/prometheus/config/setup.sh index 7c3db03a..c9bdc239 100644 --- a/prometheus/config/setup.sh +++ b/prometheus/config/setup.sh @@ -6,8 +6,11 @@ ytt_template() { [ -e "${src}" ] || (echo "Template not found: ${src}" && exit 1) ytt -f ${src} \ -v node_exporter_enabled=${PROMETHEUS_NODE_EXPORTER_ENABLED} \ + -v node_exporter_targets=${PROMETHEUS_NODE_EXPORTER_TARGETS} \ -v cadvisor_enabled=${PROMETHEUS_CADVISOR_ENABLED} \ + -v cadvisor_targets=${PROMETHEUS_CADVISOR_TARGETS} \ -v alertmanager_enabled=${PROMETHEUS_ALERTMANAGER_ENABLED} \ + -v alertmanager_targets=${PROMETHEUS_ALERTMANAGER_TARGETS} \ -v smtp_enabled=${PROMETHEUS_ALERTMANAGER_SMTP_ENABLED} \ -v smtp_smarthost=${PROMETHEUS_ALERTMANAGER_SMTP_SMARTHOST} \ -v smtp_auth_username=${PROMETHEUS_ALERTMANAGER_SMTP_AUTH_USERNAME} \ diff --git a/prometheus/docker-compose.instance.yaml b/prometheus/docker-compose.instance.yaml index f6d30d37..11e6c8cb 100644 --- a/prometheus/docker-compose.instance.yaml +++ b/prometheus/docker-compose.instance.yaml @@ -19,8 +19,12 @@ #@ mtls_authorized_certs = data.values.mtls_authorized_certs #@ enabled_middlewares = [] +#! ### Custom app vars: +#@ prometheus_enabled = data.values.prometheus_enabled == "true" + #@yaml/text-templated-strings services: + #@ if prometheus_enabled: grafana: #@ service = "grafana" labels: @@ -57,4 +61,5 @@ services: #@ end #! Apply all middlewares (do this at the end!) - - "traefik.http.routers.(@= router @).middlewares=(@= ','.join(enabled_middlewares) @)" \ No newline at end of file + - "traefik.http.routers.(@= router @).middlewares=(@= ','.join(enabled_middlewares) @)" + #@ end \ No newline at end of file diff --git a/prometheus/docker-compose.yaml b/prometheus/docker-compose.yaml index 19006bea..a29a97d5 100644 --- a/prometheus/docker-compose.yaml +++ b/prometheus/docker-compose.yaml @@ -11,10 +11,16 @@ services: context: config args: PROMETHEUS_CONFIG_YTT_VERSION: ${PROMETHEUS_CONFIG_YTT_VERSION} + profiles: + - default + - alertmanager environment: - PROMETHEUS_NODE_EXPORTER_ENABLED + - PROMETHEUS_NODE_EXPORTER_TARGETS - PROMETHEUS_CADVISOR_ENABLED + - PROMETHEUS_CADVISOR_TARGETS - PROMETHEUS_ALERTMANAGER_ENABLED + - PROMETHEUS_ALERTMANAGER_TARGETS - PROMETHEUS_ALERTMANAGER_SMTP_ENABLED - PROMETHEUS_ALERTMANAGER_SMTP_SMARTHOST - PROMETHEUS_ALERTMANAGER_SMTP_AUTH_USERNAME @@ -86,6 +92,7 @@ services: profiles: - cadvisor image: gcr.io/cadvisor/cadvisor:${PROMETHEUS_CADVISOR_VERSION} + restart: unless-stopped command: - "--housekeeping_interval=30s" - "--docker_only=true"