Changes to Prometheus

_scripts/reconfigure_compose_profiles_default_optional

@@ -0,0 +1,27 @@
+#!/bin/bash
+
+## reconfigure_compose_profiles ${ENV_FILE} VAR1=profile1 VAR2=profile2 ...
+
+BIN=$(dirname ${BASH_SOURCE})
+source ${BIN}/funcs.sh
+set -e
+
+ENV_FILE=${1}
+shift
+## Make new .env if it doesn't exist:
+test -f ${ENV_FILE} || cp .env-dist ${ENV_FILE}
+
+#PROFILES="default"
+for arg in "$@"; do
+    parts=(${arg//=/ }); var=${parts[0]}; profile=${parts[@]:1};
+    if [[ $(${BIN}/dotenv -f ${ENV_FILE} get ${var}) == "true" ]]; then
+        if [ -z "$PROFILES" ]; then
+          PROFILES="${profile}"
+        else
+          PROFILES="${PROFILES},${profile}"
+        fi
+    fi
+done
+
+${BIN}/dotenv -f ${ENV_FILE} set DOCKER_COMPOSE_PROFILES="${PROFILES}"
+echo "Set DOCKER_COMPOSE_PROFILES=${PROFILES}"

prometheus/.env-dist

@@ -16,6 +16,7 @@ PROMETHEUS_CONFIG_YTT_VERSION=v0.49.0
 
 PROMETHEUS_INSTANCE=
 
+PROMETHEUS_ENABLED=true
 PROMETHEUS_NODE_EXPORTER_ENABLED=true
 PROMETHEUS_CADVISOR_ENABLED=true
 PROMETHEUS_ALERTMANAGER_ENABLED=false
@@ -66,3 +67,7 @@ PROMETHEUS_MTLS_AUTH=false
 # Enter a comma separated list of client domains allowed to connect via mTLS.
 # Wildcards are allowed and encouraged on a per-app basis:
 PROMETHEUS_MTLS_AUTHORIZED_CERTS=*.clients.whoami.example.com
+
+PROMETHEUS_NODE_EXPORTER_TARGETS=node-exporter:9100
+PROMETHEUS_CADVISOR_TARGETS=cadvisor:8080
+PROMETHEUS_ALERTMANAGER_TARGETS=alertmanager:9093

prometheus/Makefile

@@ -8,19 +8,27 @@ config-hook:
 	@${BIN}/reconfigure ${ENV_FILE} PROMETHEUS_INSTANCE=$${instance:-default}
 	@${BIN}/reconfigure_auth ${ENV_FILE} PROMETHEUS
 	@echo
-	@${BIN}/confirm $$(test "$$(${BIN}/dotenv -f ${ENV_FILE} get PROMETHEUS_NODE_EXPORTER_ENABLED)" == true && echo yes || echo no) "Do you want to run node-exporter to collect the Host system metrics" "?" && ${BIN}/reconfigure ${ENV_FILE} PROMETHEUS_NODE_EXPORTER_ENABLED=true || ${BIN}/reconfigure ${ENV_FILE} PROMETHEUS_NODE_EXPORTER_ENABLED=false
+	@${BIN}/confirm $$(test "$$(${BIN}/dotenv -f ${ENV_FILE} get PROMETHEUS_ENABLED)" == true && echo yes || echo no) "Do you want to install Prometheus (the only reason *not* to install it is if you're installing cADvisor, Node Exporter, and/or Alertmanager on a host other than the host where Prometheus itself is installed, as additional targets for that instance of Prometheus)" "?" && ${BIN}/reconfigure ${ENV_FILE} PROMETHEUS_ENABLED=true || ${BIN}/reconfigure ${ENV_FILE} PROMETHEUS_ENABLED=false
+	@echo
+	@${BIN}/confirm $$(test "$$(${BIN}/dotenv -f ${ENV_FILE} get PROMETHEUS_NODE_EXPORTER_ENABLED)" == true && echo yes || echo no) "Do you want to run Node Exporter to collect the Host system metrics" "?" && ${BIN}/reconfigure ${ENV_FILE} PROMETHEUS_NODE_EXPORTER_ENABLED=true || ${BIN}/reconfigure ${ENV_FILE} PROMETHEUS_NODE_EXPORTER_ENABLED=false
 	@echo
 	@${BIN}/confirm $$(test "$$(${BIN}/dotenv -f ${ENV_FILE} get PROMETHEUS_CADVISOR_ENABLED)" == true && echo yes || echo no) "Do you want to run cAdvisor to collect container metrics" "?" && ${BIN}/reconfigure ${ENV_FILE} PROMETHEUS_CADVISOR_ENABLED=true || ${BIN}/reconfigure ${ENV_FILE} PROMETHEUS_CADVISOR_ENABLED=false
 	@echo
-	@${BIN}/confirm $$(test "$$(${BIN}/dotenv -f ${ENV_FILE} get PROMETHEUS_ALERTMANAGER_ENABLED)" == true && echo yes || echo no) "Do you want to run alertmanager to collect container metrics" "?" && ${BIN}/reconfigure ${ENV_FILE} PROMETHEUS_ALERTMANAGER_ENABLED=true || ${BIN}/reconfigure ${ENV_FILE} PROMETHEUS_ALERTMANAGER_ENABLED=false
+	@${BIN}/confirm $$(test "$$(${BIN}/dotenv -f ${ENV_FILE} get PROMETHEUS_ALERTMANAGER_ENABLED)" == true && echo yes || echo no) "Do you want to run Alertmanager to collect container metrics" "?" && ${BIN}/reconfigure ${ENV_FILE} PROMETHEUS_ALERTMANAGER_ENABLED=true || ${BIN}/reconfigure ${ENV_FILE} PROMETHEUS_ALERTMANAGER_ENABLED=false
 	@echo
 	@test "$$(${BIN}/dotenv -f ${ENV_FILE} get PROMETHEUS_ALERTMANAGER_ENABLED)" == true && (${BIN}/confirm $$(test "$$(${BIN}/dotenv -f ${ENV_FILE} get PROMETHEUS_ALERTMANAGER_SMTP_ENABLED)" == true && echo yes || echo no) "Do you want to send notifications via SMTP (email)" "?" && ${BIN}/reconfigure ${ENV_FILE} PROMETHEUS_ALERTMANAGER_SMTP_ENABLED=true || ${BIN}/reconfigure ${ENV_FILE} PROMETHEUS_ALERTMANAGER_SMTP_ENABLED=false) || ${BIN}/reconfigure ${ENV_FILE} PROMETHEUS_ALERTMANAGER_SMTP_ENABLED=false || true
 	@test "$$(${BIN}/dotenv -f ${ENV_FILE} get PROMETHEUS_ALERTMANAGER_SMTP_ENABLED)" == true && ${BIN}/reconfigure_ask ${ENV_FILE} PROMETHEUS_ALERTMANAGER_SMTP_FROM "Enter the SMTP default sender from address" prometheus@${ROOT_DOMAIN} || true
 	@test "$$(${BIN}/dotenv -f ${ENV_FILE} get PROMETHEUS_ALERTMANAGER_SMTP_ENABLED)" == true && ${BIN}/reconfigure_ask ${ENV_FILE} PROMETHEUS_ALERTMANAGER_SMTP_TO "Enter the email address to send notifications to" || true
 	@test "$$(${BIN}/dotenv -f ${ENV_FILE} get PROMETHEUS_ALERTMANAGER_SMTP_ENABLED)" == true && ${BIN}/reconfigure_ask_host_port ${ENV_FILE} PROMETHEUS_ALERTMANAGER_SMTP_SMARTHOST "Enter the SMTP smarthost domain and port" "smtp.example.com:465" || true
 	@test "$$(${BIN}/dotenv -f ${ENV_FILE} get PROMETHEUS_ALERTMANAGER_SMTP_ENABLED)" == true && ${BIN}/reconfigure_ask_email_address ${ENV_FILE} PROMETHEUS_ALERTMANAGER_SMTP_AUTH_USERNAME "Enter the SMTP authentication username" "[email protected]" || true
-	@test "$$(${BIN}/dotenv -f ${ENV_FILE} get PROMETHEUS_ALERTMANAGER_SMTP_ENABLED)" == true && ${BIN}/reconfigure_ask ${ENV_FILE} PROMETHEUS_ALERTMANAGER_SMTP_AUTH_PASSWORD "Enter the SMTP authentication password"  || true
+	@test "$$(${BIN}/dotenv -f ${ENV_FILE} get PROMETHEUS_ALERTMANAGER_SMTP_ENABLED)" == true && ${BIN}/reconfigure_ask ${ENV_FILE} PROMETHEUS_ALERTMANAGER_SMTP_AUTH_PASSWORD "Enter the SMTP authentication password" || true
+	@echo
+	@test "$$(${BIN}/dotenv -f ${ENV_FILE} get PROMETHEUS_ENABLED)" == true && test "$$(${BIN}/dotenv -f ${ENV_FILE} get PROMETHEUS_NODE_EXPORTER_ENABLED)" == true && ${BIN}/reconfigure_ask ${ENV_FILE} PROMETHEUS_NODE_EXPORTER_TARGETS "Enter a comma-separated list of Node Exporter targets" || true
+	@echo
+	@test "$$(${BIN}/dotenv -f ${ENV_FILE} get PROMETHEUS_ENABLED)" == true && test "$$(${BIN}/dotenv -f ${ENV_FILE} get PROMETHEUS_CADVISOR_ENABLED)" == true && ${BIN}/reconfigure_ask ${ENV_FILE} PROMETHEUS_CADVISOR_TARGETS "Enter a comma-separated list of cAdvisor targets" || true
 	@echo
+	@test "$$(${BIN}/dotenv -f ${ENV_FILE} get PROMETHEUS_ENABLED)" == true && test "$$(${BIN}/dotenv -f ${ENV_FILE} get PROMETHEUS_ALERTMANAGER_ENABLED)" == true && ${BIN}/reconfigure_ask ${ENV_FILE} PROMETHEUS_ALERTMANAGER_TARGETS "Enter a comma-separated list of AlertManager targets" || true
+
 	@make --no-print-directory compose-profiles
 
 .PHONY: override-hook
@@ -35,11 +43,11 @@ override-hook:
 ####                         # (this hardcodes the string into docker-compose.override.yaml)
 ####   name=@VARIABLE_NAME   # sets the template 'name' field to the literal string '${VARIABLE_NAME}'
 ####                         # (used for regular docker-compose expansion of env vars by name.)
-	@${BIN}/docker_compose_override ${ENV_FILE} project=:prometheus instance=@PROMETHEUS_INSTANCE traefik_host=@PROMETHEUS_METRICS_TRAEFIK_HOST http_auth=PROMETHEUS_HTTP_AUTH http_auth_var=@PROMETHEUS_HTTP_AUTH ip_sourcerange=@PROMETHEUS_METRICS_IP_SOURCERANGE oauth2=PROMETHEUS_OAUTH2 authorized_group=PROMETHEUS_OAUTH2_AUTHORIZED_GROUP enable_mtls_auth=PROMETHEUS_MTLS_AUTH mtls_authorized_certs=PROMETHEUS_MTLS_AUTHORIZED_CERTS
+	@${BIN}/docker_compose_override ${ENV_FILE} project=:prometheus instance=@PROMETHEUS_INSTANCE traefik_host=@PROMETHEUS_METRICS_TRAEFIK_HOST http_auth=PROMETHEUS_HTTP_AUTH http_auth_var=@PROMETHEUS_HTTP_AUTH ip_sourcerange=@PROMETHEUS_METRICS_IP_SOURCERANGE oauth2=PROMETHEUS_OAUTH2 authorized_group=PROMETHEUS_OAUTH2_AUTHORIZED_GROUP enable_mtls_auth=PROMETHEUS_MTLS_AUTH mtls_authorized_certs=PROMETHEUS_MTLS_AUTHORIZED_CERTS node_exporter_enabled=PROMETHEUS_NODE_EXPORTER_ENABLED node_exporter_targets=PROMETHEUS_NODE_EXPORTER_TARGETS cadvisor_enabled=PROMETHEUS_CADVISOR_ENABLED cadvisor_targets=PROMETHEUS_CADVISOR_TARGETS alertmanager_enabled=PROMETHEUS_ALERTMANAGER_ENABLED alertmanager_targets=PROMETHEUS_ALERTMANAGER_TARGETS prometheus_enabled=PROMETHEUS_ENABLED
 
 .PHONY: compose-profiles
 compose-profiles:
-	@${BIN}/reconfigure_compose_profiles ${ENV_FILE} PROMETHEUS_NODE_EXPORTER_ENABLED=node-exporter PROMETHEUS_CADVISOR_ENABLED=cadvisor PROMETHEUS_ALERTMANAGER_ENABLED=alertmanager
+	@${BIN}/reconfigure_compose_profiles_default_optional ${ENV_FILE} PROMETHEUS_ENABLED=default PROMETHEUS_NODE_EXPORTER_ENABLED=node-exporter PROMETHEUS_CADVISOR_ENABLED=cadvisor PROMETHEUS_ALERTMANAGER_ENABLED=alertmanager
 
 .PHONY: shell
 shell:
@@ -54,3 +62,4 @@ test-alert:
 	@docker compose --env-file ${ENV_FILE} run --rm -it alert-test
 	@echo 
 	@echo "Sent test alert"
+

prometheus/README.md

@@ -24,6 +24,11 @@ make config
 * Choose whether to run node-exporter or not (For collecting the Host metrics).
 * Choose whether to run cAdvisor or not (For collecting container metrics).
 
+By default, this installs Node Exporter, cAdvisor, and Alertmanager targets
+for the host (of (i.e., your Docker context). When you run `make config` you'll
+have the opportunity to add or change targets. You can also add targets from
+within the Prometheus UI, but they will not be saved if the container is reinstalled.
+
 ```
 make install
 ```
@@ -48,13 +53,13 @@ This configurations comes with the following dashboards preinstalled:
 
 You can find new dashboards on
 [grafana.com](https://grafana.com/grafana/dashboards/). Add new
-dashboards, by downloading the JSON file into the
-[grafana/dashboards](grafana/dashboards) directory, and they will be
+dashboards by downloading the JSON file into the
+[grafana/provisioning/dashboards](grafana/provisioning/dashboards) directory, and they will be
 made available the next time you run `make install`.
 
 ## Alerts
 
-If you enabled alertmanager and SMTP via `make config`, you can test that alerts are working:
+If you enabled Alertmanager and SMTP via `make config`, you can test that alerts are working:
 
 ```
 make test-alert

prometheus/config/Dockerfile

@@ -1,4 +1,4 @@
-FROM debian:stable-slim as ytt
+FROM debian:stable-slim AS ytt
 ARG PROMETHEUS_CONFIG_YTT_VERSION
 RUN apt update && apt install -y wget && wget "https://github.com/vmware-tanzu/carvel-ytt/releases/download/${PROMETHEUS_CONFIG_YTT_VERSION}/ytt-linux-$(dpkg --print-architecture)" -O ytt && install ytt /usr/local/bin/ytt
 

prometheus/config/prometheus.yml

@@ -1,7 +1,13 @@
 #@ load("@ytt:data", "data")
+
+#@ node_exporter_targets = data.values.node_exporter_targets.split(",")
+#@ cadvisor_targets = data.values.cadvisor_targets.split(",")
+#@ alertmanager_targets = data.values.alertmanager_targets.split(",")
+
 global:
   scrape_interval: 1m
 
+#@yaml/text-templated-strings
 scrape_configs:
   - job_name: "prometheus"
     scrape_interval: 1m
@@ -11,19 +17,20 @@ scrape_configs:
   #@ if data.values.node_exporter_enabled == "true":
   - job_name: "node"
     static_configs:
-    - targets: ["node-exporter:9100"]
+    - targets: #@ node_exporter_targets
   #@ end
 
   #@ if data.values.cadvisor_enabled == "true":
   - job_name: "cadvisor"
     scrape_interval: 5s
     static_configs:
-    - targets: ["cadvisor:8080"]
+    - targets: #@ cadvisor_targets
   #@ end
 
+#@yaml/text-templated-strings
 #@ if data.values.alertmanager_enabled == "true":
 alerting:
   alertmanagers:
     - static_configs:
-      - targets: ['alertmanager:9093']
+      - targets: #@ alertmanager_targets
 #@ end

prometheus/config/setup.sh

@@ -6,8 +6,11 @@ ytt_template() {
     [ -e "${src}" ] || (echo "Template not found: ${src}" && exit 1)
     ytt -f ${src} \
         -v node_exporter_enabled=${PROMETHEUS_NODE_EXPORTER_ENABLED} \
+        -v node_exporter_targets=${PROMETHEUS_NODE_EXPORTER_TARGETS} \
         -v cadvisor_enabled=${PROMETHEUS_CADVISOR_ENABLED} \
+        -v cadvisor_targets=${PROMETHEUS_CADVISOR_TARGETS} \
         -v alertmanager_enabled=${PROMETHEUS_ALERTMANAGER_ENABLED} \
+        -v alertmanager_targets=${PROMETHEUS_ALERTMANAGER_TARGETS} \
         -v smtp_enabled=${PROMETHEUS_ALERTMANAGER_SMTP_ENABLED} \
         -v smtp_smarthost=${PROMETHEUS_ALERTMANAGER_SMTP_SMARTHOST} \
         -v smtp_auth_username=${PROMETHEUS_ALERTMANAGER_SMTP_AUTH_USERNAME} \

prometheus/docker-compose.instance.yaml

@@ -19,8 +19,12 @@
 #@ mtls_authorized_certs = data.values.mtls_authorized_certs
 #@ enabled_middlewares = []
 
+#! ### Custom app vars:
+#@ prometheus_enabled = data.values.prometheus_enabled == "true"
+
 #@yaml/text-templated-strings
 services:
+  #@ if prometheus_enabled:
   grafana:
     #@ service = "grafana"
     labels:
@@ -57,4 +61,5 @@ services:
       #@ end
 
       #! Apply all middlewares (do this at the end!)
-      - "traefik.http.routers.(@= router @).middlewares=(@= ','.join(enabled_middlewares) @)"
+      - "traefik.http.routers.(@= router @).middlewares=(@= ','.join(enabled_middlewares) @)"
+  #@ end

prometheus/docker-compose.yaml

@@ -11,10 +11,16 @@ services:
       context: config
       args:
         PROMETHEUS_CONFIG_YTT_VERSION: ${PROMETHEUS_CONFIG_YTT_VERSION}
+    profiles:
+      - default
+      - alertmanager
     environment:
       - PROMETHEUS_NODE_EXPORTER_ENABLED
+      - PROMETHEUS_NODE_EXPORTER_TARGETS
       - PROMETHEUS_CADVISOR_ENABLED
+      - PROMETHEUS_CADVISOR_TARGETS
       - PROMETHEUS_ALERTMANAGER_ENABLED
+      - PROMETHEUS_ALERTMANAGER_TARGETS
       - PROMETHEUS_ALERTMANAGER_SMTP_ENABLED
       - PROMETHEUS_ALERTMANAGER_SMTP_SMARTHOST
       - PROMETHEUS_ALERTMANAGER_SMTP_AUTH_USERNAME
@@ -86,6 +92,7 @@ services:
     profiles:
       - cadvisor
     image: gcr.io/cadvisor/cadvisor:${PROMETHEUS_CADVISOR_VERSION}
+    restart: unless-stopped
     command:
       - "--housekeeping_interval=30s"
       - "--docker_only=true"