feat: deploy monitoring to data

clusters/data/monitoring.yaml

@@ -0,0 +1,33 @@
+apiVersion: kustomize.toolkit.fluxcd.io/v1
+kind: Kustomization
+metadata:
+  name: monitoring-controllers
+  namespace: flux-system
+spec:
+  dependsOn:
+    - name: infra-configs
+  interval: 1h
+  retryInterval: 1m
+  timeout: 5m
+  sourceRef:
+    kind: GitRepository
+    name: flux-system
+  path: ./monitoring/controllers/data
+  prune: true
+# ---
+# apiVersion: kustomize.toolkit.fluxcd.io/v1
+# kind: Kustomization
+# metadata:
+#   name: monitoring-configs
+#   namespace: flux-system
+# spec:
+#   dependsOn:
+#     - name: monitoring-controllers
+#   interval: 1h
+#   retryInterval: 1m
+#   timeout: 5m
+#   sourceRef:
+#     kind: GitRepository
+#     name: flux-system
+#   path: ./monitoring/configs/jotunheim
+#   prune: true

monitoring/configs/jotunheim/grafana/dashboards/kustomization.yaml

@@ -2,16 +2,6 @@ apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 namespace: monitoring
 configMapGenerator:
-  # - name: extra-dashboards
-  #   files:
-  #     - extra-dashboards/edb-postgres.json # https://github.com/EnterpriseDB/docs/blob/main/product_docs/docs/postgres_for_kubernetes/1/samples/monitoring/grafana-dashboard.json
-  #     # - extra-dashboards/node-exporter-full.json
-  #     - extra-dashboards/node-exporter-nodename.json
-  #     - extra-dashboards/cloudflared.json
-  #   options:
-  #     labels:
-  #       grafana_dashboard: "1"
-
   - name: personal-dashboards
     files:
       - quantified-self/zettelkasten.json

monitoring/controllers/data/kube-prometheus-stack/kustomization.yaml

@@ -0,0 +1,15 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+namespace: monitoring
+resources:
+  - namespace.yaml
+  - secrets.yaml
+  - release.yaml
+  - repository.yaml
+
+configMapGenerator:
+  - name: kube-prometheus-stack-values
+    files:
+      - values.yaml=values.yaml
+configurations:
+  - kustomizeconfig.yaml

monitoring/controllers/data/kube-prometheus-stack/kustomizeconfig.yaml

@@ -0,0 +1,6 @@
+nameReference:
+  - kind: ConfigMap
+    version: v1
+    fieldSpecs:
+      - path: spec/valuesFrom/name
+        kind: HelmRelease

monitoring/controllers/data/kube-prometheus-stack/namespace.yaml

@@ -0,0 +1,7 @@
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: monitoring
+  labels:
+    pod-security.kubernetes.io/enforce: privileged
+    app.kubernetes.io/component: monitoring

monitoring/controllers/data/kube-prometheus-stack/release.yaml

@@ -0,0 +1,31 @@
+apiVersion: helm.toolkit.fluxcd.io/v2
+kind: HelmRelease
+metadata:
+  name: kube-prometheus-stack
+spec:
+  interval: 30m
+  chart:
+    spec:
+      chart: kube-prometheus-stack
+      version: "68.x"
+      sourceRef:
+        kind: HelmRepository
+        name: kube-prometheus-stack
+        namespace: monitoring
+      interval: 12h
+
+  valuesFrom:
+    - kind: ConfigMap
+      name: kube-prometheus-stack-values
+
+  install:
+    crds: Create
+  upgrade:
+    crds: CreateReplace
+  driftDetection:
+    mode: enabled
+    ignore:
+      # Ignore "validated" annotation which is not inserted during install
+      - paths: ["/metadata/annotations/prometheus-operator-validated"]
+        target:
+          kind: PrometheusRule

monitoring/controllers/data/kube-prometheus-stack/repository.yaml

@@ -0,0 +1,8 @@
+apiVersion: source.toolkit.fluxcd.io/v1
+kind: HelmRepository
+metadata:
+  name: kube-prometheus-stack
+  namespace: monitoring
+spec:
+  interval: 24h
+  url: https://prometheus-community.github.io/helm-charts

monitoring/controllers/data/kube-prometheus-stack/secrets.yaml

@@ -0,0 +1,19 @@
+apiVersion: external-secrets.io/v1beta1
+kind: ExternalSecret
+metadata:
+  name: grafana-env
+spec:
+  refreshInterval: 1h
+  secretStoreRef:
+    name: azure-kv-store
+    kind: ClusterSecretStore
+  data:
+    - secretKey: HEALTH_DB_PASSWORD
+      remoteRef:
+        key: health-db-password
+    - secretKey: HOME_IOT_DB_PASSWORD
+      remoteRef:
+        key: shelly-db-password
+    - secretKey: QF_DB_PASSWORD
+      remoteRef:
+        key: zettelkasten-tracker-db-password

monitoring/controllers/data/kube-prometheus-stack/values.yaml

@@ -0,0 +1,135 @@
+# NOTE: These must be set manually to the control plane IP addresses
+# kubeEtcd:
+#  endpoints:
+#    - x.y.z.a
+#    - x.y.z.b
+#    - x.y.z.c
+kubeControllerManager:
+  service:
+    selector:
+      k8s-app: kube-controller-manager
+kubeScheduler:
+  service:
+    selector:
+      k8s-app: kube-scheduler
+
+alertmanager:
+  enabled: false
+
+prometheus:
+  prometheusSpec:
+    podMonitorNamespaceSelector:
+      matchLabels:
+        app.kubernetes.io/component: monitoring
+
+    # Discover all PodMonitors, Probes, PrometheusRules and ServiceMonitors
+    podMonitorSelectorNilUsesHelmValues: false
+    probeSelectorNilUsesHelmValues: false
+    ruleSelectorNilUsesHelmValues: false
+    serviceMonitorSelectorNilUsesHelmValues: false
+
+grafana:
+  grafana.ini:
+    date_formats:
+      default_week_start: monday
+
+  # envFromSecret: grafana-env
+
+  sidecar:
+    datasources:
+      enabled: true
+      label: grafana_datasource
+      labelValue: "1"
+    dashboards:
+      enabled: true
+      label: grafana_dashboard
+      labelValue: "1"
+      searchNamespace: monitoring
+      provider:
+        allowUiUpdates: true
+
+  # Choosing to add data sources here instead of separate configmaps.
+  # Reason: Adding here will trigger a redeploy.
+  # Future improvement: utilize Kustomizeconfig to allow separate configmap files.
+  # datasources:
+  #   datasources.yaml:
+  #     apiVersion: 1
+  #     datasources:
+  #       - name: Health DB
+  #         type: postgres
+  #         url: health-db.mischavandenburg.net:5432
+  #         user: health
+  #         uid: health-db
+  #         secureJsonData:
+  #           password: $HEALTH_DB_PASSWORD
+  #         jsonData:
+  #           database: health
+  #           sslmode: disable
+  #           maxOpenConns: 100
+  #           maxIdleConns: 100
+  #           maxLifetime: 14400
+  #           minIdleTimeoutMinutes: 1
+  #           connMaxLifetime: 14400
+  #           postgresVersion: 1500
+  #           timescaledb: false
+  #         version: 1
+  #         editable: true
+  #       - name: Home IoT DB
+  #         type: postgres
+  #         url: home-iot-db.mischavandenburg.net:5432
+  #         user: shelly
+  #         uid: home-iot-db
+  #         secureJsonData:
+  #           password: $HOME_IOT_DB_PASSWORD
+  #         jsonData:
+  #           database: shelly
+  #           sslmode: disable
+  #           maxOpenConns: 100
+  #           maxIdleConns: 100
+  #           maxLifetime: 14400
+  #           minIdleTimeoutMinutes: 1
+  #           connMaxLifetime: 14400
+  #           postgresVersion: 1500
+  #           timescaledb: false
+  #         version: 1
+  #         editable: true
+  #       - name: Quantified Self DB
+  #         type: postgres
+  #         url: quantified-self-db.mischavandenburg.net:5432
+  #         user: zktracker
+  #         uid: quantified-self-db
+  #         secureJsonData:
+  #           password: $QF_DB_PASSWORD
+  #         jsonData:
+  #           database: zettelkasten-tracker
+  #           sslmode: disable
+  #           maxOpenConns: 100
+  #           maxIdleConns: 100
+  #           maxLifetime: 14400
+  #           minIdleTimeoutMinutes: 1
+  #           connMaxLifetime: 14400
+  #           postgresVersion: 1500
+  #           timescaledb: false
+  #         version: 1
+  #         editable: true
+  #
+  #     deleteDatasources:
+  #       - name: Health DB
+  #         orgId: 1
+  #       - name: Home IoT DB
+  #         orgId: 1
+  #       - name: Quatified Self DB
+  #         orgId: 1
+
+  persistence:
+    enabled: true
+    type: pvc
+    accessModes:
+      - ReadWriteOnce
+    size: 4Gi
+
+  # Allow Omni Workload Proxying for this service
+  service:
+    annotations:
+      omni-kube-service-exposer.sidero.dev/port: "50082"
+      omni-kube-service-exposer.sidero.dev/label: Grafana