Add examples folder for alerts & dashboards, and observability stack (#…

…461)

* Add examples folder for alerts & dashboards, with initial 2 dashboards

* Add example observability stack & docs for using

config/observability/README.md

@@ -0,0 +1,48 @@
+# Observability stack guide
+
+## Deploying the observabilty stack
+
+```bash
+./bin/kustomize build ./config/observability/| docker run --rm -i ryane/kfilt -i kind=CustomResourceDefinition | kubectl apply --server-side -f -
+./bin/kustomize build ./config/observability/| docker run --rm -i ryane/kfilt -x kind=CustomResourceDefinition | kubectl apply -f -
+```
+
+This will deploy prometheus, alertmanager and grafana into the `monitoring` namespace,
+along with metrics scrape configuration for Istio and Envoy.
+
+## Accessing Grafana & Prometheus
+
+Use port forwarding to access Grafana & Prometheus:
+
+```bash
+kubectl -n monitoring port-forward service/grafana 3000:3000
+```
+
+The Grafana UI can be found at [http://127.0.0.1:3000/](http://127.0.0.1:3000/) (default user/pass of `admin` & `admin`).
+It is pre-loaded with some kubernetes and [gateway-api-state](https://github.com/Kuadrant/gateway-api-state-metrics) dashboards.
+
+```bash
+kubectl -n monitoring port-forward service/prometheus-k8s 9090:9090
+```
+
+The Prometheus UI can be found at [http://127.0.0.1:9090](http://127.0.0.1:9090).
+
+## Editing dashboards
+
+Dashboards can be imported in the Grafana UI using either raw JSON, a JSON file, or the URL/ID of one of the [dashboards on grafana.com](https://grafana.com/grafana/dashboards/).
+Some example dashboards are available in the [/examples](/examples) folder.
+
+To import a dashboard, click on the plus icon on the left sidebar and navigate to **Import**. After entering a dashboard URL/ID or JSON, click **Load**.
+
+After loading the dashboard, the next screen allows you to select a name and folder for the dashboard and specify the data source before finally importing the dashboard.
+
+Grafana dashboards can be exported as JSON in order to add them to the project's git repo.
+When viewing the dashboard you wish to export, click on the **share** button at the top of the screen.
+
+In the modal popup click **Export** and then **Save to file**.
+
+## Editing alerting rules
+
+Alerting rules can be defined in [PrometheusRules](https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/user-guides/alerting.md#configuring-alertmanager-in-prometheus) resources.
+The can be viewed in the Prometheus UI Alerts tab.
+Some example alerting rules are available in the [/examples](/examples) folder.

config/observability/additional-scrape-configs.yaml

@@ -0,0 +1,81 @@
+apiVersion: v1
+kind: Secret
+metadata:
+  name: additional-scrape-configs
+  namespace: monitoring
+stringData:
+  prometheus-additional.yaml: |
+    - job_name: kubernetes-pods
+      honor_labels: true
+      honor_timestamps: true
+      scrape_interval: 15s
+      scrape_timeout: 10s
+      metrics_path: /metrics
+      scheme: http
+      follow_redirects: true
+      enable_http2: true
+      relabel_configs:
+      - source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_scrape]
+        separator: ;
+        regex: "true"
+        replacement: $1
+        action: keep
+      - source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_scrape_slow]
+        separator: ;
+        regex: "true"
+        replacement: $1
+        action: drop
+      - source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_scheme]
+        separator: ;
+        regex: (https?)
+        target_label: __scheme__
+        replacement: $1
+        action: replace
+      - source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_path]
+        separator: ;
+        regex: (.+)
+        target_label: __metrics_path__
+        replacement: $1
+        action: replace
+      - source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_port, __meta_kubernetes_pod_ip]
+        separator: ;
+        regex: (\d+);(([A-Fa-f0-9]{1,4}::?){1,7}[A-Fa-f0-9]{1,4})
+        target_label: __address__
+        replacement: '[$2]:$1'
+        action: replace
+      - source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_port, __meta_kubernetes_pod_ip]
+        separator: ;
+        regex: (\d+);((([0-9]+?)(\.|$)){4})
+        target_label: __address__
+        replacement: $2:$1
+        action: replace
+      - separator: ;
+        regex: __meta_kubernetes_pod_annotation_prometheus_io_param_(.+)
+        replacement: __param_$1
+        action: labelmap
+      - separator: ;
+        regex: __meta_kubernetes_pod_label_(.+)
+        replacement: $1
+        action: labelmap
+      - source_labels: [__meta_kubernetes_namespace]
+        separator: ;
+        regex: (.*)
+        target_label: namespace
+        replacement: $1
+        action: replace
+      - source_labels: [__meta_kubernetes_pod_name]
+        separator: ;
+        regex: (.*)
+        target_label: pod
+        replacement: $1
+        action: replace
+      - source_labels: [__meta_kubernetes_pod_phase]
+        separator: ;
+        regex: Pending|Succeeded|Failed|Completed
+        replacement: $1
+        action: drop
+      kubernetes_sd_configs:
+      - role: pod
+        kubeconfig_file: ""
+        follow_redirects: true
+        enable_http2: true

config/observability/cluster_role.yaml

@@ -0,0 +1,23 @@
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: prometheus-k8s
+rules:
+  - apiGroups:
+      - ""
+    resources:
+      - nodes/metrics
+    verbs:
+      - get
+  - nonResourceURLs:
+      - /metrics
+    verbs:
+      - get
+  - apiGroups:
+      - ""
+    resources:
+      - pods
+    verbs:
+      - get
+      - list
+      - watch

config/observability/grafana_datasources.yaml

@@ -0,0 +1,14 @@
+{
+    "apiVersion": 1,
+    "datasources": [
+        {
+            "access": "proxy",
+            "editable": false,
+            "name": "prometheus",
+            "orgId": 1,
+            "type": "prometheus",
+            "url": "http://prometheus-k8s.monitoring.svc:9090",
+            "version": 1
+        }
+    ]
+}

config/observability/grafana_deployment_patch.yaml

@@ -0,0 +1,84 @@
+- op: add
+  path: /spec/template/spec/volumes/-
+  value:
+    name: grafana-gatewayclasses
+    configMap:
+     defaultMode: 420
+     name: grafana-gatewayclasses
+- op: add
+  path: /spec/template/spec/volumes/-
+  value:
+    name: grafana-gateways
+    configMap:
+     defaultMode: 420
+     name: grafana-gateways
+- op: add
+  path: /spec/template/spec/volumes/-
+  value:
+    name: grafana-httproutes
+    configMap:
+     defaultMode: 420
+     name: grafana-httproutes
+- op: add
+  path: /spec/template/spec/volumes/-
+  value:
+    name: grafana-grpcroutes
+    configMap:
+     defaultMode: 420
+     name: grafana-grpcroutes
+- op: add
+  path: /spec/template/spec/volumes/-
+  value:
+    name: grafana-tlsroutes
+    configMap:
+     defaultMode: 420
+     name: grafana-tlsroutes
+- op: add
+  path: /spec/template/spec/volumes/-
+  value:
+    name: grafana-tcproutes
+    configMap:
+     defaultMode: 420
+     name: grafana-tcproutes
+- op: add
+  path: /spec/template/spec/volumes/-
+  value:
+    name: grafana-udproutes
+    configMap:
+     defaultMode: 420
+     name: grafana-udproutes
+- op: add
+  path: /spec/template/spec/containers/0/volumeMounts/-
+  value:
+    name: grafana-gatewayclasses
+    mountPath: /grafana-dashboard-definitions/0/grafana-gatewayclasses
+- op: add
+  path: /spec/template/spec/containers/0/volumeMounts/-
+  value:
+    name: grafana-gateways
+    mountPath: /grafana-dashboard-definitions/0/grafana-gateways
+- op: add
+  path: /spec/template/spec/containers/0/volumeMounts/-
+  value:
+    name: grafana-httproutes
+    mountPath: /grafana-dashboard-definitions/0/grafana-httproutes
+- op: add
+  path: /spec/template/spec/containers/0/volumeMounts/-
+  value:
+    name: grafana-grpcroutes
+    mountPath: /grafana-dashboard-definitions/0/grafana-grpcroutes
+- op: add
+  path: /spec/template/spec/containers/0/volumeMounts/-
+  value:
+    name: grafana-tlsroutes
+    mountPath: /grafana-dashboard-definitions/0/grafana-tlsroutes
+- op: add
+  path: /spec/template/spec/containers/0/volumeMounts/-
+  value:
+    name: grafana-tcproutes
+    mountPath: /grafana-dashboard-definitions/0/grafana-tcpcroutes
+- op: add
+  path: /spec/template/spec/containers/0/volumeMounts/-
+  value:
+    name: grafana-udproutes
+    mountPath: /grafana-dashboard-definitions/0/grafana-udproutes

config/observability/ksm_clusterrole_patch.yaml

@@ -0,0 +1,39 @@
+- op: add
+  path: /rules/-
+  value:
+    apiGroups:
+    - "apiextensions.k8s.io"
+    resources:
+    - customresourcedefinitions
+    verbs:
+    - list
+    - watch
+- op: add
+  path: /rules/-
+  value:
+    apiGroups:
+    - "gateway.networking.k8s.io"
+    resources:
+    - gateways
+    - gatewayclasses
+    - httproutes
+    - grpcroutes
+    - tcproutes
+    - tlsroutes
+    - udproutes
+    verbs:
+    - list
+    - watch
+- op: add
+  path: /rules/-
+  value:
+    apiGroups:
+    - "kuadrant.io"
+    resources:
+    - tlspolicies
+    - dnspolicies
+    - ratelimitpolicies
+    - authpolicies
+    verbs:
+    - list
+    - watch

config/observability/ksm_deployment_patch.yaml

@@ -0,0 +1,21 @@
+- op: replace
+  path: /spec/template/spec/containers/0/image
+  value: registry.k8s.io/kube-state-metrics/kube-state-metrics:v2.10.1
+- op: add
+  path: /spec/template/spec/volumes
+  value:
+      - name: custom-resource-state
+        configMap:
+          defaultMode: 420
+          name: custom-resource-state
+- op: add
+  path: /spec/template/spec/containers/0/volumeMounts
+  value:
+    - name: custom-resource-state
+      mountPath: /custom-resource-state
+- op: add
+  path: /spec/template/spec/containers/0/args/-
+  value: --custom-resource-state-config-file
+- op: add
+  path: /spec/template/spec/containers/0/args/-
+  value: /custom-resource-state/custom-resource-state.yaml