Improved PAAS monitoring handling

* Rely on osc.edu/role label for service, ingress and pod

charts/kyverno-policies/Chart.yaml

@@ -2,7 +2,7 @@ apiVersion: v2
 name: kyverno-policies
 description: OSC Kyverno policies deployment
 type: application
-version: 0.26.0
+version: 0.27.0
 appVersion: "v1.11.4"
 maintainers:
   - name: treydock

charts/kyverno-policies/templates/add-annotations.yaml

@@ -0,0 +1,26 @@
+apiVersion: kyverno.io/v1
+kind: ClusterPolicy
+metadata:
+  name: add-annotations
+spec:
+  validationFailureAction: Enforce
+  background: true
+  rules:
+  - name: paas-disable-scrape
+    match:
+      any:
+        - resources:
+            kinds:
+            - Pod
+            - Service
+            namespaceSelector:
+              matchExpressions:
+              - key: osc.edu/role
+                operator: In
+                values:
+                - paas
+    mutate:
+      patchStrategicMerge:
+        metadata:
+          annotations:
+            prometheus.io/scrape: 'false'

charts/kyverno-policies/templates/add-role.yaml

@@ -12,6 +12,8 @@ spec:
         - resources:
             kinds:
             - Pod
+            - Service
+            - Ingress
             namespaceSelector:
               matchExpressions:
               - key: osc.edu/role
@@ -22,4 +24,4 @@ spec:
       patchStrategicMerge:
         metadata:
           labels:
-            role: paas
+            osc.edu/role: paas

charts/kyverno-policies/templates/pod-role-validation.yaml → charts/kyverno-policies/templates/role-validation.yaml

@@ -1,7 +1,7 @@
 apiVersion: kyverno.io/v1
 kind: ClusterPolicy
 metadata:
-  name: pod-role-validation
+  name: role-validation
 spec:
   background: false
   validationFailureAction: Enforce
@@ -12,6 +12,8 @@ spec:
         - resources:
             kinds:
             - Pod
+            - Service
+            - Ingress
             namespaceSelector:
               matchExpressions:
               - key: osc.edu/role
@@ -23,4 +25,4 @@ spec:
       pattern:
         metadata:
           labels:
-            role: "paas"
+            osc.edu/role: "paas"

charts/prometheus/Chart.yaml

@@ -2,7 +2,7 @@ apiVersion: v2
 name: prometheus
 description: OSC Prometheus deployment
 type: application
-version: 0.18.3
+version: 0.18.4
 appVersion: "v2.44.0"
 maintainers:
   - name: treydock

charts/prometheus/templates/config.yaml

@@ -153,9 +153,6 @@ data:
       - source_labels: [__meta_kubernetes_service_annotation_prometheus_io_match_metrics,__meta_kubernetes_service_port_name]
         action: drop
         regex: 'true;.*[^metrics].*'
-      - source_labels: [__meta_kubernetes_service_label_role]
-        regex: 'paas'
-        action: drop
       - source_labels: [__meta_kubernetes_service_annotation_prometheus_io_path]
         action: replace
         target_label: __metrics_path__
@@ -190,9 +187,6 @@ data:
       kubernetes_sd_configs:
       - role: pod
       relabel_configs:
-      - source_labels: [__meta_kubernetes_pod_label_role]
-        action: drop
-        regex: 'paas'
       - source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_scrape]
         action: keep
         regex: 'true'
@@ -280,6 +274,9 @@ data:
       kubernetes_sd_configs:
       - role: ingress
       relabel_configs:
+      - source_labels: [__meta_kubernetes_ingress_annotation_prometheus_io_probe_skip]
+        action: drop
+        regex: 'true'
       - source_labels: [__meta_kubernetes_ingress_scheme]
         action: keep
         regex: https
@@ -321,11 +318,11 @@ data:
         target_label: receiver
       - regex: "label_receiver"
         action: labeldrop
-      - source_labels: [label_role]
+      - source_labels: [label_osc_edu_role]
         regex: '(.+)'
         replacement: '$1'
-        target_label: role
-      - regex: "label_role"
+        target_label: osc_edu_role
+      - regex: "label_osc_edu_role"
         action: labeldrop
     - job_name: kube-state-metrics-telemetry
       metrics_path: /metrics

tests/kyverno-policies/add-annotations/kyverno-test.yaml

@@ -0,0 +1,29 @@
+---
+name: add-annotations
+policies:
+  - policy.yaml
+resources:
+  - resources.yaml
+variables: variables.yaml
+results:
+  - policy: add-annotations
+    rule: paas-disable-scrape
+    resources:
+      - test-paas-pod
+    patchedResource: test-paas-pod-mutated.yaml
+    kind: Pod
+    result: pass
+  - policy: add-annotations
+    rule: paas-disable-scrape
+    resources:
+      - test-paas-service
+    patchedResource: test-paas-service-mutated.yaml
+    kind: Service
+    result: pass
+  - policy: add-annotations
+    rule: paas-disable-scrape
+    resources:
+      - test-skip
+      - test-skip-webservice
+    kind: Pod
+    result: skip

tests/kyverno-policies/add-annotations/resources.yaml

@@ -0,0 +1,45 @@
+---
+apiVersion: v1
+kind: Pod
+metadata:
+  name: test-paas-pod
+  namespace: paas
+spec:
+  containers:
+    - name: nginx
+      image: nginx:latest
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: test-paas-service
+  namespace: paas
+spec:
+  ports:
+  - name: test
+    port: 8080
+    protocol: TCP
+    targetPort: http
+  selector:
+    app: test
+  type: ClusterIP
+---
+apiVersion: v1
+kind: Pod
+metadata:
+  name: test-skip
+  namespace: user-test
+spec:
+  containers:
+    - name: nginx
+      image: nginx:latest
+---
+apiVersion: v1
+kind: Pod
+metadata:
+  name: test-skip-webservice
+  namespace: webservice
+spec:
+  containers:
+    - name: nginx
+      image: nginx:latest

tests/kyverno-policies/add-annotations/test-paas-pod-mutated.yaml

@@ -0,0 +1,12 @@
+---
+apiVersion: v1
+kind: Pod
+metadata:
+  name: test-paas-pod
+  namespace: paas
+  annotations:
+    prometheus.io/scrape: 'false'
+spec:
+  containers:
+    - name: nginx
+      image: nginx:latest

tests/kyverno-policies/add-annotations/test-paas-service-mutated.yaml

@@ -0,0 +1,17 @@
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: test-paas-service
+  namespace: paas
+  annotations:
+    prometheus.io/scrape: 'false'
+spec:
+  ports:
+  - name: test
+    port: 8080
+    protocol: TCP
+    targetPort: http
+  selector:
+    app: test
+  type: ClusterIP

tests/kyverno-policies/add-role/test-paas-mutated.yaml

@@ -5,7 +5,7 @@ metadata:
   name: test-paas
   namespace: paas
   labels:
-    role: paas
+    osc.edu/role: paas
 spec:
   containers:
     - name: nginx

tests/kyverno-policies/role-validation/kyverno-test.yaml

@@ -0,0 +1,51 @@
+---
+name: role-validation
+policies:
+  - policy.yaml
+resources:
+  - resources.yaml
+variables: variables.yaml
+results:
+  - policy: role-validation
+    rule: paas-require-role
+    resources:
+      - test-skip
+      - test-skip-webservice
+    kind: Pod
+    result: skip
+  - policy: role-validation
+    rule: paas-require-role
+    resources:
+      - test-pass-pod
+    kind: Pod
+    result: pass
+  - policy: role-validation
+    rule: paas-require-role
+    resources:
+      - test-pass-service
+    kind: Service
+    result: pass
+  - policy: role-validation
+    rule: paas-require-role
+    resources:
+      - test-pass-ingress
+    kind: Ingress
+    result: pass
+  - policy: role-validation
+    rule: paas-require-role
+    resources:
+      - test-fail-pod
+    kind: Pod
+    result: fail
+  - policy: role-validation
+    rule: paas-require-role
+    resources:
+      - test-fail-service
+    kind: Service
+    result: fail
+  - policy: role-validation
+    rule: paas-require-role
+    resources:
+      - test-fail-ingress
+    kind: Ingress
+    result: fail