prom-server-cm.yaml

apiVersion: v1
data:
  alerting_rules.yml: |
    groups:
    - name: pod_cpu_usage_is_high
      rules:
        - alert: CPUISHIGH
          expr: sum(rate(container_cpu_usage_seconds_total{container_name!="POD",pod_name!=""}[5m])) by (pod_name)  > 0.1
          for: 1m
          labels:
            severity: critical
          annotations:
            description: 'POD {{ $labels.pod }} cpu is high'
            summary: 'pod cpu is high'

    - name: node_cpu_greater_than_80
      rules:
        - alert: NODECPUISHIGH
          expr: (avg by(instance) (irate(node_cpu_seconds_total{mode="idle"}[5m])) * 100) > 80 
          for: 1m
          labels:
            severity: critical
          annotations:
            description: 'node {{ $labels.node }} cpu is high'
            summary: 'node cpu is greater than 80 precent'
    - name: node_memory_left_lessser_than_10 
      rules:
        - alert: NODEMEMORY
          expr: node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes * 100 < 10 
          for: 1m
          labels:
            severity: critical
          annotations:
            description: 'node {{ $labels.node }} memory left is low'
            summary: 'node memory left is lesser than 10 precent'
    - name: spinnaker-service-is-down
      rules:
      - alert: clouddriver-is-down
        annotations:
          description: Service {{$labels.service}} in namespace {{$labels.namespace}}
            is not responding
          summary: Clouddriver Spinnaker service is down
        expr: absent(up{service="spin-clouddriver"}) == 1
        labels:
          severity: critical
      #- alert: clouddriver-rw-is-down
      #  annotations:
      #    description: Service {{$labels.service}} in namespace {{$labels.namespace}}
      #      is not responding
      #    summary: Clouddriver-rw Spinnaker service is down
      #  expr: absent(up{service="spin-clouddriver-rw"}) == 1
      #  labels:
      #    severity: critical
      #- alert: clouddriver-ro-is-down
      #  annotations:
      #    description: Service {{$labels.service}} in namespace {{$labels.namespace}}
      #      is not responding
      #    summary: Clouddriver-ro Spinnaker service is down
      #  expr: absent(up{service="spin-clouddriver-ro"}) == 1
      #  labels:
      #    severity: critical
      #- alert: clouddriver-caching-is-down
      #  annotations:
      #    description: Service {{$labels.service}} in namespace {{$labels.namespace}}
      #      is not responding
      #    summary: Clouddriver-caching Spinnaker service is down
      #  expr: absent(up{service="spin-clouddriver-caching"}) == 1
      #  labels:
      #    severity: critical
      - alert: gate-is-down
        annotations:
          description: Service {{$labels.service}} in namespace {{$labels.namespace}}
            is not responding
          summary: Gate Spinnaker services is down
        expr: absent(up{service="spin-gate"}) == 1
        labels:
          severity: critical
      - alert: orca-is-down
        annotations:
          description: Service {{$labels.service}} in namespace {{$labels.namespace}}
            is not responding
          summary: Orca Spinnaker service is down
        expr: absent(up{job="opsmx_spinnaker_metrics", service="spin-orca"}) == 1
        labels:
          severity: critical
      - alert: igor-is-down
        annotations:
          description: Service {{$labels.service}} in namespace {{$labels.namespace}}
            is not responding
          summary: Igor Spinnaker service is down
        expr: absent(up{service="spin-igor"}) == 1
        labels:
          severity: critical
      - alert: echo-is-down
        annotations:
          description: Service {{$labels.service}} in namespace {{$labels.namespace}}
            is not responding
          summary: Echo Spinnaker service is down
        expr: absent(up{service="spin-echo" }) == 1
        labels:
          severity: critical
      #- alert: echo-scheduler-is-down
      #  annotations:
      #    description: Service {{$labels.service}} in namespace {{$labels.namespace}}
      #      is not responding
      #    summary: Echo-Scheduler Spinnaker service is down
      #  expr: absent(up{service="spin-echo-scheduler" }) == 1
      #  labels:
      #    severity: critical
      #- alert: echo-worker-is-down
      #  annotations:
      #    description: Service {{$labels.service}} in namespace {{$labels.namespace}}
      #      is not responding
      #    summary: Echo-worker Spinnaker service is down
      #  expr: absent(up{service="spin-echo-worker" }) == 1
      #  labels:
      #    severity: critical
      - alert: front50-is-down
        annotations:
          description: Service {{$labels.service}} in namespace {{$labels.namespace}}
            is not responding
          summary: Front50 Spinnaker service is down
        expr: absent(up{service="spin-front50" }) == 1
        labels:
          severity: critical
    - name: latency-too-high
      rules:
      - alert: clouddriver-latency-too-high
        annotations:
          description: "Service {{$labels.service}} is slow ({{$value}} per minute)"
          summary: "latency of clouddriver is too high"
        expr: sum(rate(clouddriver:controller:invocations__count_total{service="spin-clouddriver",statusCode="200"}[2m]))
          by (instance, method)/ sum(rate(clouddriver:controller:invocations__total{service="spin-clouddriver",statusCode="200"}[2m]))
          by (instance, method) > 70000
        for: 5m
        labels:
          severity: warning
      #- alert: clouddriver-ro-latency-too-high
      #  annotations:
      #    description: Service {{$labels.service}} is slow ({{$value}} per minute)
      #    summary: latency of clouddriver-ro is too high
      #  expr: sum(rate(clouddriver_ro:controller:invocations__count_total{service="spin-clouddriver-ro",statusCode="200"}[2m]))
      #    by (instance, method)/ sum(rate(clouddriver_ro:controller:invocations__total{service="spin-clouddriver-ro",statusCode="200"}[2m]))
      #    by (instance, method) > 200
      #  for: 5m
      #  labels:
      #    severity: warning
      #- alert: clouddriver-rw-latency-too-high
      #  annotations:
      #    description: Service {{$labels.service}} is slow ({{$value}} per minute)
      #    summary: latency of clouddriver-rw is too high
      #  expr: sum(rate(clouddriver_rw:controller:invocations__count_total{service="spin-clouddriver-rw",statusCode="200"}[2m]))
      #    by (instance, method)/ sum(rate(clouddriver_rw:controller:invocations__total{service="spin-clouddriver-rw",statusCode="200"}[2m]))
      #    by (instance, method) > 200
      #  for: 5m
      #  labels:
      #    severity: warning
      #- alert: clouddriver-caching-latency-too-high
      #  annotations:
      #    description: Service {{$labels.service}} is slow ({{$value}} per minute)
      #    summary: latency clouddriver-caching is too high
      #  expr: sum(rate(clouddriver_caching:controller:invocations__count_total{service="spin-clouddriver-caching",statusCode="200"}[2m]))
      #    by (instance, method)/ sum(rate(clouddriver_caching:controller:invocations__total{service="spin-clouddriver-caching",statusCode="200"}[2m]))
      #    by (instance, method) > 300
      #  for: 5m
      #  labels:
      #    severity: warning
      - alert: gate-latency-too-high
        annotations:
          description: Service {{$labels.service}} is slow ({{$value}} per minute)
          summary: latency of gate is too high
        expr: sum(rate(gate:controller:invocations__count_total{service="spin-gate",statusCode="200"}[2m]))
          by (instance, method)/ sum(rate(gate:controller:invocations__total{service="spin-gate",statusCode="200"}[2m]))
          by (instance, method) > 5000
        for: 5m
        labels:
          severity: warning
      - alert: orca-latency-too-high
        annotations:
          description: Service {{$labels.service}} is slow ({{$value}} per minute)
          summary: latency of orca is too high
        expr: sum(rate(orca:controller:invocations__count_total{service="spin-orca",statusCode="200"}[2m]))
          by (instance, method)/ sum(rate(orca:controller:invocations__total{service="spin-orca",statusCode="200"}[2m]))
          by (instance, method) > 2000
        for: 5m
        labels:
          severity: warning
      - alert: igor-latency-too-high
        annotations:
          description: Service {{$labels.service}} is slow ({{$value}} per minute)
          summary: latency of igor is too high
        expr: sum(rate(igor:controller:invocations__count_total{service="spin-igor",statusCode="200"}[2m]))
          by (instance, method)/ sum(rate(igor:controller:invocations__total{service="spin-igor",statusCode="200"}[2m]))
          by (instance, method) > 10000
        for: 5m
        labels:
          severity: warning
      - alert: echo-latency-too-high
        annotations:
          description: Service {{$labels.service}} is slow ({{$value}} per minute)
          summary: latency of echo is too high
        expr: sum(rate(echo:controller:invocations__count_total{service="spin-echo",statusCode="200"}[2m]))
          by (instance, method)/ sum(rate(echo:controller:invocations__total{service="spin-echo",statusCode="200"}[2m]))
          by (instance, method) > 6000
        for: 5m
        labels:
          severity: warning
      #- alert: echo_scheduler-latency-too-high
      #  annotations:
      #    description: Service {{$labels.service}} is slow ({{$value}} per minute)
      #    summary: latency of echo-scheduler is too high
      #  expr: sum(rate(echo_scheduler:controller:invocations__count_total{service="spin-echo-scheduler",statusCode="200"}[2m]))
      #    by (instance, method)/ sum(rate(echo_scheduler:controller:invocations__total{service="spin-echo-scheduler",statusCode="200"}[2m]))
      #    by (instance, method) > 300
      #  for: 5m
      #  labels:
      #    severity: warning
      #- alert: echo_worker-latency-too-high
      #  annotations:
      #    description: Service {{$labels.service}} is slow ({{$value}} per minute)
      #    summary: latency of echo-worker is too high
      #  expr: sum(rate(echo_worker:controller:invocations__count_total{service="spin-echo-worker",statusCode="200"}[2m]))
      #    by (instance, method)/ sum(rate(echo_worker:controller:invocations__total{service="spin-echo-worker",statusCode="200"}[2m]))
      #    by (instance, method) > 2000
      #  for: 5m
      #  labels:
      #    severity: warning
      - alert: front50-latency-too-high
        annotations:
          description: "Service {{$labels.service}} is slow ({{$value}} per minute)"
          summary: 'latency of front50 is too high'
        expr: sum(rate(front50:controller:invocations__count_total{service="spin-front50",statusCode="200"}[2m]))
          by (instance, method)/ sum(rate(front50:controller:invocations__total{service="spin-front50",statusCode="200"}[2m]))
          by (instance, method) > 1200
        for: 5m
        labels:
          severity: warning
    - name: jvm-too-high
      rules:
      - alert: clouddriver-rw-pod-may-be-evicted-soon
        annotations:
          description: Service {{$labels.service}} in namespace {{$labels.namespace}}
            may be evicted soon
          summary: Clouddriver-rw JVM memory too high
        expr: (sum(clouddriver_rw:jvm:memory:used__value) by (instance, area) / sum(clouddriver_rw:jvm:memory:max__value)
          by (instance, area)) > .9
        labels:
          severity: warning
      - alert: clouddriver-ro-pod-may-be-evicted-soon
        annotations:
          description: Service {{$labels.service}} in namespace {{$labels.namespace}}
            may be evicted soon
          summary: Clouddriver-ro JVM memory too high
        expr: (sum(clouddriver_ro:jvm:memory:used__value) by (instance, area) / sum(clouddriver_ro:jvm:memory:max__value)
          by (instance, area)) > .9
        labels:
          severity: warning
      - alert: clouddriver-caching-pod-may-be-evicted-soon
        annotations:
          description: Service {{$labels.service}} in namespace {{$labels.namespace}}
            may be evicted soon
          summary: Clouddriver-caching JVM memory too high
        expr: (sum(clouddriver_caching:jvm:memory:used__value) by (instance, area) / sum(clouddriver_caching:jvm:memory:max__value)
          by (instance, area)) > .9
        labels:
          severity: warning
      - alert: gate-pod-may-be-evicted-soon
        annotations:
          description: Service {{$labels.service}} in namespace {{$labels.namespace}}
            may be evicted soon
          summary: gate JVM memory too high
        expr: (sum(gate:jvm:memory:used__value) by (instance, area) / sum(gate:jvm:memory:max__value)
          by (instance, area)) > .9
        labels:
          severity: warning
      - alert: orca-pod-may-be-evicted-soon
        annotations:
          description: Service {{$labels.service}} may be evicted soon
          summary: orca JVM memory too high
        expr: (sum(orca:jvm:gc:liveDataSize__value) by (instance, area) / sum(orca:jvm:gc:maxDataSize__value)
          by (instance, area)) > .9
        labels:
          severity: warning
      - alert: igor-pod-may-be-evicted-soon
        annotations:
          description: Service {{$labels.service}} in namespace {{$labels.namespace}}
            may be evicted soon
          summary: igor JVM memory too high
        expr: (sum(igor:jvm:memory:used__value) by (instance, area) / sum(igor:jvm:memory:max__value)
          by (instance, area)) > .9
        labels:
          severity: warning
      - alert: echo-scheduler-pod-may-be-evicted-soon
        annotations:
          description: Service {{$labels.service}} in namespace {{$labels.namespace}}
            may be evicted soon
          summary: echo-scheduler JVM memory too high
        expr: (sum(echo_scheduler:jvm:memory:used__value) by (instance, area) / sum(echo_scheduler:jvm:memory:max__value)
          by (instance, area)) > .9
        labels:
          severity: warning
      - alert: echo-worker-pod-may-be-evicted-soon
        annotations:
          description: Service {{$labels.service}} in namespace {{$labels.namespace}}
            may be evicted soon
          summary: echo-worker JVM memory too high
        expr: (sum(echo_worker:jvm:memory:used__value) by (instance, area) / sum(echo_worker:jvm:memory:max__value)
          by (instance, area)) > .9
        labels:
          severity: warning
      - alert: front50-pod-may-be-evicted-soon
        annotations:
          description: Service {{$labels.service}} in namespace {{$labels.namespace}}
            may be evicted soon
          summary: Front50 JVM memory too high
        expr: (sum(front50:jvm:memory:used__value) by (instance, area) / sum(front50:jvm:memory:max__value)
          by (instance, area)) > .9
        labels:
          severity: warning
    # Clouddriver-Service specific alerts
    - name: Clouddriver-execution-time
      rules:
      - alert: Clouddriver-execution-time-too-high
        annotations:
          description: "Service {{$labels.service}} execution time too high"
          summary: "Clouddriver:executiontime is high"
        expr: sum(rate(clouddriver:executionTime__total[2m])) by (instance, agent) / sum(rate(clouddriver:executionTime__count_total[2m])) by (instance, agent) > 100
        for: 2m
        labels:
          severity: warning
    # Front50-Service specific alerts
    - name: Front50-cache
      rules:
      - alert: "front50:storageServiceSupport:cacheAge__value"
        annotations:
          description: value = {{$value}}
          summary: "front50 cacheAge too high"
        expr: front50:storageServiceSupport:cacheAge__value > 300000
        for: 2m
        labels:
          severity: warning
    # fiat-Service specific alerts
    - name: Fiat-UserRoles-sync
      rules:
      - alert: fiat-userRoles-sync-time-too-high
        annotations:
          description: "Fiat userRoles sync time is high"
          summary: "fiat-userRoles-sync-time-too-high"
        expr: fiat:fiat:userRoles:syncTime__count_total/fiat:fiat:userRoles:syncTime__total > 8
        for: 2m
        labels:
          severity: warning
    # gate-Service specific alerts
    - name: Gate-issues
      rules:
      - alert: "Gate:hystrix:latencyTotal__percentile_50__value"
        annotations:
          description: "Median gate response too slow"
          summary: "50 percentile value is {{ $value }}"
        expr: gate:hystrix:latencyTotal__percentile_50__value > 1600
        for: 5m
        labels:
          severity: critical
      - alert: "gate:hystrix:latencyTotal__percentile_90__value"
        annotations:
          description: "gate response slow"
          summary: "90 percentile value is {{ $value }}"
        expr: gate:hystrix:latencyTotal__percentile_90__value > 2000
        for: 5m
        labels:
          severity: warning
      - alert: "gate:hystrix:errorPercentage__value"
        annotations:
          description: "gate experiencing errors"
          summary: "Error percentage is {{ $value }}"
        expr: gate:hystrix:errorPercentage__value > 0.01
        for: 2m
        labels:
          severity: warning
    # Orca-Service specific alerts
    - name: orca-queue-issue
      rules:
      - alert: orca-queue-depth-high
        annotations:
          description: Service {{$labels.service}} in namespace {{$labels.namespace}}
            may be evicted soon
          summary: Orca queue depth is high
        expr: (sum(orca:queue:ready:depth) by (instance) ) > 10
        labels:
          severity: warning
      - alert: orca-queue-lag-high
        annotations:
          description: Lag = {{$value}}
          summary: Orca queue lag is high
        expr: sum(rate (orca:controller:invocations__totalTime_total[2m])) by (instance)  /
          sum(rate(orca:controller:invocations__count_total[2m])) by (instance) > 0.5
        labels:
          severity: warning
    - name: igor-needs-attention
      rules:
      - alert: igor-needs-attention
        annotations:
          description: Igor in namespace {{$labels.namespace}} needs human help
          summary: Igor needs attention
        expr: igor:pollingMonitor:itemsOverThreshold > 0
        labels:
          severity: crtical
  alerts: |
    {}
  prometheus.yml: |
    global:
      evaluation_interval: 1m
      scrape_interval: 1m
      scrape_timeout: 10s
    rule_files:
    - /etc/config/recording_rules.yml
    - /etc/config/alerting_rules.yml
    - /etc/config/rules
    - /etc/config/alerts
    scrape_configs:
    - honor_labels: true
      job_name: opsmx_spinnaker_metrics
      kubernetes_sd_configs:
      - role: endpoints
      metrics_path: /prometheus_metrics
      relabel_configs:
      - action: keep
        regex: spin
        source_labels:
        - __meta_kubernetes_service_label_app
      - action: keep
        regex: "8008"
        source_labels:
        - __meta_kubernetes_pod_container_port_number
      - regex: Node;(.*)
        replacement: ${1}
        separator: ;
        source_labels:
        - __meta_kubernetes_endpoint_address_target_kind
        - __meta_kubernetes_endpoint_address_target_name
        target_label: node
      - regex: Pod;(.*)
        replacement: ${1}
        separator: ;
        source_labels:
        - __meta_kubernetes_endpoint_address_target_kind
        - __meta_kubernetes_endpoint_address_target_name
        target_label: pod
      - source_labels:
        - __meta_kubernetes_namespace
        target_label: namespace
      - source_labels:
        - __meta_kubernetes_service_name
        target_label: service
      - source_labels:
        - __meta_kubernetes_pod_name
        target_label: pod
      - replacement: "8008"
        target_label: endpoint
    - job_name: prometheus
      static_configs:
      - targets:
        - localhost:9090
    - bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token
      job_name: kubernetes-apiservers
      kubernetes_sd_configs:
      - role: endpoints
      relabel_configs:
      - action: keep
        regex: default;kubernetes;https
        source_labels:
        - __meta_kubernetes_namespace
        - __meta_kubernetes_service_name
        - __meta_kubernetes_endpoint_port_name
      scheme: https
      tls_config:
        ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
        insecure_skip_verify: true
    - bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token
      job_name: kubernetes-nodes
      kubernetes_sd_configs:
      - role: node
      relabel_configs:
      - action: labelmap
        regex: __meta_kubernetes_node_label_(.+)
      - replacement: kubernetes.default.svc:443
        target_label: __address__
      - regex: (.+)
        replacement: /api/v1/nodes/$1/proxy/metrics
        source_labels:
        - __meta_kubernetes_node_name
        target_label: __metrics_path__
      scheme: https
      tls_config:
        ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
        insecure_skip_verify: true
    - bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token
      job_name: kubernetes-nodes-cadvisor
      kubernetes_sd_configs:
      - role: node
      relabel_configs:
      - action: labelmap
        regex: __meta_kubernetes_node_label_(.+)
      - replacement: kubernetes.default.svc:443
        target_label: __address__
      - regex: (.+)
        replacement: /api/v1/nodes/$1/proxy/metrics/cadvisor
        source_labels:
        - __meta_kubernetes_node_name
        target_label: __metrics_path__
      scheme: https
      tls_config:
        ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
        insecure_skip_verify: true
    - job_name: kubernetes-service-endpoints
      kubernetes_sd_configs:
      - role: endpoints
      relabel_configs:
      - action: keep
        regex: true
        source_labels:
        - __meta_kubernetes_service_annotation_prometheus_io_scrape
      - action: replace
        regex: (https?)
        source_labels:
        - __meta_kubernetes_service_annotation_prometheus_io_scheme
        target_label: __scheme__
      - action: replace
        regex: (.+)
        source_labels:
        - __meta_kubernetes_service_annotation_prometheus_io_path
        target_label: __metrics_path__
      - action: replace
        regex: ([^:]+)(?::\d+)?;(\d+)
        replacement: $1:$2
        source_labels:
        - __address__
        - __meta_kubernetes_service_annotation_prometheus_io_port
        target_label: __address__
      - action: labelmap
        regex: __meta_kubernetes_service_label_(.+)
      - action: replace
        source_labels:
        - __meta_kubernetes_namespace
        target_label: kubernetes_namespace
      - action: replace
        source_labels:
        - __meta_kubernetes_service_name
        target_label: kubernetes_name
      - action: replace
        source_labels:
        - __meta_kubernetes_pod_node_name
        target_label: kubernetes_node
    - job_name: kubernetes-service-endpoints-slow
      kubernetes_sd_configs:
      - role: endpoints
      relabel_configs:
      - action: keep
        regex: true
        source_labels:
        - __meta_kubernetes_service_annotation_prometheus_io_scrape_slow
      - action: replace
        regex: (https?)
        source_labels:
        - __meta_kubernetes_service_annotation_prometheus_io_scheme
        target_label: __scheme__
      - action: replace
        regex: (.+)
        source_labels:
        - __meta_kubernetes_service_annotation_prometheus_io_path
        target_label: __metrics_path__
      - action: replace
        regex: ([^:]+)(?::\d+)?;(\d+)
        replacement: $1:$2
        source_labels:
        - __address__
        - __meta_kubernetes_service_annotation_prometheus_io_port
        target_label: __address__
      - action: labelmap
        regex: __meta_kubernetes_service_label_(.+)
      - action: replace
        source_labels:
        - __meta_kubernetes_namespace
        target_label: kubernetes_namespace
      - action: replace
        source_labels:
        - __meta_kubernetes_service_name
        target_label: kubernetes_name
      - action: replace
        source_labels:
        - __meta_kubernetes_pod_node_name
        target_label: kubernetes_node
      scrape_interval: 5m
      scrape_timeout: 30s
    - honor_labels: true
      job_name: prometheus-pushgateway
      kubernetes_sd_configs:
      - role: service
      relabel_configs:
      - action: keep
        regex: pushgateway
        source_labels:
        - __meta_kubernetes_service_annotation_prometheus_io_probe
    - job_name: kubernetes-services
      kubernetes_sd_configs:
      - role: service
      metrics_path: /probe
      params:
        module:
        - http_2xx
      relabel_configs:
      - action: keep
        regex: true
        source_labels:
        - __meta_kubernetes_service_annotation_prometheus_io_probe
      - source_labels:
        - __address__
        target_label: __param_target
      - replacement: blackbox
        target_label: __address__
      - source_labels:
        - __param_target
        target_label: instance
      - action: labelmap
        regex: __meta_kubernetes_service_label_(.+)
      - source_labels:
        - __meta_kubernetes_namespace
        target_label: kubernetes_namespace
      - source_labels:
        - __meta_kubernetes_service_name
        target_label: kubernetes_name
    - job_name: kubernetes-pods
      kubernetes_sd_configs:
      - role: pod
      relabel_configs:
      - action: keep
        regex: true
        source_labels:
        - __meta_kubernetes_pod_annotation_prometheus_io_scrape
      - action: replace
        regex: (.+)
        source_labels:
        - __meta_kubernetes_pod_annotation_prometheus_io_path
        target_label: __metrics_path__
      - action: replace
        regex: ([^:]+)(?::\d+)?;(\d+)
        replacement: $1:$2
        source_labels:
        - __address__
        - __meta_kubernetes_pod_annotation_prometheus_io_port
        target_label: __address__
      - action: labelmap
        regex: __meta_kubernetes_pod_label_(.+)
      - action: replace
        source_labels:
        - __meta_kubernetes_namespace
        target_label: kubernetes_namespace
      - action: replace
        source_labels:
        - __meta_kubernetes_pod_name
        target_label: kubernetes_pod_name
      - action: drop
        regex: Pending|Succeeded|Failed
        source_labels:
        - __meta_kubernetes_pod_phase
    - job_name: kubernetes-pods-slow
      kubernetes_sd_configs:
      - role: pod
      relabel_configs:
      - action: keep
        regex: true
        source_labels:
        - __meta_kubernetes_pod_annotation_prometheus_io_scrape_slow
      - action: replace
        regex: (.+)
        source_labels:
        - __meta_kubernetes_pod_annotation_prometheus_io_path
        target_label: __metrics_path__
      - action: replace
        regex: ([^:]+)(?::\d+)?;(\d+)
        replacement: $1:$2
        source_labels:
        - __address__
        - __meta_kubernetes_pod_annotation_prometheus_io_port
        target_label: __address__
      - action: labelmap
        regex: __meta_kubernetes_pod_label_(.+)
      - action: replace
        source_labels:
        - __meta_kubernetes_namespace
        target_label: kubernetes_namespace
      - action: replace
        source_labels:
        - __meta_kubernetes_pod_name
        target_label: kubernetes_pod_name
      - action: drop
        regex: Pending|Succeeded|Failed
        source_labels:
        - __meta_kubernetes_pod_phase
      scrape_interval: 5m
      scrape_timeout: 30s
    alerting:
      alertmanagers:
      - kubernetes_sd_configs:
          - role: pod
        tls_config:
          ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
        bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token
        relabel_configs:
        - source_labels: [__meta_kubernetes_namespace]
          regex: oes
          action: keep
        - source_labels: [__meta_kubernetes_pod_label_app]
          regex: prometheus
          action: keep
        - source_labels: [__meta_kubernetes_pod_label_component]
          regex: alertmanager
          action: keep
        - source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_probe]
          regex: .*
          action: keep
        - source_labels: [__meta_kubernetes_pod_container_port_number]
          regex:
          action: drop
  recording_rules.yml: |
    {}
  rules: |
    {}
kind: ConfigMap
metadata:
  labels:
    app: prometheus
    chart: prometheus-11.16.2
    component: server
    heritage: Helm
    release: prom
  name: prom-prometheus-server
  namespace: oes