From 382b03e40aca9c46f7ff3a5006f810dae81c1401 Mon Sep 17 00:00:00 2001 From: Dan Pock Date: Thu, 16 Jan 2025 08:59:10 -0500 Subject: [PATCH] PACKAGE=rancher-monitoring make charts --- ...ancher-monitoring-crd-103.2.2+up57.0.3.tgz | Bin 0 -> 299555 bytes .../rancher-monitoring-103.2.2+up57.0.3.tgz | Bin 0 -> 480143 bytes .../103.2.2+up57.0.3/Chart.yaml | 10 + .../103.2.2+up57.0.3/README.md | 24 + .../103.2.2+up57.0.3/files/crd-manifest.tgz | Bin 0 -> 308613 bytes .../103.2.2+up57.0.3/templates/_helpers.tpl | 30 + .../103.2.2+up57.0.3/templates/jobs.yaml | 102 + .../103.2.2+up57.0.3/templates/manifest.yaml | 8 + .../103.2.2+up57.0.3/templates/rbac.yaml | 76 + .../templates/validate-psp-install.yaml | 7 + .../103.2.2+up57.0.3/values.yaml | 17 + .../103.2.2+up57.0.3/.editorconfig | 5 + .../103.2.2+up57.0.3/.helmignore | 29 + .../103.2.2+up57.0.3/CHANGELOG.md | 47 + .../103.2.2+up57.0.3/CONTRIBUTING.md | 12 + .../103.2.2+up57.0.3/Chart.yaml | 148 + .../103.2.2+up57.0.3/README.md | 1080 ++++ .../103.2.2+up57.0.3/app-README.md | 46 + .../charts/grafana/.helmignore | 23 + .../charts/grafana/Chart.yaml | 39 + .../103.2.2+up57.0.3/charts/grafana/README.md | 770 +++ .../grafana/dashboards/custom-dashboard.json | 1 + .../charts/grafana/templates/NOTES.txt | 55 + .../charts/grafana/templates/_config.tpl | 171 + .../charts/grafana/templates/_helpers.tpl | 305 + .../charts/grafana/templates/_pod.tpl | 1296 ++++ .../charts/grafana/templates/clusterrole.yaml | 25 + .../grafana/templates/clusterrolebinding.yaml | 24 + .../grafana/templates/configSecret.yaml | 43 + .../configmap-dashboard-provider.yaml | 15 + .../charts/grafana/templates/configmap.yaml | 15 + .../templates/dashboards-json-configmap.yaml | 38 + .../charts/grafana/templates/deployment.yaml | 53 + .../grafana/templates/extra-manifests.yaml | 4 + .../grafana/templates/headless-service.yaml | 22 + .../charts/grafana/templates/hpa.yaml | 52 + .../templates/image-renderer-deployment.yaml | 131 + .../grafana/templates/image-renderer-hpa.yaml | 47 + .../image-renderer-network-policy.yaml | 79 + .../templates/image-renderer-service.yaml | 31 + .../image-renderer-servicemonitor.yaml | 48 + .../charts/grafana/templates/ingress.yaml | 78 + .../grafana/templates/networkpolicy.yaml | 61 + .../grafana/templates/nginx-config.yaml | 94 + .../templates/poddisruptionbudget.yaml | 22 + .../grafana/templates/podsecuritypolicy.yaml | 45 + .../charts/grafana/templates/pvc.yaml | 41 + .../charts/grafana/templates/role.yaml | 32 + .../charts/grafana/templates/rolebinding.yaml | 25 + .../charts/grafana/templates/secret-env.yaml | 14 + .../charts/grafana/templates/secret.yaml | 16 + .../charts/grafana/templates/service.yaml | 61 + .../grafana/templates/serviceaccount.yaml | 17 + .../grafana/templates/servicemonitor.yaml | 68 + .../charts/grafana/templates/statefulset.yaml | 58 + .../templates/tests/test-configmap.yaml | 20 + .../tests/test-podsecuritypolicy.yaml | 32 + .../grafana/templates/tests/test-role.yaml | 17 + .../templates/tests/test-rolebinding.yaml | 20 + .../templates/tests/test-serviceaccount.yaml | 12 + .../charts/grafana/templates/tests/test.yaml | 53 + .../charts/grafana/values.yaml | 1315 ++++ .../charts/hardenedKubelet/.helmignore | 23 + .../charts/hardenedKubelet/Chart.yaml | 15 + .../charts/hardenedKubelet/README.md | 90 + .../hardenedKubelet/templates/_helpers.tpl | 170 + .../templates/pushprox-clients-rbac.yaml | 97 + .../templates/pushprox-clients.yaml | 157 + .../templates/pushprox-proxy-rbac.yaml | 68 + .../templates/pushprox-proxy.yaml | 57 + .../templates/pushprox-servicemonitor.yaml | 45 + .../templates/validate-install-crd.yaml | 14 + .../templates/validate-psp-install.yaml | 7 + .../charts/hardenedKubelet/values.yaml | 166 + .../charts/hardenedNodeExporter/.helmignore | 23 + .../charts/hardenedNodeExporter/Chart.yaml | 15 + .../charts/hardenedNodeExporter/README.md | 90 + .../templates/_helpers.tpl | 170 + .../templates/pushprox-clients-rbac.yaml | 97 + .../templates/pushprox-clients.yaml | 157 + .../templates/pushprox-proxy-rbac.yaml | 68 + .../templates/pushprox-proxy.yaml | 57 + .../templates/pushprox-servicemonitor.yaml | 45 + .../templates/validate-install-crd.yaml | 14 + .../templates/validate-psp-install.yaml | 7 + .../charts/hardenedNodeExporter/values.yaml | 166 + .../charts/k3sServer/.helmignore | 23 + .../charts/k3sServer/Chart.yaml | 15 + .../charts/k3sServer/README.md | 90 + .../charts/k3sServer/templates/_helpers.tpl | 170 + .../templates/pushprox-clients-rbac.yaml | 97 + .../k3sServer/templates/pushprox-clients.yaml | 157 + .../templates/pushprox-proxy-rbac.yaml | 68 + .../k3sServer/templates/pushprox-proxy.yaml | 57 + .../templates/pushprox-servicemonitor.yaml | 45 + .../templates/validate-install-crd.yaml | 14 + .../templates/validate-psp-install.yaml | 7 + .../charts/k3sServer/values.yaml | 166 + .../charts/kube-state-metrics/.helmignore | 21 + .../charts/kube-state-metrics/Chart.yaml | 32 + .../charts/kube-state-metrics/README.md | 85 + .../kube-state-metrics/templates/NOTES.txt | 23 + .../kube-state-metrics/templates/_helpers.tpl | 196 + .../templates/ciliumnetworkpolicy.yaml | 33 + .../templates/clusterrolebinding.yaml | 20 + .../templates/crs-configmap.yaml | 16 + .../templates/deployment.yaml | 314 + .../templates/extra-manifests.yaml | 4 + .../templates/kubeconfig-secret.yaml | 12 + .../templates/networkpolicy.yaml | 43 + .../kube-state-metrics/templates/pdb.yaml | 18 + .../templates/podsecuritypolicy.yaml | 39 + .../templates/psp-clusterrole.yaml | 19 + .../templates/psp-clusterrolebinding.yaml | 16 + .../templates/rbac-configmap.yaml | 22 + .../kube-state-metrics/templates/role.yaml | 215 + .../templates/rolebinding.yaml | 24 + .../kube-state-metrics/templates/service.yaml | 49 + .../templates/serviceaccount.yaml | 17 + .../templates/servicemonitor.yaml | 126 + .../templates/stsdiscovery-role.yaml | 26 + .../templates/stsdiscovery-rolebinding.yaml | 17 + .../templates/verticalpodautoscaler.yaml | 44 + .../charts/kube-state-metrics/values.yaml | 491 ++ .../kubeAdmControllerManager/.helmignore | 23 + .../kubeAdmControllerManager/Chart.yaml | 15 + .../charts/kubeAdmControllerManager/README.md | 90 + .../templates/_helpers.tpl | 170 + .../templates/pushprox-clients-rbac.yaml | 97 + .../templates/pushprox-clients.yaml | 157 + .../templates/pushprox-proxy-rbac.yaml | 68 + .../templates/pushprox-proxy.yaml | 57 + .../templates/pushprox-servicemonitor.yaml | 45 + .../templates/validate-install-crd.yaml | 14 + .../templates/validate-psp-install.yaml | 7 + .../kubeAdmControllerManager/values.yaml | 166 + .../charts/kubeAdmEtcd/.helmignore | 23 + .../charts/kubeAdmEtcd/Chart.yaml | 15 + .../charts/kubeAdmEtcd/README.md | 90 + .../charts/kubeAdmEtcd/templates/_helpers.tpl | 170 + .../templates/pushprox-clients-rbac.yaml | 97 + .../templates/pushprox-clients.yaml | 157 + .../templates/pushprox-proxy-rbac.yaml | 68 + .../kubeAdmEtcd/templates/pushprox-proxy.yaml | 57 + .../templates/pushprox-servicemonitor.yaml | 45 + .../templates/validate-install-crd.yaml | 14 + .../templates/validate-psp-install.yaml | 7 + .../charts/kubeAdmEtcd/values.yaml | 166 + .../charts/kubeAdmProxy/.helmignore | 23 + .../charts/kubeAdmProxy/Chart.yaml | 15 + .../charts/kubeAdmProxy/README.md | 90 + .../kubeAdmProxy/templates/_helpers.tpl | 170 + .../templates/pushprox-clients-rbac.yaml | 97 + .../templates/pushprox-clients.yaml | 157 + .../templates/pushprox-proxy-rbac.yaml | 68 + .../templates/pushprox-proxy.yaml | 57 + .../templates/pushprox-servicemonitor.yaml | 45 + .../templates/validate-install-crd.yaml | 14 + .../templates/validate-psp-install.yaml | 7 + .../charts/kubeAdmProxy/values.yaml | 166 + .../charts/kubeAdmScheduler/.helmignore | 23 + .../charts/kubeAdmScheduler/Chart.yaml | 15 + .../charts/kubeAdmScheduler/README.md | 90 + .../kubeAdmScheduler/templates/_helpers.tpl | 170 + .../templates/pushprox-clients-rbac.yaml | 97 + .../templates/pushprox-clients.yaml | 157 + .../templates/pushprox-proxy-rbac.yaml | 68 + .../templates/pushprox-proxy.yaml | 57 + .../templates/pushprox-servicemonitor.yaml | 45 + .../templates/validate-install-crd.yaml | 14 + .../templates/validate-psp-install.yaml | 7 + .../charts/kubeAdmScheduler/values.yaml | 166 + .../charts/prometheus-adapter/.helmignore | 21 + .../charts/prometheus-adapter/Chart.yaml | 28 + .../charts/prometheus-adapter/README.md | 160 + .../prometheus-adapter/templates/NOTES.txt | 9 + .../prometheus-adapter/templates/_helpers.tpl | 113 + .../templates/certmanager.yaml | 76 + .../cluster-role-binding-auth-delegator.yaml | 20 + .../cluster-role-binding-resource-reader.yaml | 20 + .../cluster-role-resource-reader.yaml | 24 + .../templates/configmap.yaml | 97 + .../templates/custom-metrics-apiservice.yaml | 34 + ...stom-metrics-cluster-role-binding-hpa.yaml | 24 + .../custom-metrics-cluster-role.yaml | 17 + .../templates/deployment.yaml | 143 + .../external-metrics-apiservice.yaml | 34 + ...rnal-metrics-cluster-role-binding-hpa.yaml | 20 + .../external-metrics-cluster-role.yaml | 21 + .../prometheus-adapter/templates/pdb.yaml | 23 + .../prometheus-adapter/templates/psp.yaml | 66 + .../resource-metrics-apiservice.yaml | 34 + ...resource-metrics-cluster-role-binding.yaml | 20 + .../resource-metrics-cluster-role.yaml | 23 + .../templates/role-binding-auth-reader.yaml | 21 + .../prometheus-adapter/templates/secret.yaml | 17 + .../prometheus-adapter/templates/service.yaml | 27 + .../templates/serviceaccount.yaml | 18 + .../charts/prometheus-adapter/values.yaml | 277 + .../prometheus-node-exporter/.helmignore | 21 + .../prometheus-node-exporter/Chart.yaml | 25 + .../charts/prometheus-node-exporter/README.md | 97 + .../templates/NOTES.txt | 29 + .../templates/_helpers.tpl | 236 + .../templates/clusterrole.yaml | 19 + .../templates/clusterrolebinding.yaml | 20 + .../templates/daemonset.yaml | 309 + .../templates/endpoints.yaml | 18 + .../templates/extra-manifests.yaml | 4 + .../templates/networkpolicy.yaml | 23 + .../templates/podmonitor.yaml | 91 + .../templates/psp-clusterrole.yaml | 14 + .../templates/psp-clusterrolebinding.yaml | 16 + .../templates/psp.yaml | 49 + .../templates/rbac-configmap.yaml | 16 + .../templates/service.yaml | 29 + .../templates/serviceaccount.yaml | 17 + .../templates/servicemonitor.yaml | 71 + .../templates/verticalpodautoscaler.yaml | 40 + .../prometheus-node-exporter/values.yaml | 530 ++ .../charts/rke2ControllerManager/.helmignore | 23 + .../charts/rke2ControllerManager/Chart.yaml | 15 + .../charts/rke2ControllerManager/README.md | 90 + .../templates/_helpers.tpl | 170 + .../templates/pushprox-clients-rbac.yaml | 97 + .../templates/pushprox-clients.yaml | 157 + .../templates/pushprox-proxy-rbac.yaml | 68 + .../templates/pushprox-proxy.yaml | 57 + .../templates/pushprox-servicemonitor.yaml | 45 + .../templates/validate-install-crd.yaml | 14 + .../templates/validate-psp-install.yaml | 7 + .../charts/rke2ControllerManager/values.yaml | 166 + .../charts/rke2Etcd/.helmignore | 23 + .../charts/rke2Etcd/Chart.yaml | 15 + .../charts/rke2Etcd/README.md | 90 + .../charts/rke2Etcd/templates/_helpers.tpl | 170 + .../templates/pushprox-clients-rbac.yaml | 97 + .../rke2Etcd/templates/pushprox-clients.yaml | 157 + .../templates/pushprox-proxy-rbac.yaml | 68 + .../rke2Etcd/templates/pushprox-proxy.yaml | 57 + .../templates/pushprox-servicemonitor.yaml | 45 + .../templates/validate-install-crd.yaml | 14 + .../templates/validate-psp-install.yaml | 7 + .../charts/rke2Etcd/values.yaml | 166 + .../charts/rke2IngressNginx/.helmignore | 23 + .../charts/rke2IngressNginx/Chart.yaml | 15 + .../charts/rke2IngressNginx/README.md | 90 + .../rke2IngressNginx/templates/_helpers.tpl | 170 + .../templates/pushprox-clients-rbac.yaml | 97 + .../templates/pushprox-clients.yaml | 157 + .../templates/pushprox-proxy-rbac.yaml | 68 + .../templates/pushprox-proxy.yaml | 57 + .../templates/pushprox-servicemonitor.yaml | 45 + .../templates/validate-install-crd.yaml | 14 + .../templates/validate-psp-install.yaml | 7 + .../charts/rke2IngressNginx/values.yaml | 166 + .../charts/rke2Proxy/.helmignore | 23 + .../charts/rke2Proxy/Chart.yaml | 15 + .../charts/rke2Proxy/README.md | 90 + .../charts/rke2Proxy/templates/_helpers.tpl | 170 + .../templates/pushprox-clients-rbac.yaml | 97 + .../rke2Proxy/templates/pushprox-clients.yaml | 157 + .../templates/pushprox-proxy-rbac.yaml | 68 + .../rke2Proxy/templates/pushprox-proxy.yaml | 57 + .../templates/pushprox-servicemonitor.yaml | 45 + .../templates/validate-install-crd.yaml | 14 + .../templates/validate-psp-install.yaml | 7 + .../charts/rke2Proxy/values.yaml | 166 + .../charts/rke2Scheduler/.helmignore | 23 + .../charts/rke2Scheduler/Chart.yaml | 15 + .../charts/rke2Scheduler/README.md | 90 + .../rke2Scheduler/templates/_helpers.tpl | 170 + .../templates/pushprox-clients-rbac.yaml | 97 + .../templates/pushprox-clients.yaml | 157 + .../templates/pushprox-proxy-rbac.yaml | 68 + .../templates/pushprox-proxy.yaml | 57 + .../templates/pushprox-servicemonitor.yaml | 45 + .../templates/validate-install-crd.yaml | 14 + .../templates/validate-psp-install.yaml | 7 + .../charts/rke2Scheduler/values.yaml | 166 + .../charts/rkeControllerManager/.helmignore | 23 + .../charts/rkeControllerManager/Chart.yaml | 15 + .../charts/rkeControllerManager/README.md | 90 + .../templates/_helpers.tpl | 170 + .../templates/pushprox-clients-rbac.yaml | 97 + .../templates/pushprox-clients.yaml | 157 + .../templates/pushprox-proxy-rbac.yaml | 68 + .../templates/pushprox-proxy.yaml | 57 + .../templates/pushprox-servicemonitor.yaml | 45 + .../templates/validate-install-crd.yaml | 14 + .../templates/validate-psp-install.yaml | 7 + .../charts/rkeControllerManager/values.yaml | 166 + .../charts/rkeEtcd/.helmignore | 23 + .../charts/rkeEtcd/Chart.yaml | 15 + .../103.2.2+up57.0.3/charts/rkeEtcd/README.md | 90 + .../charts/rkeEtcd/templates/_helpers.tpl | 170 + .../templates/pushprox-clients-rbac.yaml | 97 + .../rkeEtcd/templates/pushprox-clients.yaml | 157 + .../templates/pushprox-proxy-rbac.yaml | 68 + .../rkeEtcd/templates/pushprox-proxy.yaml | 57 + .../templates/pushprox-servicemonitor.yaml | 45 + .../templates/validate-install-crd.yaml | 14 + .../templates/validate-psp-install.yaml | 7 + .../charts/rkeEtcd/values.yaml | 166 + .../charts/rkeIngressNginx/.helmignore | 23 + .../charts/rkeIngressNginx/Chart.yaml | 15 + .../charts/rkeIngressNginx/README.md | 90 + .../rkeIngressNginx/templates/_helpers.tpl | 170 + .../templates/pushprox-clients-rbac.yaml | 97 + .../templates/pushprox-clients.yaml | 157 + .../templates/pushprox-proxy-rbac.yaml | 68 + .../templates/pushprox-proxy.yaml | 57 + .../templates/pushprox-servicemonitor.yaml | 45 + .../templates/validate-install-crd.yaml | 14 + .../templates/validate-psp-install.yaml | 7 + .../charts/rkeIngressNginx/values.yaml | 166 + .../charts/rkeProxy/.helmignore | 23 + .../charts/rkeProxy/Chart.yaml | 15 + .../charts/rkeProxy/README.md | 90 + .../charts/rkeProxy/templates/_helpers.tpl | 170 + .../templates/pushprox-clients-rbac.yaml | 97 + .../rkeProxy/templates/pushprox-clients.yaml | 157 + .../templates/pushprox-proxy-rbac.yaml | 68 + .../rkeProxy/templates/pushprox-proxy.yaml | 57 + .../templates/pushprox-servicemonitor.yaml | 45 + .../templates/validate-install-crd.yaml | 14 + .../templates/validate-psp-install.yaml | 7 + .../charts/rkeProxy/values.yaml | 166 + .../charts/rkeScheduler/.helmignore | 23 + .../charts/rkeScheduler/Chart.yaml | 15 + .../charts/rkeScheduler/README.md | 90 + .../rkeScheduler/templates/_helpers.tpl | 170 + .../templates/pushprox-clients-rbac.yaml | 97 + .../templates/pushprox-clients.yaml | 157 + .../templates/pushprox-proxy-rbac.yaml | 68 + .../templates/pushprox-proxy.yaml | 57 + .../templates/pushprox-servicemonitor.yaml | 45 + .../templates/validate-install-crd.yaml | 14 + .../templates/validate-psp-install.yaml | 7 + .../charts/rkeScheduler/values.yaml | 166 + .../charts/windowsExporter/.helmignore | 21 + .../charts/windowsExporter/Chart.yaml | 17 + .../charts/windowsExporter/README.md | 42 + .../scripts/configure-firewall.ps1 | 31 + .../windowsExporter/templates/_helpers.tpl | 216 + .../windowsExporter/templates/config.yaml | 14 + .../windowsExporter/templates/daemonset.yaml | 200 + .../windowsExporter/templates/podmonitor.yaml | 91 + .../templates/scriptConfig.yaml | 14 + .../windowsExporter/templates/service.yaml | 32 + .../templates/serviceaccount.yaml | 17 + .../templates/servicemonitor.yaml | 75 + .../charts/windowsExporter/values.yaml | 366 ++ .../files/ingress-nginx/nginx.json | 1445 +++++ .../request-handling-performance.json | 963 +++ .../cluster/rancher-cluster-nodes.json | 793 +++ .../rancher/cluster/rancher-cluster.json | 776 +++ .../files/rancher/fleet/bundle.json | 246 + .../files/rancher/fleet/bundledeployment.json | 219 + .../files/rancher/fleet/cluster.json | 484 ++ .../files/rancher/fleet/clustergroup.json | 468 ++ .../rancher/fleet/controller-runtime.json | 454 ++ .../files/rancher/fleet/gitrepo.json | 325 + .../rancher/home/rancher-default-home.json | 1290 ++++ .../files/rancher/k8s/rancher-etcd-nodes.json | 687 +++ .../files/rancher/k8s/rancher-etcd.json | 669 ++ .../k8s/rancher-k8s-components-nodes.json | 527 ++ .../rancher/k8s/rancher-k8s-components.json | 519 ++ .../rancher/nodes/rancher-node-detail.json | 805 +++ .../files/rancher/nodes/rancher-node.json | 792 +++ .../performance/performance-debugging.json | 1652 +++++ .../rancher/pods/rancher-pod-containers.json | 636 ++ .../files/rancher/pods/rancher-pod.json | 636 ++ .../workloads/rancher-workload-pods.json | 652 ++ .../rancher/workloads/rancher-workload.json | 652 ++ .../delete-workloads-with-old-labels.sh | 14 + .../103.2.2+up57.0.3/templates/NOTES.txt | 4 + .../103.2.2+up57.0.3/templates/_helpers.tpl | 467 ++ .../templates/alertmanager/alertmanager.yaml | 191 + .../templates/alertmanager/extrasecret.yaml | 20 + .../templates/alertmanager/ingress.yaml | 78 + .../alertmanager/ingressperreplica.yaml | 67 + .../alertmanager/podDisruptionBudget.yaml | 21 + .../templates/alertmanager/psp-role.yaml | 23 + .../alertmanager/psp-rolebinding.yaml | 20 + .../templates/alertmanager/psp.yaml | 47 + .../templates/alertmanager/secret.yaml | 35 + .../templates/alertmanager/service.yaml | 68 + .../alertmanager/serviceaccount.yaml | 21 + .../alertmanager/servicemonitor.yaml | 84 + .../alertmanager/serviceperreplica.yaml | 49 + .../templates/exporters/core-dns/service.yaml | 24 + .../exporters/core-dns/servicemonitor.yaml | 58 + .../kube-api-server/servicemonitor.yaml | 57 + .../kube-controller-manager/endpoints.yaml | 22 + .../kube-controller-manager/service.yaml | 29 + .../servicemonitor.yaml | 69 + .../templates/exporters/kube-dns/service.yaml | 28 + .../exporters/kube-dns/servicemonitor.yaml | 71 + .../exporters/kube-etcd/endpoints.yaml | 20 + .../exporters/kube-etcd/service.yaml | 27 + .../exporters/kube-etcd/servicemonitor.yaml | 75 + .../exporters/kube-proxy/endpoints.yaml | 20 + .../exporters/kube-proxy/service.yaml | 27 + .../exporters/kube-proxy/servicemonitor.yaml | 63 + .../exporters/kube-scheduler/endpoints.yaml | 22 + .../exporters/kube-scheduler/service.yaml | 29 + .../kube-scheduler/servicemonitor.yaml | 69 + .../kube-state-metrics/validate.yaml | 7 + .../exporters/kubelet/servicemonitor.yaml | 246 + .../exporters/node-exporter/validate.yaml | 3 + .../templates/extra-objects.yaml | 4 + .../grafana/configmap-dashboards.yaml | 24 + .../grafana/configmaps-datasources.yaml | 81 + .../alertmanager-overview.yaml | 616 ++ .../grafana/dashboards-1.14/apiserver.yaml | 1772 ++++++ .../dashboards-1.14/cluster-total.yaml | 1882 ++++++ .../dashboards-1.14/controller-manager.yaml | 1196 ++++ .../grafana/dashboards-1.14/etcd.yaml | 1229 ++++ .../dashboards-1.14/grafana-overview.yaml | 635 ++ .../grafana/dashboards-1.14/k8s-coredns.yaml | 1534 +++++ .../k8s-resources-cluster.yaml | 3088 ++++++++++ .../k8s-resources-multicluster.yaml | 24 + .../k8s-resources-namespace.yaml | 2797 +++++++++ .../dashboards-1.14/k8s-resources-node.yaml | 1026 ++++ .../dashboards-1.14/k8s-resources-pod.yaml | 2469 ++++++++ .../k8s-resources-windows-cluster.yaml | 24 + .../k8s-resources-windows-namespace.yaml | 24 + .../k8s-resources-windows-pod.yaml | 24 + .../k8s-resources-workload.yaml | 2024 ++++++ .../k8s-resources-workloads-namespace.yaml | 2189 +++++++ .../k8s-windows-cluster-rsrc-use.yaml | 24 + .../k8s-windows-node-rsrc-use.yaml | 24 + .../grafana/dashboards-1.14/kubelet.yaml | 2256 +++++++ .../dashboards-1.14/namespace-by-pod.yaml | 1464 +++++ .../namespace-by-workload.yaml | 1736 ++++++ .../node-cluster-rsrc-use.yaml | 1063 ++++ .../dashboards-1.14/node-rsrc-use.yaml | 1089 ++++ .../grafana/dashboards-1.14/nodes-darwin.yaml | 1073 ++++ .../grafana/dashboards-1.14/nodes.yaml | 1066 ++++ .../persistentvolumesusage.yaml | 587 ++ .../grafana/dashboards-1.14/pod-total.yaml | 1228 ++++ .../prometheus-remote-write.yaml | 1674 +++++ .../grafana/dashboards-1.14/prometheus.yaml | 1235 ++++ .../grafana/dashboards-1.14/proxy.yaml | 1276 ++++ .../grafana/dashboards-1.14/scheduler.yaml | 1118 ++++ .../dashboards-1.14/workload-total.yaml | 1438 +++++ .../templates/grafana/namespaces.yaml | 13 + .../_prometheus-operator.tpl | 7 + .../_prometheus-operator-webhook.tpl | 6 + .../deployment/deployment.yaml | 143 + .../admission-webhooks/deployment/pdb.yaml | 15 + .../deployment/service.yaml | 58 + .../deployment/serviceaccount.yaml | 15 + .../ciliumnetworkpolicy-createSecret.yaml | 36 + .../ciliumnetworkpolicy-patchWebhook.yaml | 36 + .../job-patch/clusterrole.yaml | 33 + .../job-patch/clusterrolebinding.yaml | 20 + .../job-patch/job-createSecret.yaml | 73 + .../job-patch/job-patchWebhook.yaml | 74 + .../job-patch/networkpolicy-createSecret.yaml | 33 + .../job-patch/networkpolicy-patchWebhook.yaml | 33 + .../admission-webhooks/job-patch/psp.yaml | 47 + .../admission-webhooks/job-patch/role.yaml | 21 + .../job-patch/rolebinding.yaml | 21 + .../job-patch/serviceaccount.yaml | 17 + .../mutatingWebhookConfiguration.yaml | 77 + .../validatingWebhookConfiguration.yaml | 77 + .../prometheus-operator/certmanager.yaml | 55 + .../ciliumnetworkpolicy.yaml | 40 + .../prometheus-operator/clusterrole.yaml | 109 + .../clusterrolebinding.yaml | 16 + .../prometheus-operator/deployment.yaml | 204 + .../prometheus-operator/networkpolicy.yaml | 29 + .../prometheus-operator/psp-clusterrole.yaml | 21 + .../psp-clusterrolebinding.yaml | 18 + .../templates/prometheus-operator/psp.yaml | 46 + .../prometheus-operator/service.yaml | 57 + .../prometheus-operator/serviceaccount.yaml | 14 + .../prometheus-operator/servicemonitor.yaml | 57 + .../verticalpodautoscaler.yaml | 40 + .../templates/prometheus/_rules.tpl | 44 + .../additionalAlertRelabelConfigs.yaml | 16 + .../additionalAlertmanagerConfigs.yaml | 16 + .../prometheus/additionalPrometheusRules.yaml | 43 + .../prometheus/additionalScrapeConfigs.yaml | 20 + .../prometheus/ciliumnetworkpolicy.yaml | 27 + .../templates/prometheus/clusterrole.yaml | 30 + .../prometheus/clusterrolebinding.yaml | 18 + .../templates/prometheus/csi-secret.yaml | 12 + .../templates/prometheus/extrasecret.yaml | 20 + .../templates/prometheus/ingress.yaml | 77 + .../prometheus/ingressThanosSidecar.yaml | 77 + .../prometheus/ingressperreplica.yaml | 67 + .../templates/prometheus/networkpolicy.yaml | 34 + .../templates/prometheus/nginx-config.yaml | 68 + .../prometheus/podDisruptionBudget.yaml | 25 + .../templates/prometheus/podmonitors.yaml | 38 + .../templates/prometheus/prometheus.yaml | 472 ++ .../templates/prometheus/psp-clusterrole.yaml | 22 + .../prometheus/psp-clusterrolebinding.yaml | 19 + .../templates/prometheus/psp.yaml | 58 + .../rules-1.14/alertmanager.rules.yaml | 305 + .../rules-1.14/config-reloaders.yaml | 57 + .../templates/prometheus/rules-1.14/etcd.yaml | 461 ++ .../prometheus/rules-1.14/general.rules.yaml | 125 + ...les.container_cpu_usage_seconds_total.yaml | 43 + .../k8s.rules.container_memory_cache.yaml | 42 + .../k8s.rules.container_memory_rss.yaml | 42 + .../k8s.rules.container_memory_swap.yaml | 42 + ...es.container_memory_working_set_bytes.yaml | 42 + .../k8s.rules.container_resource.yaml | 168 + .../rules-1.14/k8s.rules.pod_owner.yaml | 107 + .../prometheus/rules-1.14/k8s.rules.yaml | 237 + .../kube-apiserver-availability.rules.yaml | 273 + .../kube-apiserver-burnrate.rules.yaml | 440 ++ .../kube-apiserver-histogram.rules.yaml | 53 + .../rules-1.14/kube-apiserver-slos.yaml | 159 + .../kube-prometheus-general.rules.yaml | 49 + .../kube-prometheus-node-recording.rules.yaml | 93 + .../rules-1.14/kube-scheduler.rules.yaml | 135 + .../rules-1.14/kube-state-metrics.yaml | 152 + .../prometheus/rules-1.14/kubelet.rules.yaml | 65 + .../rules-1.14/kubernetes-apps.yaml | 568 ++ .../rules-1.14/kubernetes-resources.yaml | 282 + .../rules-1.14/kubernetes-storage.yaml | 216 + .../kubernetes-system-apiserver.yaml | 193 + .../kubernetes-system-controller-manager.yaml | 55 + .../kubernetes-system-kube-proxy.yaml | 56 + .../rules-1.14/kubernetes-system-kubelet.yaml | 379 ++ .../kubernetes-system-scheduler.yaml | 54 + .../rules-1.14/kubernetes-system.yaml | 87 + .../rules-1.14/node-exporter.rules.yaml | 188 + .../prometheus/rules-1.14/node-exporter.yaml | 801 +++ .../prometheus/rules-1.14/node-network.yaml | 55 + .../prometheus/rules-1.14/node.rules.yaml | 109 + .../rules-1.14/prometheus-operator.yaml | 253 + .../prometheus/rules-1.14/prometheus.yaml | 707 +++ .../rules-1.14/windows.node.rules.yaml | 301 + .../rules-1.14/windows.pod.rules.yaml | 158 + .../templates/prometheus/secret.yaml | 15 + .../templates/prometheus/service.yaml | 80 + .../prometheus/serviceThanosSidecar.yaml | 39 + .../serviceThanosSidecarExternal.yaml | 46 + .../templates/prometheus/serviceaccount.yaml | 21 + .../templates/prometheus/servicemonitor.yaml | 97 + .../servicemonitorThanosSidecar.yaml | 55 + .../templates/prometheus/servicemonitors.yaml | 47 + .../prometheus/serviceperreplica.yaml | 54 + .../rancher-monitoring/clusterrole.yaml | 135 + .../rancher-monitoring/config-role.yaml | 48 + .../rancher-monitoring/dashboard-role.yaml | 47 + .../addons/ingress-nginx-dashboard.yaml | 18 + .../rancher/cluster-dashboards.yaml | 17 + .../dashboards/rancher/default-dashboard.yaml | 17 + .../dashboards/rancher/fleet-dashboards.yaml | 17 + .../dashboards/rancher/k8s-dashboards.yaml | 31 + .../dashboards/rancher/nodes-dashboards.yaml | 17 + .../rancher/performance-dashboards.yaml | 18 + .../dashboards/rancher/pods-dashboards.yaml | 17 + .../rancher/workload-dashboards.yaml | 17 + .../exporters/fleet/servicemonitor.yaml | 53 + .../ingress-nginx/network-policy.yaml | 19 + .../exporters/ingress-nginx/service.yaml | 27 + .../ingress-nginx/servicemonitor.yaml | 49 + .../exporters/rancher/servicemonitor.yaml | 58 + .../rancher-monitoring/hardened.yaml | 147 + .../rancher-monitoring/upgrade/configmap.yaml | 13 + .../rancher-monitoring/upgrade/job.yaml | 46 + .../rancher-monitoring/upgrade/rbac.yaml | 131 + .../templates/thanos-ruler/extrasecret.yaml | 20 + .../templates/thanos-ruler/ingress.yaml | 77 + .../thanos-ruler/podDisruptionBudget.yaml | 21 + .../templates/thanos-ruler/ruler.yaml | 189 + .../templates/thanos-ruler/secret.yaml | 26 + .../templates/thanos-ruler/service.yaml | 53 + .../thanos-ruler/serviceaccount.yaml | 20 + .../thanos-ruler/servicemonitor.yaml | 82 + .../templates/validate-install-crd.yaml | 23 + .../templates/validate-psp-install.yaml | 7 + .../103.2.2+up57.0.3/values.yaml | 5433 +++++++++++++++++ index.yaml | 166 + .../rancher-pushprox/charts/Chart.yaml | 3 +- 583 files changed, 105310 insertions(+), 1 deletion(-) create mode 100644 assets/rancher-monitoring-crd/rancher-monitoring-crd-103.2.2+up57.0.3.tgz create mode 100644 assets/rancher-monitoring/rancher-monitoring-103.2.2+up57.0.3.tgz create mode 100644 charts/rancher-monitoring-crd/103.2.2+up57.0.3/Chart.yaml create mode 100644 charts/rancher-monitoring-crd/103.2.2+up57.0.3/README.md create mode 100644 charts/rancher-monitoring-crd/103.2.2+up57.0.3/files/crd-manifest.tgz create mode 100644 charts/rancher-monitoring-crd/103.2.2+up57.0.3/templates/_helpers.tpl create mode 100644 charts/rancher-monitoring-crd/103.2.2+up57.0.3/templates/jobs.yaml create mode 100644 charts/rancher-monitoring-crd/103.2.2+up57.0.3/templates/manifest.yaml create mode 100644 charts/rancher-monitoring-crd/103.2.2+up57.0.3/templates/rbac.yaml create mode 100644 charts/rancher-monitoring-crd/103.2.2+up57.0.3/templates/validate-psp-install.yaml create mode 100644 charts/rancher-monitoring-crd/103.2.2+up57.0.3/values.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/.editorconfig create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/.helmignore create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/CHANGELOG.md create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/CONTRIBUTING.md create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/Chart.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/README.md create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/app-README.md create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/.helmignore create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/Chart.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/README.md create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/dashboards/custom-dashboard.json create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/NOTES.txt create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/_config.tpl create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/_helpers.tpl create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/_pod.tpl create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/clusterrole.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/clusterrolebinding.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/configSecret.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/configmap-dashboard-provider.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/configmap.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/dashboards-json-configmap.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/deployment.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/extra-manifests.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/headless-service.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/hpa.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/image-renderer-deployment.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/image-renderer-hpa.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/image-renderer-network-policy.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/image-renderer-service.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/image-renderer-servicemonitor.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/ingress.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/networkpolicy.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/nginx-config.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/poddisruptionbudget.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/podsecuritypolicy.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/pvc.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/role.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/rolebinding.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/secret-env.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/secret.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/service.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/serviceaccount.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/servicemonitor.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/statefulset.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/tests/test-configmap.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/tests/test-podsecuritypolicy.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/tests/test-role.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/tests/test-rolebinding.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/tests/test-serviceaccount.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/tests/test.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/values.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/hardenedKubelet/.helmignore create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/hardenedKubelet/Chart.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/hardenedKubelet/README.md create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/hardenedKubelet/templates/_helpers.tpl create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/hardenedKubelet/templates/pushprox-clients-rbac.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/hardenedKubelet/templates/pushprox-clients.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/hardenedKubelet/templates/pushprox-proxy-rbac.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/hardenedKubelet/templates/pushprox-proxy.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/hardenedKubelet/templates/pushprox-servicemonitor.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/hardenedKubelet/templates/validate-install-crd.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/hardenedKubelet/templates/validate-psp-install.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/hardenedKubelet/values.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/hardenedNodeExporter/.helmignore create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/hardenedNodeExporter/Chart.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/hardenedNodeExporter/README.md create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/hardenedNodeExporter/templates/_helpers.tpl create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/hardenedNodeExporter/templates/pushprox-clients-rbac.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/hardenedNodeExporter/templates/pushprox-clients.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/hardenedNodeExporter/templates/pushprox-proxy-rbac.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/hardenedNodeExporter/templates/pushprox-proxy.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/hardenedNodeExporter/templates/pushprox-servicemonitor.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/hardenedNodeExporter/templates/validate-install-crd.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/hardenedNodeExporter/templates/validate-psp-install.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/hardenedNodeExporter/values.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/k3sServer/.helmignore create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/k3sServer/Chart.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/k3sServer/README.md create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/k3sServer/templates/_helpers.tpl create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/k3sServer/templates/pushprox-clients-rbac.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/k3sServer/templates/pushprox-clients.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/k3sServer/templates/pushprox-proxy-rbac.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/k3sServer/templates/pushprox-proxy.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/k3sServer/templates/pushprox-servicemonitor.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/k3sServer/templates/validate-install-crd.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/k3sServer/templates/validate-psp-install.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/k3sServer/values.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/kube-state-metrics/.helmignore create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/kube-state-metrics/Chart.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/kube-state-metrics/README.md create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/kube-state-metrics/templates/NOTES.txt create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/kube-state-metrics/templates/_helpers.tpl create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/kube-state-metrics/templates/ciliumnetworkpolicy.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/kube-state-metrics/templates/clusterrolebinding.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/kube-state-metrics/templates/crs-configmap.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/kube-state-metrics/templates/deployment.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/kube-state-metrics/templates/extra-manifests.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/kube-state-metrics/templates/kubeconfig-secret.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/kube-state-metrics/templates/networkpolicy.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/kube-state-metrics/templates/pdb.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/kube-state-metrics/templates/podsecuritypolicy.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/kube-state-metrics/templates/psp-clusterrole.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/kube-state-metrics/templates/psp-clusterrolebinding.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/kube-state-metrics/templates/rbac-configmap.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/kube-state-metrics/templates/role.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/kube-state-metrics/templates/rolebinding.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/kube-state-metrics/templates/service.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/kube-state-metrics/templates/serviceaccount.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/kube-state-metrics/templates/servicemonitor.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/kube-state-metrics/templates/stsdiscovery-role.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/kube-state-metrics/templates/stsdiscovery-rolebinding.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/kube-state-metrics/templates/verticalpodautoscaler.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/kube-state-metrics/values.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmControllerManager/.helmignore create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmControllerManager/Chart.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmControllerManager/README.md create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmControllerManager/templates/_helpers.tpl create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmControllerManager/templates/pushprox-clients-rbac.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmControllerManager/templates/pushprox-clients.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmControllerManager/templates/pushprox-proxy-rbac.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmControllerManager/templates/pushprox-proxy.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmControllerManager/templates/pushprox-servicemonitor.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmControllerManager/templates/validate-install-crd.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmControllerManager/templates/validate-psp-install.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmControllerManager/values.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmEtcd/.helmignore create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmEtcd/Chart.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmEtcd/README.md create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmEtcd/templates/_helpers.tpl create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmEtcd/templates/pushprox-clients-rbac.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmEtcd/templates/pushprox-clients.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmEtcd/templates/pushprox-proxy-rbac.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmEtcd/templates/pushprox-proxy.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmEtcd/templates/pushprox-servicemonitor.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmEtcd/templates/validate-install-crd.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmEtcd/templates/validate-psp-install.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmEtcd/values.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmProxy/.helmignore create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmProxy/Chart.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmProxy/README.md create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmProxy/templates/_helpers.tpl create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmProxy/templates/pushprox-clients-rbac.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmProxy/templates/pushprox-clients.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmProxy/templates/pushprox-proxy-rbac.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmProxy/templates/pushprox-proxy.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmProxy/templates/pushprox-servicemonitor.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmProxy/templates/validate-install-crd.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmProxy/templates/validate-psp-install.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmProxy/values.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmScheduler/.helmignore create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmScheduler/Chart.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmScheduler/README.md create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmScheduler/templates/_helpers.tpl create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmScheduler/templates/pushprox-clients-rbac.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmScheduler/templates/pushprox-clients.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmScheduler/templates/pushprox-proxy-rbac.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmScheduler/templates/pushprox-proxy.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmScheduler/templates/pushprox-servicemonitor.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmScheduler/templates/validate-install-crd.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmScheduler/templates/validate-psp-install.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmScheduler/values.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-adapter/.helmignore create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-adapter/Chart.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-adapter/README.md create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-adapter/templates/NOTES.txt create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-adapter/templates/_helpers.tpl create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-adapter/templates/certmanager.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-adapter/templates/cluster-role-binding-auth-delegator.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-adapter/templates/cluster-role-binding-resource-reader.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-adapter/templates/cluster-role-resource-reader.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-adapter/templates/configmap.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-adapter/templates/custom-metrics-apiservice.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-adapter/templates/custom-metrics-cluster-role-binding-hpa.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-adapter/templates/custom-metrics-cluster-role.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-adapter/templates/deployment.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-adapter/templates/external-metrics-apiservice.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-adapter/templates/external-metrics-cluster-role-binding-hpa.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-adapter/templates/external-metrics-cluster-role.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-adapter/templates/pdb.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-adapter/templates/psp.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-adapter/templates/resource-metrics-apiservice.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-adapter/templates/resource-metrics-cluster-role-binding.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-adapter/templates/resource-metrics-cluster-role.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-adapter/templates/role-binding-auth-reader.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-adapter/templates/secret.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-adapter/templates/service.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-adapter/templates/serviceaccount.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-adapter/values.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-node-exporter/.helmignore create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-node-exporter/Chart.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-node-exporter/README.md create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-node-exporter/templates/NOTES.txt create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-node-exporter/templates/_helpers.tpl create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-node-exporter/templates/clusterrole.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-node-exporter/templates/clusterrolebinding.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-node-exporter/templates/daemonset.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-node-exporter/templates/endpoints.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-node-exporter/templates/extra-manifests.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-node-exporter/templates/networkpolicy.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-node-exporter/templates/podmonitor.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-node-exporter/templates/psp-clusterrole.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-node-exporter/templates/psp-clusterrolebinding.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-node-exporter/templates/psp.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-node-exporter/templates/rbac-configmap.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-node-exporter/templates/service.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-node-exporter/templates/serviceaccount.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-node-exporter/templates/servicemonitor.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-node-exporter/templates/verticalpodautoscaler.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-node-exporter/values.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2ControllerManager/.helmignore create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2ControllerManager/Chart.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2ControllerManager/README.md create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2ControllerManager/templates/_helpers.tpl create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2ControllerManager/templates/pushprox-clients-rbac.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2ControllerManager/templates/pushprox-clients.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2ControllerManager/templates/pushprox-proxy-rbac.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2ControllerManager/templates/pushprox-proxy.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2ControllerManager/templates/pushprox-servicemonitor.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2ControllerManager/templates/validate-install-crd.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2ControllerManager/templates/validate-psp-install.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2ControllerManager/values.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2Etcd/.helmignore create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2Etcd/Chart.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2Etcd/README.md create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2Etcd/templates/_helpers.tpl create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2Etcd/templates/pushprox-clients-rbac.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2Etcd/templates/pushprox-clients.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2Etcd/templates/pushprox-proxy-rbac.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2Etcd/templates/pushprox-proxy.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2Etcd/templates/pushprox-servicemonitor.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2Etcd/templates/validate-install-crd.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2Etcd/templates/validate-psp-install.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2Etcd/values.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2IngressNginx/.helmignore create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2IngressNginx/Chart.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2IngressNginx/README.md create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2IngressNginx/templates/_helpers.tpl create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2IngressNginx/templates/pushprox-clients-rbac.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2IngressNginx/templates/pushprox-clients.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2IngressNginx/templates/pushprox-proxy-rbac.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2IngressNginx/templates/pushprox-proxy.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2IngressNginx/templates/pushprox-servicemonitor.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2IngressNginx/templates/validate-install-crd.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2IngressNginx/templates/validate-psp-install.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2IngressNginx/values.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2Proxy/.helmignore create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2Proxy/Chart.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2Proxy/README.md create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2Proxy/templates/_helpers.tpl create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2Proxy/templates/pushprox-clients-rbac.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2Proxy/templates/pushprox-clients.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2Proxy/templates/pushprox-proxy-rbac.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2Proxy/templates/pushprox-proxy.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2Proxy/templates/pushprox-servicemonitor.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2Proxy/templates/validate-install-crd.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2Proxy/templates/validate-psp-install.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2Proxy/values.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2Scheduler/.helmignore create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2Scheduler/Chart.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2Scheduler/README.md create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2Scheduler/templates/_helpers.tpl create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2Scheduler/templates/pushprox-clients-rbac.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2Scheduler/templates/pushprox-clients.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2Scheduler/templates/pushprox-proxy-rbac.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2Scheduler/templates/pushprox-proxy.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2Scheduler/templates/pushprox-servicemonitor.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2Scheduler/templates/validate-install-crd.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2Scheduler/templates/validate-psp-install.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2Scheduler/values.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeControllerManager/.helmignore create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeControllerManager/Chart.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeControllerManager/README.md create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeControllerManager/templates/_helpers.tpl create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeControllerManager/templates/pushprox-clients-rbac.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeControllerManager/templates/pushprox-clients.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeControllerManager/templates/pushprox-proxy-rbac.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeControllerManager/templates/pushprox-proxy.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeControllerManager/templates/pushprox-servicemonitor.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeControllerManager/templates/validate-install-crd.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeControllerManager/templates/validate-psp-install.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeControllerManager/values.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeEtcd/.helmignore create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeEtcd/Chart.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeEtcd/README.md create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeEtcd/templates/_helpers.tpl create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeEtcd/templates/pushprox-clients-rbac.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeEtcd/templates/pushprox-clients.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeEtcd/templates/pushprox-proxy-rbac.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeEtcd/templates/pushprox-proxy.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeEtcd/templates/pushprox-servicemonitor.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeEtcd/templates/validate-install-crd.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeEtcd/templates/validate-psp-install.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeEtcd/values.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeIngressNginx/.helmignore create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeIngressNginx/Chart.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeIngressNginx/README.md create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeIngressNginx/templates/_helpers.tpl create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeIngressNginx/templates/pushprox-clients-rbac.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeIngressNginx/templates/pushprox-clients.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeIngressNginx/templates/pushprox-proxy-rbac.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeIngressNginx/templates/pushprox-proxy.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeIngressNginx/templates/pushprox-servicemonitor.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeIngressNginx/templates/validate-install-crd.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeIngressNginx/templates/validate-psp-install.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeIngressNginx/values.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeProxy/.helmignore create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeProxy/Chart.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeProxy/README.md create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeProxy/templates/_helpers.tpl create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeProxy/templates/pushprox-clients-rbac.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeProxy/templates/pushprox-clients.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeProxy/templates/pushprox-proxy-rbac.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeProxy/templates/pushprox-proxy.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeProxy/templates/pushprox-servicemonitor.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeProxy/templates/validate-install-crd.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeProxy/templates/validate-psp-install.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeProxy/values.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeScheduler/.helmignore create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeScheduler/Chart.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeScheduler/README.md create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeScheduler/templates/_helpers.tpl create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeScheduler/templates/pushprox-clients-rbac.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeScheduler/templates/pushprox-clients.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeScheduler/templates/pushprox-proxy-rbac.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeScheduler/templates/pushprox-proxy.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeScheduler/templates/pushprox-servicemonitor.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeScheduler/templates/validate-install-crd.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeScheduler/templates/validate-psp-install.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeScheduler/values.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/windowsExporter/.helmignore create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/windowsExporter/Chart.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/windowsExporter/README.md create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/windowsExporter/scripts/configure-firewall.ps1 create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/windowsExporter/templates/_helpers.tpl create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/windowsExporter/templates/config.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/windowsExporter/templates/daemonset.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/windowsExporter/templates/podmonitor.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/windowsExporter/templates/scriptConfig.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/windowsExporter/templates/service.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/windowsExporter/templates/serviceaccount.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/windowsExporter/templates/servicemonitor.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/charts/windowsExporter/values.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/files/ingress-nginx/nginx.json create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/files/ingress-nginx/request-handling-performance.json create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/files/rancher/cluster/rancher-cluster-nodes.json create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/files/rancher/cluster/rancher-cluster.json create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/files/rancher/fleet/bundle.json create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/files/rancher/fleet/bundledeployment.json create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/files/rancher/fleet/cluster.json create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/files/rancher/fleet/clustergroup.json create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/files/rancher/fleet/controller-runtime.json create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/files/rancher/fleet/gitrepo.json create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/files/rancher/home/rancher-default-home.json create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/files/rancher/k8s/rancher-etcd-nodes.json create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/files/rancher/k8s/rancher-etcd.json create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/files/rancher/k8s/rancher-k8s-components-nodes.json create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/files/rancher/k8s/rancher-k8s-components.json create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/files/rancher/nodes/rancher-node-detail.json create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/files/rancher/nodes/rancher-node.json create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/files/rancher/performance/performance-debugging.json create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/files/rancher/pods/rancher-pod-containers.json create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/files/rancher/pods/rancher-pod.json create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/files/rancher/workloads/rancher-workload-pods.json create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/files/rancher/workloads/rancher-workload.json create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/files/upgrade/scripts/delete-workloads-with-old-labels.sh create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/NOTES.txt create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/_helpers.tpl create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/alertmanager/alertmanager.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/alertmanager/extrasecret.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/alertmanager/ingress.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/alertmanager/ingressperreplica.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/alertmanager/podDisruptionBudget.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/alertmanager/psp-role.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/alertmanager/psp-rolebinding.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/alertmanager/psp.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/alertmanager/secret.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/alertmanager/service.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/alertmanager/serviceaccount.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/alertmanager/servicemonitor.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/alertmanager/serviceperreplica.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/exporters/core-dns/service.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/exporters/core-dns/servicemonitor.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/exporters/kube-api-server/servicemonitor.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/exporters/kube-controller-manager/endpoints.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/exporters/kube-controller-manager/service.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/exporters/kube-controller-manager/servicemonitor.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/exporters/kube-dns/service.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/exporters/kube-dns/servicemonitor.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/exporters/kube-etcd/endpoints.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/exporters/kube-etcd/service.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/exporters/kube-etcd/servicemonitor.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/exporters/kube-proxy/endpoints.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/exporters/kube-proxy/service.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/exporters/kube-proxy/servicemonitor.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/exporters/kube-scheduler/endpoints.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/exporters/kube-scheduler/service.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/exporters/kube-scheduler/servicemonitor.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/exporters/kube-state-metrics/validate.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/exporters/kubelet/servicemonitor.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/exporters/node-exporter/validate.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/extra-objects.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/grafana/configmap-dashboards.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/grafana/configmaps-datasources.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/grafana/dashboards-1.14/alertmanager-overview.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/grafana/dashboards-1.14/apiserver.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/grafana/dashboards-1.14/cluster-total.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/grafana/dashboards-1.14/controller-manager.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/grafana/dashboards-1.14/etcd.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/grafana/dashboards-1.14/grafana-overview.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/grafana/dashboards-1.14/k8s-coredns.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/grafana/dashboards-1.14/k8s-resources-cluster.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/grafana/dashboards-1.14/k8s-resources-multicluster.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/grafana/dashboards-1.14/k8s-resources-namespace.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/grafana/dashboards-1.14/k8s-resources-node.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/grafana/dashboards-1.14/k8s-resources-pod.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/grafana/dashboards-1.14/k8s-resources-windows-cluster.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/grafana/dashboards-1.14/k8s-resources-windows-namespace.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/grafana/dashboards-1.14/k8s-resources-windows-pod.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/grafana/dashboards-1.14/k8s-resources-workload.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/grafana/dashboards-1.14/k8s-resources-workloads-namespace.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/grafana/dashboards-1.14/k8s-windows-cluster-rsrc-use.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/grafana/dashboards-1.14/k8s-windows-node-rsrc-use.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/grafana/dashboards-1.14/kubelet.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/grafana/dashboards-1.14/namespace-by-pod.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/grafana/dashboards-1.14/namespace-by-workload.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/grafana/dashboards-1.14/node-cluster-rsrc-use.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/grafana/dashboards-1.14/node-rsrc-use.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/grafana/dashboards-1.14/nodes-darwin.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/grafana/dashboards-1.14/nodes.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/grafana/dashboards-1.14/persistentvolumesusage.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/grafana/dashboards-1.14/pod-total.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/grafana/dashboards-1.14/prometheus-remote-write.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/grafana/dashboards-1.14/prometheus.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/grafana/dashboards-1.14/proxy.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/grafana/dashboards-1.14/scheduler.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/grafana/dashboards-1.14/workload-total.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/grafana/namespaces.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus-operator/_prometheus-operator.tpl create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus-operator/admission-webhooks/_prometheus-operator-webhook.tpl create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus-operator/admission-webhooks/deployment/deployment.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus-operator/admission-webhooks/deployment/pdb.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus-operator/admission-webhooks/deployment/service.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus-operator/admission-webhooks/deployment/serviceaccount.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus-operator/admission-webhooks/job-patch/ciliumnetworkpolicy-createSecret.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus-operator/admission-webhooks/job-patch/ciliumnetworkpolicy-patchWebhook.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus-operator/admission-webhooks/job-patch/clusterrole.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus-operator/admission-webhooks/job-patch/clusterrolebinding.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus-operator/admission-webhooks/job-patch/job-createSecret.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus-operator/admission-webhooks/job-patch/job-patchWebhook.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus-operator/admission-webhooks/job-patch/networkpolicy-createSecret.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus-operator/admission-webhooks/job-patch/networkpolicy-patchWebhook.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus-operator/admission-webhooks/job-patch/psp.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus-operator/admission-webhooks/job-patch/role.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus-operator/admission-webhooks/job-patch/rolebinding.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus-operator/admission-webhooks/job-patch/serviceaccount.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus-operator/admission-webhooks/mutatingWebhookConfiguration.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus-operator/admission-webhooks/validatingWebhookConfiguration.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus-operator/certmanager.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus-operator/ciliumnetworkpolicy.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus-operator/clusterrole.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus-operator/clusterrolebinding.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus-operator/deployment.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus-operator/networkpolicy.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus-operator/psp-clusterrole.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus-operator/psp-clusterrolebinding.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus-operator/psp.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus-operator/service.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus-operator/serviceaccount.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus-operator/servicemonitor.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus-operator/verticalpodautoscaler.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/_rules.tpl create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/additionalAlertRelabelConfigs.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/additionalAlertmanagerConfigs.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/additionalPrometheusRules.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/additionalScrapeConfigs.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/ciliumnetworkpolicy.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/clusterrole.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/clusterrolebinding.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/csi-secret.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/extrasecret.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/ingress.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/ingressThanosSidecar.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/ingressperreplica.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/networkpolicy.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/nginx-config.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/podDisruptionBudget.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/podmonitors.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/prometheus.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/psp-clusterrole.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/psp-clusterrolebinding.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/psp.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/alertmanager.rules.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/config-reloaders.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/etcd.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/general.rules.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/k8s.rules.container_cpu_usage_seconds_total.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/k8s.rules.container_memory_cache.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/k8s.rules.container_memory_rss.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/k8s.rules.container_memory_swap.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/k8s.rules.container_memory_working_set_bytes.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/k8s.rules.container_resource.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/k8s.rules.pod_owner.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/k8s.rules.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/kube-apiserver-availability.rules.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/kube-apiserver-burnrate.rules.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/kube-apiserver-histogram.rules.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/kube-apiserver-slos.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/kube-prometheus-general.rules.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/kube-prometheus-node-recording.rules.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/kube-scheduler.rules.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/kube-state-metrics.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/kubelet.rules.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/kubernetes-apps.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/kubernetes-resources.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/kubernetes-storage.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/kubernetes-system-apiserver.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/kubernetes-system-controller-manager.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/kubernetes-system-kube-proxy.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/kubernetes-system-kubelet.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/kubernetes-system-scheduler.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/kubernetes-system.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/node-exporter.rules.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/node-exporter.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/node-network.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/node.rules.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/prometheus-operator.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/prometheus.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/windows.node.rules.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/windows.pod.rules.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/secret.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/service.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/serviceThanosSidecar.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/serviceThanosSidecarExternal.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/serviceaccount.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/servicemonitor.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/servicemonitorThanosSidecar.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/servicemonitors.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/serviceperreplica.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/rancher-monitoring/clusterrole.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/rancher-monitoring/config-role.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/rancher-monitoring/dashboard-role.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/rancher-monitoring/dashboards/addons/ingress-nginx-dashboard.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/rancher-monitoring/dashboards/rancher/cluster-dashboards.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/rancher-monitoring/dashboards/rancher/default-dashboard.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/rancher-monitoring/dashboards/rancher/fleet-dashboards.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/rancher-monitoring/dashboards/rancher/k8s-dashboards.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/rancher-monitoring/dashboards/rancher/nodes-dashboards.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/rancher-monitoring/dashboards/rancher/performance-dashboards.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/rancher-monitoring/dashboards/rancher/pods-dashboards.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/rancher-monitoring/dashboards/rancher/workload-dashboards.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/rancher-monitoring/exporters/fleet/servicemonitor.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/rancher-monitoring/exporters/ingress-nginx/network-policy.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/rancher-monitoring/exporters/ingress-nginx/service.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/rancher-monitoring/exporters/ingress-nginx/servicemonitor.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/rancher-monitoring/exporters/rancher/servicemonitor.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/rancher-monitoring/hardened.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/rancher-monitoring/upgrade/configmap.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/rancher-monitoring/upgrade/job.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/rancher-monitoring/upgrade/rbac.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/thanos-ruler/extrasecret.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/thanos-ruler/ingress.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/thanos-ruler/podDisruptionBudget.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/thanos-ruler/ruler.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/thanos-ruler/secret.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/thanos-ruler/service.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/thanos-ruler/serviceaccount.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/thanos-ruler/servicemonitor.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/validate-install-crd.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/templates/validate-psp-install.yaml create mode 100644 charts/rancher-monitoring/103.2.2+up57.0.3/values.yaml diff --git a/assets/rancher-monitoring-crd/rancher-monitoring-crd-103.2.2+up57.0.3.tgz b/assets/rancher-monitoring-crd/rancher-monitoring-crd-103.2.2+up57.0.3.tgz new file mode 100644 index 0000000000000000000000000000000000000000..c39db26a0e795294220dbb4aceeebbbeac0c8ef7 GIT binary patch literal 299555 zcmV(@K-Rw>iwG0|00000|0w_~VMtOiV@ORlOnEsqVl!4SWK%V1T2nbTPgYhoO;>Dc zVQyr3R8em|NM&qo0PMR7Jd|JiFpjd5J!{;CLKyqLZy|-OQDm5#8I74Si!DnbWT_M( zl7uW#*>@!sQB))eNwV)-l>9$4RQf*O?fJdW`#kUaf1mT|qx(MRy03Gc>pItZ&fs8Z zHw1{2_QIf%1Pl&|_K42AK4y1@tp6bMCPWZXa;0qKqe;p)KFUin=x1QHGh(dqz%fFpvC-vvR#yg)n_ z<_4+*o1NP?9o`>L0KIXm!9xo&gTx-Ec@O zf&6ZY#uH#D6doWTK)}Gt5D&OxaKD2H%J8RC5d5*AIsn6BQAjs3#~6IR&nPFW0F{Tz zOA@h4Do|Od0>gh=tbe=xf&TlzP(%>_FERjsrT>abvcJ)PWm#FJ|LXt0qlo~9pgW9+ zA^@8r{{z(^Z3-xZ2w;Oi;sGQcfB_DAhYv}+V{l$D0s({r?no3!0x$&KP%s<__`q;T zm@5jz0|X4<3IbRd9uL9+B$|K${4qovKmfh4C>R06Lm3!6P#9MjYRmo*P$20E0I+!M z<^uqLXc8{CI^YgN;X!ieroq@E-U{?U;t4o^bpQfkU_g4oJjgh3AQpost-$|>Qp?~G zAPU6*00fwaI^ZJBR6ZUhWY9XJ)lz(ByDK-`vH1dciV-=om~WcvgC z{~;Zj69^E61#x&N0sB8S1%ILca`Gw)zv@5fLHWP>|Liz?z7~VsHe~Vr2F+eD8Hb5%B~N2lygUC;$eB16%6?2rwj?KsI?O zB%0_4kn8|nih%^;3ZgK+Tix4?0m+>CVsIx>7#JK6xM7GW5-5OoLx6B13WSr!2?z`x z+>Vi9zYK}Y?F=T5N|Q#Vw_)GpuQYHHB$=|GArJ;n3aQ8zT7BzVQU~bn4!RN40f;%q zdW!&3HwH_>N3vuPBX1%M1^Ja*e?SI)%l`c$V?5~P1L6!YURaV*hiJ+{ zhK0EzQAh$3#6!)9uHaVm1vN$zwjRlNTtNa1dXnf0;?N+;*pnBdzR8?TvQT)C4Ed)( zzbEzowCeo-qWz`)_r$pVuf~CYVgHpCl;nRC|EZ8d!vEU;e?!}zH@U(HZU`A4Ifj!+ zG+Z4xh;e1`0tqlUi~u7=Q=7@rsZ#)SvpQf7^8$gOAZa%o2qS=709y$j3Ce0SZzIEz zyMdq}1^__8TtO6mGqQtWvA=_n0RVoJmI43-0iwL1c!UfBgEISkZ%4Eb-)#L$KXI|@)&tBNDNvEAiv^? zZf+n52jN@q?l2?@gfrl=pc{#r{z zh9RN}=6^7g7Yq&aAZP5G-?$NRNP@os22B9{2tVh<5z%^ha}3%FgCT5>v44!&;z67` zAiLG+hC!1YCy3idw)EC&H`hk8A^0C-0I(U6lL-9Iu|Ekhpg>R%X(?OL0r^|#&rlu+ z3R3@vVFJwKJD0Z2Xh}q&EXh(<2Ta|~F$7B-hzHSxZE$WFFE1DxuD-2H8jx{CqGj-i zU;3ooetFg2_67i;8v+AB^srczKN9Uht}#HNP>9AaKwc-|NE{%I1!O>in~WO{{_C(S z3=b+R0@CoUaSYlW=|KV`NfGG|;t5cKM*!4SSrG^X2rwKV?HAw!NE`eCI+8neBcMn* z5Q?OG(pK(2Ac4TZALE+>`(;?;ch+WzL4(4OZ38|S6wwPjOxk^T^?2g#G`{?`-k*|4F}NNLu^KazY}VGNB4UkhSX0o4bTHf6o`i!g9HHLPI~+Ye*^&n0as;35bXv80$cV)k%ay~ z&CvhL+F$BF&K2hNzqkIUATRgZ`yVQba{tx;e@ol0|720ahy(-%hYTQlZ|F%?ay9;^ zW!OAPvBIFhziS(|5`n+YuYLgIh$vDdAPvB#?uUQRj>sTL5Kji$B9zpQ zi4pPOFJF?)>E>epn2FmWtluWo|D2rlk!Uy)?eV`++!z#S1-g@9zw_)*Xk-9@f82_H zO%a~x>Iu3L$lGf3d+T2^*}t`@zvJG1%j_R4{mp#;96SG2HH;r9No4*r`+574HaD}3 zkDMz=fPG({B?fN2wUalqsQ>f|zgI1=IHV5}1$u11J_19bFus=G2aND;FcfU_uGn`7 z1O`tq2MN9y+{s@%O)U+6?XWa8{MqrdYC$HEWXaS4F)LE99@<}w0RZseAyNwV$KymS z76p2NXaWrN)A;=WJxs(C$lzj|y)d-L_N>VZiB<>XexyZSFh6xbSxHGji2(rI@teB! zFGBY{*|x%92;^{$Tvz$YieEB1X+Z2p)+YA72M@a8K!VtIQ}v&AhyTyDzl{HUU??QX z4@zV4*zX5!|71M)SNWf^>~HS>$;&G#{P+IPzom(6IROs>AdUjjfH(?+IZ4C#Y(!OVcfvgNK^H@Brjjza#Sx9tR@v z1EV_(i2@)t2oNB^a2_Dx#}yzr2E>ymL%@7M@`d1kKm+Jmn*NBjpnxUW5dm-{oExIkZ6E_K;i*6lIPi~;{#Y6#tj5Xu@V&WlM!2n?T?=S)AIiV{kJmGGdyer^@9H^ z=J*T!SCs$9`~PzPt^fRcn#dpBcGvrUF>q^>6A&;0fc%G>^h7)e2VDJs_clF)hzMYU z@!iz9ZTdnAs&p6_Y<~85!Eks488Hln0x<61@sqC=;{oysHhfd-K)_{tEyx9cBXQ)) zxc}zmV%X2>)fUb8?_`5;0J915-5S797_?~L&N{kl1M-q7%b3kn|9{M3~&$&ZWgS*8yOG}z;NI;GG&0R%Pgcv9ERNI3UfOt zwVjQiALvFTkiPxpHVh#1%XOIVuix*(00%LyWX6y(Axl!$OTvKm2S7h0o_vb5^{WQ_ zHaXyboV~#TUnBwX`;x$H_I_tB0R!Mc0^mXl#GzOmhzD^#pe_0&8sm#TfCN!+ycRiD zbkW#c6`lZMNz%fB-b5r0gadda+6|N7U~=o8mWY zDeRY4R+CebQ&Uq=mXHGQL^s5i0LW+0fGhZOYoIYWFBl5t54htnUVzK*l}EliOBa$f z{s_Va${-C`5z(Zb=D*5Ts6Xeo%$T z*YM5_1(Mi<-S*ohlCXbap*e;CsspC(znU2`FbqXX)3!MB(}e!8e+2R?0qF&jSoC`` z*t|3CipStw0gNvi#NiQ0?00^mknW(HzZ(kN?7-kRX(TB$h$BZuQY0?G02c&^^7<)d z7pZN~TT`I$*v=ZnM=tsoV51_6UEHdWaDKUl8yl;t87$<0s3uR{b-n z+2%C1R^kOiqBn8sFff>+0TU9v1PmPJF9rBxh~L!@zV-7K7}-0K%)!r|?-pbeC5$Wq zF+922M5f^9>?Fx!hy(yclidqQa+d&%9JCT~Aci)`+jURcbFSF@+bc$=sz76{Xf(GApZaTL%09o9Df%7%gHIoE6XbV z_WV~?;lJ_!zoiMq&}}|!OxB(B9V#!Aeb>Iq<+snFRiWU@WXkb?as1obX}_0iEyrt7 zV*WK(l_N|{^bb=>Q9Qkwp0?0?o%(c!$vII9hpTT=>U2Z}Q{TO-oYGh}|8!K`KOxHI zoBIiDU|`_anjyC@8+YmzH*|vM&aO9{9njqP;CrH0HTc`3krP4RQin!P29lr^uzUjNwe`&-*bP6=EJuQlZ<=c)^tSD*lyt3W@DqmOC+wf%(0L3 z1l7Y{ueZu*bZ@N2$XwUin2%XrYw-IvY=7b_P0&Ttug%>X){6G|7sZT?sRrUaY@W56 zN<4cj+1RN%xOzm`&%84&0oTL`x%(k=!8c}TZpJzG!iDBjY23VH0qzM=i{c}(`=R2g zaU4*(q%}#MrKwtr12v>4p#rPSU= z!1ubdz=i7}+Sxt3r#VHpeO@GWs^B>V&|B)tWT?js7A8zi^tE z^%qLhp4)AAM+3fagu@#daiMLk)aE8@^!i-{KKM<~{oL)|c5QR`7=@I}0=<|NBiJqO zk%5rlt~DiniTHCFXXVXG)HLDe#Hr&=p3zt5j@((A6@D3GaLeqeyOL-4r?1b?n~V=8 zIYjSkf(n_kuz`5>8_`*GC7g0aqY0NdCC)D@?`HBafo4Oh1iDojZzLV=lJvac&A`{- z#Q*M$U_%f?cEA^VgM9Z(SzQUj+D4w{d)c#C4cXmU3>oEKvcMuff4nYw#EoDSeqiW8 z3bq{37uI9fc3T`!LgyLZ(zyyV%8x6ddYTWD!P;FiP>@!0hUhKYTek3hZWF=9_G9%} zLF-^Eh1lJ>JnVs^Wu9MFYPf^=ZD3& zbKS<~tG-|zp>7OR?A8p2n8}_DIFHb5RkYP6K{8BDu zb=yS!zz9=alUj?jtLbt*ML^E5i3REnyJFSq+C!h$d?8qYGLJgto(Hyimpmz-`QJLH z#=qRmx~|L>p``@V|Kz7vg|$$6@%rnd_q;cCq^EimGsmz)b4qQ6No*cH?1P!j;&LOm zKWJJ*JNnZ0;lrjP)Sv-j@|wauYrF|jUkjR2G;)`QoI33_Uru#;Guo>Z$Pe6&ohfcp z-Y75M=~z1|V}EB9#<2dD$Ep3?I$PfhsbkE+=Qs!MpL+-Ycyuq`w@sjr7N^uU_qfIe z|NgLiH=WjrxV%qA$%U`tIt3)dOJWBKg^-NVH*0O0;!mlFF?lPAHLlYzt{dxNs|~F^ zvvjHl)SJg;47^40WJx1RNeyrXPQ_7{tZlX(@XrPddd ztZt||vuc_8Jalk2Npaw-k<~gHAJ4=P!+XTXxouzfjMArL7?(gZ))e!y*zpHlHPu}K z1boUfvW+*IKrITYaH>G=+sFFnrgycA;IG7rs7NS=m?$ z9tw$UV8cR{cxr$^VdltumF`)a;e@(on7OChvjs98E`^MdqCgE zjH$v4#nL=>-boccEK4OCP*(Vc{@6lBcISA>YpO32_m=aGKcU16QZFpF3P|!-TvR)t zs?j4ajBs?fz6!b*+66Wo^VPEpNnyR&m}QU&T8A?+@Uuz-j`_yArA?O^GC@Boy;79s zOEE_pw#9~g>PDq!Eo#<=QrO*|%MA{v?hLw@mEVisRcq6-*=N-ikjU&u5a9aAp z?22zCw;cyRUH1WA9Gie|$O{(>7Atb-(VCkii(AZD)u+}E9$Ff=-_vxhu;0_7k=NO- zp`i)&+T!?q6d|PiTwmZ-;is2wClK^5$~FvJvzTjmR$Rs@MAN4ndAx{hdMZ+&f9o}s zgKrjX>;jkB9^%O`#`_6FD~d7;9E(*Lp5}vyFN@-%-lwt__bs-ngl9GC8(sKzuwhA( zC**XLr}i7n4%>wBpz$887aHS?xZYm3=hbuY0 zR{F+U{w5U+af-94HL4KjELBD5rW&l}@(q0hHz!Yg$v@%Ibwq}4uv zqzDOQhK-0_7ZM1#C#ZR+f0oVCB|FMRY*fhNdhp@v>#u^RM;9RQ-*ko;zr-YnE=M+K32BAQt$0{8X#1}% z!Z!x|mwQUC>nt#9cDT}MIXNag7Jy2pYjBVrxY{+D-Vd(XK|0CoVe_1=m~{dS}=tBdKJ18`&`#&uE|kc@3hD}RB7T>d?@*0Q`~%8 z{af?JF=oRi)q~Hk8Y-0t^g8ELKbn3aJm=TIl;9Z}e9%tfmJ@4^(_m<@@2dI%v95p} zraO<&viDueX13g$CUGdmRx+P=HtnH?dJSeTEO(u z7^OLN`xCl430f;BJ6U16o7B${z=ot+VnFYD{8HHYH^2{frxGsXpRN>9Dz^5OSBs2!&1@9ce9 zoJyCzmNYRDv%kep`XSL+lP1aYIrVyl9$%H3dFo82+JmtG!6Ick1F3TL%Z>}DJQbhU z+U$K^ycaTa4T69yg*0B;x7JGOckdZ{g+w9o{=l2^w^nizeLEZ;IJxNOG%P7R;9;TB zKL<#_qIWf&dE23#6;X9^Z{3IWH`k6x?hTX8JPo*envbk#p0Eg(^r!9d=BJPwsoF+eDE&t z;fWZ`@8;_b(hf@z|w;$x9e~0I4&rjIpd$0M1vRP@);c#!zQcU+61<{pG z_ZnTs#~t|`*`koIJB#LASLOx%?in!G^7zTkeqg0={#xnp9n0o==*G!lvt*}CX)2ZU zH!16eQCj(VxsC8o43TkeYAQI|FT2ip8-@?E?ou0!zfq|h`(jX)Q#v{H>`OrdJmSl& zK(vQ@Z@!2IyKY0&Y>$B@3pMSsn=eu9ksTVD84wwqjf_}*Xcu?#%mM`0G-JMG zMqDgmJ0ov(*jsV$e){MgcJeRO)+P(5Ps<%RFvpZV((C`?U2DzVca^$!SL+`sq5AKBGc|@XImf3JA+m zy=uM1)8?#aIoGXPv=6_D@s2K6w0yCAk`I%c$+{;~J30R532?WUS4n2!q#dVMO{Vx4 zE-$x-nR`BSDmClYbUtv5ZmBlA>v}h!{fX);L6I;gl^#gNLe7~AWc8!@NS-?)RPHQo z!PKW*m+gBe-QF)wU_;D>Rvk{#g{w;+xjyL1M&Ok>WAH=^c8ho_+clXL$FPG=O|Kcr z3Vh_$ZJ}k_sG0x3ebi-x>mZ}*;XJ367(_64UCZk?A*ubTW$GJe40$%(Zpg8M`C_>6 zn8xER6mx|g%6yD_r2fn`; zR`gCi%6aMTmq$ylu6~#{Xr0cw(t8D@o_wntc7>V>n+5pYr7g;Ep|p&)g4v zZB#oOCdAy?XuQNQC9i2k_bqs=P9a+P>(|7#;+a$fK@;1fagm22j@P^eCf1g^K7l#z z>9a*QY5ihe&07trRK%XEd76FYN>XxhFe60yY97-W#b+{>#rpiCHz_MZK8rn=VoA?1 z|4j4?y+1Wz##Om4#1=+f5u!>_vSE^c17ch$sVYlP+bs8(WbdS$H@#GvB^3vVKOD5^kfxX3jw=&gF9Zz;8 zME8b{n}v(j+T>g=e0+521@n#0kn^0jJ=5+TE?;z??f1I3+)wM%L7(z5*}XTRKP3Ca zJA=V>d&$s<*6CE*nO;0kl4md)3P@gXExl)eSaKYG6a~G{=J#CjX-tNC@V!Li<6PP- z7863wAxo@6PmA6{9wr)S@@SZSHDKFdJ`MS1*%Y!TBdP(05=Ngsy8Fxg{Nl6GXFW;j zYK7Ccf2kRiLi0_+_s?l#6P1o@S4e!@~~08 zx$euEwN~!vOk>!HnvVcm_}OYD1A(g|8Yb6=t9qZ`#a{viD>i8O^PH|k+iQo9z3hvu z4ZsH-SvB!}c*15nK9v9C`taJblS6w6j2N{;3twl#OFgIVoH${`j`%dXwv_nrBLB5f zX?xqSn>zRe+#Tq{N?eevWec5IQ?X?e;SyZ*p7Xv(@j%r9RQ*^{({!`sYLq~({1g4L zpqgh^V=J*`uuqQ9sAW9x1eIGY@1~^(tY9$s9lo3PLp?n4D@PfVy&D%Ub3eRd``*sx zW6)&tqNLD=gq{63f%?Hp5f15(`-b|_a%f_h+k?~$ zP#{?+7KLw}M{Cw6{Jv6>+bkqtV}Kwwxi2n=3WWb?CLNc`A5X z%BjZd=I33mJsOn-31&uEXhl`w;-_)PhU(xG^w%US!azy;YG0m^uRGKQ8$Npm*2dJB zbJ+@4TWsM>lZs@9^|>Lsq&n4&g0n~#f>{!{0q z91+QG)JJw5nvjI?`%%XX&Bbsx^i?tXc6Hz-ZyCA2%_wq&tM_=w=ou!zT6XUFbS>8X z$qTGZb7US!d!7{=ep+OwfUxvI?TnHhO~LlE>uE~WS*2>(dc05N6gf&(xN{BZufPpc zL=RaRal>PS;iFH)4j{gL7G)Jts^)Bjb|+;m@1W`7P5B^iEygGLAT&zodiY?Ug%Mr& zK6BxyPG?Ue*Se%f5e)3IwMu)lvw7oQta2W%yqcQzZrtT8Hk0ct!vH(JMf8QMD*YRN zZ`V?~YRMUwLgTno{@i`l-2xuFh}~!RAp*E}W|*H=m!r!t-)T8d*>z5Slp!}v$$8Sb ze}_!bOfr=WjWbiND#dAT9kjE6TR4yP=+y!Ca@%*R8B)=t7{N0@^X-1|PyI)U{B<~y zC$DMlvRZ@GgBLjj9gurj-3Vn|J0IX>bY}%ZM<^*CEa)ZAOAYxZiDv6#o|@3l9sV*E z5j=3T{G#RML@M0#R;fZnG3;^sK!b7P;DDDpS2xv&(Z{HQ=P#gZJyE)ew#sM5P71Ko z5`r$Qq}mRvb7H`%sw7`nmG_IFdpBQP=nbBJ4=kdtU!Qu`9Oi8EfTAwfj$+DGc@S16 zcE;JJ&PGor-?+);2Dj~&+gQvT&l$h5l35&a|MgT_+OK~1QCCpJMQc{wW_o?|Fbd&Y z6bH3kJ`ZrU)!wK2Qt;LBKJwsH^$Ej;n$g4MkH7UU_AAQ3!*8YYlO3;7I6ZVQ6ObY* zIdAj3@Wz-9iDUc?K}$JBx8GfKShl!vhSKz9qCY&MU_~+hMkX|Oe`(mLEsNyTF`dG( z4RaJntZL;wnG^NS17-oM4+~|6_|b=Q;`3k%ybd<%IAymF+|MNhz9KSqYvhd^1!M5ow4OdTce`y^TWU7Q$Jw_vW ziJR9h{iVl^J)wC|HTerJvoG|cO{+vMtBq9j!}E&xny0~#=Z7k(SVcdt9PM#2(-ZPu zZrph)O?P9~HGpM_zOf0{bSN@rurgrh8COZW$%vRf@SX2<>R6^S z7}h&dP8J4z$=DkXjpFIvr`s~xa4-}LsZ5~rSXWF%9X&R9?B>(lN#>Df_I#8f;o138Lc(~i zQpQb-KGAh}$iSCtvaqup2B*S116vjZL`)f^?n3fvmwEPK1%2G;mead=L@8T|tri9f{e2so#xmAGJ!>KS^(&32Q)b4)7wLq+4x`-Nt0ZR} zVdHy_@#d`xxIdz`U&kO{N*?oUiAU&EhHua{lZv-`iy5C&4;|ShrJ(ZS=G_GWjq`Mk z`dS&n_3iO{vPSYuEYfFo?SVz-^DNz|*8vSrWPgR(C2e%3wJ~tE>rQ`COu3mbk}O}) z4%r_Mq`bJ|+cclSY$v13FNFvjrht^Q-a8vL#j4w+e3~5*M(uT1nc}U>k`}YqT~7)k zRVO_uAwnFX^17SBynEL5{iFLa2K}ECXI&9tWmJeT+a0A3#%U2@!<2}yGWrgh0{#r< z-K7sgm5(}63?nl%nQf%rgh$%mqj>si(TaK)DaaOfc3yFiqV1aqxNMo8lbRptoKExB zrHoSFoD~;(kt00KF+rFAHrFP3L40prmZ-MydEKq0K!;$zgozvIPseoLIsg89KSQS7Oy{>W zR93Y*FJP30mW-XOP~<<9JQ>V(AH%AqIytRt-f~vkeyr2yeU>Lpj7bj(j=iL|TlI+H z&a#I2UCG{DM)ud9*>!#Bnwbf|sBd{0VAE?eT4H$G918yg%^CAO1NF1LdOzXuf}EWc zUA13u6x3NDGp3#8#Ut*>oTdklk~Daef#Ua4dm%YKRVN=72UM-SIBzDDU{{jh#4Utb zK~xcJ6g}g+6ILxajFStWX5I8{YaZag+AomFe~j%MV`=_{kB?NJ6g@V2ROahIbB#uB z^fl_*^+L{QPdlIEx|+v_c@f7;WnSvXpvrAufO?J^n&DE7Pc=W9iDc0|4-;ZKQg}H!YR0=b%2LY%Oe4TH%4%h(rqpnJ zekW5MSBtCtv#8srWFyQRLNZ^R_HBhqOL$LZt*Na&ukvqP2ra2>saEdFKAV?l-Ecyo zWGq4S>Z|5#o2FYsDON!?ndjbDz*R)nx=Nogj7JZzIs3;Mk495F-Gk7dT6DVS9-5RM zTUK_SU}|>|HpWrD7ZnY3>~|G|mF^kMD$Y8^{!J>BBZ+IVc$_L%ZikY4#^n8dnzxI0Si0pT?a!54h~kmzOs1!GSr(Eg ziR4r0v(Z}QQ|M^TXWmjn;A3az&H#r(3Nm>!uR2=uxwr6*g2U9Op9)j=<@jkgPVrci;MKwEQY`7K<5furuwFRn?{Oy8rN!o(p!-ikjw$ zi0fS!Z1|SaK6r7s%Rohyj6GCQ+$44sr0GEsR(qsKeo8{+6A*{(SW1&kUX>I)c8YDc1w+?uGtWj^Gj!Zj%pMYJK@v16Z-li z2a~VRUbsAPJEGoI#nxBXXQI3Ot@N8<*cE)?nDS+Gh*sWt@JMv18GAsYK|`i7CUt3V zau*SC_x6UL%dIik?9B!`d1BjgO>6fwYmK?U+r`}Hvtim7LX)D4dn&YBrORh6>w2^v z-Z&KRkhnijPD_b5D!cR0Y*(o4-WOw!)EsO5ln>lNWFNI25SCkx7&FMV)xUZX(=wl9 z8`iwUhl+g^rjQkS#3Bg7@~M8@sJCk$gU+1Zr!~O|HZNqlvA`8syykPJjoHUbg82+* zd0(+9>_KZjv1MaWRCbFS#)_!5sV?(BdJuT(n8;^2qV(;Qc^MHx{tSjLEk{-pv6-WJ8(K}&8LK%3`zK28A7p!A-l3DO zz1}-Fx)QQ6RWe+1XX5${pU$ng>mza37aq44ubx_+oM`vC2gN5o!HV5_Sn>KHF(rqc zob*cHjb^f*c|0HRM*l3wB6N~^)QPT+agXj*=ByswtIt?TI@gifB~ z3*dfDF}`zC$zNpd4B*zE(I7Qb$V>HO!8McFI*F`=4*}HI79)UWWGY{sXc%*m)wmiKHmSDZq!Mux% zU6`5SWd=a{LnW&_)7VT~uvX4i1`hZnJn1u)p3}D!bm;w>88FP+;%<7ZJgd zJtop8u_+QU17UJ0_^V7MP4}0qZNOs-z{0$_>ctjxub09eJ$IY^klGLY+(veBqxXGI zDs(d8${QtNf^I|n-Biu)g#P4jI-j0k5C_n#iDlJPTDT13U5m{~%Ee|qr8`j{KiW2` zJgeT{v(Xr+nYK3*Km2V_!|Jfz`EQ?TUuC^7n46R7;2G@{m*;Jvw?3K-M~jYpJujT* zt$f+sv?sgjNc5z-)s$*kb%Y#^3Y~W8p|SG;BIX;N9-V!MGG3l1h#6cmoUGlG0nkh5 zHW=fwb+48yI0jOW3R3E~zTzl9wle#E&nW#dp;FKNQ%4HE!FHw6svyL6>1t3;!T0ED z=rKF9x7^y}F=c}C+>`AJ79xve-%Oo9yW9Ox3Ys^4k11CE!l(x~f!PONWzOvXY`D?H zO1ae}CRpd@tB0@hGx(io*9*Z%)O%9T@`-6yv^q*W5xLE2{>f1x@%r2L4LEo)iJ{Pl zI?%;xe|EQM)=m5AoyE;znUiwD9tsl>mUD}X0;SQpm_ygc^s(7xBF|PEA?Z0B#}8() zgPEhDoi&M!HZC_1R_;gPW(ih#EXPfUHD%8yT?#)KcCy&;XusWyjpHZ1yDFC&@^pQa zLND&)J&cay?vaYgRGXXL|CE#M4HM?0^64J)n+~Sd^T?OUPdb7k@@;C~JUdg~Y-*C% zmY2}|F$jx;5k4jlOmo#m>k0#trg9L_vX)<-pk+1{|uHH2b>bQBOaF zJ~+cKg0pU-%?g3>)jrRW#$J65YCPtvt_o9LZVv2Ou!L8izGF!X&8@IBxE<1B;?^!Z z-3))eS|~Bk%Jphq?c~tcgNY-}VT#pB6qTWN6h#?=6!sl=o^vI|4^91|%fYQv?=Q;-im|1HdmNvH@6YLKo2E%oD_fa`9o@;&bru#k z5X=&BBUoiuto@oj&2_bJ!Zh`-Iatg0J-K1~bXS~>&!prrL5M)wq^Cyz{`2ByQ`8im z!x7m}X@0yELDfH9e5IrB9cUnVrx|zyKw8Ceb$h%SeTJjA%^E`aD3I-J7OK5 z`ybDqm#cn8@sMkzBau;AeFcfw$xr=71kxDMq3|i_Dbv_PM>gx{Ts_P;yNyM&4jt3j zyRx>HInDBFF=*!C;U~|upU1D+7SJkQpVABhc4^Jb{fu?dE1`okrk3{em+?k!x7iDcyfKtl&63F z-5vwP2hLEFscbvZ*9HCPT!8Z7F>5Qwt1o!2S-c1@sJfS15XRMb&W>|`pqc3You;Kc z=2XiE(s-orCioslb$7Sd)kdD5+@(Z-(-NsV&#_VvKin(e$QE_&_32cZ9@d(5jpXx_ zBc*J@6t$nd7e4TaG$reIf4y?IgYV$I7O`ax;pN;a0ffTgL)T-+*^dk#n06VUG!emg z(5eTC6gyDK9q=g4Ks?}-N?1dm#GjuGL;RHNK%+NYXh^d3~ni3i7h2tRy0 z{m}Q*D!IOF{hekBmQ@#YQ~q|@ise}X<1wWo>gO@dY!z#n>oYuX()g(oOE<@>`UZJT-WU$0nE z+fE3vg`Le``krf+)su3~vics|Rl$fDqYkik=dL{~;n!YIB}T*an5JAB zdxd1bS{!x6KJBF5{U*ftDV4dkXZJef>VOfZSZR46XC=p9LsE0;-De@jxHICALs)ul zYFM5mKYHDFz^Q)M$1m}%`#79;+K(YF4Aci$dwKUH{7%(s)cji5XMH}sZgBjT()z&J zMD;hxM>da(;GUK=$4)#}d6#@h#QLGTe6_4;C+cyNOq#Xo=NG!tfi`zKpbtgPzMo0z z&tybAY(LQ=^`&0z;~5+3XO&Ok&1D75#YHDhcsaX!exjdI4>(<+DlNmmej~r&xF^!T zr7!JnzK%)dHtU4dzDZL2wX?Tq+<*PHWS_0P-NQ*s0j|(DuIhO9 z>Rl55qQjyRR2>+>1LcP4G!+d?JD{(Lc3tbSaYE6b@O*U$DcM5vYw7F0qBwt4a82K% zp_%+IXO!2xu3(;L=NBY}i|sS=?pe1{WemKyFRkwBs?L~sbeAWp|Im5M<7(;gPQtC? zHC;80BF*|1=1}x24WH?}K>h?ruG%$Np3Pa{Te{krfrRkvB*S^s;1E-x+oHjvZX!r^&lb$>wa)8Z zpC6w*Mle=5G45)+?k+Z^eb*|~{NON$Mz5$VZEM`p)Tm>dN-iVsZTrio^_S`>&W_)( ztlKf0ciMtFaK8bCb|S^W3`;G}`~eDan}HQY&ip%+n}r_=Vx|#2XMUvq5-AwnA+yVW z$i6Y=hUGd#hGh`5Wp2GthGiXXhUNM`-5YD@)A~!6xq=K`zLdv-vy55TTL=!p*ViDG zx^K<6jsro4OuL~MBQ|rshtx08`GRZIR^m1TG<0fu&Gh=_tT@jjRA_B)ymV85(bQKL7En_M1D3GKQ+sbl5Ng5E&q3)c1uF?2;*($moL z2moQq@Be%YlCc}`ac&(!QoEeiUy5h3vpp#?v8%?h!ch?kvzfbHP+?yPY17DCKGvot zf@uxZH{6-Xy(Zn%b~4KRWOSaPM9XL>-K9+GrM`=`0r<1x26rB}H>~N;oDq<;PV#Co zOThWO9~YhI)6XvS9dSdgIpItD3M%(EjldU`WDvN3RlAAn-R@%7oa_5qj%(+$v_ zUtPU6)@vWKKK6BTpP^e4Wnw*BLi;RLLi_1?UP_8#1)3YI$NBHaA75^)ANIT5-Co_(}K9p!7jWqnfG_o~Li=c*lQtxt4(6q8EU z#&Syvb)t7?%+EGW=2h4)D`|B{miKb+p4>Um!qM{4v~OzSW5K}2lce^LjSnj)zI|Jr zG4Gc$ethjCZ()0Omt3k~tXGnq8K_a}zbD=e>OL1Yokl;@G4MjF7dcW*Z=dr9 z(^PrAGc28_dqVm}`PK1h;wRpFNO{~))aMBIkM07NncZP!4);~^HSf@3Y(pvw6lK=p zZ`gMt4lH5OU-inDJC-=rpMje1V)wn{qc1jGMw~?<&ln=gVONaPy5VE${@ncGX^ zCeneYo%Rn0^~@B&XeJ9oPmbsP%3Y;V!kw$0XddYxjM$(pLb8{!!*2K6r|PGfOqx%J zGc^mJW@@H5m{BNIn%GByknCMiEKMxjZkS3F&z2YeVwegi88Qk{%oo?*vRqsnr!)!b z3%>wa;(Gg3|5(qrd-}&5#8-GIET_GxEh$kPf_Buh;wx{=xDrkU8ZzyEs z^92K{j1iK(zt|=((4N8zyis_21^I(oAIj3irN3>P-ad`^yS9mpU4Of6Dld*_d0deg zWKA?aA}S+)JK~mTDN$ zM_CD4s6XB39+ydedWGO?_aG%(j(ur;e8g^d@1oZWiyvxxXdi~jXMHfS&|+ixRPQO$ z+r@q+hfDHw4wn|iVaZ_PVM(lBUCVR+$f`$7>kfBrJmWu@=l3nH=Q`WYYp+%4_A_*5aLwKMxLi5Ad(aVDxd2gtT5Nl z?B3g9xsxv%MWM(!c8ZTLx^sJxhA&#zfbEwejU}AA!!nBoA^wu3!?I~dhb5dNt*r|a3rV-`1ktWXzG|%)!E6LZJs${OnC>w@IPxj&$fG5L z?zAUYCY|eIIY&~sj1NjOcEN?m-J%ZcoN@(1lcI|qj$e+neyH`%t6P9O&i04^EQ_w6 z*?$Fvb}cv`5+x|sT=YJ5bXkq~U?jGL7cvvWq>%dhRngbU9@d%y#=0=Hv@cx0AM+}2 zn0EbxV0G!&yO7*F28@oDvCdX0OjRmUK`Rp-uKPSZ`W$MJUi|N#oFBaUz81}UdNet9 zLYSAzTW?r(aP`H9)gzA*12xC=XQ>aNL!?P!+D~ z_bpM>TKeu8FSZTxecTd>J?;eO+O= zjTim>c#DMnmw2msBA(^bi^SZ5w_PUjwas_=(l5QA|DdU(69ats+BVnXe$6%Rz;$iC z+wWcK6>hRs7C(9z!Zv}vhp37EXegb%>I-B5nu6NdUEtQx7kkd`Q6FpTzyhkCxDRhmX5XXr z)^+gIP!%_02J?C;c5@!JgpvPRUg3LcW#BMQJYC$WL_US#A^qK%kWJ&vRLkS%Ekhx* zD`V$Vt5eMp#hsQ{0(df{%D6M6loGQ0cZ!=i?MTRWrWH41?tb*X`i<-hY#6XJc#jI* z`|6`Iwpgc0Ho2pf(A`^1?S5397ckoWC_{?Kzoq_q&3Q)j3E9q+3E2_Fo&C3NND*ap z9X!R&`lv0hyk&g9Ok1Jj7pgo6^Z(OJe&S|(^!M@zxL{8+y*(YdH|+lV(QF2$^CXeJ zr`s|XVK-CrBhgqgcIK^A>mm);Y09To#@fk4E1#6>A|F;TS`V!#%HW%x2I_}bArO`} zNnheD?f{L6T}!cf`x&(k=(N!rJc*?9NHK_F9x*GH-#a!7H?L~Gr>>Wf*M5RYJHyjKhhmjMj=XI}%b3c^?$3LdfL7C*{Q8_ai4jvlj@4&%z0vT3?SDWBT=^1t*a@)3_l#MEk_i z7O!OH^9)5AK%OrQRzYC`onT0Qxp{Gr5&Xgz@Ft3)V09&6GP!vuY!9noRwwh%Y-|r@ z|EEi0dk72@_`h6I-ZVmPo+8rBObJnLUL0(0-ZX`fXpa4#FoBsFSiCn^aE!G-BKVHa z-sr!UHf^aIW@Rxk);z_2U?Bkhdu1n@7xO6C^*4|84Kt+UjH;@xOd* z54Zo%a)S33^8aj+{)b)Y|CEz3fdmwdOIH$`@oWQNI|iw(x$&ucDJ?Fl&X6x9v~vZT zp3nFx^q!9xp*Kc%kBAqU{Z1`r_fWQ8nvvYLG>QA4v#m+Yu$8qDsk2zYAO4NsK0#^a zrjYh0T$5{~=+dd{uBQ=ftZW{x$vU34o@ZZF(PreT2#z;l&)&8EfM@o84 zq2wZpn_i2?uE6w7UV9;ATa-!rU}9~BLW>hQVrk0sFU)=F1XEW705NcLw`hU2&`m;k zzJb%EfHR#+6J~fQcJ-n=0!08{_Y%bti`$#!S6ohRi6;lTS=eQ%Q)*t&`WzcS7rps3 zGTI_+I-`}$T)z(5!n{R!Vp8;hcWXHhPeu?*xAo*kDF z6QE&WF%y|p4XZ5sR-+_cK->BUALuV_7#A%o9jaFicWx7t{nzco>2|DS_xHO)4t7P2eJ#J|ju(aHIwde!H#^n}P?3&1N#iz-p=2U%yEau+|YB^m< zGC4q)`t|PX?@!m9sl-Y^B4RcXxQ~2As4F*~ti7c7xFf_fWc+%Ls!Lelknw@S{<59N zKZH3h8g$?*Ox9=jcf^NAy<0-;OMNf8yr|5Xd?dPqk)SR@SHV&B9D);*lm)YHu33!82eEH7!~A6a>p* zU9mK9!sE6KtDB~s^hA-n+E4Ut#LFKYH`(FQT-&ig8=hS(_ATtvZW+vViorZ3LB_a{ zdmLHILWFk%NBEH=_@h-(;sm8X;L{N0>6@_cSK;kD&=O#6|J1xLBC3k+$x0d_=5iHe z&F5%g>{7CP{Bnw6naC#^hJ<6!tK+oZD}SqYt~EV$Qn@8YDY0*JwJ>DQC&u}hhjZ25 zHWxYx02?bL;Ueq+N4KcTP`OF4wLPB?uXOC0d-c`pgds*?77tf1@}1;^Pn2|MC81nW zkoL5H;Mc>zC7;uF9x7X7B6|1y2w{ZFNIvol3e`AZ^|vH!@KE~+pqYwDbh|ez(Lzp- zw9xxEOPyd7ZD5d1oIW8Pt33AC3>J3m5nL$IY(C-M(iR}5sF5R-fd@NA=uLwk18J!? zIX=p@>(!sP`H>)_{nP7VG3xteGs@Br{}49l$Wyqy0nCvq#`aU$< zzC!m{4e3mNtz7YyFo}(Q62qo(cimYex}C1qLZDnLX>mX1+)gC5qZv@AZ49J}il?!? zT{9Z`?(}}vSf^ZCs(FL1#P9V}<+I|}Mgh*ZAAu!e0TZ*!$7F%1LA`WBt3q?}_*SS@nt%!)@ zCe9MVVO25j|LB7lkJfRaEc`@QuE)72=gw~F?anoR_Y>fIQS{zsDJ(|XE8K&amU_(G2IJ$B`rjAy3m=vt`1X3-277$%-<+{= zad839hiOmWhO6I1-7kF~->;g6pI0yUjQ;l@QQLmEdoQopIRTHa4zY#=0Zt!_4l}n5 zFekw(G<5zhzCaU=n}eoj?y?(Ko`Uk$#zL}(eTKTc4gohnb^s_TJPo%*16RRRu6CbOPLt=QhoFrNg4TXGLN zN#ruz7n{G0SALqoyP0}{bw{E7i`7bH-Q>G|Ku0fs?d{Ac8*my_dUqiVG zs)-ZRwAGQYGT3+w!MVvbzeZ(c#WW@9T)(*QE7n-6BX^9*1cKPl=v8xE^0um}aEXEI zf=U(xzVAhf!v;Tzrjza3)$oO%{cHUb7NBwgvNHTFWAzGg4bl z%o+g7f48dVzY2e1uDnRvHFSG`U#imcFl2PoK(QBLYxtMRTSoc;jEQO~n&uBZg;!1o zU!QXf8~rGifWj5#(YRbf05niPYQ96>-tl4R|C+=B0Fr)7i0AGGzeguS(tVYEc^c}~km zS~=4DV2s+3IbLG!94a}=hh$oP_Z?S|4Vea4};i}iS+ z&wM@lz-*?m;==kYc~i5Pc}x$RgNkb6ltZ!-nKm(!P7TwCxd^v?p9bR|W**BE@+s-3 zpR)DO&`kDa3E3;PS@o8WtS(E?rda$b(agQ4B!_6oQ%oD)c+k6bw%9>KSFl90VW!Gjt8`pXh2#$2r@tjuIY z-&^bRvo_WZ29?0hIrcWNtrEans=&)>YwoQ&ipAU}1D}4@3r}d%8$3ikqGk+4u#nCr zDQe&Ric1&#(8Q4h{XW$$JANCN-9(f?mXwjr`J}qh%(cDBu>gR_!Lf} z4`(z=@4!3POVLu+E=pgzZ*Z=v(s3*72Sn4I7m>-L#Kh&$^rt0pA1D5{!|wBS5^hZh zjbLQ~CID@>ly5U&q)jV^mf*_7xYKOdk0GhbD(5uI|CPftTE)ajg_@S)**6mj!(k=HP^1IdBg8|4>kZq4w-we+5YV$(e^Am4D zqRBTMRG7^Ci7sr~Hrd2LQ;i#Ei{m1h>x6!DM@_`*pbxxhbl$&;z^4L1vCHIttMvkJ zdC~r)4XvnRSQ{3J|2zz;B;In^+e+@VJAsf??C5Ui(V<9x8axyN9H(`2Yw7X+xvkYO zae4%ikf8hnH{(p~gfkwNbO8h^t6wU+MrUWp%Pa=6h1(58ZS^FDst$~0IQ&2j!>Q>8 z2(Uz`S9qo$2(yK39aEPZ>F-VEJ4|qTAHBLAXbTK0Sd{uVjnk zS}Pm6@odTBQukmny_wH*ArXmL!RPb8m+T6DN0-)hRapct$?!Bl zcT@>u`mWbag|SdU*O|zuYxc2~kHH}JJ?GU_t%_B&edc#|SsYX0P}>YVJ$nmI%0H&= zcUDdcwa>uLF{BBIyWpwH2;Zh;T+v-z%R8eF@sL=kj1dcT8 zr3b{$=A^8qp-~x1s?o2SFkiGZ>qV+sOv3O7%Htcx&U#tB`l^k-zk#Wg0qhwz#PDij zdhtLhNZJgxUMZc$fRv+U5+xPws9za+BcQxMQSG9QTo`TQ;{n$TcJn<9!%$OT6*W9yI zrW2uD$UtN$9wo{@$WT0}Cqi(pzrXj13!;TO{G1TZdn28TKaD=4&$mEtb5~($OT4r? zx2@%7ca2Vy>Z@ZAAY9>T$?3Q_zjn7V@z|76w$~5wnF$UTl{R9t@EAOS(ca+;ITWvE z(WUinzc&)d=SUXyj%H+r1_Q7reTf5Ngt3HhO5SO>3sL!kJc{c?rjzz@K^X zR+Ry5Ox+n?8v?0Eup=8j%tiR#IlX~@@fpt9wldVJvGlg}e=|417ZgW5IV8$D#nrYM z{Vj$W(o_;Q#Kv2gzFAXv2b{UWB(Xxuep;^nOn{bGZ#fc_wt}>uN>Va)D*~i84&Gn3WX z>I%z>!?@$1>BPTxzT{3@k%4O??=;)AMHJG*s_$xM;x>NB%)J)#<`YjYP~=^O(sQ5ZLvuF*!jz1NT^4SBZBFfM4Q#FeYC ztxeTXC6^x3tdD_*UCoOzEun=imE2zDt{j?sBYdR&2+DOCGxoSYE zld3~zY3LG5AqvaT;jq1`AL*?D+6FU#=wo||L8XTrQ3ET&hH>Y1nuQuyp!3}j5W~qc ze-)oh0#);!U>?9%$?DyY@H$F_D0kDc&=3KzOw8tfAc0n2r6~ftMaP28NZ713FF#pz zhH2UmOMsnm{?V+#FaiFS<9Hfd7QYr*>%UKde0*W;%Yai=93fNqJP%=69NReR=PGZi z7Zs&xY~g`_b)=^=Ijl3OmQUGi_Ilu$#RzTc>pYdc62Aw7Qtrkz|8Vs8bv@}Oc)JK< zxU5On%QBd#+Yr>hxb%`rw_mW~pi{$(w040ESO{DH-Te00rf7TkQY#Qo1I)XjH;!N# zlqV<2&;W4sY2?}bu+^-o)M2v#Zl{K>)4T05kNwOJ&%W^fr{b_>qZ?SubYl-g_@moKa$ z$K=aWWLAC9zC^<%1vN+SCOf<5(mh6mK{qVHJ5{KJk5+Q7t+i37@3A0eDzy(~Nb(#O zJ6C8nO(#eFJ#{L^9*ZY9+xX|mZYZde(b)JjfRkmB7=F9Kt|zTeqL`zu>e?O36Rl8| zCh}7q8lD!KhxVvt7qN=Va%B%a46xtA(Rp0CJ0`3eq)rW&v0L30Naz|y3rH~GSCF^9 zJ&l&Z94#&DV5~712D|XSzQ}M|JX#T^s?bI{UJ=%U#e(K~Qs8UGppU5bfj^Zqp8 zdm6-p5eljl#(^sMjRW0FB`Cx+2KHYJC&B&42;+Z(MHo1V%d6XW=mHH3g@%NU2CfP_ zz1X*OYAxyV*xUqAiw!PnO-%W7P6L!c6VB%mz z)RajS<0GZssNaPIQe0BWub{=P`09S7kHz43n_AfI`a-rttmA+vJ}DilmqBOn#A_PW zQSZ|k?S7eDBd|^pj6dV2b#bUYG`B$hgBT1Lh!L7Qo({ji@ zx@*xpR58#LqBvMf-5>%q23r%b&%mQ5$*=&h9{{k=F?*XUXa@^@YIYVEGb-Z*$E}>^ zjjd9dOa+AwVx?SMxf_hy)BAiUVAdll*oWJ=W94S(LtomsSAb=fRBnO|srR6ggO*G2p(%Rw>4a+gC&-AROapDNz;3w zv&wH{b#2CYRnctP{LES0|UYMbq)DKMes7 zPKtjgL)d~6fUWa31O3MxLt)L2iu!l7njatvwX6g~>7nPnU(!v83gTGk1de7ugzuG! zW?u}}y9wn^ta{lXh}Q@1;)1>uonWReFvCdE-H3^ji=npj4y&$0Pzt)FW#ZukUyK5i zuLrw3&Gn(PFkiAL4W~xbPOIGF4v#hg7%Y3sa~rZ^j1t;2Qup zi_@;B!muyhJ$U@N!$k%NV?Q^xBRbmAQXl{FunLA>0anPy!In)=vSKXqivoW7KC>^W z(nH3|_2|dM_}~~cJa?9$wFLK9gNJNRu1Na_2)W=FpFAPiXYh8FjLV~3diGjJMyjY^ z@{;guJ{WyOyl*$8=7Hd+z~)lJnTz44z!c~`a^Yi3;1#yZg(-nwDM1QcGw_yCc_b$x ztWZk6m5{kFI2c61d!|ahbU$)mv`~a@nW+S@-jFef5I>+Wi0;8Kh$?oL^?iTv?DADF zA+^daP;Pnnu6&%hs>?ybzg!@fN62vHK8gg|CF{zqC6QKN&57H;5( z|5s_IIe{>FVO?x=D~riTp5t3wH81IF}OHQTabqxs+1~|H(Un5eG9Rqq6*o2M_u2 z+YF32*!WL;;IR`Nt{vF_BxX=%&|$sz*ucH_{?i61Xm}Cbawm?9ROImAE-o4)cjjFB zt@2MNbGMaP(dJpvy@l|CweX*lulmFDgplv4lKB=Ys| zg=aie_c|WHgh{Kd{_8+E?zFg&paZ+T7>u&rX4wt!|vw_j*)066 zKdnH+-OJr@4X?lcGeF){QU35L3@pW4pwZ`J)X&O# zme!DID{X84hL_Dasev(jnTzDyfiA|)HiME54Q(wnVlZp?5jlAKG8naujQDi(V?^wP zG@lAtI@Efyl)ak9e0o7o_z4wINa=H%->9ads$eJnFUBa@rusVz%E0}vECiAQ*k;6E z_2A~VHr%f>gGhr8W)X+6p2(P^Aeg1O;9}%;dU(1sf^^$LP0J`^qK-i?!oMu~o0Ha_ z{ql#!)F_h{^917A1cX&)Gym=ik}2<~cIk*i6D2cj8L3)%+xGWT>sO_SDwQ#r`B>J^ zPgL)_d*Tebd3`*B6y_Tc)KU`V#w%^i1SMK>x6$)3gG60rwed2rwC5hq<+*2|6Ku`) zv;6i<2-iyH;TFi-GLvX1BsXpsBD;p2bpp4x6tZf>>;|MLftF;y5&8HIndml|XgqEN z%w66KM&+%lCX$NH99-Cg8Q2i3xam4zhP$6qawdw4iupfN&J29$Q-{QpS*TN3~`Pe@c?*^B5xN@RLOhOXHB<`Y`JQDu{K>Ey!y~Y=RF3B-que{M(7+RdAZ`eDxg$%mTjtCSl2LZ7r2N!rRzKik-NDopV+S7?+sCzL>~J*xp26Sczxe^-i- z`VR&CsiEw!Rw_~Yka8ntA>uX@hb4TcXu65*=@&{0+b8-N#QBYCGxa)EIh*YeAz(AX z0wEcNcEBKDGaZR2WVLKEEDVeSMbD4`x@Gp`|wV|P#p`xR9 zbo(9C{>bK+%6`@y6zv{^X=%}8dgj|3BFTv>E=37B8X}2Oy$J4(G8n8g=O0L1KK!^D z5IC}#bvTjx@tAJH^0^v>AdP0TR`oe)k0z-UD2IxiM=VoYkp>YYYw~_DYw~!FpJ4GCEN~pT zFC1Ty+a8NqMEIR-3Vk#08}`-Xt_xA^MvzoC z{RsOcJR@x792Rw2Rhk|)lq8Ve@ub{sr*;q5watbkLPf#uR&*?}QV!mGP{pep87-Rz z)ck;LZGS$GlucT$6oe}yooFnKm(w$c#J{**j4)aEc@z>)EZ2b{^7>O^`rpNvaRL5s4-Ufx+uxtf)2Y*4)d5|$ zA3zTOY)=0ve8KUf>qAgYj-Pu@3^^-s!=8S<`eFU|${QHey8As$P}O4!KO9GTMWAnB zECG0uiy`J%x3`PmY>Q`IRL1>rF<(TaF<2Oa>Yt|M`c1kJX}Smo z%#imn@aLhY*~)qCo>!&06E1|<&x3w?IB5p(QeiZr)5^1!NcDr70v^fyNIkD~eyx6K zwCCC=)K8!rUXfSCR#C^6&Q_L;8(P>`s5*bXoe8k|e!i8jjsv2%oIAC{+R}7hSPi+$ z%CGQC%|ggcw5d$iCRh@w=4aKN$fTxH7zO*44j5{#OFQx>>XZU@S(@A(Y*T;y?0i@% z<2;0r?o$zrqu?%Horg->^kpY=U`F{$L0E>g9yiW9jlj0k2qo=kEaCfVVpd@hB z#ddyV8FKmh)0+r-bfL}pTz}2y*RdUum4?J*LWAD5(-k9X=S>3wT@(#59-_8GmV=-9 z@`s%Rxbaso8lD6Faoe3-wMDZwF5&?lTSZcEZJG01doZr*@@^71Xc8gSySnEn5mSrt z*mgxUq04L4#iph6ZkrI#u@>pOLm~35zkW$@vfpKvi-L~yg3`)0;g{NA&0Tny#bx8E zKwX)aTYl%O6_E#_r*?$3<&3grp4F0;^XT%!)X!gj?cTbF&skcVE4|u7*U>A9DfM>F zi**m!xq#Fx!UBirP9q*VfV*yoV&+wDrnK^Nf=Obi2)V|eMxX9>vEB=&0UEy*j?-Tv zFlhJKHnw5gD}R|e_T?s(CYi*<-l8Xj&|P>T=L420uEHioxwqOK%>joa5bn1tD}x$N zLpxIYHDQ`iG|QVs126_dMSI+g>1}iPti&bK`QPR|dv*%q?_xmYjwmWwz+$0yW8H1c z%vARHN|%m0PQ@CFTm(4^Z3fQZn6F)tJ~kgaKSMjdsU}f{@^qaTXK=r`O!C1Is`Mm? z$7!wlJhC8@dWx)dR`5=Q*nmbAebIQi^7Ub_W3?@>z+`P^31}oe<-&Jvq;e_#f}RSc z{8@M@xWHtqqGIM-RwDf?<105qF4R(A?jaf@=;TpGF`9{+i-`*b8P!|`4fr&-JxbRA zQ5nG{kF7Hyh)0 znwmM}WX$>gdJV|!=KtpP)=5r^rkLK4Y@0;+G<#f%yr-RFx!6di0BlEq{^NV+X$kN$Rm%a;h-(q0EK!7JJ-O}i+R$d^Rg-pJ%Y*A2%gS$#6legg#RvNuWV`r;sc?kE znuHh5Sw_2TKwa5D<13!Lh*njAIIa+W9b{?Fo{O=$@;b+}8c^lG5b(YqbK15HMA^Tu zgFH#**OBKUq2abEh9z3GI-DJ%+`u4W+dffQrA4fzomx1@#8x9$C-hbrv~B1Mk)!E(hGc=3oI_koOGbmUkKGj*0Df}4 z8QjJ0RKF?t&P;TJnMH1AgWI*QA-YYzx=!7X;(gMlDm-l2=6PMxR_hB}ep_!sD*)A# zia{&CH^m^^e2Iz*2gtd>y#!yl4C#PjG*(#wchZTY!M^xdz;eJI`+vt+ACl!t$dp}$ zdGKFKg0Mq5(Cjk&=a1=gVsgapJa~=$7u)>jxVx$6DMx1N);MTGINrOXK+?(RcG2un zV;wcKwT%)*>QyG61ow6BrW5QD4GW4=kw*heYsm^}c#56=6`$JZo8p%X2K@0Z9^CmB z42xk3nji-S;c`v23K^|wZVj^~>y%@$i3qHt(T&KkLX18PL6-$Bx-AioOoD#NxuZB#I8Ek7tGRY3~>AV)kF?%FcCu z{P;;lRnuW)f?~1;?wMzWN*Y9*dafczp2A+#O0j8HBye5gS_Qgmh;&nT3-5fygzjcF zBQ}>is)f^z5~vIfm{y!CHq}V{vIhC>Hnx%V>zf6mc@tueyh-xDUCMM8#>yqZs?4-5 zeU%qXc)Y46)+$8hgsQ5Bxa*$=-DwU}HuGC{);8|o`{f!h!zVs;rD(rW0#q`ga`->L z9_~t-OB@XN&&BQOQy^HY#8;!q7s;-7gsz*RF@dWXiT7oKAIMqUPt230>% z(|7t}I++1d4C#H>il~H1%-(m}`-+NTW%N+XWZKoHlE-)O*=%*uv$UUpL=LQK?6MdL zbb3pQ3Ugkupov;3R+1wAD_ZOdkGv5dniR1i-29;v^G^Fm@meO%=Ds_PU)rrKGIf0g zL)!fHUu(*J_XHr2@@Dq;)eEW>M-BJX z_tLE!bG|ER&u> z`P5-3r2&L2Go|}UuWgWM?J7JJO;)}L3`luB5q~7Eppi(W2!9JVco{h-;z+`Q40q3Q zZXuF}7X$h1z0UMTqOC#!&*U)q_?@=Kw{%)`Q$GZ4dkgt;brN_uMW zZss!y!zIyH)rbre+fxGh93_OLI;Xi*SImCzd6j2mFc*M9ugf!~(58a2Ox$)Jr=L)3 z(@`q;&&I9a`x+mz*CKgDDP_c(VM)mh5uh~Ec_KxvuQVFmL_9b2-u_4QkDY^_D4coZ z56zx+U&{Pj;qbYmyKm1jgiN(mQ}wL&p40Dui%&!sg629-b&*A>x|ujxdOHFh z4wpACOzAO8yPhU4H;|^EKW|0p!Q4J?R!g*D*LXgSk-u-lL5x!ZtpUF7?(W`ipN3n$ zkGp4&aUU1+&-t4hJKX|V{?i8E|D+EtE~0Ls+>OtjBp4p#I9P$+V*Hkb9>CmpVA}JQ z{mH+A?#W-q6^gBqdg1>3oM;gG$VGJ(s5a(eM-k7(w#>eU-Got;1M9P%H;k5iTv&=Y zcF)IuwR1+Lv8_3r`e1QBWx`OHEVO<-K=LRX%BqaFRh9m_O^x;edSr#h8q-|QpBlk> zzT5vas38o%ZY`zhgJzNYeL+BpB#KM8vlvvn{*QHLSQ}~tMm)JV=E#JdGJw5%f73i` zMlW7o*V;gGhStDdlv8w(MI^uy!((DiH*VN3#kux~c z!15lJwW>wfoY!Kc=hE(ob=p5$h)@A5Q7m6Tv07J%~gE!180EJhZZ(V5u| z8hmp2fP9sTJ-v`vtKrVlZ>CfM(vf+eu(`)&`&ZkqBudswn4yr-^7x40@WHbGv!&>X z967O4Ay@|EVtiBON3l`=MSv8j^5S~r$X|#^z%4rSHOLjbi5i#IewTpq30|lqK)@Lb zn27&Nd=NilcjkTxfu0SKa;p#jNH&w%t3F)j`+GE_W`alKB2`_(>^BZ;-QUp^)brTU z4>M7+Fu99eOW_PQ%JM`EQwcn6^-A1p`#}dBFKFcl{HP$>{+m9h8g1um(YWfh^0q`w zMu`}b&&8k+FsMFRTTg((7Q7KBwvyI9u*1B$lAAJZyjjICZ-H|vThwtfv}&S`jPds& z78wHA=|*eZ)Z|83h~z0mPV2snQitEo(Ho%>iyqpy-!*yZOCsia{unO}Hp9F*w&ea3 zxw3B2>QLyd_#*;?RKFU1COq~!Q9Lqi7pn(IXeKYNT^B}2T~qnCr22#1mP2o5_hqd*ZyUZr zsvsOI)b6)|jBJnDN=_R(q^3BGCUGgmV|!z%nNxujEnAJ9y|kAoLTefAN_TDNKDtv* z)%CZ_(evnpuQvau{xkXqu{+~&q`(C!Ed}|L*JlD|wrXyMG~&cA}1 zDtwMmrs+!Vs?GUZv?z3oey;1Wrhf*h3(dw5zBe4)q77NO>j} zt|ivve3ByNameXj^oF!kzA?}70VD<2J<d22_XE4v#hwyQG0Pp`rxgO54KCvoUt9o$+!0Ev1jW%?+&6NRN&{wC%$ z8rP+}8i4&y%;a8m@9SbQ&9&jzZ^M_7dKHMWxMeUn``+SaPFCP(lpgzgLMe2{F%XhP z_xJoW$0WWi#Vh4#{3p5GrbDBo(p|n&-8%q$mr&&Azv4bEj zW%pj7Dx4g?3a2wRSA-F$2gViFK`2ptubPN`f!_4O23a?J4(LjXFU+#}l)U#57zJ@1 zPC=9^VEuL>iY+5I6I673Au>;OKa|IwK@)=3i9fvoe0na;QtT93J+ zU{PFJyLadkPn;S#u128I@M=GAJ?#zn8V>ka516RSjQxpuaJP3kxLw`t1K+F6UGZjQ zhuM<}Y7yOjgRco)4bT@yR`YjZc9hsUmDEWG`0acG$JJOO(d3Mf%T4Gqgt9K=%e_pO z&2`#B1gBn-@(Or@1KAOT6-R>T&HG1!dVhM|4-pq>4kA#GorXBZjpU6azhAFcGwI{J4zD?Q+e86e`U}hTE$O9auA`dA2mRx;g+>~Cm+dHoKX|w$ zl{$CuEs5UNbe5wjy!&k~gR!w2T3QR_;WcQ>UxPIHFrB5C$i@4?3h2p9j&GEtGXFq? zqO+B)2$-Gd+zpLc%8t=8#Wg|9!+>iQmUgByu}OUIy={&19@pckf3hq_#9qRb8zN6! z8p>tdG#uH{sIhQ%4rPqyBD#?0>3yHkZ!WY;mzD8wJFSvt zc|ahGw<|7d(&BB;F_Phj78KPj=tBOe1gx~;a`pB4{0KB$W#O4uwp;&!|J#;Hs82%=kwL_AC#d>E}F z;*x5)qmJJT-|=G%{1bCYW#+MW$p=)#;nZWS&DP2pzJ`L0S_qy;Ey7hm;|4%Kw3uOT zl888tLTj<$xjl_AP=bIrx_V67pjmPl2griKH{+|-g1B;9NutGBNw)`OLO-?tz{dGl zf%1Yl8wc0_gqTeQac3t5Z$3_gnnv}mnQ{0Z>9gKD&PJZEc{y#cgA^9AS19x zz^^H>JQ=mp1_8;-Sg9z*;DaV8#X#l5#`A|^GbqZXE<0yxnGJ^-14VL_^S-^7`>`SQ z<(r2zz-UvFwg<;_#MqM>^tCJZ)_TD?pZtwu@$r}9L#~W*-j^uj$mMD?`P6@yE+-YE ze)1_$T0ok8*le3~xWtkTP`%AQcpM?ktm@Z=D=@FawYxOVYHC(zeopdTYyVGYoW^^| z`}iCQ+jri6!d?A{>Q?vKLDL`%$^s)Tq&L<&;wDk}O=65zV{K%Jy)#tC3ZTpc^VkUd zN+)@9aM>2kS^xC1D}vzP-xg4uvXKF5v%3kPe4c$Sj{gZ)+0~-LZmti)&3%b)jYjEJ zavA98(jSLY@2u!AqaN*rW<2=Vrayt$>Se=$`_(N@j;+Y4X1#<{)_Yf} zZCvVDm*Fr5dVmf(6zCVx5Z>Wl;k5hHUB&Uw0I)T_Viv}!qyyS!X*G)d$Vk=);h2Rf z-Z_0Fv|4gYGG<)_Pb8Y&ea7W&GfEze-SS&(r=k1C&VR<#Zdq&Z`F9P;jqWo0Mn69@ zD%{g=v2q><`bNg>D)O~3I^}bl1)HKD7wW) zq?r$9TJ&a`b!J|`lEF(+txueuC9lm}RzkST)q!B!+1#H(S`0oeV zDpY7z=C7YO%-d3?Y-IZ&KF$g7c*^fL=Yz0ynzLDX)(|oE$XCWU#1VbVn>QBf`HDuOZ#-F#k^6XzDaBJfGyvz3A8u!Nob=Tr86cC9N7zF3mjRq zRSiGY9~qW+mrT>`$tlx$g8t8{%0Ka|>sVivGX$^u%(!jdO_Dt(is^WW42CenMR9#7 zXd`eHWNT=PYxbp)#{*1mDf*dT8L75De1n8gz3^fsgM_SW)NdJz4rXraZJepMbY&B@$o^1HJjlcrh#-_5V-0G z&bt<3HS@J8~Dx|5e2htnA9L1@$ndBjJ6UN{s8t0?*&En+Rl}5@sDU+G2H%&NQQzL5lP{@0@= z03+}=m&Gv$*2?MZ2y=Ovpb!qEq*qz)s2HoO;f&bpx|yMLk+=V#0H_<5fJwO=F&`te z@W}71>6|jQ?Oi0C+X3>C22c=<<_Q0Noo-o3L_r!^e6B6M4|#p9;66N}clLtD!icU?Gh1$}Zt^Vg$U{!wKene$b!@)kB5b{SE!>j%+=`46L>FP)_b6)QI(q#5}a{rvv5Ut&8^9 zi;H`03@>@ScZVDk3l+M2K(>r{TyAi8?*9bOPHpu;g3lrK9hK_;4I}r@DTu+B5eiplh0Sdb{-+=k(@ zS6EIg;-4Ip3m%U&{n5D_r&61Ab+XACP8<5P;waACAf?UC=!ew1F}tuzg{v_6A^`R( zMLOTIY1q1x3dL12z7r1zZ$+AF!ZOsHzr%~C6;W?4Nfboq*v37sMhi7Z3p`d!hRr12 zZP3T`DN2rCMwXwG_uD6EHOK$s;vt;-W0+{Fi|hU1>vnZ@_4M>P!0Uc7ykc4K4y3pp zEDPu+z1L&^WOH=|_>+T9gV1pe?? zg;YDlZmpKS>aAbN+!y$axloNV-B>TvDCxddaN8hYgZ)OTj@WPnxKfhJKw(R7BsUe< zu7IG4f2o-&TN*Yts4DM0{jeMWVYa*$SLaf(029)KC4wS3pGhMU>69uhh!BTl)<+Gm zl_%?r3R7k_Cowuq$N`6JPBU_Zd?_QhV6qG{{^R4R8pgHxhy$77a~EOu!k&!~XH?Rl z^_Csuhy7O8zog*fI=rMyNl|C9_UdF?7qY8=pxwin5@J~M!&N*C{eEeRg;P|-Eh3GD zaPo%+B$wh_!$vVxph$RKs>OV*&DulcC)sRW#1X|Dq9R!P>&dXz9g*6#nuYqKVH<{9 z<=yML1e(vcuaT+?o85|Rqvmq zHdU#dFK~WWH`0_WUmTbJ?wo(Fex-T)?jVpfjcK%|!8_F<9R>-AW^)J-NKlqKNCs2| zS$99)DQwSzgzb;^wEd*1$y@|3pm2Bz8^hCR7gcs#*txP-ny%^Y%k8Z8<*$qV5-MTp z>Z`-{9;0g4tySltNFsdH7e*jJHB!aYCUvaDETW(NXMl+%~v8iexe6Zfn( zGf*9VwvwhfNbO6_>XysW*`VzA zVkbn(JjVufsyNqc0gCCRJj;vfV2(-51T{#K1qH34NRg4DcJWqueGuu&ntZg_nnvNc z?}ZJbV&WcGsu|fGNSfG9!deTT4>04wyL8 zAiZ~X(x;9ee4O3;5^KlBPd?ctCPJ3goLKaOX=Gz*AhZgRoX>_Ry?QhW@g^I6>b67l zI_t^kB=l-vOJ`Da6kLa;zJpcod$18weAY(bE04~2{0LqN z^l-i-#Oh140BY|!(*WIQIqvZ9| zCx^ueGAgn7?fADq%Hp(_qM%+KkgYtTW_}-QR2uC&EclHaCRv!rU#vY+QS8DDOc5isxU9v3 zY3|db7@E-xmSa;%b#7dhAckntOTPAuML4zXDgt=$5G#z>X%gL>*_Amwg6n3flenXuAA{q@#hj23qYVj}fdF%AYRkD4t1`^^xD{AG=;l-=(2 zHfBn9^_Oul{`DG7F)dN5p*Sw3s!e$N*o z@J_`Bku_{TQ$J>ke-QB_HkfP@n@$S*41HGGsif}~6e`vE5x1lsKI0$sV=#f=<2a4+ zgu9!QRXG{up`2elQx)_4%puwaiQ?MC4)9ZLfUU(RFEf?LUnj|i0iFM?*72 z#<8zTYX4!iTpZnWpJAr{0@$I<7%jM)Si=soeJ zpAGtdlMzbc9dPQ{qWWxR7+Fh~U2)~)qn!-zx${oLsZxVS08_FCzXJ~O%}i1`CZiEUy-P-00Xkv>-~(qeZ63tC;1 zWZ1t8&rVP)xa*Y(Mrd;OZ%-RJI#BpnNaoETw)upF;9ZMh$YC$e27VT$iywKk^sGvj zOJ=u2DTA3IVrLU$urYJ5qR#Oh(r)t@xj1`z58Z;nEuJOzDYJ?qpJG=jhFsx?36S@I zXabKNY646qkl!0HFcacJ`wnCk>>*fBzGJ(oG;avw#la1pc)nDD0yW5sRH}kQqS}AO zG5}w-Kfr=>KY1<_0bM?_m)ov9M1NhIOnCl3{R4bFu2oSJK# zin|r3>K3Q!Hk$wc-VlFg$m03bHa6`4SB&?KhAH`?pZDR+|4!lj{|%-GS(G)^hecd> zjtU{#_nv4^l4!S%*lUDg#Qh3ygHSrC&06@sXHFAhOnq;FmiUBL!p;QgIzuQVYlz3s zopitQ80%lUa7h({S=pJ3U2b371~VSF}V&HdNDMmP#r)qSsIB!qcVTdv6=@ zp}37r!f2!thplQPS$+jq0Df61I+`~UxyV(xPr;$?PjJ%waomH^P~O4uVfm0vqH~A8OgWmW(`*A+QDMi$7 zsEsZ0l=TKScWA(E(ks;Asfn>K$09E#JFu@~Q`YXVYgvOp!g+P$P|l6mq27yQ9*OL6 z9^%W}h($39qDwPH2R4#vIQyCa9$aJ3lqfUsQjK7r_A5~AaS%%lz{|4KP*vQNNJDGy zNi_fxrl8mU{;%bKB+<2dSR~{TtW(w5Xx}5x-3nW?z}}| z)%DvC$WrLdL_sA29b7S2&qS_A5_^w&$yuwA2>It4L|S=STbA`c^%)IkZc@?+^N8w>f4} zTa8v8afnHJ%G`j`=Oa(-5R#hmb8dIvrfO?8VZBG}JnbGjcd=(Dl^j7e6VOs=>?ADx zmlt^Gbst)Q)mve_*Mt3REOeb4g?fC1?;5%lf8Z=Ke&8CXz^knyLyXlw20FUZw_Dck z_aLV8Qi7vg5OMrsJjOCOgOI%f9%DHR(sm(4OJcA{S*=mj!&3dhsbNAFkcO{i+~D!$ z#0k47XB$VYiSM?UjoO4c?qRp=*|T+dZkLu{4H`k4Gj1XPM#J5?OUg zub{4DJl|FrN>(BK@3}Mjk%8-IVb$Cq0SHJzxlXr5x~ljlv_M0)7ibG(ZA3Z6p#$01 zdNwYewJ9^|OEU^K>)Rm1H0Z{kKAz0`L99b}uNzj|^I8u(IVJja==LCey@RpYC03?% zo8;e<6@fI>tEd_8$_bD#E14pAd9f!8av8T>4C<}W;dy>MrX88x!}<5SA_2GFz%c|l zj>+Z6Dfv8u2Gt-5d`_LV4*GvJ$dSBmK&Dof&mSoU|D6(1jy~U&KXks+x0SWS3=MjS zW}DEYrX6FAiJ{e!$!tH0MZqFx0l>H^Ha76x+{yQMd%*J@s%)c&wQWAMPonNZi6i>Z zAR7pYrLD&f*1+G3u#+uC%T^brsRc28X{gj5wukS-b26p$EBXr}PL8tl9Tso zZo-G-P-OG%z7QsVL_Q*t7dkIfO=0YDOCc`E&_%B04FDp3g)3(pUvYO?ExcgqQ@$!T zi_KqdnGwXF;1zB0%J%vG!A=i~j$p;6qGCg|7C0pMlGR9vnhZ3#+o=sgm7&-cEdNG9 zu407tC&X?5oO)aBZvZy=bS_bzaah^qHUx(fPN)7{-mn_|h@06sM5Z-Su3J+;-ZxpL zn$2Kq!bKqVZhSf2lV)>jzjR+en#0G`nY7G*uVybMpr~w~uRS$dfL|?IYUn-$!QG76 zqXu_?FqqD$I~Q3+kyMEW#uz7pjI;Au`(xu`}gU^N+h#xV-g)$a!}!4(fN8^KK^79n%IorkWyG(x%M|Gt;##yAFv@F+JBF z^i|L107t=QSR1che-B?fqRO78+$ z5H%G<4Hln?EEPh6LEL2}3E82ouGgI&V@25IhPY2*fl9ELN*j&d1~sq%rUYy-^qSQY z-4V=%*^I{~xJPYg0$)%}{em*Nj9|zvqscZi4VFNIkiAsZRx#qw4Lq;fs%|W{{95GA z7vz(l9fv^InXgxB4pT`=uflYeOv~h7^Q}!`c;NnnAeW-)#+qcHdUOLsZ-2%CF!~0ETkT5mp~&%56ZyO6$eQVHQS+XwzN`=s1t;XCNDf25?N!c4(F% zR@i*S+p}G3G-I>!n6#n#oBLSq1m@3}+i09A$ynS>XzaMl3Vort2dM6IcRG0^Tf8C| zsY8c=9a?D!hUqoJLITiOM-0>4`Ybi&HXK5OAbDZ_$)|2Roz)WT1TC=DrtIYeg+Hun z&)My5m0Fsa*lNbX@PZ=t`0Re96k&onpl5#=qX zrRV1;Pwp;Mc)cTe%Gbw61N84O_b9O9bv(U`b^2+vv zQj|eB3trDl^(icd*=mG>Jj+!w4k zv#9$hmrXN9B_Ce;U`V1(9Kpiv19&x!`H1smOg!6@ViaM%=VbMaXUo@f2)fZ=20A-x z6ubLdWJ_SWAxCJP?dTQ}|IOmk|Hndphfs6VW@FD0W^A?FU?pu*E&j+d$N(KF3UrsF z>@4hw&s(e|pft=w(^is_AS(`sX#{g5daa#cyJq%jf!$Q7-DH8$OQ<5uU0no!)lS;_t;eHVE1YKdFCX|a?@bB+1+WyX)z(G3*Mr6La9rRK2x;= z&^3zpPKR8&L+z`)B>6Ar=W(VQ;T1Q?95=vUuCKfNfz{G&r?p#;4)UdJSFuT#t2>NR zbEgZ1vCmz1n=5_Kemk^;oK(R|9 zQ5?0!%^1EcwxV|a`3QRG%tkKnoA3QXvZvjZST>y|4oytmaRy5(k(@A<)6=#O)()@T zp;!di;-pBs&6)Uhp{%AR$OnBtCK-B!s9hNJ|K$-~UazveW~eXph@Yc{C)6Q%ik z+UBzpv$dSK;_a8)`ZP=N4~1QcoWu${SsrgsC~=%-o&UMMd(m6~F$&l9gNyfV@Ywl? zXq&*BpnlKK<8M#>j&9FS!YNpB5t1%(7sHu*@M?G3w7tCn$1@2JRuY6$8do)*V^Gl)w=WRPkAB+fO{Z zj-;#A5Wd6n3ej5nUU&FL4-ZO-1#uJ6>R9+^u69v)Vya)a&+F%P_#{2Oeh(i6Hz6^o z(!W{eU@P*#MW@#;BXLKy^+oeNjNOc6eHgAAeiez-0)TGjfM5Yd#^MmvNM1JR+^`;z z`_)^I46=!G(0N}o3%Huh}FkdPlH13N`F48M{q% z?o(QvA(zHTE2kkr41d64+g@3AJk!KD z?9?y2Lu{7NIo`FYZXB=u@hiu~q+w*VHK_O8iRgHhf+U%$G$C>V(O!d#@iQbRTp9NB z2dvwi3!m!(1JNkVS1;!=v{~=;+FIQkbkaZf?GJLd(D^a>k2pnMbuT~X`-8sz@9Qrw z=lflG7VmBsoG$+L7!Ps!j=*FB#(TUFl3s)JvWUWjnx)6;==#*U^7$n7&zyE@s+y*2 z@QEtZ#ga<+j^iJ1+_8J^5?<`Ob=x2ffqoV&0;s?%wz_UkO+WxYapv<927jk-p5lFy zjJcHb+J?6bAc5k|yW`rIG@bmcYi+Q-j*c!D$3J2icPMo&v--oi(d-yGLz5nQ{OMwx z(%y#V99;kee{n%Wm5Rp?%Z$aSoJ0IG(VbHq=db55Ta#v9_dk=DEZZ0=24yWafIUb2 zankF2@x@-b+a>w*Zvp<0mwZ+2u?^yfHQnsd?C+@K6zt(u{p)nz{$1$)H$lOM1*1-) z#PFnmA_}`$Aw*gHrPj#nisN>ji5lCCOmz6ImTQk6_a<(Bik8ml54IkxLE_FA)W*hm z){=#LbGtJ;k31@|fZ8&xX0aZEKj+V3^L#UN{hU0%KbzNbe17i#TOzqGv`mkofkv&zWyMaTSq!?c5e8UEn{)=PSXccyDzXq=&IS_fF8E+dmtY8VPzOgbc}_Qb z!gdPtOsVUgISJr9J~9cG@xPqpJC0MaO7HgS2}`Ne1d!5+aY>Rqx=A>E@7gn25^H6( zVsxjRw`x@dLH$T=bP$!ci+=NY=EL6gf*;LW?MB$DC8CGp!RzxRr7Q|o+SDu@SKrqo z2*6#W1Qe?`yT5t#&E7lsW8$&ye7{_Q`x@TCgG7qZqrm$ObD=X8zZOEG(N}>8=s^v-ESI+WI8%fKN}O(ktkWU@32Jmnz~~*V?_tj z&Q+CcWS5!ejy0Ojx8%Vh;p$d&W-jtOkm31N?k?3c3{1~m=nx$9Il>MwNQ6ET`WUbx zmW2!y4vIUfY>$vqYd-7+;DY0$>??PHiExUztjT2Q{p5CY+27Dw!@6z`taf@xGhgRS z*CH=a^b*6WG@(X6CAvNA<-%o<6@oLd>e9tEv4zRG4gV9pz@FunDvb^5=>W|!OEWD| zVc_Vht&ruIno0`lRe?qI>b@5gVf}BC*}jFl>2=2&WJ7myu(~LiI-xFz zjymlGG-Xi9_BEy51R=Xis8@QMyLXXo^DFlg&~LuXO#13_!4trD;cLBFRNr-RG+G(C zS7)_Gqwzu&;WG{;WvF&w zHaO5JgDbGKoxKYO^$DC6mWhsLiu}AEE>3b8bhG2RDCAgW&T@PP9zB2SYHbR9l=gZv zi>s*g9d&I3V-Yru>j8!*82<-*40AD>dQTf7^nG? zDbpTyzS@*S2xY!3P8P3Dlp}jg&r$OD>Yb@b&zk{zv@SDGgt5v#Ta_hOZl4MHyg1fl zU2YbGVokc!6wj66R*6@X&*iBoP8P3fl*22Y<;t9upJ?&AIu2l6 zyIa}vAt#)u@0TN6HXv8#jaI&`>G8Tg-e=vISS}r>)RQU$Xe}FVAq#FHTeF%=ZhigE zP7Es3LSu-K-hE0^7aH zDx z*SPoXIt;SkjM}J!rrz51_V-dd!c0@pK}u@_?b=94>`8czNMUbul(j2B*3v?a7WAbp z@hgk`+1`4eJ9qQP*Qykle{Y>lXv#)1SC`G673%l)dtu%yP~kwI9IB}?0HfeDr9Tj2 zSjAl0RFaZ8hEvO(i@Qu@OlI{IreEv9z)Zy$Izxr|xo^kB#L0sjgk=~wJPp9DE4Uc*F z{~faI9r8LmgGk8@w zi}27?>#;=ZkzDPb@DoT12ua!@vQVtv0U1o@nKpVOu-47nv#*+i|M8qKlz(~qxjbgC zz?J`x{{2V){-b~Y(ZB!b-+%P)Kl=9{{rivp{YU@)qksR=zyIjpfAsG^`u89G`;Y$p zNB{n#fB(_H|LEU;^zZ+Y{#8Py07^)<{&o7DPTorD|70XaN3mi_Q?sGD*jOS_N|2iN z7L3fyYf*OQGk6V(N}fkK6?VfUbi-l;zIHv|oNy|HMNm|t)DzaUqV`}w zw#4mdrt0`c@^${xnMifr4NR_HBZiN2-hz~Jx>+zT!j;(NCzgLT&ND01pPKn?nj8PC z0z^U}UgT+BE1Ak}X90yhP_-3;X4TQKvVXKrbc&5VIXzr&;*3+xNt zmux&6)RQ{vFB56#wvvX5!NOCl434rrePRf1TFGFwhkWVd+Er#zsRfj#9BgKfa5yh! z^v!+jb9jo$S#5&361nYV5Xhn`DhVm!!9a!c<-@L?gBOxK;~}3r%ZK-DTTp-T`TgkM z%EYd_@UN{jn`-Ma`;_%>1haUZNAvC<^Sxn&?@l1?HbdQ3{_-Yi7jS--JGX67M*y)r zJE?Uc$fedKx|c+?R%=7;;2m4v+UHeNb|xlk>zu(h)@|4~Eo8IW8_gF;P-PgvZfgCkAH!5o|b6C2x^+QNtw8Kyv`gC(Xi-LA+X;%xQf*LrE z={&5fY0adO^t#wAwg7cONe%6c5F715oK1#3XNO5B8*~lUwvEppz2mhlZ|HdaWMbaXMZ89 zgvx!7rk%-1!zr56;v6jNqvl3U#r~f`TCi1HQtZcBC6$To8c5w^vq97a+Kn$UOKR`e zAp34ZS5sAJb(L0Uh%Z}m@FW1XSuJVZNN<|s_I)St7);=I?Y_S~usq&BvbAM~8)zoSqfv!H24F&paPi7eY8z0H5-2XM@sUFihQRAH`Z|b zQtAdjmrd{M(u2<3bg^ISwo1)Lw|W9eXKftf!$3Xa^?IOHh{XkkzN|hBae!PuR@CQV z=y$g8bX4lF1|e?zVS}&5&KnOiWh)le9sjtoYTOsLOWB6DOX>0D9+6+X?^Gb*$e3N& zB@CVWx^ODZF1_XBhv1#JH!05!{rZ9sm&NMyTWQXKy-6-r8^*q*()e&aI>gvQyI;+8 z0q%)jD0J=7o!&EPBm~mZiBFCqyy%nqqEpU_(1R@Co^6D$G7-7MzmP9x_$a^M!{9<= z2-Xxmo?Z)sO%AJb(;mxQQf$)owuv%z}`AZ;-n)%&qPa@@|3r)eiMn zH`C9)&RNRfC)b*(!)iaqL(&dgDBZJE9z6Vpu-lBFmSBbVXj@ZFT5-@HAUXbShn~=F zUE&Zq1NJ>`V&V4t{=s@-#>9L$M&q@1hg}(tLg0t)QbJFRE*%9gLeR!ct`*OrDsEDQ z1i)iIqPK0H;ko+#2y_F%!xswfzBu@u9#0AG@q9_lbb`YzQ?(E$=GUZRi*w5stJhep z&%>^+RXk9_oVa0l|A{Y{E5s!85mtc3)@iNrqnzN&vTVl4((-T~>>=xyD5v0^M0@A+D_6!qtsW_Uu&yVAV6A zg{%Y;uAi42YuWKU&(p*=t!m?@WZ}0?p<9nL0vjB03wz91t7uogiVSkv_`xYF#6afp zLpSqHE&Fxs)pIX88>C`(%#yaP(0(_OgD5A zcfnUb+;dR1x%q6cW_q(OkHRX~<(xO9lINxnmFlf8Js`1Z&{QYWgu8+*WpNH_J8O8EGll`P^$4(S z%A#s2j7-*Dm2-C=!`O)453=^+YHJbN-p~+%8op%NY6w^RmD@->Kzk?WYoASotrkzV zP%!Q{9miJJTjmCw4)A!A?@CwhIe-#fBYXT#fRgxN(5iNU)4bTbj5dzv;lJ&YU7hX? z%Nt7@oH+RmYr-61*Eq52jaZq;>`@VezFG&nb(*?SE5k1P22DO8o^mz%nWX1X7|G1l z;aK)6sg%cNywMP2*5`gZ=z(h=)rK%9G;-3tNh`W)1EL7(+saEYS|6XA*`r!B_=14s zTw2dY&H01G~BGVr`L}&J3$fnt`kX5QioX2FTmJykKdFM z^SrH5CB{xU5RyDnZ2dXXM|3~)y3l^Hh

g@xe}bIlt8vXF;7mynHZuF13Rn$r2{D z>O0fxD%($&rSu0OU!C(xz4YMty-J{O5I-=F^c@uaVAK}&w3pWD4@+k>9u^O$yi${g z-;Ov|8q}LT`r(mfD@>M?TU1b-Xz;w)%}swY6u*IAIroU*yG54aHm^AN1Y7@bfHMF& zoTS6m0s-`FuB03P&gNXhc-kpyiuz3YwzvQLCRujh?fHg&<<-mW_m3;2@n3hRSpJ)O z>Ba3`eEqbu>-XC`;QB26%yR&yQbTc4);Bc#l|i%Ce`GzjH4WI|AzAI=vXTRAUNEF? z;!j~84^nGeetGw~o|(9{7EDxUI2zuoNvCc)HjoZBw>wV`b`AO=h``TlvU<}Au>lvJ)_k^G+>tz)2pRV+$T@)ty*;|~ArNxd4#bvn zaW#Ez{DrbT9fKY$U1x)18*(1v?YK5D2}hQVL-;!6T{%EkFSOA*rDVB6$UJ-`rKjJ; z6TZ?m4#$*`0oFLH{kU>congDN0V(gP;P(l`8+R_X-Y^3c8%`DLffG^s3UH8?o~U(I zY~&#Cgw7Pu{LX2sac07dV-(bZg?+ScL1Y{siwOKVpdP)d;x{!5I41qRzVC`*KP0Ny zT6%>g?r}L+1tu1l#VJ|71W?40BX6r(IA)~wxhvp<)DsZcyv`jaEk0GePCNy*2n%8w z1YVB1I*q5mxs#fK(~t9nYKZD&ep+Qg9p%~vfvR{1*PH?DeBn!$vfUuoP*pu+hx+9B zw02}*!w8L)w7ba^!^p@(PAJ|ap~nx6Znn*dNUwR!6}$&YTg%Tx#@#%kHHq!SrEtp;U`N%bB!7dKo2fU} z1n&qasrHt9*cFG`cgB#57SSBBs@8t@3J!Y>m3Aq}HTBB`1J2)>rU8YH-i|$En->G) z=&m+za$gXr8M+lAnOXeN=I+aYI zL5@}h`1zf++XR<-I?|_im&l%DG#cwz80+%x%zPw}nFRZ1;f1ycZNnC(2uW!+IM?X< zLZ3G?VOp*S(8TyHq(CJw0B@h!mn=);OfxNJ0o^W1uGCGIXFuCrL52hDpt_+BgYV|P zRB%PbROG07x&p^jb2k))9eX%iOBgQ%cV1)t*1b^#`7qII*Dl@^B<9z$P=PK=sLhB4 zx&@4K+sE*a=1U&-d0k5-_{a5xSt}-laWtUmN@X)S`Kq2?=FAIVO_vo7^fhTEbsG-b z*3JE)7TMLmDU;SOE7^J)7h(~6!Nv7CC2U%)y|~jJTaB?w1I3lq&lTX(h9%Xof35zeIRo)^ zlML&^YJV#e)?o*6hIC&0J)mE8w<+xV7|@M>u@Btl@;()G2q7|DtvF(j*FKEHjOD!{HtU2vuHxao9YEB1Sp$3Wf1S1#Z z^gaI|X$>*26!#Et-*k=U9+M|)Vi?LsU z({BAQ_h$o4%Ag>Z)$h0ENxf&CNTd+HZ(vxSZVXZ`{}U4S)uJoI7%&-SK9i%{efSr> zs=Yh(F%6cu7CIu@NzH_i6)A51UxN#?d^te$KaB}cms8v2u8T=K5X`tUc%e&vk(R=S;#<+;ppCA%M|_Me$dgi7;y7)-f8I zwK&X=3l$5B;6th2&Db6Ex;}5ry)QLZp*4_@O^gx4)1lRA5GPRd8n|_5oA^&o?zb&3 zS;W84i76jVW*#HH4LKbPw_-M!`9EMZt(gh!4I2Ve)CK_RwlUc}XfY7#45gNhBG14t zoOlcm+b2q;z>Nj~lF$#5qahB`Xt7;}17`ahbDm2mhXT@-QMS>41$A~Rkld&Ax_&0< zw%6V1UnAN_r*6}vZ@~dRP-1?(;=xrf9}s$?2+WzL-eS&|VYT_B*vv3*>o=3S_3dU&V?Wi50U&y%y5fYeX-3*%E@R+M+q_EW- zp8V=gmNq>P`v^Bfl#$fHSyDdhp4m8jFrbus<4iy2bu2hNmu!8_Qm4}m1ALTr31 z?)?H&S-ZRXqt};%oRW|3>xGy;cCS2Y`8e7|K$>ez)dwN)@#W{2SqB0XP;$E2w==V( zcv3AM14?`Bv5vzAdI|i`A}G{lP+HZkh&%z{3CgB_4DHbj{U5|N3V#nOkT^@f4djTq zaCx!JG&ea_pCUq@+Y;lj>yUbdCQw8TG z)G;b(!MpFj=*$sD`8FR+48GB{a);Fh#UNR_V6t|t63#zhTQhI1)olt-8i$7st}eUH zq(gMx{+q>lf=mAV*Q7e7yo=89p;Or2^bMP681T%s4ynrX<?hx5#no;zoo^73825Ckswj6)5Fat1d( zNqp)&H5#u6J6(Na7D}P;1^*zFt|Vkyuu^96|XydY$O0-@I+ z>3C4hPRuvR-2$85r?WS4+%K4~r*Fg=#_*LhT=BO*RaY`=ual$_0HtLhd1XDe=j4_e zx{6i@nr%b_0Wo>3rPm#AA+HU%8zN2Kksc&9t4JEN`t~r+FxVLb2AVA?!;?2}+CXXT zZY`9l#@8iT^&Oc1Ij58g4T|UFoYRD)TkBj)`3*WuSTeACBN@ig9gr*(u*tH2`^|tb zuP7alQ+C*4xBSaKTBD|n5sK0U;Ni@4U+a73VBp>1Zw^t_NjO*lS3+=O3qAyYP6e{) zG>o~ARZOp#=d(8goG6(`<*sb@hPGju>#rl6ZrK8}2&tj+On1h5&b@AU&B&Le(?v~r z!Y$;^t}IdTWWZ*upFk_TBbktQq1ZFGpAHh3mt%$m2{YjCSf|#_#3pw;@48 z1}qA{`Bk*%y{r}QZQc9Ld|4G}>O4c}Ras_l0uAmxLxdHQ5sWKM#-J3`HYfm9+;?|( z;YP;*9U7U`9Ng=!_gk5_U=Ure;T!J;dB0uW5z&J$KKuL6^-A0~zqsX$j8*ETb}s^?!-!O;Dco~(9M@m=3;JI}#G7%TDO zdy({Goh!L{w7Z_S<)jJuykJEYW9mO?n#lx=4zy*Jg2>P_930Q8UyaidwAONnolhv6 zR-G4wT2N`PLdF+-1t9+h%x&=-fu|)Mr9!xAqf|9|-HjC<{-Q;v;QLm4{gL*|jo9_; zM(=ZPm#ypi zI@@r~eMEo)VS8!elFu~xS}E{TrZF8Ft}A;MkzD9Ci{25Kl&s*0GYnGm7)L}Z-O!3k zuL;aG`%$&*^v!Ajl&e_lYf~W`zJ~Qo7wMmcmk~UjeN0B5%z{w7mdt07Mq|4e!H3^PfnZqjugeHrlQ&Wdx)F8oS+1br*Ty{3ZV!iI8P{PjPz3XA>w zn_Io{C_X(NS{a3>;1E{3ZZ$jtC(*d;6{TZppth%yLASpxE zlu+B?-63<@?@r5?Y_?68w(0NDl61G*;%2C!(KjtBrR~XopGm8^blem;;+DbfYPz=1 zebyL-XNaA*h?_!5s;=Px)MFOhTH%h(TH~*2NK=<9N@HhrbO!Xc#R)cRuFzUaQ5evc z%g0_D-mQ4=&z%O!E+cm4=4dng-Z{w2K4|St53zaNeHTkpuIB(rK(@cO6U~?FL4T(H zI&n~}6g)TaLt3A)Nd=p6f|G=85(RO$E7=hbgOEgp??-e;$m&AR^)a>S%55N(=j$I~ z3iP~lHzy-W=)J6&WT$1BJ7^X2o0ECw^Bj0>!_{l*Y0BO@Ww4J#a(66!$rKRx%r8-4 zaGR0WLcM~=z($t)K ztJ@#Bx^4D1Elcj<=z8%Onl{L^eyvzTQl-%hD~d}xiP#gE(xsEF@?KD2y^4_p=cp{< zIE4kfpOm`)7qPpr4xA4K0&|YF-H(t8TZAVDh`q3PdSWSxW@}D`({ZGx!!4ml$eyj{ z(CZ7;6yB+4Qk<7|ou(>}sy}0)T+?Y5+9{s5m^Eh*00h(bB%2wmHn9)lXkqQIi<_7F zN@~oxOAwJLPc$ z%f<)pHcaIf?HNmiEFN0^vP<{;o|<~*0fn^jkN)gy9iiEN3KtKZWKHl+$T}HUFtrW4 zEf($1!B*IM-o!Vp{N6<&-(Z4@oz7asN3&TeHcq>)<*Kag(OQj)G+LpKGW# zt?8m`Y5c$gEF~N4N}XOt^g*K#_9dKU<&-$i4NjGzYk_J_X04W9Q~6+8#MDyeY7f=0 z(hu`(9lJ$fo6T#Ot?K^Di;@@@bf|O}Q8p^;8kz!Sg&FlmhJRj_(5f4);3CJ9bW*{0 znBBtlwJ1txhR66IRJ&`rGW8dz#C#qZuOt2bCawu~i7?^kRmiB|6`VEbpZ8SPAv@k~ z9@f2SfzS)H?FaX7&b(B;X}6ZKgCiY>r?BlfR^crl2sNTkD27`IqD!uasP%9vc3vBP zP+=$e%&e~pL|X6B3R(!2qLvG498KiV8R^!tQt%^)Hd_b==*HTPXr3d|wke;hyCX$o z)Bn_Qs52WDk-+(l*hiI9ZpdBkLTyt4sx_9v&@)(dEc6V)4qBD%*uXEu<3u*A?zGv$ zS~hMo&IPMtWvt2)Pey{f+HzxsTZzdk#Ih`$OTp<^0F1!;i_@B^i=1BknUytk%VM;GuEr3vl@^okBi_VH@8Dsw{2;K6jsA|3D2kCNh>6qRU~@ACQCTH4Xo3Su zWJSZH&bk*YXoLfZc7>3rd2rC6ZKwu#v7o4tEvQbIMhqCd1nz`_4TT4?M%bt7K<>F$ z2i{j>%!L=!cNKyYk`lvswE|wIB9Q|js0Sk{N$5EV?8X9UhI&W^tf=N7s0MhD&8X%; zs77#ibch=T&6&d7i4a_F0 z04vD#638Zs_%z7%0?a0|fD_=tMG3j@Z`CD4O1Ka(kO2oW!h#$^nW8TO%)s*ae*g?X z^S`ASJu?n`sST|ZCAIy+z)SH`P%Plmwn>N4;4`-)v2jPh}A%~)A?gyG^^uUgT zB#MS38j0COLQq5w21op|9EKt4zAKvM00dD#tWY%7KKP+tNTFzoJ3$YPgdN%fa%dd< z&KRhnSeT(v5JOu)uEg|h6s*v;n$M`&84D$(L4qQh5eXx-2@n+1#UO-G7aSIDXhAlnZew39fsZ)tk_V5)H-K%)VGTJSz`*2rHN{3mkrZ3+6b1>jFup&1SK6GrtL z3G@@;KsgM^&;9{^#)JFdpa#f}0rnY$?Fqc2AJiuR*%RE9Z9tz0xSjyT4wz3IR8NH4 zMcaU)VR`~2yIDff5Iw;GMF5{M@I1lZ^MSafL1>=P_6Ojeaj-nWX|H(p=|b`x4vuH* zpqo)AQ5b?}3;3O|nl%b~#{lbu*w`+xJEI0|r8+sb^CvApxe_0BIn<56#Rr zYHw>czr#2s=fEvl!uUOtP^qB<8@WA9ef( z#{b5V)Xh{# zlZjCT$OKO}9m5jtf5Swkb-r3p z5_qlk_`N?0(7h;VX%dWP;@%ocCWruH1BB|_wjgMWgPfBVf7HzfO2!q(QepvI!T_I3 zIBAuNtTRWIiaRCm^WME31yrH~%f(!;GmBmctKfy*g707f#`5M5tf%FYh z4ekS%v&o8<01hSjPx`2dQ>3{CIGGd~&Dpf4onKO(slyst5n)7d+o-hXIXtgkwb^jR zOGDd>RIFU_oMxD5pqltrqzoIUHIe6?F$(LWorXX|dCuh$SCJh@dx{66*5-8ytkyp5 zO28y%EVT*gBp0a~>j1^3o6cJ4t_HO&6raa65YzE?&)VbJn(R@-si?N6+q6t$LUpDw zK=Y=gxdf;exLs+pxlUZ_wieURG}{X8B|azYLqSpErXY~1^JJyuF&kYtaV67)hWbBa zPWD&t-Y@QxDV}=Mfz$DOV+cL|d*E&VN!}j~^OE;RAK`U4?gx3*#@0UYTMj&O-L?Au zs4w3Y(n(l#MtCqZwq@Vbd<>lwT;!l0=SoR}9vMj~K+}<<-sj}PWsABrla`y+W{JXN zhaGRiezP^H^SfL;tfz^owl~N>} zhNIzNuFT|v$y+=2vtmh|EMjZI0=rY#`sa;QH}bj6t&Ib`mq)j zJ=5WYWteH9ixTH)bGxcTMx8@oo{!zITUj6g1?C#%0Jw)2e1pTT){1@Ukc>dSw|`rs)NS z?a)j~bkK+x=Xj+9Ul6FVVXIyVG~^Mn@@bp|iqcnTg^d<9)gB2D9Vu$6SH|k#(W0gi zCd0E8EowHxWPs>MQF9+@kNXid&FDS~iN8W2(c!zT{9Mh~V$TJ=vFbc!WwwUc8Y zFgZqx$#q4QD0J)}*C?N6YqWZUT$;uh&u`G%rW5s*%vo1x?1NFh57kT@lkSmGSzVQ4 z#%_2FenqFNt*0&QhWh*fIIgp(%rl}R>huW3g#FG%xuVr`qE48JdSS#fqav17&(KD! znzZ%cv`Mr^D7K2X9T%l=YGO)WRAR0YAkmzN!OC!eeiGxiVp7t@_(9XpnUPx7E1FMA zMpLM#pnr`TH=d^evY#(HJ4wxpGnO&KHLB|+qtaIgXrSl#7)Hb+N6_OR{kD*mdMTes0!9DbsiVhFdKyW4#1O=0 z=uF~bK2Ttdib{!2dkDoRX-TO!D05_JUx-vkI1BXCO@QbRM>Y{h2fHJOgIj1ZiIQn@$w!>RdvFGG1 zd(iuYMaIq%G@CUET=N<>zS>~2Ub!VQmn^x9u!s`K7eTYHZHwYebILQfKvcHgZ#mq_nbGD zH~eu(DE}Cl8QvQ_KUQ?HISB0&@rFWp<3+~G3L$~Y226?Y6StQt$5-d?0(!O);q@BgeHKer3V!mTa&fa&fb6*VBx<|%pQMfOLl2o{y3S=J&Vwb1TNY$TdH zEg5+$uGQ4k89O6akSo@F1LxXX@#+Ih>S|D04zd*wu237qxxo>_{OwQq^lopA$O?99(a}}w8XFpU5H!? z9g3s4&mcv`iw)~(R)Vhms8}#h>Wr4A0PK3}d1#jDwUS=aK^3dbJiAYI&;NpzTp-G< z&urh>C?{AD$gEj0H+;u#Y6nTsL)48b*5~D^!^!QAosuusCfVH^cI-yb=r-zYqo=50 z0&`J92IomCh_5a=yXQ%DIwk*yl>(Ms&K9)dP284o2Udu6pw8YWi^%6~hbBj@oNrA?nDQX$|-Ce zVwo1A@*isbQBtx6>@nOuTkfz3@w3l1TZuwsVzIt1N=8$q{Z%D!fUN~4q(>wWgwQ=r zacco;pova3u%&oFC~bJBSK28=DhXW)Eg4}nzG?L~au0-+UI(BtTOE4sK4Bx?DnaH@ zs6w|MS~?|HBK0gzD-0qt>OePQ>|~wm2-}olQjp)-n%AotE5Ws{h6?Qiylg|fH1nK_ z73G6o!x&rP82rt{AaCvib2B0^;xIUyqY*X@dS;OBIBd;$vv3ri=3$UD$6#oVutCM+ zW*!PLa|BkV#(In(x;0AXh@p!`#ykWjW;_~ZH1J#$4rYUI89^}u0keZ_8M6-tu`jp9 zw2Vfx9Pjdv82sp>CIQsSNW98O3ujF0Bhe|x$GAq0^C7S(4}(HE9)B`~g)|y@GA>$i zADEL7D=gxc@d%U6^nAtEaXR|Zo7OW%c&W4q*__y5;lRV%tEhwA3Hb1+0;5>3xRWiK zu5&_|wk<2-UTdC_urnd<>cY$^1Py|HIZwn&%iP#rQ0;ukgLiB_LB$X6Z>AN)@>^;_ z({ZopjTQ97+P8Ku_w9CL& z*9OgnVhmM#;@fGyo{m?&!5pgT?IFC(uSD9`h==EW5XFw1m=7~+mpUCx*$5o%5eH59 zAZ&3eHNY{f>cjONdx+vu$8YfFPJxt^ElLJsgWC(Xn=cJ;#z>vBd#0v6b|yoFQA%$qCRY9nzu~} zW-FkFyHViHIk(8fjAo{0%<$QCvvIF&x&E`UAha6b+oN!>^996nphZD#!=G3wOg3CY zv~#0}t|S=)M?1~Q8=k+!`&$4eXYh9N+2kjYGm>PrtXMfbO=cM(a;jQ5Pf=2U5w<|N zz*k$0V34A=Sy4%zoj*H&`dn!s=sTg3o%PI&diT>Q`FW6{>d8~`GyMBZ|N9yFnfy%1 z6}ckrzv>_IGnuGC)?M?B=F_t_BTZfJIci^PSy1Dj9iJ-+P_KQxCj4@B=3*f{Nnu)cyz2n zqXD~{tZH9vYqKp1R;o!vBM_ATy4V8Cy~)I}at++k+_Oy8S>s}5fS#kbsccWv6TLy;#*m8>+WB3VKU#BE zOKt)w+Lz|-JIEy%tHxxj%Rpk1r*7v8W_kK$=L&(ie7-hfkA5`o$X2G|>UyJ~g~rDy2Zph=d{nK)xe9v=uu?fg3&~HxxTN zCp}7COL_Bc&ux{u{hZ0UNZG}F&U0R^uSCX^xQHeIYHL5|oY711Kpq$lJ1o7y{$GL5 z*=@wR`opd*IH&AUr=?bNv~s*D~b- zrgsiz2{G?rop`=2b!*$mGTq+rOWzNliQvny2d#VDz!}5QfZ7gDQoq8mc?4>;0;Zk) zG|AXK%j`xlwIZBF?%Tv_g5*1pB#EO%Tvio1)t(r-Hh#GZvd5Y<;K#T&hbNHnx;*yOPAOB0uK)>@~- z)`<27F#{2?^oL2l&$nIX!`lX7V76K3!=@;Vy85W=+^nUqA|qFCn){8B=#Ihp3$ulW z*eUcu`m28eK$xi@t7R}QZGZw#SzhskX0DXlz=j15P*c@j2`4rbhl))>VSWTYG zO1;IW*iomW7nu}iirpB(o5naS%LJc?wHi=X^r6UDlV@c;UwYm<2fq)=t;<-agiROV zQtvFSqjo)qrI1xlSL|q5VT4gtD-U;~u{!D!)XwPS=@}$JG%xgN3`h3Kc8ioad31(v z{YfL8IdAm{qpj9lR(+P6ZTItreJakFzNxmgPcDcxqk5kXdp#82$FfYRU@|{Gw$>}1 zrr3!fRN_!eFC3(NV#Hrku`JVBk47yA^|KO<{>_1M+cGyiA3X$JV%mXea!UbS4$d20&cC1ynq0P2>@l)fnF7twWp7fxjVy z+MKX^R<21QB<5uTi>FfZMZWU+_U9%p4KzsTol&-qw6=8wRvgh{&-=6oI7WQG-72K` zlh4kdoucP#VICQna^2b-})QTL3=mTmqr*ARFQqD*zHwVuR5vI$fIG@jj+Id5V6 zIXm`20J{N~0E!zsf5m~IWh8mKtrCfGPTd@aAf+o&aLQ~0Dd>;i@G%SZYP|ob$`zo=^lu>Qt}%V zYM9sAHLFls(jY&rmp1wJN`LF2iS-^?o}x(#7kP_>mPl!D#%G^QV^Mb(EQd^y)Ryd1 z%7mDks;<`sOZ1d)9g*)e)p4j(vQppZS{zj01$=WrYNSn{wu!ul*Im(S86~x-8+Buh zeOxEdszcRkr5YnC09FdP3=mj9CFD&lD-bF)Npk#r5JfL2l3}MxQP2gf)~LR9A8JDV zQ6O9C-B1Dy9o*%M7cyZ{N{Uts)EO!rETH)^T@zZ0I@bXZ9x-0$G`;CG^sR&WtMy`= zP^ENZU(CSM>C-1$4|DE;2I!>kat*|7@+=s6t8@F!8sEAVN^1Bz&*GHnH#(C;Jv1>g zS5?;R48M9PdBq~q)=uU$lUNA2DeHRL&O?M-{N0)fk+A_uQP#7q_od!|un3!NX_w&f zu;A0$x~(nhcJ1hhoZy|ul9WPJr&~{-?&hlxMO@+!K}(?rDkgrmdDK$-ka_d-3jvw-;|-ohpvv5yEwEn>=*4 z<<4um6EwVZI}>Nxt!4mtlj(=zbgNnkquH9U4`3d<-*%q{gfw#s_5P~(%pN5#tU?(j z0YwQvnqa@69xeBDMIKkc>7#2z9`p8JKzM1V+^AVfohofqBQ`*GUB4|rbrEWTqANn_ z`VJ&VZtMWt$TjpqzRIYqc=AOik~`J+NW)CuldYcShJ5$|-dY40rSWJ4_T;2pACfmb}_Z!Rh zys4XPN-{@6UcpGOMNJ+k`VxSrOvw!|o|9L(tT8^M>b%BJQ@2dkg|2a-9R@IK8V)UA zn5=E`3QZ<~q>>}{q5Ayjj2wNKOZ87)&E?UQT&{|YC%jTJN$%82XZiszO+I%8k%KG> z@B44Qe);ut5R+4ZK+Wvlsf1?$vYOdd$F!GXB34ByR@}i~flcDWy^2cL0VBUpD)HWr6@!ayUGwe~sBMVT-Ia}2k5M@O=%LIA{;8RMfecoLik`oha@O!9-|D3!PIrFD*zhg8dYUiHe$L~sBF)ufYI)c;^ z58*1dI%FO&oq6j3Bc^Jm)D8M9@*ftNn3-Z&`{>0t?=DAm;+uDurVr|)m~S*6n&@Yg z`)U42LA8-nzwF4U%~Ada18=>pryDYEq;%<}`s%~L$`bS++|)!Xgh-JVAqGZXcuflxK9tGhVHgd~iB7(jle+!!Yd3jYf++R^r*Pc9IkI+c{WK zPp7~$gK`fbe0KyU^WdORw4?kIK8p><_1^H!yUU%+_J(VIuid1%p&@uxN>k+;*h3bc8TTxebdwACE@f~_K1 z1aO*h{WDLP?jb{juZd@4MK&qCftvvyzQwe0+fAIXVmXh?q*J$kdEK^{zF@_Ye0gmp zY{y^dvm-#WCE8byCCT`ld2kz}(8EN|D^bRyh8>$H)a<@9b3xuX-%^r@EMrN~Dj?=K z5SC<*>ZW6n}KzY+@+Br_Z`MXyaFW_)82x zP2q9mksfnXwNyih@#QpoPQH<7&Idb~abXHk`L&mlf``BsFP+y}hGSiojGeibrt^<4 zMKp$!R#yWuMNnM>iu*nW$j4cfV@?~N?65i3%`@%MzHWUV(jH-9Rstf%0qJTkO0_o2 z_F7_90)uc~OIA+t)t}1Xh7LMt>^tTyYuY!o@7TO=WcAn9^Xq3gD1p(sT4E$VSYEX_ z&^y!!SoX`0Z7gn~FJh#BbX`k_9>r?3bx+(Y;g~&~KZx><=Znb$ua*<-13ib*?&rUa zsf=$N9T#b;-LJJ|^E$(T8#yJk;5ugS43yn7QuFjLp;{fIPMRQM+IYY!b*a4?zv$K7 zP~_V^qDmv{l{6I)k5s~p`%funIn#8-^G}8wgk?-4vTQ9s@;n_L^^Qps?rjT@`aA7} zu|8&BGKi5LL~gv$Z69>+-s%qjYAt`vCiwPAkv8N1={SztMmfKMh#kx*XFPGEOmm^e zBC!~k@6Pi^K7#c3N$UMk=jIN>87JdyFNZXIxe$=esj+Q?qL?n{r|k}q>p)AoVihYJ zB%MU$xvntsb;0uMiY9knugIf(@T->(c5wKbWrGQ(Iy0>a6vGYlOYX*B*l&x$g>PsP zXGt=5cVq$kH9}ZnFl!+S0Ja<)0!-2Ng;&MwUCqPCIcc4T+AM3OzM1jLG914D>Vp-|bakY9go>Z72h^$it;1!5X z+0pf`v=|5-J|h{wV`MJ>T8g@8kccrH)=N{v(qSW-wJ2Hj5#x}8_Q}>>g1b8sq0W5QeGR3?*f&5T^HvmUl`=)Es#V`6bfgieXuWd-N`?=-SQO z3byrlIz2`fB1$4FTIxNozTanUkTET zyem){UI$5m5fsOq$l)+&y!L!9md#PO#GI@Nu=8nCaI-+LUs)?~6oPlSxI^c|%BNFsThB|&m>^1>$EnE;~8ykg~?ChXL;FtB3DR;={Eotj`s zlk>Kud8$DM6IKEvVIL}1s<~*2{Y!H4g)d26gRTvF{-ewj@TT}cWw_n2X154^(`@V| zb}^))S_{imtv5)=1Xj2fb*X(`ro0Fg2u!M>l-K5>l9X(rRFKkvphSp}ox^mu3yp4E zF2m8t-aeuDqh|D;r$)8`7k2|9SWv00iQp=iQY739#Go!`+EB#;>8Mr8(YhgHQv(Im z!Sg!bqsBTHDZ8p?8JElJ(MJEihcWods_Y?HMPj`TJ+^+q1QKyTNodAtSz%+!$ zr@LVU9oX1J4{+javj_{d-ME{Ku@MSD9>6a> z2rbZHk<>~NLw>C0tch0$m+p*Mx)BKy(9Uhe+EhH`5421#t_JhR+9FwVzbllQuZjAe z&Hz{$a~M1huTAgRFX;vAGq6=cv9b+MmUl_?dbxb&J0b(8*cMJTw3K_89zX`w<0F$LTT%G0qlOXATu5Y+o8= zv)LORnfIw58pm7(ttwXL&&hwCy#M{rlhePPoP7Ue^6!8D{mJ_&eE8kzUrv8EfBgRR z^yK9GuipIi=IYho`RUK!=k@9i|N8mld-m$@kwi{U|MIU<&1*?~Jg+9AoZ!$u2dugX zB4&WL*)Po@Wto_j(hsoI$B5WL@YZ$<4lZHG#;<(5=#(HKaGAa2ktvMk?snor#a4wV zX}Kn;HjS-E;9ocZ7(0XhLHkl(?vlm0VkxgzBY-sEes@G30Hjuf-ifoO_`KnRgCHQt60wy9O&+h@FH#(@YboQVGyAHJr>G>SkpTaCxLH#lYfuW%jCZ z1ga&Z-iXp4m_;+`H+a*c$an(K_n@i5G}~o!1DL)appJYk)gl+UzQ;>d?-<*_2ZP!V z;3&E!m2>;NG|5rnHFY8=EobS^RD^0M>RA z>Rbn+eU$CqH~B@F=u+YzD3vNCcVK+T4xGPiAb60V~71M$rQOTBpOLEMpa9urLeTkI^$LajzEV z`<#3u(a}2Wo|!)<^t6>7Ds`1;#!}3WajFBBbuQ1y?9_w^4iko&ndQo<)c{tclBod( z9Gg7SzX=|tWRLfG-i(&bV+NYC@ zN;A5>bhVZ&NxIHyICU4}>LeH1`vdJDlw4WpW%M__$S7Zt6Ui8PWtKGla_twF(HdE{ zW5t9m6qO?un8TDqK6eekuSTG^k*9W9_5M7(97>48GPMO*o(dFCG?I4%REKw& zgoCEic0RPmLs3k+2HZRy57`Pa2`JWjMfR-#=;np3n5cwFFp8_GmA;~_n|*}xk_}L} z{Th!%tURprfFZp-Mt;9<*3S)F6&bDKV>yn#bFAi$_f_p_nx_+*sbzQd?Tb*PfZeIB zP^_D#Dype(qy)85dz9b;V5=ERQ>W4lm{J~ctfs8~xcc_R8R1hlHGg?Y8Zzb>sKf$e z>}|>iEO_XGV{i7-RO4jm-u{Q`xO(!>lmBq49sK!Loj9esA)-Wxe!`9Gwq@FGuS9xX zuw<&M@!Xg{h$!Y;FY<^-8r84DgMAp$ff(ewT~tcXsKU zAVMsiPw2hirh=7X0XZ1#14E!^dA24iE-QM+rsP@)9B$|W&SXviqo1Xu&MWSZK2lHG z_rox_LdxZw$U0GnVo|b*5r;yHzmK76(4a!-D0cOxCB?|DA`NyPi$3RiOLk+qq3SV{Ua(3w!I^lTDaNR! z7j3zL!w#k)NR?`1MZaQzlPHBgm(=MS>iifN9oB<;Bhom3UZ-?h^Kg@Vfyq4*%v*Rv zl-pqiRH|+@Te770+gq9&>2+c&%|MeT@Vd#<7?(&yM4`DxdfH<0Rn4V%)OEC5sDu1t zpVh2h$8FfiposER=3-p2py%XwEdfUh!&almgq#zj;!pL280@I_?J*8((s?0BX8l^K zP8^DJCK}0KN#Z4#)J@8yTF|A)GFD=NcGUjkmO%@}1S+ne>+8&#f2^vK_tr|^yo`KC zC89~1xCAhJnW#kb&N#f-WwR-86-Y^!IPqdfuF&lv1!9D24?v23| zdbhEW!5zB=P?I*V+(m9utX8%Ip<;VLS|!qjn*?WNU9dl>x1A9_Y@u~8N^+4;bh5s6 z-e|eoL=dA*f#z|Nofg`lwJ!OKt4qzKPjZnX+rxX~1UIy#^(r?=#!4S1pFH-2bzqr{ zc*OHmod<9%f+0iJNx~wxk~SN_h{B1P14Cxqv+qFL6|>edTDfW)bN zNv(WnuP?Gxiy}m zOH+9k6lIMj_@#)8Xq?gH?tzx61b_otaje*OhEkg=VCBchaa**RW}bKDOt;e~&2Mqr zv@4^YTG>}JTLY{aggBygC04ZJ30MI6-0`lhY?TTVOA)t4K@v(2b_Hw!+o};VuJoGMgzK$*G-b*A&2FUQ50X4Ldv&>?2sA6bRpKjV7mFJ zi9c%MecgN1#QUy)22I@IVOw|AN@s23qSNWEH!yiCDjlc*k~JmcY@lRAOQXO$ALx3T zvr6Mp8K1yA^WMLfGyp>GURo`4HlE=M1~F@kR#yU#)Fxg7otEWPPJzoUtcI19%Nt8*Uy(_6P}BHVb(RFhxo1V6h0 z`SylrhgF~3^>R+$u|(Xn5*NU$4+YI77x|ZzXR#h3vs4V)-@U{fYUv{sN&I)2v@gMicKkxo{ zXJ8&WI|NVW_{p_-g_}F11(IvtH%2BGD_&L1ggM)t0r4qmB)*<|X`TQM6pU+TA8f5n zM*#p$`+V#K!O>B9}bXxhvXR7o!P`6;r@X_v(X^jREayEB(4l zqzz68LlM2?>@d0#D+Y_UWY}1@nz}CDzKqwSMuahv4NIY(_r6X~p;u$hS1(V^)#d zLsWJz^E2{RRO+8skU=tXU8?J)VDhb~;Q5a-N!a}zrbzquIQZ2t%L|Y;Mz*7Ai;bnU zlg}l&%!w%V=x(tEbR)5O#=X#Z!$(r=X`h~OQMP7qZ%ws6@Ib21=7n^?ehLvJHZ25- z`O%6kA_A@2*NT-3K!;1B%3n3t$TJGBE!Xw9@OyKOF)!j3E`Zg(+bo)R33;rNI=<&1 z2)%|xr;nrAZZW5uk{dP{rH&OX6xaE`O1OXx^#4ZavAmR1azP}|7a4PD7_*-9x@fYh zMM_WlvF7(QW0=oKb6fLG%s1rEcy573UX@5-bNi0&*!t+K%a@`l9$n@~4gP0~2U!^o zVj7O%w@CP6qBFI+BJeoT_K+e$NDU=m8PS*~d##yE_XE%({ zQ6zUXR0HHjToTqOhR`i3vcupHl5#S3qZ3#l+-=Epwp53xlgJ#oE%lsj7GZYlZ#BtA zQ8Y!V6Q(!9lcD}sQ>7(bkFU9!FpZMQLSTq)?;wnjV3ttQ+Jx;@q^Qe6NL}}-yJzA! zC%1^kCOs6hmhA?C6jGhG97XY>jnSqZOaMHkeaoVMAfl+7P*POeY^@t?Dkhd6A7k8& zwyDRiRHaH)ZVXr@+8u#(_;6)wb4i9pxSe|G_AD6(CgWa!$2AjmuGhUm`PzL%RqqX^ zfNh@M4NYQTTqEadoHSy9(F`2_<(!oz&fRR?&$*&rlM0424R1Qbx$BSN9FrZVBXlXb zVz_N(MJslO^$_QoDVq{~2(3iQ=Q@J$+Hr=SA(({5gK$BZqsW@!-JcT`|7{pv5`Y5#0tA+lQ^7(QN53p zyknH?3Cxb?hKP3sZ&mb!}R`Je%UKxb2&gwQoW&FHlac z`+QS%PqP6S8dPWDaDsdNi*~qJF1cJ;pC!+obD8M}ymWa>x|!-?nM?6NQb8UV1R^Op z6*3ECOvN0h_mGsAh*eRF6*p1xW=-r2z?;?Hl1S|t&gxuEZ&PXJ%NL}g<$_g;4$14) zjCny#_giCZbm`7xTf4K>i|q?<67tSagbXB*A?Kn@Oo*o`6^e!9)XaaY&&E2hr1#K{ zE9!~E|J`?Q_Y zjOwcffv8a+JY{*MHh1Zl(@uceDS5>ak`HJ@d6w0T=MBDNGXWtlklI(q&@WkqpkV;| zAgU#id{t)^&6%iWw)QTkk2wk9!!=_ca3jPWviA1&VGC(nJTVg%R2xiyN|SM$p4*ex zg66RgEYV;qMOH8PAT-uTt%u)~_O2y>eEX-;sf54l5wTr~`d_CQaMN3!o}{c`c>vE# zR5hPxD5Dhc~gmDKz4M66aykhS#Ygyw$M6{R}t zOBhIh#lirVxDR9~cRUQ{k@hIGzK+u{O+q`C*>Bh7hn>lC$)R>wY+2PxKqYSr#R<$< z?=0B{x<0FAHNJPfZvZpFJ_Qu~>-BWLw zfXF=`lT+7T8E>7#$K+Fuwz7$PR^D^=aIQ1HCJ($?PIR*JIS7#FzlDE-)&^9nF49!z zY$9H&MrW8hC0@@Uwdah~JpD_^DmF^w2^uf9aTgxC{QDdLd*jLm>3cgC4gIASvK&jr zur*`Af}!`UK1~gjoq2&au6wv!dCyHECn@VZAD*-*(TrA9vMOYbYs+WNGm3k-dqn>P z{wJ0a8ekb3dzMW8ti`;%MP)8ZJ20l)ke~mW@LG7c8sm~B0rcaj)vsIh%{P}X#}1Qv z#t4G&xjj@79zSsN@E>be1p|nGJ2;xwZoLfdqD8_Qlj%z?@8ZgEZ9LZAT7P}bp#~sZl--}pls-L! zCd6dxw);sXIeL0DCD(k#Gg@W{2xd{y%zb{7xF~HyAgJxh5jhbhv?!rD$(R8a(;Q}v zT0uZ(`&ev`?A`obq@zTaW4zxvQeE@X)yu|N6;Mn5`r_4;e4|wveJmqaQ~g(K8t(X* z0~@tl%^?h4OSaw8gZu}9&tDSx`4h;7>$ zGGRjVrj8<19-uk7%z0(2T>E9eBony~A5m1bwR*uJ+=u1*CF#|@i*A|| zJ;l!EG-}|x*Sog^cjJgw=n`ML4T)H-uo}(MLY~C|Tw2btTOY!lH8}NQ?uu1_=7U?d z!(Ywq9Szlfty9wdp$nka0KXblIa-(4k_Hb-A8F^dQRnYoUA%nrYPw2yVNonk3&C?g zL_?>Jwt2*?9rJ}#$xGue`DG?Mj+H1xCKmlQm`4o6Z!s8oLMLj3cUNnwqvu=2b)MG4j#*T+iXX{QV zpCgM6k_8hwmm0G+%hpXHKy4kN3#<%NYEMZ~@_U}Kh0hSUi5G+tfXkpD*g z)!0<4MTe{N4-4y|_jo7ydJ7J1tnF4RbXMdvrc{#Q^OVT- zD&zTGQ^aU4G)N*N>IbuaOGf?q1>MtOrxma>_pe4-nZcl}_SxSIo6x}?t}=gg1icGL z?7wi&BUJkxcrKTTEku#IFIh3 zer|{U848E!-G0O8TYjv7<(*br__6#nb<0NkvHbq4j}VH#<4FTM`oD|sis!uIv=Zeg zzd`FMYeK1IC04k@z>LX*053q$zql1|!I*bB-z3=Ng@O(QV}6+{tt158+>U=fBRPA} zZQ|y6@*mo)2-QBaKR=HyU*41|G!@t?e&4)=ti2a#AIQ*T1*-9GI-m zU2z55d4tpO+UCIcnzUypAgt3^cc1?2~F;=vqfNFRp-!B>(ouCm_jv_Uw;Oyp#WN^G-%v zl0K?;ac6O6d~9(mh8VCZl$3Ri;kAmF>P-%dFTfDe=G2In(;j1Ef90{UCcOz1c( z!y`9rLFszxc2%A7`QQi~$lmGTPq&m*H7HOVuACmyHs|Ru`Gwq3s^>TwA zhwFV*LkJlZrH21`ww2CcGk`{Y*hnUhhOV4yTEpvme|SS(m??l?*@<}qg>liL_K|M# z@xX9}6q3!MBvhJHSBI~;S;~jt%sY=YIT+uzZ2P-lmZ*17diB2j=zL+GVb{#ooM9;{ zk#CV%)AwN8X@3QGya)rfH7f$+m#r%a>I~~MQRK#{)OIQDut)!y((3nV(%*Hf7U_sY zHg&TZ^UIf7!Zev^QZF(RvC7EiN;~+8o?gD(dSi;RdJyFpkdF2vod_EJTQ`YxSR47Y z_TDw@y-vNop*`rAh^^8(atr$XAlC2J*@B0a<>eY238mjx5pAq zD_UR~c#?{T{DGFKr&x4UngcGx5G-ap(V=+;g6qJitNxhrN@G83glI?C_Pa{*#wn@T zh+-M~Esk9*CnBk6W=dPK>VYx8H2LJo6SO@(`O}~NM4&)e$`Zcn5+QikAD%v)lK*}2 z#uw60bh%uKh*j7km=Kq+JR|?(`s=q1POARZOIRb(t2JVWb4p%1;didBk5r8bLqT?9Nf@in6ojLyC(Ygv6!=V=xJ9NX&c(TfcV7~88; zGfX+_DEwhHN4HZz#;qE9w@pdg&6h<`)uwZ`pJ+~KU5N?6(3@cEDS4k0z*Rm2+~XJ^ zb1gw_w#S)$7fU$?>-K%#L(jHeZ9nZOnwhFvu}#hCik&HLYXFYhXn~S1a#o@}Ca!1{ z?bK|8A^ZwuXm!S@T`vh;ETFo`eatN7icw_zrv}4t$;j2KHxL63BC!(1Dwbn%V;m;9 zT7ZohJSF6n6cr%)Dav|+qy+|tV3NsXyi;=~4!FdbiS0~m+qP}Iv27a@+jb^4-q^{+ zwr$(qI{&k0Z?@`p(NFg$=&G)V4lR;U0i;f>+EqT0kNg%{izO@Td_jhe1~yFr#_KP~ zK@Me86peX#o`RM(ZM*hK1OqmT**G#mfC6fGfj}@xD`pP3ZnNWXRZ^M_y8h%QwRz|h z42dL`cIXlg@y9shb@Cj05Qwccz1*ds7Hrb{)COy+!7kuib0twNRb(BLD>2b(PJmP|a27BpsJ6AYjxyKjyOw!Iu^0KmiP!9Fv~B>k&`)qCgeoBh%WzXE)*T zyf8@>&a*tXZFFk8HLn*zgFBH5PNCM1=W1 z_@fNg%|>4A0;yWEI}L3JL6eEWG<4CJN~9qgvNOp_3MyBu>6?n%^80wl$#jG=>9nyM zC`HrG(6FX7zA3sais|HLBnSrwSNF=~%IuFOw|%3C+i;2XHNMWol>T5IWoe!KdD}Vx z&tlPZu15^EVtEw=^=FcmqnY)kw?fi;5@iEkw26_v{9H9S@0vwJ0pb+7mOAFD?jJ0r zZZj2QVRK;N5x#7VDF&}B!1}h@tB@_6Kx}W$4bG5tJHdx9m*Aj%4Qw1++$6>BZ*%C* ztVp4_*qDf6BnAkg4*dbXv*%m{kM?zQ?3Y*ZmDlKX^+$!r0+#HwM$|hEki*fTF`ihW zMBQ~%q9%p~8eve;2sEb_nrXWH2&n)ltWsnt0ihOKXc=4w`b$GcGYKlFA?Ht(vRLkJ z^#nIlvOe;rGTT-L4{e*Y>oBjf_Q9vIycmGU+6GPn0br@_{Ys;;F2KVPP#)SqQG z5fUS_5e0I~OM2)KM&5x_qI4_wjg zi2H}9&NsdNj)`(X@87_;sDcqKj~mrq*(c^p~KB|J3QTydu+1G%#@rzxSw zw{}s%!@Qxc-%LzdU^J#ef*|gT-RqlQx!hk(>KHZ5cQwPZSe}`W+*}9HQ!dIzOq1AI z`;y$KkqD&j4?tPU58IT>HoIzd7lBLkrb4R><)pEsno~<|Sv`Zw%=Y1Li^$Mxvzy$%ZcN49wB4<4&cJ9J~zm`Wg|ESFK*Q*G896e_AGj1?@0TH)BsTL zF&uwcFm&hyGV@vLDG>teiI>t4%vI%btsqy1jXIL6HkZ;HRT#cno6#tm1T z#WXUo9Y>tmHqxnvwn)l{j(P6QIh(EM{zO8j_%jVFNvu6J`yzh=^&o=tc+pnNY#5hE zH-yt9%c|ak)RDd;-ThdzD27T0q9rcAY~g^p(ZmMRvbpB}9l(fLJO4TE!dsIfFXrR` zSt*3DonQAJzQMZO$w0JsYs!kbWc2u*fy=yB_$^sr>&B>)BsMW2KkPXS(6tDrb{{{K z5*&a*&7ctXD~*%~*piPwWs;Zs<9XGy*UT*vtw)f#X#u;&mkG64S`RP@Mdo#{V471A z=0wcR>gVncHZ=-UEn>MP{Qh?Z!|1QN)7`}I^_BaCd~LOZe()=wigPiE>|?hB!lcc!|BEvtCGE-&l>X9FNy6_xtta&JI+6Y%qJ z_{c~}xg(hQ`LPoaaPJRG=o4$@ z*ZuB8mbkFc`*|@M*769N2S8I+aFodx1wW*6HgEITSecsJNwL5#BSKa}MeqEP!u8;N zV$@6xDEI4)c;?*Ry02(cl0cSRO;6-7_qFWnst%B|!brY<9n(q+h}2{caf~m1$0~oO zMN}Kew9kYj`nHwRAuU{F`ld^=v`xWjYb#3sHc8Qb^^Q^T`43#t2WFY8|AYSce}fIt z6dW2m{s!yB+=Zw1ETQHf1t)72`uY$(vdHp>bBWD{2#}OGdZS>_l#KKB-^R8uhJLGL z9;?V4+GV;GcRJO~59KNUY8L7ucdlOKni<)*?g!w5UfVfK{Y zx+{6y;Q7Xcu=@Xud(9(97wq5xxNJCyFq=km6lqNic9rk^Q=a2h&9Tq}S0%3TL@Yio znW5C#+>}jJGQ!E^$qExQ5|b@q^{u-tTHU+cG!Q~s(vfRfDTRgVkFe{Pg`Jg?v`exi z)PN3WWw-g%Tyno4-DlG4A;w%zo;&}qZw0SG)bYg);z|Xq}lno|*tu@reK6Rzt z#wC@)L>(BaN$YNmpaZpz^(cXkVa8Ox-^bU0By&P96U`JP)c1zpjcyo#O3|H-u;Y*X z_lem1z~C$QaoF?wK>4S>z;I*JRB>u+Mk?VdM#t8BMDx0cnyGitI)#J>hK}b!QWnnU?DjgOyORnh4V`tpD? z25}0>-*Y_AfO;DYkLP?hYpUf!3M6!=-)={pxcYpQb#*n1Q#PUF-kO@tl}{A zrtMB}a(u-xNz+I-?PC$^Iqzs=)qY#-5j1c3ccLRD%g9-eQEcLV{9fAQZ3z?VJ7u_l z((W)I?;C~Np*1?#!es1CB<@Tk)yQw8;a1da;9FU2wUF>3$V(mdA|HQs78^N+;e$X+ zH0#ia$wT`36l( z!iQmOo1W{`;@K-8(4g%5T@ts{G54w1RGkQ-SjSpV5cIwMz_Qq_+0<(Yt?6>yvG4Ch z$LJAWb5u%+(nskbb@JeSnK_EL^fBKDRuS$+Po&dsoXeKKZP}%!hiyuc8+tw*&VGvv zL)<)Oys(iiv(&43u<*R=5ZK#XK6o^Il_?Pb;oBN<$&Q^Rb2diLAHO8^%x5ENt@(St zVg^r1G|W0Azzh>D$ZgTD#0$ry?4dvf_DdoOrudHfg3BJR6zHNE_6E$73I7NS*WNYgT zf+AVKwxK0v$n9b9!t+m5gh&dLMk!IkHHvBTTxTz{+e45Z$%b=R=StfL$ zCblozf%RxR%qd0Q3P!9S?}+CX_|p{=F03599*`^QR2#h`>W|CPeXO~_3D=nX(Zj5> zF%ILCq{82(WxROOh?J>oR0$nEjt|pEGrM;@mzJI@ABO zq8FC48X#w}PV#;b-hOqq{msG4_QcP z{RO9I+(WEi66mWe?5l89QMcV4OBVG1eoN^(BuB^EZwYcg02Yvg8KF1k2vH(5LeLNT zC{=g^EB^V9BEl}!uBqyn%MO`K4nbf+cEj;wpQdlayC&-pV9k_wJtVfT2Cp6O@i`0X zagO-btsRg$4{zBzGEQa4-A&=$?z$ozg+sCQetp*EV)W!!l5H%LL^?LbPM*e1F2*UI zM~she0~nqo>NncDE>3MPP^40}@h$?p&Y*!l{4sYteTgwE8Nt|%xhBKxAsB1ZPH_cX z$8>S@f~_ZL*mb$UjFBUTcxliGS&)cn1p=_rvnlEAN!GZ6)#OOLNrpHg8-!m)&QV3! zdb;+)TsB-J4~7n7;7*yPqz|&$lghoWtUaIoSSIgOqQg!Pa&=zAyAlvXaQN|SLX`lK zyFM_R5i-iiBUAE7!fzRkBGb@GRk06b)3aP%{!pt?89%zhmVjtD@U`+)j$zJ=~gZFf1UOT3_4qNXyB`a?w+bO8zT8YiD4Swnw{=Z@93i|W8n z?U)6q@Lk>gNTSN;=Af!tNLL$Ja5qr|bq^FGTnY;OAl9-GLOHZCXsng~=|N_n5gb}? ztN*z2yu~D9Fv==hJH%}uFv>^pDSMO9DznqL`Tk?CdPEI80>Fs4$g#n$!O9s@x$eA1 zS=dV7w3o{k>%uq}2k@v^9BIGqb;p`@RJl8ANwgPlW>d6=Gn|K)5lD)OO7&#Ao=Fr55! zG;zla_L{@(R~(C}w3EzO4qxBT$yNr(#*(9J;CFYdR)bzopx1NhU7%+LXK?HO(lk7c z9K++sna9`1bV=;8kYn(7UX7pfL>;ppG#Ct?BaV%Z?D(mA89Ec41sh9!@cW#$xAB+Hp?lzP}Tu>q*vuYEp7 z`tYy|QKcV&S7ks3=50E1a35HAjxlzz6q70w^L|ld;a;&{GRh$AYCJ(wew zLY@p^h-#&nvr%LsZBLmD#gWaR$u6BWyp4#gF1q;huil8u6kbCHX_+V%i-t8ZptHT5 z1cLr7O1)Yv;7^t&p|RTQp)N}ni$RNC6@8r>w%ix`wWXa|nN^ep3LE-~$h(rqev`u0 zQn$IZ-o16Ku2YBu61Yh9Tji-7{NKpqy4BSB=*k7!dwB+}5`{%Ai3i>vFM!JhVf>A!bxk6qYizG7ZK{NlWwZQA$%~p-EYmTs+_p&&pz<1Dg1g zSbw0&%=ia-$t+2jZG~+|u6O=o;MMDR?9J~PZnV+za z3tvZqmxfJ&Q~JMTMy^|NcbVBG&%G4p7Sudw@|Ff0QM!K|QcDo*r5H@7PaXykyaVF9 z>P)<@V(HXD`w|T}@)KJ3wcTtPRX<92DBLsEi0Zks=O*zPe{2iQsLZ&BZ_2wVWO|@W zX0A9|YvS6N^7}7Z(&zmR0RP1}iIQl(FTRzFB4)GhYnJ%l32E z1L#Ik>QjEdsecIUI2Qi5t;{K|>vtg9FSY) zoE!9EcRI4j>I$!BW;+9qMUMpfT!Be!uUF#c&{_4|O4cZ;MA4~Mis}_NSy#uak72@T zywkxO5y~4z(exM>5vuTsB21h&AkU@O=PdE z=%x0PHiSUjhollv0X0}%@%u~h&$R@?ZE>=butYjzHBgRy@4~$Ipg6O`XSkj&>BAcD z4nmdLX|)+LcJRGc%lon^@Q`g9uh?;z5dFUw+~4rQl>`vw{0F5*f=6G}ylf>O*<^9| zc1b%L5e$9#G(UTSY6|akS|PIID^yn!Gy9%yEQs?FlPx8sw9M+?J^y(s5n+|YjX;cm(DHQy9X%M+*;0nXBwnIh*N z4>#MIg;J`u_38Q!|8IJ&vV7U%u3Lz|cZ(A83Sc%ocni|2?MhCw;T~&(@(W{(4YoQy@$z6TGEy~p16_wCr!;^d znJezt`_NFBVeI2(ou7;@ZsSCd3g7tA^%GUPAVC6gVZ8HOk?uEDx*#xj??Rd3XX!Qv zs~vB!I=VyqofK(fV-JUnR>kl(khHcLqJjF5?88)1Sbr7Luo}z4S*9n1PT8?aJ;maW z-P|VSKZ1Vfw>R;slYt8X<8DefubEmLwWd`g-&C`6mmqLgqA1)ixjrLB{cg{ga)8_9 z)yg>`pFNW6qP^3k1k3mB-8(g54U|+(BwlOVfb*v?%!r{U`ygIYgIT&rc+rnx3=euM zH!|QcIIl}e!f>A#pHwv%0aBPKs)FNI~X?)*cmA4m8!McUZj zAq5@s@yB}=)@Z8awJGy!=~6}muvEWt2k$99nH-Ng9S}<;Edkg07I!k8-Mdfbg8&oW z9P)$X+=bKZ&X&yr`DVuMJ%v&AJoS}HGv_iXs3sB9>rqug}{#7-O`uzc#f?zRR0q1p4>I-k0(HSmB;C$0Vn;fvJUfgrLmU%N9?Mok@$o+fjp0vm#uXnjz~4Yw5H5*sN%`8SSUENA_pa3kTl7 zcK+hUhr;pI{4>uwVAu?TW))xOMQ*QegHw7J4$!0n(_SExgMfE*gVSP~0+v+=BVP#_ z7N5wYsFf)k=*7qIfv9E24nm4OZQn1`e5T1mj>AM@g3&lgKUmsrXUOy+e(lV=M~C@V zYX2I0@Z<#?Jji4=(az zn8hxC(+e72i%Cs}m?)F!$X?=;)q)wEs=CRS%lGH-1C7t9mCL?vz8(^OjeVv@Y1Qe* z|A{p|sAP{;SsZei4?s&|ttrLO0a+CM<|pPCMd;mK)s1{aPzhyB-F;U-3189u(21&M zuv)g}%)^jX2EDa-m`+{JBA>x@laPJ+yMr8Z=DoekQ8BO})=h(h#xL@sm?#4@rNxb) zhD|fj)${*YbGx{Rlbdfc=nN`m^|gEe6W5y*7O*Og@Fz`bT<3e%TB4)RDpvTHevKpR zrl@qW4`dd2T*Ds3b8J>_Aw4xsD6NOaQj=wZj252`05;K2OGwbR>`|#2D`w)pp0h^( z=(8lWoNxXOU+WX4Fm9Z{>AX#AF{Cxedx_w|#i6Lyo`EkPiY6~(7h^V8UJvBhzVWE5 zb*QFqP$XKqw=f~FE)_z+>f^JbdQ_?YLou-+a@*zX%Yk1A%&YMv$CXNzksE_EJci@%p_ta1tO*GqSE;FW zGO7Fp5mc;8v7eW!NABO^E;qWM84kKz)O~g}E1?r_ztOt0&aa@#mNz?z7)K8_S9GE& z_gl%zHDHmG=RUD(X3T6!Xqp!UC*WJr`N#HsEx%N;G@JX{|G`5*Q?y`7cGM;fEA($u zAnVH$a-ey0!v;iEm|V2{6o^Sxn;)$Tg+nZN;STW7_WY7fD27<)o) z)rwqdtTKspt0Eu*TZF*)a;%1Ub47VYGFptD(eckT9{k7#ZDJNmAI7|BGX|4*Vu$r` zjaN8NCyS;vxuRpcFSSgeSrtK)@AYG_;V;6ayN!0_*54^+<+U4$W%7Ah z&T#Z=1!=)b^Q|rqO`+D;(AdL|gQd~b?O#Z;M>8k4Z!@iB{JhI165S@%o?20bhGz)( zp|kTn#z$fldU=J3Yp<%C%*OFj~x6 zvPp2hx#l_p%ew>;&t8J04?f_&?zEQgHcL5n5>?pLc09HSAWKZt{QQgiWS6~6AO7wM z`m{#x>=T$v&KKVnw`~gngZ}KnvkOiZngWr+uNrq@e(|6zIR;})n!_$Nv0H(}Os(9? z=~NwD5uN;H!Ivg`wPpQ@HoIi_)HSnwkoC84=;)8>tV4~}tz;9M4CNF?FOWq^^`;8y zL2ydQu;cafC?MCew1{9>4S$izZiTsMfUN)u#U|kdiNR|HU0X(C2YBb48^=D$W0W4) z#Gd|z$5f3#k?_>BLgh>{YD=v((Qa-)D}rkKx}^$f%}H!2g_c*HdCOXQ_kpn~#NeZN zSmL`HB0oc*C%QI3R{1`E9}FwbUT0>LYEaRquXnl}^5f~`~>1+9UzH*APIf#Tg z#*%Q9Ozk2u%AYx?J7%?{4eV4t<=DT- z1GIVL`8km>OQ@QQz32vFT)vH=?{o9#IyjEd8l2jkQF7^WQ?xu;$OhIIDuW)ucM+$R z7fHyFI38#;P1PmY7eD*`qkLNJn6zr&n^2`q$u%$^60tm=DVTZwhk_T!dv>r7>Kw1D zWbc3zec=*cQlVW3HP(&$H($KY@E^?9PD^=W8t( zzI(H&ndOrEYpj>4!{;!v?;+>A_aSHeXK9^eueU$iZ#-e4ZWIr?7mI!mh|y~o{bM#d zp_azeM4Ae8EklJ^BCD=;&NN|o)G2J)Shq-Cc0?Swf@d&$t{h0|c|LLuA4$w^=js`Q zY$bB^pXem=CgU@g=1xc}_gF=~bc~`ig?ZaQ4+J3>amQ`So5q{iZ`4j}-7c%;cQ|^S zK8m_-4sO7Kp{lTUOK>Rs8GdY{?VqXe#2u8y97=onF`@|G@{qt1DBF9B0&=8_KK}KG z*>~vp%Nz>~+MT8LeW24OexU}FF2tg0qRiDWiy4_LSvcZ*e;gAcZS={Z>^N&$WvH7T zC5%bMbd!16>GnLsV-tg0Z+NVDZXY??gwjcpx&_Xb!4nn@jVyn3MTcF&v6R5yRZ$8{ znwh21@*jB?0gEx7vDG(+rTn2J8IHjb(o#9oOi#}d6@8jzwDxx%MfP9n^nBp1Gg%4k zd+ZT`iBf1bu)fcGVAKdt_`eTT4hut0Gu!gGtL~P)p3{Wr{)S1gpimKLLR|&;&w9ey zBt_62P}=}Yo$Q473p8IpHp44fae z{L~BhAZEH7gW<7E0Zy=-{q2uIn?j}ydY4!q!2J0y%_15%XUn~1*4veVBye8t8xL-Q zdn2h#?e#o$29nFEw*3Nzwoo(tDiHcNF3unF%JnRNlZ;l6u59iMf0ox;wJT%xay-v6 zb_Esk<;JciRM6}EtF1~6ZZS-rZ=T^m)i_O_@+vW#a>Z{P#lE%++p_eSlUa#x3a>j% zs177tHt$({Cz8gGWL{%A(e62E(j9_4`lr|AsBX3)=ZCX*?S$wHlZ6-x z%Y+z>PXqM_UoFfXLQciE0omP{Z8zDYA8LTib!`;cEA>G?YZ~LzHTvt;tPyN307*@w zQ88@Sy4)Rt04OCzccTP+^K6Y7_BZ9?N9a z;I2}38lfg{gL<%*aDN{UETNv@0(ve~c3+G-QjD;xwFqq*Qjit<3cGFJihhZsZU9~o z=`Ve^xLghM<_e8;9Ej!!zg-|~k%J8eVD7M_LI^>bdAcn@8;4g-HJX`C(|wZ}AC)6f zuhmY){0Tye!)8J7U3kc55K+8raZuk5BG5YedvgM zEMp$`C{rOyXMDN=M2>oZ^q(ZwQ zLff;->&y-sVTT~bqITTr_?*|em?FpT4Q%TW4upH)_Kw`{^?Tq`>GdDF z8_XrW+u0!4E~p<0#OFN667jZ*<4S--tJ5qxR2!EB2%EHlPSE?JLlOi$TsNo3iz9IT zxgRQ1A>7#qYo#yeH9-l?XG(D-nBC8|KgD8n0Fcm?CN`p*Acwg7sSb~n6YT7h-sn3_ zK5*XXS7~LlB1qzASVB_)PX0KsY4AxlC7&Ex)mLq!z(6(PB+KtY-i+}zke)ZMdudqj zZ7ml&jA(1q&5qd*m3a+OYprq@Bjj{kAm%WlSADEpS9Y z3>M6qLNAfbFOV4HS#kAgXxQ}kHZMRnGld-8;Bds`v$}dh_NWpT7P7`Z9!O<+UXR*l zrt(j%N54?rZcN%NwD%()w_dHrxdu=RRP*PQb(Zcv6V!q=`jq&GmK-4k2&e$D!@T@7 zP2B2EX7av22;&r%f;7uX%g~>U+pWe32x>C^$j@-4kQRZ(+fS6Y3B~+8$N?kaT#Ib! zJBseaduD{wIM>)hmQyw7qA1xW!W1$323@i$n2U}t14pBdsp)mf10~d+Y{FbRv}75+P5!OkC}t= z^tEO_@rx9ybyNonk#85S?+?dj{WL2DxmL6`E?7oG>~3ekE3qN`FBch;iAbr-rSUr= zExv&UhhVgmg&PV(-rH}kLBPLIyZ&jX0f5|MqC3Ek_3uMeYPQy5}*gAPcu-Bht26h-0*qR053E1LCYQ-awXMJ}Vx23sT^ z!~3U|hrLj2(oIFz|9<5CRxhr$MNWv6 z=|`msCVPUJk?~zm^S%kxBN;hQJ0Pl5nV_G3uqGLYiAz=OZS+yk6hpII%9Q zB+@vaHI9oUxS{T1o2|u+`kfMO49VB6U6hWk?LH$T5bivB+I5@fEiF1p%fb&C4plp{ zdpmL}d*Tg`7yysB(?$?H(5>fZsiy7qrR(F#q^j^Le;w zv!gR=3YazsEIQDNeuR+&QFfNm`D^E16_BP7Ni3&n(gcA5k*)R3`}xAx z1=)-O?^79TpcKnrXdt^sofz7}bVSMCu9kR}Z7;Q;fVKnveA5NWx`o!$(`T3eQQJF&&ZLe)=*!}-kE9a-7*)xh~V_6D=)ZEo)* zPW3Gg+P!Nxze8zrlxgKiVG0M291bwQ-5PyxQV>4-UnC|W#cu2^B`6^8Y&d@mdl>D% zx2c)67D+|g5%RyK^o{AMPk(6d#W0bWpfSGd@6=uppjaaGktHEYb=nCw8S3q`2m1Fo zJj76_sV@!q?C~W3GBca(NlPp{y}|NcOMcTu6XucBS(^IL$`y0T8H;e=G5D@&-17MH zeRl|a`FuBFbk5sWBj)%!JsI9wbCUg*YbNczq9Y75d6*M%olyt8pWGzJyFn4rEv&n3Gutak0YzGB_PJ+wwPx+WG|sC%CMs!kEA1hdI@1c2 z7$iOh;*uQS8R z%<9p*jAmxqpDNKB=hsTlTmSWm&hjReCQUoq+^;^RhKtb4OQagsCM6FdVm}lgQb5`U zL2Lg$ZUHW9_CqJ-y=eajjZ`)hfHf`uC^+(s>p9|&Gm)1?**lv^oYAA6Jwjc02$-=gf4 znqn6oh_~sjdWD(>r%T|?1?~b1>^~V$M%RILX%E|kMz$~_*s0?(BF?m98>dKmcPkrv zO-BRlL!t(q1jn<)@2^Xw(9}ItqCq}l8N*%cbs3fh-Nt_TjmOnjdOB9cH)s>~X~lPV zw-XP5nU8shuEep}iaeiElTtmq8&s(5HEEwQ1-Z~%H0beRc&GGbSYK&mDUJ}6`^eRI zvuxw}hsmCge^#L(%ZF$Oe>FQ;GVE!IX2=VgsK0jw9PL*M%M9#Yg5$Omr~ZzihDq&^ zPbpJMS3rh6stGWn`TBzlZ3c3_dWd!=)G;| z?pCO#;Qc@w#>6~2&Gf^ZJ*rse5*XOi-F*slFWjv9c|{2iY(W!C{B6SC+`)R*G`?cr zrH#{^v zyC1;9b|-qVRqV*xOg6ekeb;9nFs9j$e+cFCeno!c9%aCCi=RlTw2Z#r_I`)i(Q8B1 z?x{pRQdc*ZB9K%rm$MpD@C&{9M(gAie2}LPY70xJS=U2(n7TO5w-NIrKMkZ`_SNUr z$ZFkFk7jZI>BX08#j-`Z1lfpHs`B~f8k8_?)GcJ2TyJzkXcVofrcTsj$!Esp@6cZJ zi=-q!`66p8+CZj2Hu{pj(J{1=wLzItaSFs7{60oH@H~}D)1>msia`H%NP-g}b?)e| zvUS+u;^pQ2xXAyp{rUW&82FKb+|GeOnRad%LeQ@b`Z{oM8OEA0nL2`+T~b z=>7hcdu0cirz{mZL^wUIGU=SR`6NJzkvGHdI6X=A^N|>%C~Fbf_p$(?K4GjOMPiF{ z%C=XfO`a4B0LCW%2DL2 z*(aAlpQLoDhx|gz+WtcL6VNn_0VF0)#AetZ4nDN`o_iP1$)1yFthweRvUih2eknlD@<9HPL6{f zCSd2>9mt-xT-uEA6(j>^=piXd@r7-+V+)1I#U6|S)4L6fPKR+qDV7o(MR6sw?Sw$i zYRClwDtPrZL{+~Fj(Yfjt8i+k9*jW^ti;kWIsph-?pUJDlxWgcm2BMNojzKlV9*=*&fq+r^6?5 z>TiAgz-^}S<6XoOptDU1`#~kwpBClnjGFQ!vmawBLO}Bc)jhh6k(^D*%x#qEOg&k= zKZtkPk(h(+Cdy)k*~cFJ+G=85-UG|G8E!@uvjDkDwEyY6t{g4iTe1VESTi<`o5x1XuZ-cT zq7VmA1%nlYjlzh}vEdc#-U<1)AakaLmF$!3pYz2u&WTQpHY zeBW4dd4HbS8!Pt^o7(B|2F)t|7xw<3h@N>D6oS&}E81+O&`T17PAGY7Nha0Q&-4|} z5e;LAQG6FWl+Mx0Xo~*t`Th(NZ3e439g;hm8N+;r3#&X!DS&?LQ<8QjBQyTJ zMT_(Yos-9VK1Op_9|T?bnimLj)T&(0YHXLy3 zO}E?G+9@>6gqrytve=@=(plU$@HOWlGo(HX%9Ml0OubS;FvrDo=n)@bD|vF z4hidZA4a($C_q$=heU)MMU~i z9GuVZ2*viDmGQ3kLK56+X02*AAX!Fce(yl}&KaKXHAz+{Die#yCC zn(h<7luwdITpFRym5`MRO1F77v{H_lxROR*iK5P#Gp_oWj?|JqV|k}D)$fdI^nd90 z1;g%z!{vu}L|<+K$SPf=JpEC&MBH!C_IUyy?VoQEiI)hTKx1zZQ0Kl(q*nS~;|L)) z_@6I+bqj<&-LG~>OS)D>d=KVE~_T%Y5cg1-N+LwY+EUot4By;Qao?;yQO0(Y%^VIg(Y`l z2UIz*Vn(^nuWR!i0yCNmXBF`y4Dccrw1zFpXz8AM?Oxwp8^>E>SMt7&+mt3|z_QJC z@;kHWcY3{0jL=Rs4Od&xBj}s_I*K+Yu8n$qGNUUiO^{kztc};En0Aqn$UA&{8soVO z@6Q$30Rv`k`!bFYaE|=NLDr_FW^HQA;ivA-2PosF;@3L5k=6a|ilpa{2o+RV076<_ z{0ReG+DevX2Db*0R5`+;urm|=t#H8f#$WqwDQE#mk$TmVt{pH8=s)a1L3MZ*WxK}u ztD?4>XK2Fh>>fT^Q@zGAt_#r2 zYJViqQM#qocd5wf+{Q8}U=?iwd$=mL?Ly$!GY;zqbRH}laq>zmjulJMvwP_pGHa() zE;E5EA;0YjPMfLwUwSc_os7a;kM}qkTc*!HI2j+mXCi?T6rRAD<2q`a7nyDhgy5^FdA4T}Npr|BC%jjb|tBoj@_1 zLlKn23G3G{ki6&4p5xBIV4A{&eKRZYm6aJ8(`S+E^f-(GXl38Pt&8)qq$0Pa>ogYWvIaxphKICTsAcP4)6M<*MaA88QmI7- z8x%P*QfTo_V#SZ>n%K^n-3p%of<%{eHmR+TLIJx5%9D-q%Y^DmLW27oX9nQ_e9cr+^VWv>yzsT_fAtNK8`; zf^mlomwgG_4ekI$88vG-^)z6$^0j9W$C!%3Ig41}{mB@`Gp6Mcj-9OBVnlc5B@ zB$Z?-Bx?eUYB2VfDV0a5H%p4BDnIskNkty~(W07Sn6phKr6@Oty;!tuU=A*)`(!uF zJh_b{VNx(!2=?@x6Db8?B-U{{4tIMKpOb(zd?a>NH}80!B!Qfywamruj$5F=S-nD2 z)Y;FN>m3aAaw}qc)DSTNm-hq|0)UrH%jTj^jk*L(DRfCBDL|aka5j0VCg+57jf-ctEoif);B+6ehGiHt=f6tY zLZ8u4+;7bJC&UXZ*k)voi&8SJIUXXwpF%gf1U_l7@d2aWw(qkGqIn zZSPtoh*X9H@9itlk5o3aJLo0m08T)$zek(EMpzqkL=xwOd{K$fwve0=bx<6lD%kMD z?C!=WGJJkAxV&t@f?)1IDmrV%f?!>ZpNo~EMnxDZ^Ax70=p2L(h>P^0m_@JW?gnX; zoV$R2c5^k6n)PylMd9r==^g1i(c>KL_ahXYsPqVpx%$ESvo?p%0z@(esqd9Ftv+QD z8uZ;+Q+Xl^&iM28{t(*-gL}3IE_Shat|i4^NkZcP?_G3aHj<#6p@0O7=Z-6m`9b*t zbAx+}7EUSoZwgsdlx;SgX;B>YG?3+a!i!Q5KI3NN@8Wtq^>3kPNU#`%%ehE?o(3Be zSK}8!PM!uEq`3dFuCR;Hd|Zv=|3_~LN>DzZ4ie<9{^#9{L?TdB|2hiGX?GYa;WV3w z-JOcpcBn74Z7Z-kAylXMp!++O+S_KIMXiSqmhe!iVb$Dn9W3jx4{dYF`t`lpRMDI} zx-@M1r%!HKY<0g9wPWVR)>0+#j9AOWYzq40TT|&~fT5gIV;PUuCN7m-QiMrkd9xWv zfx?Wzo-dr3)}dwuE*jCy5+ps|zW|4%p#4=O`O6eth-XAn(L>^Yl*BJnGVmf0U=fH& zCYdzx=o9`_0LX?!2-kg%%x(W#6oFd<_;s4(IrP;L{+ied_D-p$p*bfo3C3-bgnC7j z^;5J&!DQ-ocv3JbYj2osed`Fn`Q^q(M{h(^L%s_n9X3Zs*@Lri++bv#=cm3q7_Gei zMfygke({l7rI0C{76a2$_|oC~R+CiA`0t?=iOOrQG)8qeW&(Okf7NB+|G$@p{P=$# zUS2*33cd{_Gyc3EomeRLeC{(EeC}O7P%8GkP7uznOvc>z5{|t#759PeD9U#>i=Z0$t=CN3pke;odFy;yd#uY>oY4hqskNYplu`ub zC=WfcXH)1RGduzO&jz~`DU!!1G3@*+&)B@Gpo3jQ&bD^`=vC+Y7Fs>4KEl)3bQ2FR z{OGERM2vo$ilZ-ENAx5p#cv)aWq!bv9D1gqFl!-mEU{T@%+isA{^?Y#n<-W7sdp3G z@-Ch7T)Ee3M9bGSk8D|9ZimRXMEzR^emz=N{z0rmIjewA3*0SC;iM=xS=eJV52mU6 zeEWA4Y^qjy)A9Wtq&>zt0~^?QWriivd?>PXevnDMi`rETAB5hb|1RTwWSH+e>Bv(4@~Vtk}M-@EiDE!dpd( z-LZ2Di@VV%5mqIsfSLY7#JcvdwwyoZk)Q2>yHgV$;w4B2t-#qcLsJdQP0zpS&|!Wn z0e0sbfUEy6lbiJI03iAEqShPo{qiFb?hui-Ybh~5aFBmBP@~um_5Q;B?Q&wT#cx3S zL-52b$y6FiBt&a}8YLA9@IpH+gkf+qYt(^yiqBG_r1KK_SDU}jatI$Bo&h)LiP@2b z&!otH?4ogHbHeRfp zY+)pnk2`?FmR}N4oZf!&1}1k;ku_g1uZNEvu({i>UZwpRIZ2>E zykAAv3*Mw=x1t5AJ%qEPz)surpVpDvYME720nNxkoQLXT1^658bl-!{>k5LFcH2Rg z5T`7KQx9DJuXAhu&c%p=^U6wMA5$RLldG3XQ%e-X2+G;dQ7}B84C$_(SHwRb8$iT% zm%WGhf2j1w;7p(QdjO0lwrx!8Ol;e>Z6}jFu_sQR*tTukwry+c`}zI1w)V|^opa&b zU0q%0MOUZn{Q;d~v#s~1?`d7PERpJIYwa$p)h817c`@3t(b`!;IYlV2MbrpeF4yVU zjRBT#gZ++LleUp--as&IFR(CUO(0$IV9&~z3C-Ta7|S+V%kpoY0iWfer}8#Ah|Oha zX(ZZK7SDX0_0XTfWKyT~*t8Z;@sw+G?75ku3EbFhQL5H*C^}hsuG}v482MU|$uvxn zho`ZIU1ilgF2MRi>#zHT4Faw4PaA%B}bW6gJ(j>)PkXl>tSsK=7nt|u$lsQmWp_Rgl zh59x1P*l2zqBDDvT1Xr@bnf|#P0+^g+I05AM%*%K)FbyMX%qDX(+3m2n-4P!@N(T- zb~#WR%{JMWcr;oBwqOuyY=IHBiwOHccodRHEIYHeXhux(tu}(8HXY{W`7p)n_m*qH zWtDwW)0@XNSiFZ)JlF~Zz=flsPfY9lV@3oq&d3=A_9d>xFTdkS!=M^y9>UwkQi>!U zt2aK`?&=p;WiQtCHok)W%0YMG6bj_ey)iGBNGv9=}Lwk8v=gy6BU-51j z+1jYwtL4~k()8jtDTj7Pnf?p^D)Ldap!aej%aOvCsSkEk{Q{C|BZVwgYbgl-Ezp)X z!zH1CU=)!IL1hkDTfRpDGRoO@{TbQAPnF}F`f}ZKEigL zv^eGFz*VklO#2@@#mhIFz7R`(8}7mQBMLD*Ca8)zzEaAqS}X>guTM*R8nx(qQbQH3 zt&5qPT1Rkf0*wFo&c%^;Kv>0tGnq&}S*H%D_v%a;zi8YtVV72*-Aq1Q|N2qEk67>LFtyO&B`Q-25)lMQIsL4Bph`yY2@T+AH^R+V0f>m^t4aO zA~XGIuP}M%X?kK$nC-OBJ>Nk0O}*2G3w4p3p8u)u-f2|sw{1kA9T)V$1C;PKu;yGyR9AOb{=X8)-7tPh6FS zk6#Grhbf&Csefl%fI%Htg(7onY}8Q5%CjbDhwE`$3}FTF*g!;p8DD*U(rHsWR>)$1zG?f#+W*S!C)-t^=&h^j2*S0}`EzdOUS$ ze+Drd%Dhbw=~`j2)t7aaqPT@LrLz@lk;M!napWFZb-HKTO9x;N+!>aN_&HK_sn}7N zgiscc5sM~XVm)mH-^6d;8ZY+5g+!Yk5HyBjAl6E=J_6QT&EfkXSV=DqfpuAsAH6JX z=J2#BHp9K=nU;oox~$r2{Dk`KE7N?{?$;pUYRP&!+fRjB{8)Y!^7*Lk6?dfNr`3A1 zw*%pfy*A<*OHb%KVyWNnc2!;@$-4xA@tE?~uMp2)0hOtj2uTM7AWd4WEfw;&Ex((^ zYjGI8FTfe^a3KUr%eg`FF`yJGdwSOn`V1x((4Ay@RYMwqgQc<%m-C}yDK2JgU+c%UBr!4p+e+2^yWX1j0!F{vyBTD!2kDI z{3DKInG`6F10BPn_*vOx_{vS$>&FgGmlUp)Iu7d?4F5gwY=H@3Jk zu;F95c_?ZG*!XGI1jMZApoG*~r)8W)X09>?E&fwMt3MAs>(r&$pM1w;6PF{ixC*pg zF|o)Jga+&bdppt}lCx>d#^cTS?DpMPz}F!-iXiL!BxFUe>$EyHCB@W>5eox;PURGh z4_@}-*^SD3wVH)Yy5eZEc=lzoc*bY4cqWn6&3>a)Lge=Y@{i#E?f#+9{lLG#?I;5>b8hA)%9iLX(238FYTon**Anuy%#v(;Y?DlR z;;H2&*RwQW_XzI4r)9}bmXsH?M_P+R2xmCrqlB7pxutP0Z}K=F;!=*F=;ycGRz{$Y$`GdSWLRdq#Fs6Q_xwV$jxK4t}~pleXdy{xh%AG(KDHhbs@?b< zXQh*}fuOp#xvtcR0}r~+#GkLY!B1`pIAi|_YN1!K^Q@|s1O?a}`wn)m6Uy7V^KPYw zMQE1D6XKv@+hhm{-6)<#r2QpCUA@$d_hdaa&~|`m(N5o%4ue-_65=A~_8o`GunYF( z!Rs-iDp@6l-Qh$K%KE&>kA!K zu8&|1KeO>nnQbsg#@$diQirWHX)jmq1@yD31$pQB6g4m`qM1j_qJU^;>pBuw@(iOH zs5Shi7te`%VRIP>W>&}9p&M>#`qAGt1{-u$X+M6HL(#nGVb& z1M&COvD3(mGqmamyBHt=cIdz*T&`Ymh4WD|#>O z(^!O!PZG{?g{k|HUaLxTrq_w%>2XrH=$bm=Sx;*&NK$QTHncDK9qUfY!srf7C7HzF zu~w&CRTN3-fKwHp@+R`e9yc3OnAutRY2>AwgS&3>7V*Pcjo;2I2Tb2nRIdrsA4T){ z_TR!sw@r+23L;7IPZG&;nUtQmkEZ7}$LtQO68CT|0wlDU@@szWC zBhTksLoKl=z#)VA6G{A3dNdd|sUs{E($Pnab5@Fnge|MsowsYa<=PMhSRAy%7DlKo zo2E3Qjr=avx;!#KnIz|Syu%2q2RJiT!NMTIzVN;a{P^j%=R2k>QF3q>h+8b4M#IL< z;XseeZr0WEXs6)$4$0)Q0hwAaBXzi ziCNJ5xCrB1ym{l=@Ow4E(?deG-L>7Mhs{2@(n4XY>7K;>Dx<4rJeR!h8m3%lZO33r zqNGzFTq`;S>0t2Rn!C=QTj3aFWWz*h6rnd2m=*}|nTERJM; zmhcbf2n5Hlp$#y1MwG>yZCo3Ot5eoXH$|}}u&UvpOqH7+)qf)-PoOLCV6y8|O^@Ah zAXj5qjej+igYm}v?S5fd<)Y8-hP-sdNr%mXq!F&nY2r{Fk0BMXOdCqfT+3$%o;*=c zXGu6-@&Z+;*7Hp8+2>L&B`xF|4GcZ@>_F6eQeou>NEd!tAPZ4@2Gf5l_0JJep!+Fe zb6au={R?6V&3`p?ZC14N2of~epXMK%aT>`J8<2n_47h_L;S=3Bhv1?H|et<3H4U?#3O!6T7<|gj{e$OHM&n)KN_HxeMv$%p)R0ZwT#`=1n|8)Pi$~)c%p3C5c(Wf)Jumc^_7$ro@u!F!; zU*O{oje!qNAOn>2cYY+aX5=;Bj8P! zDXhjCBOdn_vgW~{|T?6SzJb_#Kcul#h%1*l_Z3-qs(w+EDSw+>A}FB2B@|08bURvU7DB7&*Ol>=oF|s(y~~fbkn=LMNR4%^9Nql9 z6%=$i#E3v8L^ck=|QG-pMfEm#PTZ^WB0F=Sui`I>>v{Cv9kv z{}tgfosm427%haMfoxOXMF+MG(&Sg&2EL5fOvuSRN7-|g&t9_;j-`wKgw$6Ok}5r1 zb=1&v;!6C8IK9JiOGtQ81{A!qXvK>K&c>vzVD%)>Y!f9sw%9s5z!n7qnYOJ8gTX( z*B6*ltGQ zN$ct@?UJKmBIJbCI(b`M~gSW&{t@Svpa~ zwU6Vpxd=r@D3T=>XtY-5NlZ)7bc*9#kva{5QLeRD-*o1Nd{+ z6zvWfdwtahiL!I_H)0_}{t;TCB-*F2P<3(wRC`+gAkbiH0A?hA_KbvpM`4#S3X<$R zxxJ`P1KI)j8Qrj3Z!+(B1D&RVVHi1eb|{WE(_yG6DL@8fvs&d9-IfCuyV(FMs6fQ9 zqS%0nuiI9c&mqeYR-mH(Xu^$2=;L2K21EzE2|8EPJ`Z&t=It+fO22SZQb+r7AaYsp zDUjMxc;_4;{h4gs?nGiXZ#YK86^X{LEx;d>Qd)~9b5rg~4t4(MQ;8Cz?_H?7IO{8N z9R_dMpysa)r6v!S$c_Dnx6VhKqqHW{$5SHr3`NGBtBc%U3dYk&_fhf~n(JR%zEyWN`#h#(}%OvzuDR8qH4}NM@chv%A&O; zr>6ol+XP>p9B(Z-Kd%RmCkHjVIe)pi-> zPJqQ~3+*-`$(V%I5qvt-GPxy-Y8@6+4gCz1VlC!1Yz7}M=Q-~VKTbCJpEf?AkROwe zJiGk>`3PxwT3j#f{-hGSS9_Za@+NRwG1Hds)r_Requ{Ufq@xmHPnX+D-+41ZnLS|3h=VqJ-7&sCv=6}a#h9?H-&`ccmY1!Dfx9=LmzE~XNHDpcj@P#9z)9hE zLW8cNgij%3SzYv(hfJokILjsEnr7QeUdc{p(tk=aGM6)2%WB2;iwOnN%dg-jj(DnK zRfPe;jmwl)!S!-4S-my!;{^a3ZshVj3PNTL;-5(6@9{metOa0Zi;xC(} z01Ej57e#(l-51JMIfc2dJh_|{V5lm4?t&c!&~LQdE#IV=S4zAFbtgRL-%W{@9n|k~ zb52#sM879!n=#eF{HFr%LMfqB#(DJ?Of*<`2^U_=mqxarES>lj(Fihq6FI9SnM|LW zdM@2{i0D~=Pd$P~rx3!LSJSoH5#qSu#3N2_3Ps747Qvevc_8&+TJ-iGWz0~%#)$oq zRFs&}c);q1cp54mG9-dlmC-9K<7i$<=ypTu%v0GwSJ;IitTIb=reUJ|9FnGD?QxQG zwP+5(m*e+ws>TT;<9h}E>4(tvahjFCvXsElBcf2ao=V>u?BmhIpVNn9n=Zo&*i3+A zc9oR3@=6Dq@)F$?oS9Tpk7gOA-{fq$Yh?C=aLseWj3FeTP|SENe;|1V|AHO89jkh| zCS2C>dRc#igPc=#uSFqm51ae7^0@^3$JV9qDQQxStzx2-6wKl@7tt(|;ju+Zo2OMj z*Uu-bN@E;sgh>nJ_5FjQ3pzee#{q32Z^m`;qG1}D4=yDl)|ypdn)yk zop0>CURbop3SL_4{CGRA1WjVGY`?(mF>g(?(A@hZ2}KwbJ1q7>2ECY|);}(ezpLLA z`g{V-^8(1Dv%j3*KKGZVjyO}X9G>^vKn}K%SB8+=2{h%!mRy7yFBux-w5ZDQn5|0n z9l#CjB6j4gI4>b^Bki$A1uv^$K(-k1>TGu6(;<38A!ZO$Py?{({g<(2Rx?;dp_?u# zKo*&abdEqz_xl2=aIg?MJ$InHarZv}{N+V>#c&x&X07&+)~L3$-N^))k2i7z+Kkb! zX~pxkPsF`2V`o7hve+!7*1_s!Ok#>2&wvE z`Q(!@B4Znc2WF`G_hAtbo}+cOw?O&HUttkodxe7{nMN=S30391UYzSk4QCv=T{Tfi z6GQE8RHqk8roC~3R%r8UmpO~yNi*60LZnW^wA25!(mwh0^nAV9(ffQ5Kt%d`Ek{-z zckY*aKm5H30w%vpb3ik9NCHLKX?RrXa^N4Lk(X~zQX<2!bforXGQBWSLEYMPViwSp z-v`l^kIbVh^(Ds%H2!nk$2KnX z`y!URh#Cqm_tat#GimO@)4|g_e z&#(OLL#k1@gGwiF&J&r{|718wT+#P%U3{WXL;x)jIyatc7yQwmqW{+^E1kzXXjW+r z*SMFyspgZX%sbz-orTfT)Y_YCc`f*Wp9W}+#>7s@bOXGSIW#3mb$R|8aBB4xdWGMD zSe4H5J)%@p*(i@%CkgcNok5d|&CF8GH1at6G+Ei6mgf%%9fbn5J1ADOU4*Lr8YMJOp{I^}B>? z7N-pb;!Ci{?koZtmR%>ar+$dz&v&*jxrw^#%k%Q`lSDqmw^xAtRajXwK z#Bu0^dG%p}ce-7LBw<4==UOe7ET)iE1mF5jwJ9hw!IJzdJIkbLRJ4?Sr;Y+rT=Qm_ z$!rp|;`_|CD}9}d_NUEDk{RrkDVz5#gn)~jzYkVTQ=zlXL?0D63p^nBsJw$&xIIR;K|s;tp$^JBO1LrMaF19Lq1 z0nyV^^-}M&Dr|pjjGZG?lO+yr=PBSff>mntg>3$$M6M_{X2#6r9fB+ohZY>^mfKWW zzDe4mZB%d4Lf=o-02$V|>hL$i&rHJs~+ zFsq7IQV(pUhb}SsQ_Ye#5dz6|1-p~>krR!6RJX7VF;G+?)#w>3tfpA9H>FgW#8@-( zXKobxHN2o+;ZCo1!u1q;K2Hh<6C=LsdA}Op(LVb}Qcc8&?o-G~)HhYRb`lDgBf*{P zE|K=KdoA}Sc>2WR#`J~n zmJiXCs%Ch(CgE7)g@;CF500}#I_0*avgoS(5*7hcp~WP0O5!Vru2PQ0DZ@C^Tp6Fz z5OG@@fVCDAe3ba9v^hr`Pv4eLOC?Q}4hInm5S^8$8%|D*9n>XKTHJ&v7LcZt@IP1; ze)Fl{gmB$!&SkQ~^?T>C7x|H#nPYRN#3wZEoxuFR_rf#2p%!b!lQzksdEj4dL5{NB zdjFK)c3|X5Tvk;F!W|@tA~>2H;Hf4U}DW!N9^8xOFHbFEg#}Eke=BuSK?UXQah#{|YX*WRv(1uVhc#(lKnn;=F zGNd1l5jUG1ujV^>VRZL$cA}V^H{zLn*mf$@l087=raOUNYTv^|axb)cr$3hKb`~mLZ%UQyB~5>0>>IOfvMgx{>86-q;nc zp~|_{X4+`lz1+=+PZK?#-iu*_5U){|Sx(?Ah~7Jzs;%T6V-L$^^)vSluL+~jE7djU zaHcG&kC4+7&;f$c90yJNy`ihJ^HKGP`4NgNJte|bLf5~j>3xa z5J&#i;A7!5JrTTFCa6Vp={NSq!&&^>=yU zGR-nz(R4Kn&o`kMC*Lvi3IKi4DhYcr5y+V;P$}>vaYSoJg>h)#$5^r{g)iZK8-kAY zzG*)0k4kDLk4iE8<#Q73@1^2IOQ!|aG5_IabQP zCqO{4J$W;Kd}uX2^`j>K4RqVl1bv4Vf$xJ$e|oW)kE7^`Fa8{1IzVFElaTg{KmKU9 z;t#zEm3^Ua@YPff=h#@0y%#EFru-=W!{1q8d%q@&!rD*`8}>K^Za8WP9qTo1UCS2Ow87T!ezUSCd;9ppDa`RdcI!tu*a)pma3!jf+l zKVFqd+#g}QS_y)spX!b4dvvt}zaSOySx9`0`R;q9eTPx}F!GL1{9yu=e`N1loa?5R zt}dg7S=reWp6KBEd&sDJh``3>daUFX`+KnK1N9eB1nL#}WrGCX0NFN&Cr11-}jb7%CL+Q-S z>jPkd@ciau_mK0nKd#|~d?4eZcA-K|+J2(T>~Y@Q^%nJ`Ex!RvyCs{ITRRlC8kKco zbvm26zWh;NHq`QFybi{T&>Q>j$xHYQhS6y_iwEHdyHR^R*K7P@exv`xSUULQ1`w{d zm*mL5k_mHlV3G-Qv~v_#jBSz~(K5hWCaumN_?EO9rxj`2E^hPC#k6zNC;G=a!&vn! z4MEG*KKl(L%n!#0iZ&cYenMFTzTrR!apI?MUoP0>-h+C(Q$Q#;#rvVE7p84J0 z3GzdAvQ*SnT#{H_bSIJp5*9V^SS-aZ`g=`MpY;#UZTOFHl9yP#rO-E=( z(T#(S8X*7z~$a+Ss zDYv5rC@O9@k$~qMER&zHkUQ+j&jCgevBtxiypPW9im?1dbrLel>*vxFBja>Wx941( zGZvTD^2Gp4a9}Y=mF7OX5H?2iw}#P|kpzWo+y_nju)X3vfU+?Nx2G46V>v(uEcv2^ zoNk*@C5?U)+l$cj546AU_xnCBfW-y$&kQ9c=r^K4K<~DGZZp9)dR9Y>*O)HFZzb^} z(%z)1quI7Y#OdPO9{(3yd6hh|NfxpOZix}^Xk=*%Eju!7CFHU>xEBhAD?|`hu#B?3 zO=XzMQ2?c-*dQPmzrr*^-7^T4K{Vb+l)a0k@)q5108+cnr@AHule4;GD{l$1<|k2F z3)$=a@X{gYXN~NafHYbfODX*3Nt311 zENk7k2BF5SStOOjBW*}@qDCfb;2AJcAsK|`8v0GpCH8|TJ!$Z%EaPh_SlMj>Av$kUoG<2a^{z%GB3kPAh>NXq$r^gNZ}ttdi|d) znWu66So5ooa~)baDKKO_4+S_lWwmfdrE!w}zCj?pRWC-z-$U3BK0GRqKQ}v1DfNVG zk%h4mE7A^!yji%<)l@0!rcyr?Aa!9M8^kB`P%zjf4w9&s37icvI>9gynRdF8a=U4y z4)P4G`!Gn2+;x>tR&=%2**t_St%n+=UWV z{KaAhK?0AgXJH;vR$8%*uHbsw*Q3R$fkSc*%7o63VN<3Wts4x)(DG-5WHWlvnlLM5 zKwB7VK;6k>zyhO(-bmAP$cMM~v z^8c6pPvrj~!Hj1skT~I6y4q7bQBJjR5^p0xW?e%G@g~JvnCafFBzKdu$!a;sMK6en z2PdbH#}QapUzgEWVI%|yd*b?W#)wR%Rxs0jVO`|@m3e|g;T&157jG1Fdru>MzYkG0 z8W|-g7#`c)8*qO#k{vtL!+g_5W3$FS(-5sm8`?9-U|)u)~@YSGUIa1lo~lD2&;)4hAtCXzik z3#j1OKEXM;*>sOqy?c#uOmsd*@#h{V{_F(5z-R2(FDpKnU1BMh()QXGw`p5c=>*4s zsEB0UXD?0{4=t5yUMTKwjw$)Q;-ml+avnR`+feJ^|7ZgG9n=% z;Qw;Jkx>zqb9UE3q*d3>D$vFA9;tX#Cb!BS*`>+xY(BUU2cckxe}Gx6o#n3~>$Zr`7Iu%PO4^+- zeIc)%RLYFnLsL~`v)TbkL#UZvJZ-izFD-Vr!J3IOL@|Uz=o>YY1IbC zI64@^C$0%&N2YRxL2D&b5)ov6-mijkqflE&S7ZakeZ5Qpd z(gW)4Awbut3tmB%)y1%A&-n8?Q96P5CM=JKyE$fW9mG5yX7~+2Y8vVUpkPgh>!-G7 zoV)apt~Y5-X>Ms}IL;})`UJKos~e%bKlV}AWR}nI^61(Hc&I#OOg-V5{3#SZ7np9_ z*8D-JrkLba_ z35OTZig|hl)447F>o$pvLB-Q%t6c`JedvAuHbR*@$L93k_i(GG(Zi9ySvxR(N|*%{ z?}=QcG|(wG8#nq}mQvs+D{L-vzI~BjL>$CB3(Dn;+h(vA~ zq}TR9n57p6kfOOvDpW5Q64oU?n!89Qr3_upF~g;DXjg;QBlT31!N`8v4Ib`k?>y(r zKUyUpzzT-r^bQqBOcKE&0j7$fdayj=W5k1%dB^dsA+*9pCc8we7g$h&c$sGxg`VPp zAwiXxS%LvB;HCrKL+Y>;Bx}~&T@9hP5KT84BI^i+D%x3b+W8ACGjcb5QC1F4&R=M& z&7`lou{SAwo}vw$E@nn!!r>3*J4EKsG8|UvPV~cMu`5{nhS!>^!SG+B!*8rFiil$z z{otUfDy@pG<4R|m7dz?EoKi8z;J{H0#HaV6zmeJa;PxgF(y#snhbJSBM>$yrU(R1M z7Y5TgJ)OQG`c7iH6;sOVlF}d53Y7B&uJE#LnSd==<43LWo1&FuBkBl#DAY<&f1lmPT_-BPHKHM=ih{7SYet4qVE)FM76$w!S`R0?r1vY>ltF4 z9IISrm)Y*_x+b{cEI-c#*DgDN*s6ZAN)~(kGf%gvlm`S(xrSgt7^xYT?uRpJ zLHj7W;U#A6zogD|5zoxVXCE-{Vm_K^Y_3|Yw{LF-BB^5pPXJn(f=?irV@>1fE)Wj0 zjAo0~LI73TFAVqgUP8@PeBjsWr|T_34qH`UiW4^!C_&*zFFuqhZt?O5o{8_+rg|tL zDwUs{L&&T3b3XwGh~@cwrpq3i`e{Zi8zR!+)}G;$-_M1r30O%O4sy=pOsu0%ke!jJ zEh3Kfl&I`(%h98TnG;aenUCQOf(ErPtYo0v^h`2u);0;+$doapYYR{$kwRTcAfCKr zisH-Uiq8#tG`c-`7JIR{g1PWBlIA9)a*nR6qD;_cVWM$&wwjpVc-bh4ar>A>LcWSN zfL~#iQWX(X?189XBvob5x7$_hR_0r}XpWOe7II4Bzoim_Ywl_Ln#%u#|60|c{RJ$; zV`h5AwzNP)#%v;ItAtl_OSk;dD3k?H%AJzk#KPt0L|FRn6IhbGU!#ooXPqV-qf&5$ zavK>4i&+1)?%yUo&6>^nswW%F&4S7bwGhIJ&61q}yjUFk+D;>9{L)3vpP*gbob;k! zfN~wOb5hjdB8EmCfI^01c)_nYf)Ud&Zg>8_`2BKwT zFG-wyx*M|bAqZo|2Qpl-!uizeG08z`Jqa=MX|DR+TUjcKZ<60xK-c8u|KtZ#-FWF^ z6g*Dz4m!TF5w){fovb!j|?yXLxyW-VT}OAEo_mO??O6W*Cd=SzEjPJTH@@%^$Q3)(};} zqeeqwKl)u^%Erc`FoQUIRoG3~{=1dir^wRJnzLbU;A37XXITTtt9Z+1*4@YD50YBF z^P=n4Fcr-S#IhnSvE{~(4CLBQD{H1=JLg)6f$Ok5W|?y58#-Tao{A3q;4Wq04 z7XiAn$@59I+XBCye_62T-mJ|l(P}an4bSW)Jryun36n$1RAQR*r;!&>)L|!AOtLNb7FcHrN@Z6M_HH(k<8X%xsEX- z+LVD(iQ^W+R-GDK1jmIVLbi_>7yTiWPucYwf-edQ$VQ6Xb!Cl* zrKL|oORQRqyFp%0h}Ls=&p^QKxQy0w2k>yaubuf=U)Hm_udVspU)HPOnWPe6o1|t{ zGrO;yRzCF*&~Copjj@g7YqWZ7YAq5w zP+2easc=%i6A;YF5-7~0xsIZKLvv=J^drwixRn}_)99E&7!^6zUs7TG1b}!o^r;n= ze#NK)c9YjEh1Ao43dEVv*Dy|c80q<5Z%*Ve5M_9TTK4#&`Km@mRL6|nWDOGsm3kVF z5Yl=z&P~kmPR;1W|6nO_Ca9J zBJ%>&Iu9ZtZeQ`>`~BUFFj6(=IG%gAuwFu|0O`7~e=?|oXB>}moDbivHf7vE123s} z$M6vX%V(-zZk0pL9|GI#j6eG2(e24j_?93Ufps`;Vs)dU-?yqy0F^7VY#D~36!Ube z$YHvboUTAqvjta<9ta-Ft(T2fT$vT(vl@LT%t&wxTY8K__aw+(oROYu8hec1-$1;3 zHkhOUTtK70&^P=^Oo77mw?e(u%`-eyFBcV3rnbCLQqrKnr7=M8H?84cuV;t9LJcv; z)m_LWs2|F3y4g}bk)5w!v`liFypE1L>t98N&SG-x%ABP)CzuLw8Z0WKx zNL%Vom$gf+GzDm!-!D12Cd|3oI`$Y}g}cqMgdOKHeL~sQ3@|UJHGcO1)rYZ`Z7zh9 zJ2(~(@pUNoiw35|5?dOTC2&Z+yTu{P|F+~n>p(1TyuYob{$*W(a0>i(hq^1hA4De4 zEdv@=JEmCC(^{IU;Ujj#)p-otV1)J0kseJ=2{di4Dqtj&uYC@hN+toI&}$TLvMvlW=@?9K zY_^rhZ%E|>RaaZZyh%hdDrz8(vQLEJ326xk^w9n(?p!>knNhm=`8@yuwQbru;Qm_< zaRYgNC2xti_M@#GQvE!8y&dBnp3czq_0f^>WsOe#7M~H|2dQmi-=nQiw$xAbNiew+ z2zi1DB~<|iqL|Rnb-XFwUd`X94+*_>fY1-XqRYaC6SAuIS_3}PPD`%IFoQI$*`dML zCFM~qP+9aqT7|4NN?ueXGy;7ZQoY)=yTOm2=ZWpBR2kid`1drlVaU}gs(ZVM0woKJ z#!jD4He9V}AcCR>BHi=!_-fUDkYYjjAf02|jzU_h#q*Ecu)vFOB^WV2# z-7{_<2Lkk~Oxy5n^tL6Ly1SqwndQD0d$0@=DfH1Ke-+?h_UZl}nrqU+17g6UG#3LJ zFpOY@urKMzp1vFZdp-o^JDAtWn7D4eVj#KVSJR`Omcr=^dF&BpLsS6FtvX{=j8DYH2Ix5$VXC+f-uX@w?0;eG$Wk9Q3+oU zrh6XfLxSh_aD?vH<@B-TrWoCEVh3!msQp0`!K$|LcnkhR`lTyP!Mh6T++5~CNi3@* zRFAM9^4rDUtY3~S*r%kmis^~17+f?S@-=R6|H8p>l;n`IV{!Gx1|!6lLxp6v{6T#Z zqR&HHc*xdd0IO8{9!ykj)bArgh;uOcK_W64A*!^qEWU!znV2ge7O8a0Il_OqX9Gi9Q^{PikK+8%3poNXabP z8-^j;E;Fh;SY!Bxs;gX*1m-YQ(vtpKHw!Xk`o0-YJbQ1j?hg;>H1V+zCD_SU5(LDn zu;8woosSUh5H;hWGs}Q?s)i^GP%`P3j*POk)<9$n$@^PFXfU*Xh-UiitmJqyI*Hq9 zWb5uKCv7`_zE(qL@$aWY#0WWkm-p$pVT( zyU?Nt#~kSa>n4UL#IbH_JIPs%U=Hm6hiZS^60T{&1kAE++qP}nwr$(CZSyJHwr$(4 znl;XCA?ocCShokwUOAmBTDKA5)GkY&9trEa3M!ZrH-9>=HZ@L}3{y z2r7w<0IIyuY0a78nncgQHgtx3eSH-WFkAt$tn+V|`7)s6*XQ$Nk4wJR2E+F` z-~3J{`;7mW5@!oo7zc z>dmt4XfI58EHJJVyT;;BfYx|qPy$xMwp>L4-5r}6AUNi`;1PZs4!=K!?5$_QS zx1Vu75*e4F3qpLAv61ik8~{_2tFv$Mf@BBKc{6~Y2PcSNvrwWWR6@;UBN9LDim{s; zY2$uB+(^;eC_Y4ND}MtW&sOla0725YW7<*3pl$&JdfEkSvmF6Dr>^1xvj`RQ2wfLB@I|-MZR7g3l0*m}dHL$xq=qt4C1E*dwRqmU z7={<%$qL{#$WtvBx6Zx7is(z72df|48n1=-(cz0M&ecGLU&$5O&m0rr+e^phlI)f+ z#>>!n!)lgy1UvknYXGn2iGoGDV1jgzVfND|CKjCy+-QkNx=msF(gWvV5FJIeBJh2# zaB(QebiKdUs`RV{!@&6J)@+AME*=n()^n>| z9*maSeDPJZ;01-{QXi>EuE2a3K@vLAL&=yXxEU#1ATf5e$wz>~9+H-h(MWacv>L6Y z0j04l!&=IAzE=6eabDHQFjC}_lh)%udMya$5;OJCzkJ+dateFCZ8jwJMYT@xqH4vK1%rFwKmRe*w`=v=QjMSO&%X{ zD_%eWC8jE4v92n4&)H4e!8alk$c{iA|1Q(LK2Jl_I;zncvw=iY)U~(`ZUU`N4dX1! zgQ7_$a}q)&qsE1?YZ_>Z1EBb*#L zgqy1OMq+o%mF@+0v!=n1<2W0x*S@E%Er97Wwf3LuCaZJg#lzk9yA}kR-wWjYvZlGNPD_{l>HUIz>3ktO5g`DX7Yb_b{l+U%{0{H@VA0 zlXeB4P*hnKZVZe%A@mq%rKq{JSrHr#8I9~naQ5l+7Q$^VZZ|ZBEml^!^1?M5Pg25= z%B;~!>?iG7U3W!OD&(QrTGL{**>2OFn>D|$8}Md7)3vS*>hAM|i|n;yO=g5;jg~W5 z+pe9ea^%u2+4!6Q;p@nm29oqo=r;1w2P?(&@?#`-Lu`!I*o{A|vWA84neHYsA*I>= zXVO&3(RxXuke8kFvg0Q#tTl(gTZ}$;z-+YECl!^Ti5QoqBdSq1tE(|)Awv3P8MM|n zG)B5sw-WNRQWmSMv~;(m+G_*R5+PALX?2&`PUyxQgflM~)@Vgl{2{INNr<#uTNz#& zs5Ea?6HR?Ok7P-p+Y`|=&Z6V@A45`e{Ja9;$%?Fu+B}FP#~h@EQY`CkE*t)}b80W$ z=nv=w!>?J&8p%9AsK4Cv4Y_HXuKU#y%rX(fEZ1?$ou~&8T${TmkX+%N^f>-*BHhL? zWEW2Vk(E?}CTc8})9pG*qQL=^vuQTtTF5EcuFB9O6NwF6K)reJ%-xV?$UGO*$Wh3ndGrq3*MAko9}M(p zQ)1(m@{IK{6bVE&vmP`3NeRnNfnMZf6oq$V^*z5EA z{Gw);mfd2gNixwjCnz1{{25H{K{^fd3xdk@Z!~s@5+oNBbXu0JPfSMCSuhDitee!F z4h;a#Oph$rlm;y9%47_W?s|A>>O8D6U|4aN?9*_9IWeE7e%T)eC)L>mtmmM(1xSp`1^CHcl_D8f&p==FAVeI4B% z&FMx&h`3u>Ii?Uha5UvbmmLIJS?L>PFtF!hGV-<>UDsx|hjUWPWC_=Q}kN~dJ<(kXn^Acv`BVS_oy9%_- zS+y%NI2p~$?uO2ph=>|mu?E@?wN}&zj~?F)hu*eW(nHcY{^D%*5)IUH-V6H;zFqP zU@a23mtYof{dos2=M6+6Wy1y=V${BuHR+5VJw#B}zQ96Ma#{RA@)saGX?-k=x%_x| zzhCa?f8X&#L;3c`4&7e(72S=3>;V~W3^C;hSV+K2Aut%6mpqal11Q~AQ%>^fI#pBE zk7GII>FSCO)^k$^@vVjbGX)d)RC~RQ*7OsTEU=$q!~t!3TFwfZsVn9R_&U;LGj(U+ zSLAr5U3_3jr5)MVaM}Q&SJ$7gtLZXyagjY=hlpw1AyhNP@((==Wa#E! zGr^WA7N)Q%Qi?EW0&2>cm_|q=CL6O+5~TkZCSZFpmD`Rd(N9WM10^$s==c(#*?;r| z?qr;>eJ9if*W?#bf}kZg#TMV-y}_ztjA2V9f3y;xxYZnjfB!e^tAJqRc%hM45kmk$~)dN7RNYk-FT=0pY zc03_?nKj_OFGBWcqL0-TxGG^Cq2l@V*qU|ZTlFD_!y;XET-dQstqsZdxH<=+fySK0 zW#LX@;Oc11f(J5ox9wptD1e9_A$CzZt`oE1bYCzxtg_{IEwvd^>l=0=$6tW_a&^nU>t6Al*@G6&O=`3^`)@SJ%{)q02X|cfc*5+``mlY^6oau4K+A zjBay7)m#?psE>S?OWC&(@S}079UHc4srYmV^hT4aOpEhgN1LziHt%;GjSL_qFQQPf zarm7|+dA>^c1#>;JdbbY;lYX*crx0!1z5wDHv`u&7fzvxYk{L3YhJl8Gb@J%kh!l?qYglrBFnrJ{jgCPnZ2LV+ z5iA-}*z^qUeDOvQ`bsjCwO~G8qhthLe5JN&aCewxJ18Yp+76i}DBU@KmXXLhQ8R9! zOr^%xq4dyH3W09g*y;97Bauv(4*E;sk>*M+mXWx?9h8^MbCjJL8zjBZGAOlIU|sat zVFp$**f`*P){oCd?4TA zQf#C58ldp#cl?P$kQy1=Q}(!zXyPK5B%3s%oBA|-j9f|8JbF-CJAZ`b==lOQS;dDK zi|2P@69Y#IUIE64LwlyAYb`ppSJ@J0tA2}WVI1H5|A2vl3PT-qHCSLNw$9%d5L!lQ zW}*V@P}M!GsKTpL9D=>kJCp1q;+%G*!6;#wupebt;{a6=1ymK&8P%mAy3VH6V*iN` zT&Z`eIRJmao*(Q+=*#rKoD>hH1^u-0d^P_d{r7jLnsedZB$DAMZQ5H2rD|TN5o@IE zFX%Ty6D?dLQMeN0XSv5r&$$=mR)Yd`c8L)(QR2xJc6PmW0$#&!l0X+BJ5;xeLtHa=wXSPP!)XJyoHb|8(0Ye%?+k}m8-NCprJO5QzxU+Ns-1Os+S0HtZXa^u>rM~cbdcfn8JxNIJ83Aj8p zQ&Ug%nY%{RVgJ(xsnVKoC>EpUF#ej-sMi;E_N9v%K{0Ke44oBewB}AM*c2vGv$JQ% zeI}p~aW09rm)&Y3ESwB8xFN=mHv#|XerGM9GKYjL#e!daCf03_S{U;f)upC#YHZQB zC>h*pk9?fX=Gxo8l2AQZXm@z=t8#myitSgfMW`>+LNIuJ%pHlk)8~cm~--yI!*C^F!Ij&BWb)ZY`c;YgK{me)epS*Cz?3U=7JS+Dgvi=9A#;B zB~_qv;YXDkp2%&!Vzi=!;i;@|BH&iB@Jm>@B`b;E>Yt&H`*zf>%!!Uot8xBMxJfQz zeD10WjZs^=b29M1@8SI4zwpl2WyK<(oZP+sHXH_BhAHi|^KbR)P0?s49%Q$rj1N2KT*7BFU;x37-eGKYDN+_70rh${qr#5TkK>}ajUkU@!x={_C? zk*&4;lNqoQW!1lr30H!H6jDmj2ojKp$t~`bTdcVGv`}jFjhJsaZb_X>OEEXV8{xWV z(;%U=VlK3g`CWPG5f0DZss9wMLWJFVQR&NIX!B>b9AqMa>BujM%sX3aH!s$smU1gR z**EPz@#}~8%7K3+!#(h|sMAhTgQ;c_kVx9M!siO0lmEH zpQ}pB%}8wSb}+>zDw>%|$moa9kem0oaulQpvoAN9;%Ag;E3t}Qp;*UtN9~<8_3~Jk zi9a~6P9jsFNO2DcY4z5E%$?L0M6($Qb(#wDh-mlQCpf4I6TspF!Pf!+VrwH#oQZ6E za$qvrcIfH%Qd3wh+=b-xdbwqRS$r;!z}b*8rJOnkBL+#=2_P+28H>MmVw!^NgM}uu zUhNv*w>3big5vGx<^HJ1GJjN*<*S+yXZ@%k!(TZr-t_w44|C_ZILk-nm-)ZBxryrk zX}I$K|GQTFe`brr$HkwpUll?19OCdPl`&K6T;Ram@-^}jblRBw5_Y=<#cyy0f!Yq9 zbgNHG?^#^TVPQ*(x4J-836{pkZ#7>Qpy$Rgj4<>jt9$v9{AoX8S&#TbG5m`m_>?cA zS$DomKGQweOgr|OCqqd>|2B4+C%`L5Gp@Hz-|u6>W?r_+-Z2<0$5rs{!{Yy;#kId2 zPGczAY;=y(uaPQSXTs(+u6}q;fzDNL+o*%CJlI+Jdn@zdW-K(2z_`f>hm^zYV^hv) zSTSR-J%dpds!9%GEU!6znn#@9ow>qW+q{_zQ=eR5_W9%GtR!$WQ$>5=eZGW0rQ^Yu z4mK*#*_nVb3p|iIW8MW9jz)OecBLT}Y@%hMK4UQ^<~pEkAL|Y#(_?@_4AUWDmO^X_ z#$+V`Ys+@2h45M~p>9Vgght1(v9244{lUqcoMe%%;n)0KN_dV0tLxNPkMJaZAU99k z|IspacqA#8YFd_nO*NZE77mVB<&}>eK!Z$4e1+C@4?8#5uY>q&`!jTJc89K;gsmxh zHT;M;4Pbn*Qgn$ijzi=hAH5SB&pE~DaczQQl8z03;rsUK$Bd&MdB^n9i@)QHePJ8+ z3^fh^_{1hA4H>^<+}-KpoN0HHvz_JnM8zM-Z+Folc#kl3c0h3Vv=$t@ZEIJgw-yJlxxA~nyb4{dI*W;W7Y0>> zcw=gBB#RVD-Hn(?yw>3uk3X%CymxhzI#p&8i0ib!aJ+_Gj2<#{OWvfh>GOGCLi}`| z0x>vHwBS?6kiIf7$B;JFG__KK%`iW>WthX7oG_ou_k2}KM9QwMXMGE1^<58&a9V8{ zsoZ41YP!(r!%@S1F}=p9ArR-JwbWts#))&%{DqEWLyR5w6?!~PK?3Pr&a9Eju7C8{ z5xTQv8RVZbRk3#(&>&D7$%*Cyfkp~UQBAQ`__rplF^~i6Ve%ok;we(2OkC*_ccPRI zV;dzD)jr&;&;lxoG8RJGnle%LGASEzc#Yc~kJ!B90!~V6lKCXsj%S&%+VPOC5&Gg~ z=O;s#E%-#LdytJlfyp#7x~Vq|&&x82iK>QtS>9I1)<~_*dD;@#UXDColQh*#?St9C zfKBSu-{Jb5Ie0{@p@Vl+@xc5RrLgmeUR7%gQJ8W_pftQ%>k*V>K(-bUXlal|dLwNK zxMxTJ`tp;2PZgyKdJQ!e6=)i^MaG4YAPHqs4ZM(0yArsqbBro~)VnV5*OV$-(lZ^PRI*Ii}}rz+Se07SsXMx(fT%$k(G zl@S|)-jWFn&g1XmH*G>O_?k(&L7n|QMk(E^VK?QzrtQB8$z4^YARxY9?u?YPm-~DF zl=)Bfluc}og`z!cEAjr zL9w?hm9E-|A}29FiAoH*G?8`cB#fk)s=`26nI5kwbAC*vk-VMl;M5Y^Za;SmX9M5< zldn@5Sxz>W|9k$12X?wIMhUD8p6vI%6^&MAuW#=?C4gms9vPu}dNb)WC2d8JS*k@G z!NZcfTSoJ?DN)~u%L&~!bR*yV>Rjctp=~dUWZ@u`pU@x2{eY%m^v6llIur&rdf}zH zm5GAiI`pUW8zG+kI848)Er)20Dxq-)w3CsBH+CD2+i732ifc z)hAuLw=uLZ+H1c#2}=&CV3W&BOhd6_<0GmBi3o>b-FgbBe*Md?zJ9l%)zNJFS7T3k zC0HzRd20dhRh)2vQ}4di_|H)KG#y|hZ)*>XbB#@6-SISpWPr*OTD!e;SfGDoG;|px zjq>prrF5AoF;I#8)yfy~-%<;%Pa0cHSAVTJ`sJ4T8F?6nietM6=45)RREy zbH&0~%DMFp{7?GxP<4roW?YO5u&P$wM%f!s^wn;S-tl=@V>9PXD|AXsnNNJ;8J$>7 z88<=JHG~!E)CsJ#qLgz=h6MCPtjobz>6;R6oO=G@VtM;BC7o}S^#5D(x3!m1()IL( zE9-osr1w6_I9@30e9kkEl*&6^yk`El`|DoT{!CcZx8V>?N$30azhSO zf=EPNg`Co+oRATUifI`Nd|>bL;s*Tt?{v~*ueDG%!0w*ORtTNGw{1sIZdYl`iPvcJ zm$@tXH5?z%qvoI3$M3CYbY7X)#~_cJ_bf7>O5$brE90ZfyeXfG7HdXm9Lf}8_Nh(* z_;_|{6OAU+)#7}EP*2QWG|co^Tf;%RNHI@__*z@H{z$vwA&Fv-@Pnc+>vB2D-x@>$ z%1v+iga^qndQD0U5rD;}IRw%DXh9%_nvHlVzy<*HY(Zu`57WL_rc99Npw`hsbh_5l zE(G$_v*?)E$juQVWlA819ceM%)oU9!>uwDGlLk+p!Ih)q=48`g=|PBOE**^3PIODq z%?!0|z;&A*`Q|x&&TEW|!$)m{vX^<{?-%U>_fK)UXSYqxnOJ;qsVG6St$`Z$IPc_d zh%;17iUsC~-+V$)?P6fi#_rhN?Vij?-XMj4MI}qZ)T2tjYw@zSRChNbUmmNe<0Iz zO1{I&QD?qQMAIYK-%2-f>;ybofUM{xo44Bc{a%kEnInk-r?jw%R|hQJq}i+L% zOufG?Xu?$_o{p5VB3WY_PLfDJM;hKYMWJ)|vY=_QcbD-JfuP30C*ziAh)K0oF<(;A zw}#5}DM0OTdF7(&ZeW4l@lwsiHe`6MUPrG;@9_d;l# zqiEl#mt^B@(2&4v-X+Td0v!!PzC+T$8I67|%hsB-G>0}PiuJ_S(VN;>^ciE!vyWtw`5%cQ03Ratt$`B)?+sw^xH-a42V6 z+PL|F&-(2gC6YyfRBUu4z&HfQv#98iNVZtM8rC2%Q@l{5tCFr}q$ydtu3Qh>I(|jJ ziqZDmK_Y1w&~3-TJCq?E1_+2}a|jSfP?p+B`c(y4cdfe9Tb&0A+aB^!`wB6UH;Y~1 z;_&jNLhd(iFQ?8HiZGz zN>ehK(z6h=!a!Zm;y}9=qnrZgNkfTi=FhB;+k(=}Ky&`w66hTILQ^tjFG6nYzD$*U zL%&JZZ^)ODh^Aud1_w@8Yg%+qYaA)xWTnRQEkqoI0w>tvT>tj_2M{<&hD1)gBIr7`n%;|TM$Yp`mC@MF$d8-4#Fu2G$910tkeuDo z&;Fieq($zed@)Omnqy_;a$4!}Yg`9en&Ib>=Z-TO{a|*~rZn2Q_T! ze*(EpM11I2z)?k$+k*(10XP+eiX1c7lg_BXr$;u*tY6*s>X5~qWp_Q*)`H9L_);72-^tP0F5kShG3^KAPB4RNJNPt(VG7krrFd zr>iOSh~d(dUuKoQgIDjb*a;{+>Y~#NkIi|03tSI)INlRu^|af}*kTN~^7zD}i01+xkWu|}{2C!+aM_-e%BiZC z)`Z;^;09V)dTo1TNNTw933Z+xxS%_7%hF+;`~8lhOk@~IM?KuTju1TpRRl7@i}K5Cot1BNry z?nxfRnLdwoJtdO8I{e!${0|$11>ixhs~q6fpsX9E704Y}`6|$i5zNE$ZAgCh1K_`+j zM&TF0iV z;+B!aGzaVOfXz=DoQq2E>lhD=%jowAW30TU0G1u1U5&NyexjGY?% zBVZ2ups-U28#ZSp^KhKbh{;;VL#9~BU&^TJ-|1o|bJlDd17SZsV=kj0PBoGwdOGY* z5nI7=1z(rP2FE%Si-F+-+ua}&k*0@?wKA!qL%#WoOtB0=w$Uf63r(AwQE1_Zac52P zwQC+XWS+&_D3`GSYd<;|A=!fN{Bcaq3D0AG*Om$45{%3Yx){WS36;q_;~Ht-la_g8 zx&x)`>=ts)=Bzi}EUGI71#jsG6ih-P3xIU!Kv$`kmzZFh*7!&Mh~?TTyguE!K%m=} zy;IWB{|>eYU=m8&Bv3a#fX)5}2x0jI5bJ(N|I_bI#fHHeHo)A6mCoOfxPlEXo3f@; z!ZtUTpS~|~)rn4_GCSc8pnu00iGEy6@N%4}IhIh=>Jn|%&S5{- zIjrgCw_j9oUMd3%obL1Olskzj2GJD3T*TdI+2|J7{DmwWV8d7v{}U`}iH4b3oUnnt zX3C@YwCqs=)*v#mRc!E445=j2=c)p**qv}et8aY8|ER&p|ER&SV{Jgm6w-S%CPq;G`+wBHuYz&U^*VRT zg&e5E9RcmEF%0Q`n2WGh=>(8&^Aq2apkZ&5pW(b?Cm9T;V!&F7yzU72vA&vCc})Kg zcwHRVVmodg9TJ;K?~Ea`O)D91oENn7jmbhLN^tGiw$;U87uX|Nf~&U8xI zCd3I$bL!_CcStLVa%%3^OPGwRTNxGi0A_C0^>Yua>PBYO^>r`1>c&G}jH+8$-hb^FpK@w$<)qvnw{mJOVNsvv{dwi&>@D;EZu$R) z1JByXcV$%FI{z0iuWI_=;s0L6y@MHLE$v1>q64EE2K!^jtT$1lO$YWKW)QJ(CD$kv z1l)QhWb_5HAo;O|2fX1mv6Z4Tm`IfYYzQN$-}6_!OYxi8Gt7XQy;Fc8e!l2AcZYqI zaZ)!5%e^8=w!&xvBq!2TCd^RoZ4Yy31~m>E57G?BB8`ex9ferQBD?Zad*B5bKLTM- zEPADvN{>0^SRkFE93z8}mEuG=%u>%h72V-bgb7-Hh^rxoBT~$2SVaCdwV1fRFIgW& zYm4riD_2IC|x9(;J|K}GloZ$hTp7Th!4eWY!XJJlsF7kGePhxxBy_2RjR9T zE0Kd(iTfBB?D7mRy^H4_h=%eGuK%3zIFFPK(DPmYj8_v=e-Y%c6m?*4O0SI+R6fBS z(MxJj1nFY*&L*9X0R87nbOm$UPd}?K+HIbQ*_n0iepVLnAetW->v@0}YfX zhrkzF^dU&8CiGXUbuh}<&vFgmsu>w_DHav-%-Tyr4N=kr^qMaJ_buMEp0U-EUWAC% zvdmIV&S-38x(cBIyD#2WP}~Tt@M-J;l*ikr_fIsOoU+UFuy%_kx!P0wrrbLi)ZJ#m zqsAi~Rb=%M?O~V8O@l+YAwe{?aOIu4x!G`(YQ*iyg==nUK89;idwXnjv4c(uJKC-` zG%Wn3nW%iuPqf)aF!@UQL#%Gu`=OJ@bmP`r3|3uV{m=`7s_#OLO#Oe-fDFYs?uL|U zJbWwI%)MJt9MrI^uL9`f=9Ss7D{$r?pa17c{(T05W)eSdsE3Mc5DcV8RR@D}mZ{hj z6qaU7Na)?>`rGarmmRctbV_>qpXu?aUtI<2R2q2(p`x`_xC11v)-4z%W!JvTTd%&I ziQ36EYS%&I$eETt2etGlY5@T)Q>ISx@^1yWqh8jbrFWgx#ajdFXW~-PzESAIBWcgb zE!?HMz|gh(xbCjDsx%3YOR9TJ^~boZH{f4v;6^b+g+%gzVm#s<2D7+>(3?UpBUwc$ zs}Q4tSZq>IY83U*WM439*q}wAAsbm&I6Qe7mT9vXFg+}o^K~HzDIP!oOf2`(8B+`X zK%r>H3LGIV9UOlaXgufg9;>lczU5Q;r^1mouyoH~F$^w-XVL-8R^31Q^|S;(WGq;~ zuKXd|2Bo*{L$wUnhLDMa=%Jn*a~vB==uW6r)OD<<*K$9pTEyRdcLslOFdYpn8e0GW zfFvMXXFDQY)qI0mAR)Vpuz4jlqTJ%p0=Bj8oy%u#N)26UMu8SxTV$9Z%c7d44?S?dh3=w{|;&{?{)!v3>ZD{Q#VU?C=a5REUsx9XsytHFr^fd24FT zx&pCuaY!=@x*#K&FTczD=)OkQ9>WB3-0Ck{o**2UxUkP+`m#CgWA6o-J?XnG7@u0U5ptofO+SmCSHl^SXP#x6wO^E z)hK-%;QkSbg_qbb>ifNsDF6ayVsvVc{xTcFA-8-gC{3@=aVfQ#KQ_LrCZ%QMU zWZ;KQL=FODe}Ai|`Onw7&3&(z`ejz*k!mF@-;<;?>Q&O~R^`-LO-^M=NoZ^7YR8H! z32hRP7M`fyPkK_|Z9$%nf0`-QY5~e*=A1NN%1O*rD2Pbe!9?5Uy3(rFUuUJ$nR?Aq ziGSbt$EP3PQ-6H^n~Bsc*AF;ORf21URHg>4on+OS6a>YaN8<5<<$*|7K2%xgnAyn0 z_yjqu0z53ny^K_OC{aB5{vY8V$}+5`ZMsUIz~=qqeD?k4UV9x4WLsu*3|!OM32~kd z$}V$|7e((D${+{u-g(3>+H1+njCl6=zPHIBqYweZ&@!+TsIXo=Yp&C-4JDT>5i*lT zg50voEV9x7odeqpeKg-2AB9eRcX7@Vc{9hC)Hz`Xj@^mW-1-eHr}+_pHoqsalg&8v zA69BjGOs4u8$`)nJEldK6aY-b;9cDgW}ki2?jM&i-@>jSNN_ranL0feqoTW+;f%n0 zOAN}b6>ju9m?3^H-ydmdUBUTrN+VP^YP|6<4tFmNU!L^$;c0-f`!|v%emy z%S-O43SKq||6q#&$)HOGY`F%4Hmn?OV7y!u*6q0~yq2ZPs+Cyea=7H~zn0ibOhm=l zhk>&zqZQpISYR_0YNP*shxq$Q#pzu4qMwSrBbb}~)lj5lydN!b8RX;eegRK(RIn)H za_u+}#L?dW1bsWE+rmBc=0wBCl>J1K*bYpI(njVYRF=8`DlI#l+XyG=X+RAfcPAB; z`Of2-mLjIPLV}WFAFxqYY6&;=(ny*gtmZoSdkK4*rF6Bvey8A|GQB_}ZozkI{38~u zAd-?2@D*N1Jb!9L1inJx6|_*R(M`xx^x>9ZBh)_L`2KLlhJUd6{g&qT+#Js}A) z%bi;N9081T^3svQzBFQ`gp||3J(Px3n*Rq>5*Ye(4mhQRt0QX3RrD4|q-VW2fWSzYVOVZynLWhVNo7E~Ydf~xi+Yk58- z6CIzy5|^ZH^uYsJeO}6y@~rDgC5dI3GIDf!u!(6uLM(2R2JF)hdc4e0|cW(23bY78J~ z*FilGH|7ct-AOBzSO~dXtS+Hr&*iWu+CDEfhHZmdYxVe z0p0%!Hqm42BI!ibEOFI#G3?LGemQ!F@g|ktKsxKj@dZ%rHHrjfQd)?W+*)2dBl8|*|t89+ZL^_k>t^rp6{+5v$**T>bQ*)9gxh-L> zRCD*tlMXY(g2A_Jpza@%#N;saBj|_6JVhfue8Uzw+jiGk+`Th?58k23Y#}%PP@M*swhfWi12~^Bh=Qgc&v~&e7fX^AKcn?a^;y@H!Tj_{ zT?fKNr`%jNnfNn{Q<^uHqP138Vr_qyb^-x!ZKkD^ODn5xn)8g+O?&gfX~s+K=&kyY zdWXA17k#h>Xu|M2#unmzIo^^$Vr7#;?mkSH8Mbxtgr#6g+7GI^4Op4tb`?|OX?2br zj@J92gZ7LR;3+0CuTWGd&4vJl!-UJk1}yPE9S7Y-CuL#tA=w7NI5Un}erh$l`4WOf z!*S1=E#pkIZMP1Yk0w$bd*WL+@=(>N)MnKY<=0{D;aiS}fGO6ih17|ox<75LfLh31 z_NiER$eNB9-4?@4@2u35D5ziL=JWtFYRxQvyh;%?H`)Gv$7@-}9fJzld<#QbB7`)% zJCpgOn%_ju8lURF$_Spmik9J)_BE#0F&tY{p-;}|)cwDjZUU_UP-=_8)8m>tmtF7M zX)v&kxtaoQzsrz$nR?R}Eo4&X)o%cEB!N?}YPC%fQd(?`K~`dFG8#Wt3fsZ+wNe@> zVjIHf^3*iW6`n%PRcP0%O9K?rIm?M=+1%EkePIS9N(gyPJ&Bo+7Icn6aKD=slC*$0 zFr``CfJx|~S5WGKFLwBdyw@buY>E_L6_euHJZ>gsafodLVGJ`BBW z;U+oz#$99QcUXG9NDVsRmUI{pm~|>MnDEKf$7G>ePedT90s@AK((IqkWt^n3 zA9pgjaqh~QzwX{6Duf@i)yGtm%iQJr*I;t+E;L?NwSoq`qc5kk zRvyAOgL9UewzwJocC+l@4tte<3Wk(7dJ=S0D&{4mqwc4Z;Q3 zNPAYSQ|UWgv;x}r&>KF{9CzvfV#X~swFCODnyL4}JDHtaj{K=kiqKu^4eB<;`*psM zO@2S#@q|k-*5o=hrDO&-u-5fMwyWKnf-R@N`{Bkys-@`ROKX#bMCde>+~~0bxK2QT zj~M#_U}mZkh3`S;I`%n@P*IVdq4Aw}gGswdEAlu-_TFQ#w}UtPg&$$38$K_}Yd(6+ z?wlWVZS+dS;{sxT=nWFGJLNjNECY*0U6<-|e}vNJGBr!|h`Qaa$RAu=nXrJ8CV6j! zm>LCI+jOu;z4vT!omGJcW1EL$uEt92|9!5RE3r+K*hgRO{~P3Af|c2cf-2{=?S&Bv zL`ZCUd|4lC`!`FI)fitBJXWEdGDFgm`ses#e*v{8nwg zNgDAHs8IGI>Ai=^cozeDZ>B@Dt)4lwmyW)E5)fz>D5`Or(TMrNdf|U!)Kl~B-T1ES zmP_JiJO@hR-{(u>pZk*e2FQH&um8$^_Lsn4%j4hEo*DDPYlu{>UEDiV)-GKj5I4oT za>2A?eEuL7=fK&b3xh9sbK-8&h0%o%Xp|47MHdDiYm^JeM;8V^Vw4MqM;CV08|T1r z(}mH=I@c@_&W_Bld`w)-FRVyTWTeFGm@6@Po;^`8Q!O3oC@`#QeGJi1S$eYRE*W;=ba&aPC5_X)o`+qg$Oq=8S^NlqzqbuClZeO1& zHQ!0s34g8RYmxIFt${GQYBMv94y;(63NIB#2a#6zbI6NY&$bMQam(``c5f_l?^>#|1J?Yxqfx$f94~MG`Q6|SNeV(#D==mEFgK=@8cSbo%!eRht_AS z;5i%RmcVEP@tB(wl)R=MR%g@K%Z;xDQRQXKX8B7n&!4H2i`{k<@vD+@%hq!(x3+6C z07O8$zsoBzNuU+^uGx=m7gi2L&7na6t9q3maP6%$cmPX!-7W^=uDF0P?@;%!{mFTC z3}h50H4HX0FH5ToVfCY7l%=Wd1;W}YZUCDg;`9gMXmPF45WL4xpy7bQ-pQOa12;R2 zd6BQtsHiIJHavzkZ;_*6ySq)jc$vu>C0N(y8Ng{d_|fh4dHn_Dlu;fAW=9)ontEJS zau$OA!Of78A7@2Bhp6!UT(eq<=(8AFrY5l?dl6UX#nmbPi-hzuKw++)2e^X2v&bQ-xI5@HnvBv0XF2-TF~>f+;FDoU0? zke;y<6Mkzt@^4^H?-zP+Zd4U(wJ7vs|G#^eS97kf3zyeEvnPE2w>>`Q4ZGD&JaB!N z=syNY#&@Y&q|u9Zf9HDpjqNs9y%Y%VFTN72<=0c%mevK0uhK)`jUwo(FS*%+@pX?ZT1LlTQF>Pg!j5*2t);kr52-ZNR!$MM?rJgIxiPt&YgZ^X!ri-(b$p7^!dyMEIZ*1NYnQi`8}Uq4UJUmmNl zUUzW4O?krCiw_6djY8%wGZs_RyBk)rOXjGHmbEX^Q&oZW^c%@sJrb6}Ocd&8IA$(| z%1%^AQ^xSKWBZtS?aL%XtH)D(g#svn5pT;+#1O4U8%ZRq_~s688SSjduP=`LKg0eg zd=n+O5u9epqSo21RdPQ>h!h49D1cMxA`Q+I&iyR6!g#mG)HG7v7p^8_5YnY^3&Q!1 z>mAe#X5Gh<3*2GzsJu^vnPzSv=U^yI#2@@ls2PXZ5VMn+`Geq=^X%RQ zSa{p%&xAC>i}Q;)adCd0+@FuJJzV>^KdryF+PA&Rw>|5-KIgl>)IWFE@3kYs>}5V9 zw6Kx|wyTb9Dw>5u!oM1g5jlvpS*k66e1caleYU~|3|b-@InM1Lg{I9K0-z8RrxvUv z=njG=nZ%iLpGYF6{Sx7^<)4n(W`;FP4S;`wBQx|Ky7~Vfw(YS;7yz&V@Y%L)+qP}n zwr$(CZQH*0Y}@v-Wm&dlpZXvAwBR(RZ_!uZciu5q+l|d1hD2Y`cQN-ZZr+-}WUae6 zz&;6U{iUOA$_uq{EKVL36C2gN=be$(4k=vA8?x@>Dp8%j$SP(wD+38mU(z4A*1o=v zh_8HqWqdq(7Nix8rpR$)} zVh*u12DHw7hw=s6^Y>sL8!IBiwL%;{F~is22Un|Icb+u8E5j0Cw2z?tKQ$@G`JqCpx+yG9!uIL5vKDj`q)9 zV)oWUwoshxm>Gy_RX^-H={v#czj*%(LIz2jM7)jotM|m8mZU18Hq=Gf0o&MpvGy1i z-iC*+N`3)FzMvLcj){H`Na?rrzX+0d_8NbpYJZi;_UG@Y9TfMJ<|J{XuR2Leg`W$R zu@FpV#8jfc0k^}DRWu1FC@8(z-Zy|!+iaSGZew|zr6rbHR!VM35j6yz=iw2cW6_3X zNn7sjUp&^LODAp`*e2hKR%Oz)lbB-Sq97GJt=RDmz#4g28{7I!zKQc}t@MP8X5jeZ zfQ$Odk^sbTWv>BgXQYZOIhw=$vt%Tw^i`gU4 z*i4`Qv(cynlgUR3whkI1)9YpK@yb2IaNOu1ngI?al@@cQ$PbE2IreE?CDBxFG=j?q z5w^#>w19^a;mDHCW7d3`GvE3QxEqRVC2HSvDnHtW2|P*AUn|Dr5en1n>ofLDT>O2{ zZ0khY zNOYCG2w9MkbfP`B{Y$7MswyabE-(QJzTz+POqn@6dy5#2<^# zkRiL8G}-x*A-mdb!<#hGO>yD`%f7_ftG_{KF{ivIkgmY4+b@KYoKyqu2 z&*&4;k&%ZQ^I9!)J*SYH(Yx+3b~75l-cc@+b$ir!|Ex9;n2J%B%F55CZoF104v{Bi8rpQxUQX_U^cFo(~I4iLj5Qlb|=->o!3 zF#Ze_bktM%IUF*e|B7lzlc3g!LCqHcGUc9WS&>Dz%d!KR^%Rma+q8Gu1WQBUuD}4x z-D2e@WHB!N&_}PmHcOBC2j>`27P~htPk_-HjLAkCVQseleIY5vs{AQwec1N_b+~!-0GPi0@}FU#CavV)R4I*fQ^6^M)PYg$I_~P%sS#?| z3?G!!i>+b&X;fQ*@lxEMbxhV720DH{JpB+0w zL6^3~WoE46YZSEK048o79Y!6lxrXD+;mkyMMSC44HBh||vL7j}>3OR9f%>XStuv=j z?d-%}rB**|A1FC_c_CS$&*#>@wE(SAf&iUSi2%LPWuV^3xB0D2$hl~rdS^4b$KBS% z+l)dcrxv32jq8ZF8Ku$rinx!4Z3OaL0G>H_wr%mb|-5EpiDU8qI#qP47`n(2N@ zmo&_N^+TU0eN&fHoTADV*5^bg1ia%AwRg-XGR8e)PL7pC{nt@CVl#%+FtG=6OvVVj zjjCx2$#oq?CX*3wA@x+~65RBrgtU8CNhI9VNCdzf>aNIeU?DNo2SE93CPYR+J88}* zbGXbd7J97FL5Zf5`%_bjLwW7Z3@#jxh2k&8%t$kL(gKi^r&?(Z6itbgJ;nS?8}xPV z#D#KA0qIH;4x=afEjGG(Ol<<_a(fJ<1a4^-B-Snxx&uJqGME{u8xP(fqHtGDCLo^s zi43^MH6u#yj$`Stn+6GhM`sVnG!zP{YOSZG(_a=&-M-1n6I^GxN5rM?7TJlLet=0c zd$>72ke{iwm?7k@W`}{f;zJ1Qag`W34AqVYk#iT}l7pUgHBmV_{rsH7Sc_RHSL4Q< zST_VVqn&0NVwD7`l3JBW+5z*$3h}U!Nxomp--FBRnLNK=V|kZ}<2%dTCRj30*v%mL zshCZN`=m;0AeVVOqx#6<0C*TkDj3i2)gyS|aCC}LC};7zuBIMW!fLp@FJs<2)p%rm zd`^oiC?XdNtt>EW^auU8LGe}ROYK{NuZ38ZoIviN*jF|)^(6A==iIe()(W^s`^9UX z6PnbW1GDsSo>Mw}GE*SulCJqK8$1o*;8=SFK*S6=;R`v#Qdwp6{8)@xnyW2nK+4*j z-Tb4F{Wc$e*Qdw*!~Ug*wx$gXXueH;*YjB5NxQ!5{d8?)^#kTMdV;rHa_eY%ASE?Z zWO)Q!_2xTSp0EtIyGp%;ot#IaV_MSa*l$z@f~=@yB{%5{n8=|{SriE=^P0N%T;7db zJbBmeujl-+Yjic~a5M*tZ#~a@<7YP5Isn3j=Ul7P(63>0gYN73CUvtnGR&4oQp{uD zv|}{`L=WU@51FgX0aybthvVzXFN$Ax+dE(qk0n%qfodM?YMAAi$?BEUPIYDmw-!Pc zs=J_zNL{-bU3a==Xy%vNcKnc8)+#=n22F?h z;A8bW{AORR9~u*`yLIAWKU5x~Gb9ZvmGp--h^6aACeW0eKe?3LJAEU=G~tv@UdkXx$U*UkFQH9n3&GwXKEnEKqE)oEH^ z$jN(IiNCr^XHgTo6h7^o4#Q<%F6^~AV^A_RZ?b{cP}h--TtxoPIX1m8PBGZ+N4@IS zG%=(D&%{7ENN?{`6QYK(nQT3B4eId!nkz&5u1%e}bb$HMn3isMPNww?%XMpCL+X)( z_S{(;kZL=YYF64{a#1bSV4HeSVqYxmzD!dI&rYp^iXc!`$NGL9%&tI2grg{GcDL4z zNyFd<_)&p4w#R@*YIJ*wGCy`%7K$e?i-EF$pM}s+LDPXeK)&%z=s zcXNI~?Y9in!iX0r!9+jnX4)p5!-#L0Hsc$}o|oSO_F8MJX0uycoCKvZyCfa;%9q?L z{w(2%}=Frn=V%foj#d#kw9iySzF zi=r!LM7A-BGm?Y_jqvqN;NvnRlcBUwjvMb+j2ll0I^|b}($Y6CM`?-Vm8G<}{~q&L z4#`Bon-9sn_uOGh9>DVZm9-C%GG9S@h^WB^tmoIR`vBD)4!*H!=B zic`nf1vlpu+s|_Cythb7p{@|^%1l-coCL#7J`q4{&`n2{un41aO=Di{uOu9s^oBRumf!2W zoW_fV|K!6xhjZ)x6|KJrpouaHXsQe%55ZbBCN2SCM3%ut9JpR!3$c$7%#61kbDZa9 zbP^OGvydHkRVD*F1Eq0v?TN^S!e&HjxZu;EaBfE1`SHqmeVGzb-{UOElB4_3CHJBW%8=Z%hJ!GZK1A`rwKq(XCG|U4n z*vVpTM3D(<_2iob&nn) zACfYXQlc6uYMM${X;vL@LSudsl{j?YS*R#VG9w1%aNx7xKqkUpow>5?$L~n^kG}&U zY^ia4BoW-)=8bNsEHVGAg@OC;e{+Y{=x9}YMDJgysU8eb zW5{-0sgYWEOD8+{nvJu=5*Hya%IviHVL@cIE$-B3UeI$D@cWHHpY!1B0-R3wTdW%% zx#1U(PRX|0{irp4$WwG$j~eKKpc{m{a8Zsb%9fK3_(pDgs7>Dw1&njc zN0u0_?7B9Lx$E$qOyw9nROW5ZwI9_I8vrEe=(6rkeuD1X$?ep$<1-xk*)m&;#gvN1ZG|Dw9_o zOVSFgy!Ud-1+C`KNOVx?&(YYqv+(njFwxcmPjOeqXp^B%=``u=3B*yB0oQfb_nbfBG`ceF*rrO9v`<|4Aga#$1J2pq`b{FIoXD2t zLv}yB9lBVjJ6!7OA<(6mvfPKRNF;Q7X zVN!_*I-0!D*~zaolS5+_6A#xU4)B^L4H`~mtYY~kIJsn8o}wzm=zGYnKUQEm5~wf)hkEC9G%3h`zSIyDat$g z;%H7kfqs1HJ2;~4bCtAnQT|l^WUhM&cj57|@qO`qk@``1fed4GWgmB+uC9x>#0Rm5 zxly$FA*cRltQXP;u2uWBbH#tvX*%{XPg99&i!r}UI-gl1R5=05kZKf(LYcFuJ;hoX za;oXerQ5~2*s}#oR}HuL)NHJ0zyrZZ%fe`5nf`+R5T?dro;P2M9N{pX6sZ0$a`Z^q zItq%ffGZ%#zStaW{-RT6C8wusa`WR{mL8B`*3Yl7(rlWl+hB9~9TL<+EK@aCPnS6< zMd#pq=GW+N?fZql{vOT!h+W;adRjh1&S%7xdt6>ige$8B`9jzg77*;kX~Y|kBBKA$ z>q!Y}_BIIf>!y8D%dCpIqfCi5>u;5W9gP(UUF|{kSA!Rp(#kIp0E9 zCcb-q&a_K4vaxTE^ouKHbzV;}ZGvaWLjk5vBU&D@@VQ7dKU%WfE`d2gfC=F;XJ-Ou zGd5zHI3Wcb%xkvQAX9|wWpM=O8$sPJ){xjm8d}5%BB*rz$3{^xqt6ZsCf3H1#CjrNz=^iTjAjNpwjF#VyMZ2x6(4}%XlkcvQ7Zc{fc>HS4Z>qWZ!Wzd zFz$3rd9NC6TgaNDW~`;+w!0}(yKO(!9{Z|X%C*}}Q0=Lu@A!?12=7I7OH;x1Ia97T z#dm|;QeoC`kepS`km*W$?P8kqvMJO&2VMTnP113KMd4s;ujTd@A*42)CJ+F5jYcz0 zYRlEzl&U1w!pqR!e)$?mzH-reAxb5dnUWqCwKlN;Qt-P`aLO0$Isr+(fH!!#X6-IT zaNn@>W>Opk`27jr#r;#*>kqstU-H4dC*cO+-q)xmr1C|aJqNVm_ZF7DjUkS%Z39c3 zGx>_UnxYiQtMSSA)e7-aM1rSNprbVG*-7m#LKfwv*xcxwFIPJ=?|@DxK{Ta$_d3(& zm~Qk^^YyFZi`AB5dId!R%vJ`&%0*(^1iTePpW&|t0sjO1$hE;$5Ap*T5u3PXU4=|D z>Hy4t*O&+lp(xEtuXe9ejYzeF0M{+ffExgAJszHN>M;s8Zc2N!WZJjFb&^zr?ON_v zTg~dnl$=aITw%0LB{&I#a>yd^=1I1YY-*{EGOZC(c zBKcu}5&yjN!_4;Hl;Lv22%~lVedw`|{A2d~ktH?i%#PA1p(!R&HKZ3duPs{b&BgDV#k(f(BJ(LJNx(HXX`q zHv_z6tzNM3Jv;QN?%O6NSIu0~339xn?k8gul#~*udcu$0=4U}KdzbJ2q6KBrge$deMQk!RhH&eiPLK2(@9w2^ z_UPG1)%Iq``;GRhcd2d5?@nK>-*D@S^{ejy;K-5B7^DR{*4q)ngsa2bIZGwAwVK z#Ef1Mr8SS;V{VKJW`Q-gt%CA+VI^s2>d71aEim)P9Bf2{8%IM?p_jjYo`HQjletE5 zi%-=bkhn$O)(tG8`x98k?vPCTup-ZMNpY^%jAI1C(bXcxw-^)S zYXPG$KO!@BRmE>u4x-yI=)Q08ignAtrJZF;AW60I3Tu3dwZ8(317jIY8IM^<+khiO z##W{13utn4SHK6kCm=7n!wtMV4sA>ihMro2UBLm8ha=UZNDC*m*fw(VVVX}LQ9y0Y ztS&~4NK-so0r+yq53Sjl&N?5Gbgc%3h8UFAa*k!}0$vTmE9a3LK;K?cJEy4-nwGDH ztqXfEFwp$($3uZNccTDuKky*OO**#u$R{4Z6Pv6O_wuXpgadF2M-CzAJDOo`|5Suo zTg!3*Vcpyp(}2KGc)xBD%|noWcCjpN=Dhw*wM?zEh+cY`an{mJv4KR%<@fVu)bdMs zU%IUVHFytFwwB*X!p$tYHD&F1A~3Y3Lr*>0lgJsK-K6YYinr0*#msAG+z(PhRa3Wl z4lS&9dRcHhsq2g(mkq?6F~Y>K-xeNg9hGJ!*d=Yp6ho{Pf+n;Y{A!KP`3E24@U^C3 za$gX*8LAyInMnNU?G!pGi<=_6^jP}uU3=EZfChS%V+;;f)+q8bQ0|yzI$<(@`gvhk z33o~mz}HXKUIRH=E!wwsm)M?T$fv`vFqWmXawZ1YRLaTE-0O7t$%ah?%RUdO+Hyf zO-MXtsdxROAX>9+@AAB{wZiLb8k!?iV=+(hIb{6PbKHA)R^aiy!dB#l>}rnT*%?E%P~M*L;b2qQpGr zX0OCt#%t{NIfj*Xncy*6zqdD(8BjmFRkE{b@)7Q?&sagpg0p=n%@B>J@G|OHwT#p2 z4IP`j9Qc6R-Pfwp(oFi?eNoqFAy#${thWd?r-pI?Q zL{XbzB*IsPM2zI1%Pfo&6?L`=7O>D&ZZ6~^;*w917kA(kBRoSD+8gpNgLh2yMCsrH z_5@ScC65|lpfWeYRsZZ*Dal{0W;;QZ485$JoZv;Tt}-EtBCjbVjTZy_mGp_1Jz6iw zp$@Z?w$sd&kx>eX%tex!zUHl%0Nu=H2YFC6nez-tvDSIVl14qBZ;^#n{A0PCy1esL zW?2)Trh3s~-t;JsVcbod63q5~nH)muGeiW|^Znm>Z0=)W{N1yS`sJI^rr%%BS&N%^ zGbOEb7gSfqeJxiDl-kCxEjqGA%^UG3lZd@3>9c4nN4XbnGW}oG>aHF_XnvwpCY|fK zV+swmZmRE_;3Sh6Dz;4g?b|~T$79sE2?1VT+npb{<^_5M>EVGBb0=so6N1q(D8@tF z2!-dcl#{CYkhV0)>2UuH#XP;H9>&W_6+6D8&I@)od7c6C<)nYd$j7ZGtThj(v?n`@ z^4j*{R^N~V^yOTc0R_QP+~^t@2n+hTY_fHXo|Gr#)DoEe8W+57VlXhaSw1sWn25%i zUhBhccsMIx^i6y^>cO?XpycC35(it%1MPHSIJ|cAKvE(HqxwVsI(FVu;G>3$%`W|y zf1W7F!RH<5X;rV{~YOp{|Z+N5df)=-rlf+(?pGG-qP3v15`)w4&a~kXZU@xB?k)iOhK&fMb*v4*huzv7mKhk0KEuTL17jCJ@PsA6*zkfIZ`V%aV<0egY&_z z7&zzBH1*Ig}DAb~=JSbPiOy&sZitoAdNNAZbJ;D`ecN6SO z*wKfO#82qfTwq)me5{fev8NOj=$M2cRbXFGOW6KqLJl=?EBpp7Jq~kKxw4_i6;bje zt2AvFS;zV%+FZXkXU|)$RR||YK%^;3c;;|AGx7wgPQ#}0ys&pmUURaSc0DDT6*?tC zXs|#iuaU4p*9+04*S|2p)jt9Gws)A50>_K1; z08XV>c;{}2rz`$Js8)UaDKzJKbZ(?Y#j_2LxCfLY<>@1f$nvm0MMui2=h7J_wIPv> zk)w=I{Vz&PgZc4=%;YifgB0*VJ7Nm13n)TSQS(im?oo)M^>WS+KU-Pia--nn_o*$4 zJTr8`WNq3do&H_7>aR;X^l2+#N4R$u7rqA4PgrgL%|blEX1l#RQk_!Tr6(?MBD-r^ z5Eb+oiJ&#O?aX)b%CM#}iFsHm2z+ zK`uKLT2T8>Cc5Wm? z7`j7}MRde{tUf~2w9ZwYK(9=FBuXO_lpS`QT8Fa_&UL~lzycDEi)9W@M0a(%7mdbV zS^i>>)g1&o#cw4A5ephVbc^axALyjUA!lN`MMB9b^fxc%R_fOkbGI}Ni#+}Mz<4Ij zunQ2GrjKl=Ea%!A0fz>Jpq(zBEacik?kuXqHM2)L6ir~de3wkti>+diZw zlLU_2n2$t7@eiC*(wZo%DEavI@z_CfQ`p%BCeGkQ=rBI+5xbNS)Y6)VD;+zt9DwV~ zRblCPM>vP&E#8$o^#%dV)|dHOG6vF7HPF8(dKEIn%q*f%rzHWX?+U2~iO4{4<{Y`sUl5K7|N3^J{us3=J zcb_7>3U3L6iZq>1C&E

zL-2NtLjXghod11&DroMteegZmrVzHw2|OrkX#=* z_4iR%Fm!#ZrnV8ugG+z%Ep|)Ntp@%JZC`>XYM&{DA#oS ztYfTCm8}uLJq9JR>eu3ShM=(=UzB<>x@7|Qp>hBsHv;8_6s$!!D!~s4vzTG2T z>!sSY0G-JFp5N>Ng?I#EU^Hi4*F-Y-Bk_Tl)CA^rAgwNIzPEEh<6l?V=Yx4K^L?}Y z+r;1PXlu1NQ9RY*F2LQS8Y<8pEqIU_mpYr~5=z@a#gy-uW9u7$+q|O%zn(akmG!ut zQpuRi?Q{ZLEzpbiZ4elE9ljla3>yBjZA?L=uhwHJyD4w=wjO{7aR3xSkeLo;5v1lB zfrwnPp&eCL8i-b|UrbDl;7BBy zWOJHB5R&7W1xS(uF`9d2whKN$7WlW}cZldFhCs?~!2>vwdvk{O=8i)&|B4*Fz-)D| zZNbMI!CNc=w%{v9_-@1hmUYcAH2;bm`_61-w0SPj3n5T!AyxqTvL<^;jv0v8l5clg z02xcf9m9hXP1hpq3Nzn+)IHyagDi8C=i!XSCu||weFsM1*p{CdwYB~LwiwrzAAast ztnNU=rPN}#HlzSmKq`s^Ye3~%L@t^XH>^>$m{OFQVlh)H5oKHm-=06dM1hCqWI>xq z&m3hJ*obz0bH`nUE;2J0DqiaN++k(@#nwnxmYD&Pfhdk;nQsMcQhLi<*c;NrGmha1 zMIAB-;)~dzEE>XI6_*Vj(#&AoX4kBtV}E+b;jcD&JbeBSl86H#x=JG3CS?`wC;<=S-3M+&#D{~@L_b_t;wN0>>xEaG5CJJm4}5NPxGm;4b=N%*NSxxYrY9a1K;`Odvj z2Ew=g&Ag-pf%lR^lD(#7&Vbd+KQ3hU#+MLFw!D?5o@ZqA=}vDo6{h_@32g8QvM7Q> zfYiF(OvSN`rLz4*@`FqB15vZ@_T#<&9O0zkqD2{0Y(F&Myh;f2cqu*9(0hAh6 z65TOb!f{v%c94{M&>yR(xDlKc1p;G%wZor~8e4>S28gYwfApkM1jW{z>bm1dO{Y7d zN95F#v}2bCaAR19zDY%1$!)laJgVNTMP^l}d1RYd-eS<4K>!eJ&oggKq*}xw=wM;> zwX4fRZ8I(6O4JGP0AEjE2ac`^6)d1~gJO6LWtXNV(~B+*gNUa^){2FUOsV3^qMp$1 z8A_Fw0BJZB4}tdV*mQ!|>K$b0PM~%soyB}Zx~fZ%@U^oB0g#t@`mrGt_GqF7?X7k6 z+3QM5c@-?XOT%=M$t}1;3>aBlRQ4)Z_xzWddhX#Ea05nv?yZqfY%filXAXEx@J`5d zFpi2CQnz{5Al2^exCKDAP0aIA9=sMXt;Xp{fmld5i3xZWEF~~HRVF=Whe2o>bd40$ zKt@NsV?~HeSLSN@{2eSpR(R#0w}~r!9LlzWJEXLZ>&W6t!L=GV)#$`V^&y4#p>3pr z2iJ0TnuunlfB)7i8qH$btbRkxQ*TFW3ZhtnV*ypOuA;LoKuU;JdvfXfNrJSg!5T7Y z%(0gQaLWE-Y!%&=C~7FfW56JEt81|`)fNz-ceq6S7F4IZ*CyD3O2j*Nfx}|QFm?a0 zMnk!@56&JQ_I*aVa8pA21=nB5qHL{6tCq2ogN=i;oXs>=!5u#+6{1c^hI#IQ)73uFzqv`dFdy-#$rY+8pl0iwyMSX;A1A9R|qDA~-+QqUS`WTg>P*1ca%g?j6^|9oPMYM^%O)!&3M417fM7;BS_#1v;!&+WbnN@@4gyiFub6?Z75DiN({sG z3S5P%L=J?2-iw3;p{F&#Z58+o)rh)RLET|M4bX_YVcnrXjnMY!FeetU0jd^L$veir zYCNFM3q*k`F<>=MCx1nF*kBc2i^Y$CYBz(5;tVT@5-S)iF=<*uS& zavtl2W^x|t)|x~|`FNlg7T^;LRwXaqAthfUIn-_?>^6!ZE5OY^z)ciE7l_qLIn2W@K46z`F(Z^^DfHE*WO0oGEV0|lLYEensex`e+9uySl z4wt=>VGD=MhXJNyK~p|!flTn=Ngl9@C3ENs7N|EmGzrViqyAdnKKh0dMFB}TAWE@@ zNFd3_{dgjaFu>7{E1+(k=vm}0-+&$FI5hGO8?5tu`Vx8L!AG6$3$qp36flF%@b9?|S*%cA+8Y>?qaf=6JK0#R^&nLJQ!x?_L_MP0Y^_f(MhDIaNdomFSHqmurPmr+e;LdMbufE zO&R2vcSH7Rpn^WDF7-Z_&hup~LO?X_z)J#t_*$7?6}`19-*`tUJ<#nYkVot?vEi%2 zeTJnf2z&=dHt`dmDMiIK3yme&)?H&KiW$J>ek~997=PH}OK!^&qL|BUTnpih8p?@g zV%t!a8E?bB9Ot@@A&&}_=-THj?%T{i3`C&EyaXn0t0?V5B|1VUaVF=_jV8 zEH6M23qzZguKnmnyV4o#TV-Az0(KOI?eiwSCHpc-7i?}GzB~{LytNjVNHJ)SWo;=k z34tVV2~;Axzy-B&j&oYxi*NW8oa#8zlCTCyVeUJ_vR5l6*qr}WbT~gMy5!wZ6d+-1 z(51WOubihgK>~r38W-qmXI8coGH>u0!5Y1W# zBx@!r4xZxvd`CUb=&UY8Ll6h;i5K(IA9yeSvm1w{hsC8Y(#&)gWS(JR(Dfl#4$E!M znuxCDgt{u$#tn>f8eGA0SLBU|^77b-KW14hX<^zt_K zX3sD-eCN3h%m{23U(K6eHHy!Y6K?I@t}{nNwz9?qW}%m+LJ{a4xDv}oWplLbZqM*D z+fX~Fc_1}(2NjLX4iYfCP2Qd`c6B+JNtZ#!!ukvH+|bW8@9&U2^|+e#wj0OKA$-0c z7qI>d_Tf1{BgKdJ3p{n$#V@VQE&iFP<$0X3y<3R~*ZrtVkQ;i2JQ!S9PxGCBWOPw- zpcLRm6O#-=WG;(NLz3?qD+v z+@ZAj!H!XJf35jwVEIrOU#Ti7nmSML=yaaK7o?Ac62e6f;G%keZieOYRameTxKI&E zXX#X&TKG+ZHy+ zi|1|7*^uPXc3XTltzK-gG%NAi0@^-fFi2XUsbug%*uv#**F!@V(r^m>P@>_zzzj{~ z2U4L0h4wUZ#0;_VTB)&=mV(jKV~vX1(VR=Y5DPTqW?@Fcbs-w#th3#R24ZEiaa90< z^%SxCV37k7y`XLx7K_e0jRA>>Lt&LoZshW4skjK4ytfjK&WdHgB;Zo|w=KSTD_ z-kqHAj*WuE+qX3}-|9pAs@DKsjYLV^3_G%0Fw&9^O9pzI%a}KPN_?g)SA{f`s@J72 zQ;*%Rp5xjP_w+cviSSU9yWv>lL7H+}Vx$rMSD17({>txwxR?R#? zdKvTV1J-NHrl>EGlR9>cFXJRzVRnS^0uNL5vnK`GZPNIjx-D$;vny?R89-VV_@x-;jE&cY=$U@KiI?Dh+cY63XzLE zlY^7SFN^oj&TW*$@$u{7>@hQwx;(?BCy7nsG6FVXb2Rj)Kp<-s6)BfZVG6!UDXCn+ zOk&|Lp^~GN6Wd8)flBwqNcR3?~YQ4(;)1 z299g1fF0(2m+lB(gzfom#!2F|-y1uDr?ezQ^yiTK9;XXY!}k(woNVJqM=nFZsH!e{JO?%S$%a(*dh$4uj<}S0k zFMVob&Ui!nG!c-8E&>uf6pDBS!3rl*r!s9#8*+oKcmk;vG$x}r#`;e#;#aI>_SqR6?g`L4{2=~dT}ji9xEEFgp`e*-y?R$OuHKSX3J4Y}5= z zswaMtGpy2e#56@9kD?3L+Ha!{WuzyW`UF&4*X=otWhYTP>IDQ_+vEbXPNf;3(P`v0 zK1k0T2=J6li9`n*^@?@`heoj|n9;1N#>c!K{$F|SY3$#P@)UohOw3^V81IPgH~>&L za$?p%blL1zFa%K@uC67cQPB14jn$)D zt*`TE`oD(1C~@5e?YeB}=7tACe-G1n7Z+^J)|s6AS_XbvFWIJ5a6k5t`f!29Z=0-J z)D30|`!FH*(^(u7CR)oTw#*z4=mr5h=Q&`3_w}|GqGg| zaP*;(D>mxkZp!oVCAsr?pWJ5IWb>*;iH0_jdr7Wlj|OxQ!cP8LDeai_9RhQk$0#;h zHl_aovPYnBf$wzPk%aF@YBB%tA;nRo{F(>ucLjv=pcJnC)mX5x(^P=94)eb(wc@uc zaLkyu^GG9W89{{5Qq`5fRx@swrxzuD<6K#~1=SvDEGRvdOj%q+b=uwb!p*YvG9*cC zR@1d+p1rDXxqA*u(IGOrO@Q>fD#SH(#BVh_UA)WrR`99}02cX$Fm&B48^ERH)r_G?hy*mf6Xu`_Dia;>!$ z@v9zFat6(Rer2A~h)#a5pMV(u+V6p3_RZXcfbpI!oK)mK9 zkH(8&Zxvfxn!T>h`5CiB-inQYVFaaYH*fs?D06@V2(m#2?IexQA!xTAwM+=}z|DO% z262bAo6lpT5lXpjXi8^J%ha{4c@A1su>gQTf4_D%0YEVb*-e>x)!>j60ib~FV`_X& zDGC#bSvH*R8dj6aNH*bT>v+;M(E@MNd(aY6>yK5(itkC^{pw0jI1$bxN3M zpO7;4H6xKhahPUwptwP-S;gfAG#cI)X@-@J9je$-+`ZV0tuv4RgYeC{EPz);XCd-9 zaaOFOtb`wJ%kFHwLT-%Qcz)&o0R`q2*#d}^o+it%#a~Yj|FoZ{RS7J+Ge*lp;bol8-|2#PsHnXN zy=vz9UOnO)OxGaRp*3YRxSRq>VOc>g;NkxX-?C?m+tbGgyJNQN2gG~#BhYJvU`slg zA`@UEdZa(d1;w!92OM+GZO|S(M!-srE~0{Y{2QXZ8x$7!O4e~XrqI0TcmE~m3QbB?hd~L_ZTz{hTnDq<*@}k(pXJn#8ilq~%c*zp z?^UTUx%4I6=N!GO(WqjNk8-Wct5Wr?3%JpmQRj>JYu0?ajB-#9)J1U>s^`SHT87TT z>}&J+sVqC!d}n6%uZ9*P7Bb|Pg2AmXC#25ia0Mv7O zRNALc(c}cCPp&WPhXRyXQmrxx^=I5<5u&==YQ!@t3Fx|k`V`uwRS@EH*2n_1B#MXI z-|hFgb5&r0Tm+T-iUafKrri1QO)tWWRDN>rpnrbGzn{pT+@Jh7N(b?;|C9gt&ZLZe zgNC*xTYf$(gryLOvVd$GY&>xGkUa31glXSyLE7)t|GGOPQ?7&oAun$LNp6x?1CIR} zt#%1L;kbm{`*G#k(qCPYGRxq$<6q2D-ShFqMNgwk?}+&elJ~)8DZ3hMbD0rz{&hKc z8j4QD4et&d5(ssB&3mpjeg-;?VP$n)#L>BHf$V<^v+Wrc&?iQrL=euqt;s#(iYxICu4uDFM!Flg=`a0xi%wn@gX@ek!{w{4uhkrl%ZmJXD7ob+Td_T z4Mdy$9Pjy=r4YX?|9H>=o0Ji-<|bAQ;fOjCFp?lBrDA=8ol#xQ!Di;{V>2FdlfhLK zt)t-Z$$oURX<&;g&shC=;Ab3XS=FbEybNd{c(_hG!@Rxri(+PT)?>1SBH_w&L9^Y# zmi%1LZiK8#&tLr_51-lTVVJv4!>L3WtUv6OaVwUakvV_PN_B52MEEp`=j>9PhlQcs zvxhuxK+%ULARs3(kZ$lg44R2@X%E3JO!?wN$x0i?o2wAx`y%= zx}N(gbq~Xng-qGSa>)fRw^uUbNnE5B6q9Sd=ke&ZyeIbzw;fQgcl}r3QTHctuK$O_ zE#Om*)fuVPvYa82T){DalVgY@E!cB1BY%=Y7SB=Y6;NK=v9jK21T9m3QTG;ORuGpD z)`{o)LU*=}EHm>R{^$?=JP~{uj?wz;26zld18OtqO8*tMEg(>944Aj>(=20mEOQsZ z_OtMiJamaQhF$7qDlP@0Sm>H9GrH=%D}xpey|XtE!ZG#|_-}=-0DPjK4q*655Zyb` zI)pqPx+CUj>MpC&eC-UX3at!7)IYHl9}F5I(a^GElf&jTb-dA815f*{5zPx?HzES{ z`$7I7_C@plYlEd~_QCUhQxry%eNsHPxb)}9h}@g{d21xPWx#*ow9pVchc&2hhf*kF zX1m-P8H_6zpukfmN}kZnms0D}W&;e=cHY;*iCyBHd|yK#j3#Rg9mfy+uK(ibV?A0_ z$t^3b7pd6l59L)!7Br(G(HEs1WfSN;1RqC;Fe}VYAkHx}@&F%DZt!N2>p8chqo zRVnC(9rY`Wu&WyL@FrTTqiK`v?T$##v_R>7Fr+aYxi_0L(&FUN8GiLgQ;p60YCsrm z%;vJ}vfS)@pLgt2amMsrwXMmyAl8hUd)n{y&~zVLN2QV}aeVBoS0FaS`O-GDQo?k1LUS$72k9 zXbfGmEGGp^t3Z1Wm?Uuf3*Q<5`3A1&<5A-vkFfPe|0QoQZN3EhhGpKCgx#@XOLD0& zcN9oGRf?~~Cgj^++PE~(AYpY@**Y@X))827WP?0Eq&>hh;)m^4LyAB7>ip^X(;WrB z7pSlJU%~&?R(bp1%Vme1BoNtP*3`Huk%s%APvIHZ(Z4#a?fT@a^JnKzPxP9fo$A$l zmicG1r>7)PKe7SQtcx2x~E;K)ZrM{B;!n!H@%f~8XNNh9(E2M_A!@* zNmMIn!ql8xSGlR68%)E~lJ19fS>_*zEXapdktZLz1%m!a`5CnC>iLs~*sTqOaXXP4 z8=4ifANA81z@^Em+W`+2e6}|L?Pkr8vZ?0oX_Wb}`jO{*?2UeHpABFl%_SE-xt`5T zPxB}S3K@?)A_8RAT)X6nypVT1?Nw>GFqS>?TP*|s?)h)^E_5cUf|2(P6RUFloIKrU zfon!n)|W;yCSq|=P;KXVn$RnVMKnqr+$5^(q=wrD07GBBzj~qX&7W>=u596o9^@K` zx8p*wq$*xs zp!6J7f5Fz6GOZECSsTm0W;Q9YkPEut885eln&usV^mQ3ckmGt4oVUhXM9DIf)D%vC zLdta^tJRu3?SAhws}H+JRoIemn5<$5%UcsoWa~g0Mq?XS-%}15lE%hB%idZy_}<`# zKKZ-dhtqAnC#Uz`zf)@F@{SdEoZX+_%R)=MGtCZX$lcUAw6!?@WlUEPJ#M~v`OR~3k)}prs}x&S8LCEV zPH4`*!vZH~F*!3mXXX+8X39U*N~43y(eI4rc#fpp7yD@xR3B0K-sWXIeEq&b6k8OP6_=0t!`=xZKox7n+ z{5!BeMy~56`Z#RHsV`~*EgFy{V=(8IZ4cXMtQGND?DH+#UJr1y0Z!Jnu3@2gaPDz@ zZji@45`Wu+w?$IhVV?Fm^RuzMEP_)-?pqv(8qb}8_Y7l03}=etO2Zr}%#ETr(LP)V zoj>6c{Ehwn)`Xd${|L6%hR8l34GXTfe=}z;Qex*%0f?q#`at~AkwJ1OUtim2u-1Rd z@=R`R5gAtks8XGm3j)sdMJ_g&H(N@wVxq1$>%qS+X{oQW zE|*x>wl2lpC{9F`sy56P!Yi37NtLUzV6?06$zA8t=^f=6@MT*V8=5HEjNbaC&MojJ zb5E5Gr%F3g3If=Iol4%yYPf#t(Rbdnm}stSdTYI>P=?vXn{-UvhXNbj0P70%=Q$Im zHYpNK|FI6RMrb(B-S!^W!ogYHuHB+Zih=Z)e%!sq*VvI<6GVges$+BGO6q&F_X?j- zveCcj3jo!tP3I)dXL_RhX}zd@D&E8AwCoiXT@FGTPR1Uz5w>XQWe=HtdjpNHp~nvt zKsCYyTrUHX96#^v$T~>ta8ZTK>5BG8*0k%ZwO*=AvSQ%;#-Motu`%#esIHSOU!_U8 z##wKxObD5YpQT$u3t0(MsHPX9tEv6AHPLro|9F$X--s+O>FKShxT{~9+1U89sMs0)bYJk2MU+iHS<*}~ zE&o))F@)1*9ej8sysVIrqpHQ;_tIQIK*IW3Iutzamrom4w{b=Nr5zoS6MXVmkwVJy zbnoLc%Y5}Qk1Nm&ZP_&+$Ya*ryH0N}0&D(*DbAqyhG(?MwxKxuYg`FLvomqW3kkUu z`gemgBPQJT%|Ce_cZ(p5iiqZ>{+11v4Q2d#DpJ^XZKkBv)&dyt(0MI78 z4o&GcRawz2+Y(irddy{7}g^p*>C{#DbNE`evi2zgxokZglQDORenKs#n>DyPe; z%&W2?dx9=xn&sNh)h+6b+|vPZR}wN`AC_r+38rhVMy5GPq-hpR)@PTx2D}10dDzf;1}Q8cG}z9~t~L<+m`fE-n6-N|*0P}xFU}1n{gU1*hRa>( zWF^zQFSVp1maHfcTNcc`5QNE>zIgH|ND=%#nz^*_oJw-9DGF#mMDLOW?g0{wY_t z!Hu#^SMQYO#3vUU`lA%&)gP|Sv`&AbO$$<VXbSB@&+wn$7rbOaY81tSG>Q*(?rki{zF-&Q`?w%*Mn7P9QJ7g|azPBT zKg24&eSbM>5#PSQv}@4+l*^sz!xY1W@?oYwQc$hs)YmN;wPxie5iU4;x+CL8N|!;X zuRhK{gVo_cr7}QU-*rOFs zpi<-|!=fqabK5Ut5CZ{SJAu8$E^LOnx~6Oz$%i4HpqWy**Nl@0Q!^DSPv=%T#3l}~ z4F^-B(IStNcy?@{6eO3HvDxIN3UP8{LuB&w^2BLN4eml z5etmLk6aj91hDlPhe~Bdk_ezQ2u**Y&2tSj+#fqHL@CB|nxc?D5$9*qx!t(Vp zt`53>^!0VqV)}yRYx4TqN!XqjF=R)8X)Cm^9xIaZB@1*Dj_Um=1rK#1-C51CeM6t_ zdwUk-t@kS>iOe#rnh?ZZxx|fdAUjkyQ%OqAZ~qmPH>0MV;{lw%e|7Ql?W_4F{pDKT z&q_I~6r1rf#2F(RrA5YB#k3d81MWT2VQwnddJDg?5nLEJL`{DW>)4dC3~Mh1HBnsD znBS@_!@VvG#?E|8)A^^DB5K1)W2pl&#h|(d5FeTikWYh@W4KK?;I_Tj^*hbozHaP3G^vk%0lC2(ZnTICa?+Tm14ylKmECE?hvDc$-ZYjby)hh_B~s6?X3CP z`Gdm*2PLp(j-M3=I=33zP5$(I8=Hd@lb`ZANAo^PJnu+OGni7bUqXextMN#$pVkN{ zeHK+%S+Ag}geau~PCU$|Jane%hKtXJ8Vt*rR%F>*e&izU-}RPB6P|4fkmfhzgmFIR zP$^$4JxF1_&`lq7=iO>=|7NTHmd)_%vplW$|MPJiH;r;(0}*>Pqn!1`jWW#*O%{R0 zxO@*jH}VOjzt2+dkK&sLY-gN|x4#@x(`AV0SGF_xpeTma%1>v?X+bxvWJTQv&D6uw z|NJIr;<}{CZO|(+$p>G&d~krn*P;!!2QY~*b)Xntpr7(K{=|OU!7?08OOmTn(3Az- z&)9$vLtJy2LtEnBEx-mFq427hyK89pI47<5Qj=tD^e+ouI)W2+2eJ~ov|{^lE!iZd zQx-6FuMg9_E>$^TlS)e4#KJXuee&eV6Kurzo8SC~$bwLkvV?Ei4+a4vgnxPZbWZ-? z#oIR!{+O2v8#&`w^#cvSfp&X({QLDc@9LIT%{M`7blceo91c(O*I$1BrhsNHFCnc7HMbaq;n&S`~^@nqC zVjwK|jAZHL71kgC{CirMa9Zb7>6`ypKRPEc)BGKnl2Z&jWWHV zX5z%eazN+@=9M%VYr;~mMv6_>$_x2dK!LC$Ss3evRaTS{x^2S}&sAXC$!H-Y)UtqJ zuMdE#R|v6mt)#t-D~+Ad*K*Gp*OA&9^A+M!C3YZV|GcmP4e2#Zl-ic>UL-gqGnm$l zwg^3{%gUDTO_4gD^L=xYatMp~!Q{I#I6ZM8Psz@;me{ZRpm| z<{F&q@qB)aBt#ZOm9#K-UjH7F0U`|ct&+RBMlodVw$c-ARl#*;f?6*W@PTm@#jF*cpl+3CUfyW@j+a^t~r9PHH4>2Ux?lGX^5n$`yJxG*? zNWNKKETMa-%0)9+YlLffP}7ca4G)<-K3{XkgPS^{PCBNnNn_9K3pVLt|B`cv;y zim~)XAkuhqO`>?5B|&j=^1>zD*#M$TQL=x^AE*qEThc7ND_2h&SBXm; zQc<0SWv=lJ5;TDnZe>*%pO>xvlE^~q++8`*|E`xY_{yp5Lrrms^EPzY{uL97gIv<8xTB9@Xn)!lMnXJ{J$Gc@zk=Se&bt_`|ps1W|?Z&!?0vOETe0Xpu#RJrg4pgc1Iv|1Eq>Nwj*Za zohXWIBqpB`K{moU~s z0muXRW1wyeOjsrrbX){_+-%Z^riI~?38HR9f&|QSU$Hio_u`%w>BUuV{#a8atDpCU zQu{N}ztaV@#m5|mfWvFoJN8q0!MY4=rVMu5lNCgXPA^x;d`Hv(7u&C4gwz>vaL<@5z!(t|^*eD{G zXN|bLqr0aAy1N}C+)~6MOsM!>bL^JFbEa~QEOp`Ph*bD`)DBp<*0{5eu-rHUH39Kz z8GVY;h4#q-4L-=8Zo~ok!dt7bfjF?Y3SxDSty=ksF>{HPU%s$9Ht{y0zaDu!m5 zr#EaPi|tw6uRY1D23+&RX$-wA7)h$4V4}>nk0^7wof}GKuSfGxYIhl!PaRJOT5Qsi zV)l@Du1swGE$+x2E!6Oe$#^j<<#$}kD0B(-p>L0vzVvshCgq5jlhPjn)yIg~gT-5$GkA0f zJ2rmh(?zEY0s@!WOCFiRXrFE-E|hGO%YqhLk{Z+4c0~OPuK>ouMgL%aDK8GmV%)Hl zSDO)qG~juENA96WttozMV=C~muoKr5HK)lfYDsQ8v9XGmXM1#H{NNi#hirjpiiSW4 znKHfK77)&RghxL<5?us<|H$5G87JtB8sd`iyfK<1wi5pA zG;O3xDM8OPU#!ARwVXkaHM$c^Iokq?d1o>8>qqW~XvwTkIX(KySq53Kq zOEEttV-9>#33Wymr#3{eAFx~!p%nzzy#cLA1=AZ0b!-Z`Axh9(ttIrjOK46LUIv=V z6*RfkuY%7Grguh?Qxo3oA6qlUr(74ZTJ_6s)V-Z#RN39_gln{9MbfRH!>PL%SEsl% z-XE9;q2$VyUPgb@i;VIOIZ=#}R~Du5FV|sl8Lg2;GgeHx_E7?Bfq6uE$mebf@arAu zoaCuPtIj|7qeB64ShlwS6RGs#Ixem*m9Nd?oiPQfgNx6kSo!s~>Km%rfgoIHJv^;w zhKv9w-wX1^^(BJkt93cMXW-dIo?g=$J4xOSQ0>2E3K&gk?0jgA*Gs&oTd137;vpL$ zCIgDK!N@KQfMs65g-xgcVG@ku`e;>m^EDj1Yt)F~6$*DhBTH0tgI)AG8nxkwk>BrX z{JCM9JfmfNEXUFJp4Hs*zUn(oMLMIIM!T!;UJOMFxSQGt#k#R%Rk*H4T2LExM+qVT z+bmd`*1c!YlyWavVQ9BUqO%-jAa9*Tw#p8OZfnVhb4IS<{(VHPZsWn z-|B_yH-9(#t=H|~cONvzDf9yoEkev2eq6U}-uB^2rq?-3=B7W-2ULs+6J;?_Ve3k! zUMFRobHV;R3$ggY87ZX%x&yQ@2=hV^ z_K_>7go1kuM?*2#+zBZ9lNu9j;hQj`N`peb60-c2Lwy@W2oi@R^iFcy*GiKB0S5bl zEik9N*pdxbCB0>Ha;*gpKXd_4QV^)o&r(u}l82*@^qcPcunoSDa=9d`N;Ff<3pTUj z&=B%>5mZw&D3>ORU4Lm!F|sR92Nw@#Lxs7L?O1MDyA(++3cU1^QA21g zFn%4u5~dU_wr9}OjF&bh8Y61MbJZuXp#r;y6)cfLl|_}nNl~&WQOn~)Gl zY*YZAjE!K0&<#nFH5DsXr*dy-3OMp-rvVIEGwf;5rIxUH3CFU~0*Ui@{UBUT)3f1j z$94;O*2{Rv2tUP!l{BR#9Tl1Vmb%VaCgKG)%!`DH%h`0AfXq3hm3SKl=L~xUwUo*^ zc_$-1tr(CEqB3+@!y0AoYe+0vO416X@tC&sIHE1HTDlvxqc?Tpe^N);!d`UAn#0aZ z4UmbF7tGc*&Y zsHEXbn)?ndevFF_`@y}HX`DZ=)w->IxlX>o^qCJ%zjB66~ zocyXG;22@pY4n(ob7EEexj7J9JF5A3jKi9=J_wRox0Y%WhvJ-xR`S=9cmXDLn=+{p zx{z7M3fL~A{l}p}qlyW1TtU~@nKS=5RYkD-D4140qYA4@+PDNTd)cT&`^`AVVLdNw zkPI*`#1=R6rH3?V@QH%D(#z5+kD!Z8GjtF6+tZ+W*Gr5Ir2tJN#$p+hH7Uu0?yn^p~PR!b<1R zCXkA)3DPUqyhvv`Ey^w0XwKa1qXGMAlaVitjtLoRCFGCBdBfIOkC}0^y4VCbz36q; zr~Fu}jLFfoqp?cA^@g>$e5R|6kJ*6g9XwFo*ne;^h2EcRWN^n}0o0^PD-V&IG^th1 zK&aRikVfNl=_kQCSr_aN`fF!|_giTFL5YxJW|H-t^Tx>CHi8&!3RJ{Nc1CD}+4|rw zt}YFcJ`qwN*~4e!1UIy#bt*Rq#z`MGpFEbsdS#icc*I4j`2ny+Fl4AINm%4nGG;@+ z6LQT0%=T<(nXH*2N=BhOAzY}EW?9{<+aQ1hiPQL!d_gphgnQZd#sZpW)R?H2l}Iqu zk(b*sL3%ri0$4Q^TIu)_JqfjSA&#V4L;}m)7P;u!cAf=A*;EsJE#o2@7c{xOr$wru zzyU2ec5FLCsm&L##PM<56|Ik%i?*EUFZ!hZTii8m%c!SL_EpUF0IN444ro=%4J~;B z7C^rAylWfRrNYKi#9dJkML<|Ghh^?$x3n6G_w9}JunUjI+k?1Ma{Gj(5RE)$CVYb- zB;ZLeSmfp0Xr+!q8{DO>3E^C(-h_7!GEPeP75N@~=pH0AuV$INSERg`&gzBIU)*sm z4K-3{Ep@=a)#9Z-%4DdIxPu7b&h%ZsXIZBI8=}|)rtiS&`AT4RKUdZ?X_9MB(n98W zte~WE{j|M$2m>kZn1Wy+oAeWE&)|<06Re;mI6h4UPrf8{XR3{>u=hszXn;;;d!!>N zwRh|W0+XdwITEIL0;_|Y4>;rK^5ZVyXK7gJZVIFah0r~EMXoppo+x2ou-m^sBu>uj`)yJF) z#ie*nc^2ytvZzu!g`ibkGd^LY7t#@QbKD!V0jBnshG_yr^Mc)RS;g&xNn@x=lrU6j zboDh~xz4UOR2f2ViZosu#*kuKR;pQRC4qp32Xt|}vwsYU6O%Dw+12vjbqaMc+0C6=b8u)BqG?n@2f@S1y6)Uzx%Y`;8BsPX8 zkwWp*+;f8sU-agI^7^yr9y?$a&Oraj}%{@VO$Fg2=+`?jBdbOcJMO-3zrhd?dt< z`I!TkMdJh?Y^u%&K9K6OI}i@oPa%TDrG-ETpR^`>T44ioAB zF5m+F&j>x17ivx}h~i?EF&~CC>v^Ax2CHYJ^`yU5{ElV}^BJjdJ>TSVNB)eb7TDxf ziv%vW@936okIve3DH`I@r8uhVe>P~4li?ty;RyZ~S-zN=xT@$`$pQR5Cni_|h4kU3U!UnCv(cp-agX!)vQbTCy|jhq%ma+mvV~w2>)anh3&c z&l!4##UxBU2)_t(6j-~x59dV1|JDRAi9U&TG-G!h<^DJth@^ZZPCC?K$&QY`UQpfM z7C>v1+Keb5V+Ze*SH1&!DwpJSTpLLbBHtj<#H0aP5-yUgN-=&2NvNh(F?4>9E!|5R zsb94+pv`x2p+&I-zKifeVui!mB(BcFsMSYG-Z4t{3{EHTK*YBOA6fPmv1m4AO`7J% z*L+bh@?uSeV3}8l8C_$&bEaiUlQr@JN;KWdSb40-TB?$4XtL(QmBrOMTUyza(D^_YaL{ysv3l=qf*dAk}TX!D2-kptA?B95kkoOiM zWFUbIxs*j>Lp*J(P;4BhPyRc@8!J(&Ak&U(>WSO`{rB(Uo2?z!@ARGKy)3fyo~Oe_ z-V`~}fKEdzi7@Ss>I54$Uf$pHbi`)BiUlL0StM8^u4l%e#J^G8U{rlWVt4aZ>p@Mg2_r{+aNl9%0n1tt{MA? z7a@+xy2qQ$7SgtOVh?OkZLk3VzTH2CpEZZ=wwb?}yiiZJPlQXL*j7)ZZhVE|j)4`e8}JPh`g<}NgTj?*x0 zLOZtEZ}#Pfo5_UY&^s))t!gQulea6y3B=gkEY&P@fd)#xBqAKR*qmJ>o7dXsIu{VW z`Wvj_P6ednBY-nVM8{wJ0aIzTc^?O8JWoe}dy1IkhsZewh_ zp)mcm;kEE>y~hPh2C5%NVZWKtx8GjA9J@{W9V-aJznfbX;qe1E5C2=`t6)&#-))Y8 zb=WEgZ_z4YwaN4)SGRF(xOQG^UaddActy;wk(x`lop8ud*bi3L72^-=X|qaL?Mg|x zx;><|*Z!u*w!UCHPt>hpg7~(T6}dMB#*LnG!}I6ll~7f|>ZU-g&kW&kq}D=SVXb-f z-0CMmz#~nVZ8K#M_CsSs@MBT-(zZ=>NXvK}dudLK64b4D8Omx7nB%XL`xOEM`; z_=tvTtko+H;XX{5kLW))z7hb+$Q3qqbyM#(G(>4rwgqaulHhvK+QLNWe5!v(mNaN6 z7Sh>ih%B&Ko@q5dpC8X5C*yd2Yz48*21^Ek8s1b-#_!+MM7%j6-`M2Vu*Al9;x%2E zW!=KE;|oKy1?*PU!k?@O+g$NGQ_JySTXa30=p*(nr&R;rzd3v!cpFE|!nF7@b4cW7 zgWYJ>uH;!iwnbm<<1G!<{cKXw!=VeH)&RX4)j8U< z*iyD0lwoP-Zc^v(UtPR>`)a;P4FL?gJe)sq^kgOH_z5S-6Vo} z!Dg=e%#vpes&`vn+wZ#-wW-Hi=BynL-_EwJP(Du<>jeuYbgoRz+AQ1FjR1{xgs!kN zOkq4FdBN{^##SLi;F^n-S4GRO&1LGeZ0r`aj_o|*4MdP7<8q$Kxi>Dr{i+J;XQt}6C>GjE@32wMI`_Sk^K zHUGoLdgwhqNWR{KLR)LQlL|eIoX50EvUHvjwcTV~+}2HumeLeSWJLeL;%~udIKQCH z9S*{PgQ^NaK`Q&`Qz$g0Vj<`-4De#sRSVT>10cHZ=IFtAgw@kS7? zH#&%@HH;U3y0{{=(z{ohHjM{L2K5@RF~@oI3e9^5tj|JtjMY7C_l zFibPE(SEG3|LPNj;vdLq;70$W_^!C%C8wnC_T_b}Lfi4x*oqP< z1WSgR=Efb#-{Q$0|C?x8N^z_qEhc<1aTl^TflqQb1K zET~ZW+?xK9W8>sJCMRW*2mN~wo&%Fr!WUO?o;P?LZ(I&cXi5+41O%*(w`iH?-TCmqS8Eh~^JrKQ>ch~t2T}C- z-}%Y9Y_`|C=ux}XM60k^?Ya^BK z6ZaT@Yy0lk;IWBVr=4iI>jZiC`nut%+wsWtTTt4bx51aF!_nxQF`UL8$+I>@s6lr!Q$ZHx~Lr;7F>!k8JNfTyp>t>{XybWX!5TK zKT|5E_F5b3ep1Xj$lyCO>)4mf=Ut_U;wW6_tEPmIt)f)ZzreLJ8Em$oQCDpw8%IMo zUN^0N{C>E4L+vtCP=4hg;t3GOMTfdq+R4W~+cl(+>~1BY%5q&3zUF5sKL%w!c(2K$ z(QQYzzaPvJ4Q@)O-uEy%-!RQ^II}lpSj$q1Ju+*$UTiw;ui%Ln!v))FjKKJ1=SqS; z!-gkHVVz1%m(l@u^yidT|BxpAL$7L{j!0zF58FMzd}$<1n~A0jl9323BfFUP=o~%0 ze7X0SS$mezUuHo*r`t9}I!LUT^l-7|~Fzg3$ez(>Y z{JgH-{#agSK8?nEY*LkF`c$Ow+hCc(3jN{s*rI7eb8G`oQh6`#X^{q+MMtGMAR-Qx#cU=z z)bBuW9r$$h|5)(S)P6P?#qpQ&N)=O)|_^JiC}6GAU_hTU)a7p0Thr`Q*tH zv^_rg&2N4~pg~y56255zA^6n4JbgMR|L@{$D5Rg5cDV{GR^g1`KzzXJjQsocH}C2? zsphXiz@`$t#u2xjbMn#)zw>QaB2ho5$OjTvD9VpZu0?; zO?=O&1f#QWib|D#s6?7Y6pn4I_UOe91dQ!9sTsDNbyWRfeMTSVP#L$Z;eDuE+U|ZV z0;(~c>-$6np;akoP=?+HThGZ4fG|2hjn~&0b$W z<0x9#u3E8E6?DVSG_|z_j+@m26<-Nfpgkt8X%ubsY=bTQGs@7K$LL$H2wkn9yU1P4 zDG5y|GX7&-!*I>W)vLD<0}djw7R5>?Fu5_VCb&kxS`3~O@=D1PD*9>2=75wd3=Y90 zvqdJ8a5(`SiQJf;A|fU55?M2v+V0a@EXs41&Ix&;FXNIYw2W$IsNYUmT4}l3@=~${sfUb470*+jgWm67Wc{MhIW8w z*CYB%w&)YS3L@U_?pY7 z8yxQ2e{)O+&S;P%(7v*x2j>X5qgxg2(c|nfI&nt!;v5C>mKj17>c%-1A^;rQk8|IS zoO|@-e59_N57d|QfaZAn_2zuE?wpU*pYt&~bRN>9^MC`}r%&hOb?SVqUY(E8t@9E4 zbv{bR&PVClc|h0BJ^OYZfb0A9?%bz)=h*(8dv);KtB2=4d$30z&qF(T?s>|O*3I+L z`guNbN6*LT=^5YEa|}q+|MLO*dhT&AM!#@ach9)EaU7jGuSLSWtY!pFN2W@((JFgX zQ%{YJgsicmq1wkC+sNxKZMpIBd#Jf~QL!_UN^x8o^G+^PEdEsG@hZL-!mpX>?qQs$ zQ;ojjIF>roJIXV<$gG|Zp-yPygXCiyT-Eil*m)~gp+QKXHBg_z|6qOQv4qA=uG9Yb z&X^yYcrkvyr8K6RA6X&0j6P#fZOHdpkD+kldNlCtJ!l?x#(e~>n7pc8g_O~4w-obN&u&bi6s&TxG==s}L&}bcM?$yho|UMZls4?V4}V_q zLTNBSxBc}f9*7R|w=^H=QgZM4eH;i~$f~s2vH16h?ScyErL^9+2R2TRP)(YttPpz8 zjD0yHj3#R>g3Un4iNq-_a9YXorCzM_m9_cKh+QKdvU&j%#_!@{tdt2diiHR=rtl(w ztqsoHHTyHHKUR+0?NFU@rKfw#wrY13vHZlYwN4$^*TaN*RKoklnEiylT9Cr8m?^GI ztzFMcQeX2VmSS;i%xfT8OmK&N7F*Pz9*UJ!j?;w_1A4VgDQw70Hng~9sgtZ?Pw7f; zo-{1hg@nT`PC zj9TFUDizJZVH|8 zUX<22zgQj7H4wEm3T!CCs};pH3T`0cE2bH^9kC%`JIjg+32TpdJq$-PI@GH>S~N7X zQ66FMNI)CSJ>oehZw}g4iwXW}J1{^FR5xKfgTxtTx*n$(0U}MTs;P`Z=M*FI5d*aH> zH6;1S1RJ;8h8i?t|RqD8?A zU?MN;WqmhB{E61+_g9{FruJI1$~ulCnr-+8T1u0$P&41UIcU67$m9yx=M;$>VJn zv`aZ?Yp=Wl(rB{;6+0WN{1C4ifU}@!VtMPN`=W6iW|8Tz z#8~Ufs8b4AV$Q7CGgi3@Sz@epWgN^0NNwAJ8I|fb3gnO_2F_{}#9>Q}Z(SJ$I$(*3 z&X?PT8I?ab+9DOU#E>M)tz3_vmnFs%Z!FhrSY^f=I{XT6)-+Ss-i@eh(H_Q`F$fUFNjK49szA+2j zi?!N}?AHei%(a;8s{Th**=i5hlElc%b;fwcH{4`{X>!WcNI$~~0+=Uxb!LFgH*6z| z?U@6p|2UGd)w40sAlir|RZ%ceX4_9JnY;aq42O|SbL6xvSs|X2|NqGk|NQsa>F-ZY{`1M~fB)A% zpZqX~fBfq7_osii|Kp!ePft$%^UrVpcysmYzxe6j|5H?(Tl`;tKlxAg>c1j^oSy#v ze~p}ILoeW>oXKK_NBSHhAV-0-+kM!*u33NGo@JbZKtM^*N?S*)TIAwPf37-E?H?&lO=$#6*hP}aN|ZP9mPG`hv}_4G5TM0W+qMv6O91P` zF1W`HVWU;bwHm(%j@eiJ=8hs|sry4DiWzEB>ebH41u0m>F2f8))j5j6x` z?3OTYqBQ5G_snC^DfThv1t9x1OQcBmFUT8bDKED!A`GvW z);$I*v)98SrmD}J&^#|BP1a0#Gd~+~+NOlPD{h-rD=tte0LWTO%(kSEB`u>dXm^P| zR@qPLk3>R5k*zGY(4Iw_>3I9Vy^9cUT9R36TYLWN6RrOy=C*4d1I@-tNwWC30nMG3 z{;V3Gc=k0}%c@ZGm*ad&r6f<`O*AQ4{kkL>qpBoNpO6g~RT*2Nz*AX%{q=}z7_Y4C zS=7K+P~hKAONp-BNf~~kJi!nQbFEo+UluD8X*;HA{^UUo| za1rk3twbP6zpHv54zr zb6hOaCJlQ}Q>n-wpj6pQ212{fYkT*l-NJXbRc~j5QTCB;@{FnZ?Z1Mp&D^Q;Vv(M| ze|7Ql?W_4FJ*3Ix$R=9H;SP3<=FVYWr?+A&D{@b<&Z6mjcf<4NobFL z8jzB>K6vDn+1#*1`3bt=9oky9ScLU#G@-8LJxL|GhmeCit-FC&Us-F4cbLmUhPOJA zo4k-4t}I(I?c5IT!O1stH3A*sac6XZyNN%UN@Z~t?oqexmuq=HE9IwUdkfA z9nrCU1G!)#v&{60kxTM|ip!xX*f0uyP;;q}^)WyaM{U=V+u=xn`o| z30ij5aKNEYy2#}cwThIb(EFy~DO1GJry2=^AO+7p#qP$sZ(?rF>iw%_s{3kWDxn#>{3c2( zGtV{^vupFC*PnSn4e9F<{%40gM98$?Q84ufZ)!vy)??e^Lz(u9mh7I!^KA3H-9A_y zP1AxYWhtn^lHd|At}e+R5!B&*$_rVd`1w+9C=UiIw0r0_UbKty8EnjAt{fa<04-(V zmH4f+-fHy8pRhIbj!5+7w1yrpGSw23XP2`n1{k@c4=Lt&f$6wN2BMfy9Grk&EQ@L3 z!&RLhfMQkHGW$lZxNQWc(FZq{@?bsMNe=qr+3h*NvVX<1%kU&yhCs$-uJ~Dci6Gnu{*Ytj&1YCwylnB+vbgv8{4++bez2U ze}8ZA-sIHU=d0X2;z3je8BGeY^uiDUbvew3$p3I&xN;>^cPmqdwt#`PV{u?dRclwy-Fp3fgkwb zkcldpF6XuVD-cGv!J zvk@LpH3H3!>uXklrRC7T1EN^+AM&QZ;ft>YI@-oyW4ry`5j(#l;@yokU*RYR-7<8v zV_iR{+S3AB4_WFO6;U$VrhdE=-@b>p7jzM1c@SR~k-l0+M) zqMsIn0R6Ccft9L+YpNi>QFY8p(WbJ0lc|wYv5^Q&1dog^fP4cO)>&J-*FHb-qnxW_l5=~DQq=1L6@9Q39#*We3$|SshdM)|;Khg+? z=9tOT*vZ9Mh4a7T<9_A3m#9Wwq>jTr|7;EssDV{HYW`v`DHgtaCQoYF?f!3Qb8#LTJUXi9M>rM9P7V*^$bBKXqv z2*8(dA;ga`B+xtBH-q-7_x?@>|B#{lJHbYnVW4du8&;BacZEERvmk-X5_{p!GsW6! zx0vr?w9kBzcwVx|cAL;*f)-u|;(+?7yu@VhxOCakZObmdHm9?|C${WGcfHRM2)TLu zRV3xl^IQMz;c>r0+*Zyy8gd`pI8%4fWfbiE;(BLji9!k zUn+@L1H6ZPlLp%+dJmo#zF4MWiCZTGX~q!tU!`>K=DzIVyN6N#fM3^$@75BvP_GjC zgQVSF)o=M0t((`WpC?6Sz|^?+=NH1k)Lj~C2wVb{wgLdUQam(5q1~)R7*(iO8I6`{ z&SF&E5K`3Wby=g!(AsOnRE^|NIg#w4jBJXojYW5RL%Z}M_q#r;@?esHLkuTF#S+Px z_4J_^a+)xWA?XS?jYXW?|)RnjPB~tp2Ei3fMM!tX}r z+6A^hSWxH;>Ek7e1;)IN71wTmvOPoJd-|mK@BmW+&s-bjRz>D_dyq zYZR*&qg@h|9P!N#61)ohNNDWgX_He4E@huJFWH4MxfSh}%z&a^m(#ym(`0H{#4*Y6 zy{ag3z}K%Rd^I8qIFYMtEF$o&%pFV>AU>rG!STR4s1Lg#*pkVU*q(luxuO=hDL6CG z&+X--$gW6?VR+OZkKnk&lBl)xeU}J<2ikTJIZ0?Ph>u@umOA?NBnaNq7ddNHPiLh2 zddGyt-mLBGit>G{f z6t2)@^W7wPIGf4Jj1Lpnq%8yL{(_%%uU<$N!nTACd3ilAZmg%|-gZ2|Ui zcRcl!?bB-}@{K?tj(@Bco|K1s3tqq61Gr~Vh8n+)zQDG$F4A*oEH%dbsa{YVuT`oR zNULC}g-JZG~>l{#2r;f=SGRF5rAOXxd4IC+X}#KHU^RG?N;l$hPyT;rF`a}Ux;fld4Dg_nFT zRBN*!8$}tN8)%|>#BFYM8eewm*7esX56#p2)&p^ShH0tzh1eUD{kCq~Qel89N3{!i zHLR+ZwJfMvvihW8=SW5p!)Yy5G}NbPDWAfiyQF_SvU36C-lhIo%h%$n^b;GA!Lq^F zNa5lDhHrZ@A$Ha6S{Rb>fcCUL)cOt>Ula4*4Fz<`-E+D14YX<$A9Fw|uth#lNP z7H~qGb$RKs@>g@_e~AhnJ!V6$LlX{_`Tn@o3?1qjDDX;PFw{w@W7jC)qN~xE2MP}U(Q z-vr94Sx2L~-7!*(&m-h%3W!dyr{rl0POs!?3VMcR<7`NT2rJngy!S;FJeDv8t2*VvNMWy+BVQMpAR%kEixRn}=Zi)S+ zS5}|&<%bqrR8@qCe^*T_M_S(L@OfRRqHMGA=I`_V@lxCJWzF$iul_CU_AXiNhRlI4 z^%A~zaWbh*{L7l!X=gwE$U_{?#sLT4I2~XkQSs{-$1GY&){!`@y?}UX?+;o~+i=t+ zZpL||`R>z_SW9g-#<2^heQll}`ymr@;aFHy0)mkH9VFxKnlOUUa>zU=>JFB>V&5yUg*ZI5DWQ^Wfkr!Qh;=^ycBJyjVO0Gvuz$Y#&(Jvm0VBZ`&kc}NX9~5P;%fDcb&!PL%;NTWBH&CtiL?L#~?Teojrsp7z-(Qh} zp$hAo&2vJ>*8v)2R*JCtwb-?cdYuDEp>64U+;%&*zA^ftp?&A1$8a36>4A8@nJmW8 zb`mAaIL)aFxqMkk|58Y6TEHZIu6wr>jWw2;Mo&`}l(UdMgGO`RjKL6^X+LjJnN7h; zJC8fvHD>!ZglJXL^NeV0!fU6c(jHGS)%6$mSzyexl4%QnhG!$+MIVlF3npMP~`=&Gu2wxZe5M{9v#}uyuLD+c! z827HNIk;i9;j$<$%47|-1S>>jdiyoo80+;q%-Fer!}rK zz7Cr8b~e$*_}M>yRV=$#>B6gib9=P11s}@X7hKR0zhTOlGE;6Omd#9$77on#rIW!- zS>xU1eY5XuWs#hI`z~ukXo#urRt&TguyTs4OFbsOS0_c8jyc8mH}g-5AkMpsJG zdmzWXZ9x+e2zGjS&e+uAe{9H^-6*7igEW#1JZw=e77Unt+YrJS&-u+!<*7GI%5viN zh}^!U{ClFfZ_*N1m3vugrDyx?)>;o>y?c5PAOzbBt{7*ym+mjYSp5x>D5?k*8z?Y} zd~914D3sL!vwXyBrr1gBW{~xXduPYhvuUv<&c=~@fNH?cb~@s7qKmajyNu3N1p##B zNojSP_JBoml=<~}^^tR)P<$ulTHN1~;@I>m0MUuTMF3=rG*%fhO@UTE>V1aqqI%k6 zY3oSPyJeDfv7}4KVQ~(PwJxP~rQ=V0MbTh;C|0UEi^kSodZk58ysW7aT z3WZ)j{Q z@Yda2H=EZp_n!J9*bh51jT|6rdX9XTXmjCbLnBU6U(^LS_}D@zmnVu=^Y=$qTx;9&|2OpDC8Ea9lP1l(R1#+;z-nYkJj8uj#hw%xr7 zN8>QlZNS{?##Yj+eeP7=s(!ED%w!@dKWC0BP;&}hi>cyKraZG)=@&W71CCQ#zB9YYkj>Db z?gT?rq&W>8So`PWR?_Q2-Eq2~%?eAM`KAY!`yMtWWeD5y67bQ$0;0}3l*cf znmEFDjR8vaueUBn+6rLVl+GNJSxBjcV)2?t5@zCQ&p6w@Y?>pL#t<~*WJ!n3_-7X1P0uUrkBr~!9y7MvfsGCZ12ono^zlNzk#!V z+*|rXg^-f6iIzIW)}dT}6(!e6y*1(Dv%X>&W=XRb#Kp;ChqR&xdE;$n=H<1J=-{BI z?yKMtbebYh6rZi^bX)ztwfjqe^s>!CViI6_aA@hs-9V4i&kpVl36Agrb@tbZL)*|4 z**4N^D{gJRIe^jid`XZgO@r=}Nr!FEL9KJYqsz_-j?hI2HWxh_sMH!tB8G#J&p)2B zE6B52tK}#z(`wQN*+Fhol*zhsR$=%?SByr)M-R4$JJz;eVyf+$KbYH(DsLfptw`WT zuh|C)L6xrs4kfW>My?4qXxoeg%QmU3R# zNy+P%tS!E9%j&IiAjBZX;(lHdcHh`9V;sy1ZXm`X^j7^P^p&OKUHp%HckQXi;bFqS z-yzM!+emlJ_d+|KMo`65@2sVA6WOvgr^cMsMJ;uVw3^8 zils}r8+^LqbKqiym4zCOmb69UDCrQ|nI6Y5JDC&k5YC>l6O-jT*gc-PVs^JK^>1E;7f{4_s5CpZp6W}D>3y~hoX(Q&7?l0lkvuUED{S_Q zkirIJ2i(pM@O1_!S??V|MoEKY+C@2!d;KOb`6Ol<_BS*bVE*(TzDNFZ5f1Wx|K!saoh|Y99V#7?5*VjfFhWUctR(V+;pahtZHWc5U zXM4=1m3t}AbTrqzN4L9d_i(bpg}6Nfk@UY&~2Xp@Dw}?S>38K)mJ)X6PykA^eQr&|r%Gi`_eFY2G!^x{R3Ks7r)#EM ztaoe@1*19{az$$gZAhD_hK~6^i3M6J5Mw6O23yy*nPSm1>Fo##X|QBvD297h%n#{~ zYEdR_P|Rp*G^!7d3z%MHWdig;O2ae=BiLJY+(FU02zF^I*gTZoIxzVeaf zGslx%N*U(TEHf9t^srHT1ty#g`3+sj_dPOC>E>P)6%ngQ=0-6q6_dXBH!laTAp|A+ z=i+sZr$2>?>!m)_iH)`}U0C6`;?JKh?uZ5~zf|1jcuQg}&%PU&Yw{t z+ZAK!x^EljSKY1ze1K^4pC9LBGL?#E)*4=@LB`^Y!^q$i%>GPsT?Ld9vxvlJ6@?TU zGKV0$JXzOptOcaOudGx5_T===ahb)CW#uNc$RFSaT?c5I6y*4>v`KD4{0yQ2T{H#j z>X3(I<9g{_E}xcZsXak3ZPHaH_@6-UVT}vr98>{w=S4Nk`4zW0J1TiokS3gR7TF`w z@SYw=VeQl4k3PJ-*Nh?vTitWMdc`$9b;^eYR*NXj6hxKQW|U!>4X{>S>_+Xqo^~%n zmc{Q10S}wN^dK~kkSvcw_ziXZBG>~vVR_y9X4J?;Xn1a#Qo+=*H?{8h?}NUaauxS% z@ObMU?G5Kr=qgq;g5yUG65Q&KCDr&4rNDD91|rJj>Q%8zFkvy4ikx}@JG@Ig0t$su zI1-xQahSS|8I!#CM4yhrVn?UqhQ`|SjXPm+Pz6DsLSyCJgX;gPWu_6;Ol+?Cjnid7 zi%~om=k2^;$<4FKZ9e6-=00o6jlr`Z`%8LuR_Ku|nSWApP_l}HY*3@I2K6@eaaa*R z)b6C~9Qu4E46*&3-;eLjEkNe53&bY?;xk@(v=*}YHAFnLD2E!}oL(>ss@53cr3$(3YGgWcAmcsibfb$&sJ^$%3;)y1rB=a2) zbfuw928$1)-HvCzNQA&JrqS&OQk0f|#Mk(1neL$N>d}(f%H%c46%@wU?7coqEecsl zX2{%Vwdt<97_hMvZWVscZ`Ds47&>n^B;KLxv$MFb21J?htGiMYk(N1G6F%%J8$@+{ zqua?Bglah=#05`q&tTP|ByD36XAMLVi1r+^Fj#XoBN{2;%@{LSs7}m<#2OC~j%rO| z^Jf5|zq@!n8-cfOYx;fN6sktYTI?R$^ixuiosi3s`S)GBS`f)29z9Q9Fe1Q4CJB*k zOz6KO$0RY364xRzaFiSpNsy>%95vwC_EWbEW^e@qj}ea zPI)uHa=QH(fmS`5FNfdv9ssLjQVG4rtjh)`ffzQ$mj$?G-aLzU`cAb z6YeXv-wwyu6=$h$%bM`=aHZa4{tRR?e+FqXe@2zl!+s-Gmgn*&Rd$2( zeaiEBxRMSa;iSZ+W}?KUb`nlP1Pqfy%CXFnLx$#l@B#7__E-L9$je5p5Sp9>n11!+ z_xuc)GVRbnlVw~G`iY2FhA!E3WpDGN)Ddaa!y02N|L8ONgBqGQcoRYrn2c|x+~Smx zTfUI1c}o*s>K#``kVz9x$rx_7wI(q5v(Hc`5rUP+O~r~YvJh_j8`pub+1XaWNIXVd zBUm;5Q@`j(EhBraRjNMw3dSKpIjTvlYi48eVRia1TIt3^(ksn`twq-3`mU^V`7K zxwbh8yw?}W*BrS2ANQ7gG)kpwH}~)D5^P5tZp0@Pwrra zI=fiNmy`q!%j|Eq3N3#u z$w#{{6#&He7;D6JBO8UI7Ty@Rrb(m+SIU4c+sMm?|9pWVikAbQ+nfi$10wSdI^bT2 z()sGFu1e1L@&L4M{&&R5buzZJHL3;YKO;?(xM!e6y_dKyYK($ATd+VN%DV-dG5<#E z2d)Cc^@?)AVX!u<#^L|Z8z_>t{}aiHB^zTcm7sse3*qi6mJ!t>+oChgmE-O1^QyMk z(e-O{&BDc<4dvDLz?DuotyJa?mhe!$SAS(e1oUHFD4MsZxJ?#?QMy7w^)A^+t+ic+ z3A*d{(Z>8nc6@`jZFPUWP$;l*HR~$1g6$r!IoAjsdtpP@AI(EPn+dw(B%HIS@iS2I zk3fJd!zdh>;HRR!Ms^(7q)#J+2#C#{6N`MQVZ#x{TYfOr)Cs>rSOUJwi&vqX`n96z z?!E9ZC^zhs!R`nqJ?VS6Q-;@}-%O+on0DP#a2M|5K$w!V~ zr@SEBF?h{dq-w3!%4RUgvszDay{L17{Qio?Tw4Mq>kB{zTo&=QF|+lDd9qb zYDFYfE|L&>{?dwXC^iU>D>=eg5O!!AXc&%qMd*OObxoAVx`oyHeJ;u)KT8`qSAylY zsRVoxm5jPatYlO`K!Na!EAk)i)Pk!c*5{`YohZCVx==bOxKlOJd+zwz{0#2F5nCmc zsc9>RWy$MU9hNzy|AkZ}l0b%U;8<-%E%DG1zgzAwll1+Sr*aZmsC6u7<0LLOT|S1( z&PQpE5j}>>&Rs-lzIYhjX!8GpC4wZ@^F1QLNnEum?xGZc!NH?fQ)OZ*U&P9>Pbl6u z=5qQ0+hM6Xe?GHB;A6|4_+`UF8hF&RZOv2cT%pBY)E4P+l+x%)hFk`)2#@1@W*Pc$K8bXTtAGd?*T4N8@ z@rWU_6Gx>T}v+B$x=bXZq>pgV(z9@p;wscft^7(zkx_?%dYv(_4jdDMo$v{IBDCap&k@OXR>dNGGfXzYz zlZducF!sb*3^qW2cog9+9tYmHB&L6{L74TqvZ)P6c-}vhN3*WaHT5Q$AA0gfI-j5t z@Wvf4M!22k7g4S!5}r*){6o0>-4)Y!uAtm+sS9{JMI?oVj;kLMl;(OX3Mq_(7yW@` zj&fKYT|g~>lf!U$bxksx-u^J1elI-g%Y4%BaW7oXe|Oxkd2bZjPPfb_@L-N20F@!= zu{_g0>WGag0Q+1}I7|jzfO^TMPryYY?c-uLQv`V8GZUXllNWJs^q{1DgI1iYwaFGv z*IVNe&EN_aa<{xJ6=M4h@R-N zG+P!cuw!bs;N-DDF=M(oj*re9Gc_DJWWm8nnlSW`WegrOWdb5nrU{Rv>%!D90}-`N zoNzxt?TGu%{Nl%+Y^n`hEqj4gq=>3wr{i0Hl`}e=cp*ITgPA(j>I$-BgzMjrTs~Z2 zLQ%O1;*xrqS=`AO&y!rl{le}!m_}I!9&xgo_I_ypv>#CH5$9cG*&i1>w7NVrLJZ<( zQy@bNK9~pFcVZTS*0iES|f&G(@%R}XNgfPV4auRS#K;P8bU@*5)$OfA=^O`Krt;B03dQX)YJwJc3ne?!VgnJVGVzKQ$6NW{UE|$qD!WGcf-* zJb@!rY5mjbx-dx-`Tnx}j-|O=U+G@^|5Hydjb8guUy`g0?SGf9MR-6tF~V}aMr=II z0eTip__U{yu3UWsnVe7CR|U*+!J*F|bO|EK-J}|!oApjpw5IkSiP(sN&q{wf<|lh3 zwh$7#{=${$hqMoyf$ZL0=fNAv|g=6EH;}&p#$70f^ zlh3bXKXi33#*7*0>XBBI&oFqf(hWsJ$`+fcIXh`3hQQ0M$6eodsCzoz-#{PKw}#af^fa2|Vq<6Z!;7nQhaSTmP;EiFO~r;s zu}Hkrnm31Tqg+)~(4BR0D&IHc^m7WB|X$s!>#D71%c{EBl$1|g#)vUN%mAtQmT zNTWTT#U$A5>&0bd@6`{j28W11^d!~?SRoY-hGSLlNZLL&oAK#?*^S`5*}BB^p}}N# zoU&-4(VFznmfts13BWx!nEKq5d^kmJ8BNj9=^+L|4_4bu`2Aw<02aT?C>1bI>xfs# z8bhBD-cFGH9?Bw>UL08)aZGVqMrB~-cDs-yJ^!bX9-h&-F?VkP?6~CawRiS5py=bV zI8HR*!zPv|GqPmpVdM7}UFOH{)5gbrhPNXn|Bt~!u4}BMD=LH=e6+!LU*|0d8PAD7 zx7%zHcizwI?_suPQQ#8P=_zdRv~|$y!iE~Yrnxh-U`0=6X!;1 zSR9T93jAgW;lR(ielgS8`^$yBYsx+C%*kiwlB?SPL^kUh1`Gc3X@Ca%;JIJj~RL>c>U;VepBn zVaxog@J%6=DN!B7+Y=BxeSDG;mR1@FAK&_g)6+iYJf~BM&@{F2r;i&`>)i@%!Yor~ zD6~?~yXM%R78OX>e(65WK1ye5H83QYP`9HtlQM6=6 zPmRHc+s56AQy8WpnpUZX(KApVW-27ahCOA(7Kb-FFyT1Uq?KGlp zU@_<778OMu4gBJ+AmUzDS4ayGX})N_Uc`@WM+^lrD|nX~2fWwe1?nH zD42|G0BmdZYydx5b##zo@BCSRQ1#H{ezHQOaUItrG;GA#Xk#Bu?+sWhU~arD$eY#1 z`t2P9F6U&J%LycdMnw-DN-Qt+vJ3oIi=cW~m|Vq02<($a?bpxuClvT}@OU{`7~Apx zdbDtU{PXls^Xz>7*>&$X-JMtR?d|zEG)IN3Hm628BQzDeGqs{I+O<#rLv*rg&dALJ z#hGt`N%)M4F;&SHF)dtF2VEFD3n!8!LK8hO5pP!?y)j1a0ErPsC7boR^rlLtVPKnC z43$8VE)f5g+Lqu40HuxPDpC&Fnu3)ClPPuXT9$d#hAgmfCy1szfMPPiTpJLJ*}oT^hWP8$4RAv$uB!Aoc#= zo5YnaJ{x!Hws=u}?hX?nNl^%mfzygg^B=8fy!iX{s3frbu>*>^Xz2h>aSFpj0IG>9 z#M_9(=%hGBU#t=wj+FVz3xDFY5*%!-fdm|h2)`tfq*VfLMt-=hFNt_6``bVf4Meh3 zVsen~)+~had)Vugfij$c?ZxBW5@de5(!4Jp%y#^gKvwpmuAlY$ADlcEf{{oWwGx@@ynXRYaf=5k zhZFDbRSh3Ah%;VhEnfc{hP#xzjW%mi7F~>FhTcgJr-cFV)=J!# z9#fUnVgdU<=-CA!C(s-vvFp~$8VZ>mAEl&>rbOuW3ZL2lj|lbln0w~fQDbm|P=p=* z_nB|unXZ%}MFZx$_}R&o4F+1&)W%7j zh`fEtWZ8)Is*>Xn)w~p%RpM5AADsXW5|O#2zJN>6sh1)Oai`dJH^B;n&;u5MxZ zF2pM7S1Pk&iCweF6kZ5GawS*!-pK9yKKn-XWqgquJ~MnT?Kl}a?EThRt@?2|n`qFV z1DCIm->2bYS?9~LRHF>t8QlVT{IzP^p1R^&>EY>CA|Q9Z{nRN=gx^w}FtW~kzdEcz z6gU!gVY*eOtjEBAh!tb7EGBb{b}7C~>0U^PN7RtvXI9?K>m3Qo5MgjNkx|Lc zBMi7AbV@uX?_b7Pq;LxU92p&Jr-sEsQwmFrHe^;hl!rd&`vgK>_eH9xsa3cc{SE*r zm2`z++_8uY)?@M3^3=bcRXxFKrsneyyKV3oORw2Q=cyl2i2lqrsbohRU67dQ3vdP+IPR{$i1vnAd zR?dmI`BDFJk~dl9ajP+>qfZujB_mdjteyPzsdzV)h7Q@^-IOqQ zSxgjQ@#Ekr3Z{F_;VC+BO8l1>^H?uA(t|<7pO$0rY6FsML0>RxM1j)}?VI}7L;LI7>sjj=_Y-DoM)^`QM>P3!~)Ssl37Q6AXX(}N< z!%AWUsy=timh&%1rL$}-J5$if*9F3x|xMfDf5TVcgw;C#*mt;c#p(5A7txkQ6TgIJ}& zm=YfaYnXTKwOSh*#mG<<7UZUADZB(eKYlNi36bCcZKGhz&$)@eF3x_aguTuP%KHnv z4X6SH_&$z7S;z*y?OzUGMs8xgTyOgmsc)0~yeb*;qQ$i=v>72B$x0Q?9{b<9OLL6A zV>z03J6gfaBdswhx<=@ui5Ccnf%Lp9UC@`&)>4NARCF^Cr6-xCVS0f$`StUO6POL= zGIH`IV(rj}ndYX~Uff-r-X$65g6G6fJ72Hc#W)_`1#lfXeh9jnvv5jdQgZnUwCVQ% z{GJW%yxg{c8k&!3;tEpF3$+_l(3Vv5Qdtmgt@%dT7O6elZ$s*W>iH}q6 zx+bPvd;M{h%)p}p`IG+g4{(M}EzFnF_E!gd%2FEHVB4}7jG!G`+T#Hgei{qp)&U#K zd72YI=t{F|t&h=uY{v=rpY7K4&&(tYVU8Lt%MF`%BU1SB0r4GT;Y^JNGD{o*(-j*B zj-%*&_90m7?7m2POXH*ngz7|0v0S}Q8_>TNZwX7ZieG3lGi8|qSclMWtX>8O58j^& z!T->OSz=X4*W1tY=Db@5q9-DXX{72s>dYVAD72u&O(kXdn_GM@kV>vAtVS9u%gE!J= zg=#YHlz$Cl%B=?x^VT^u`Qpv92*aReXNaEQNf-)v`L(6fWchB6pFX*e1k~@=bgPM2 z@nKEKL;KT7>CjJPGpWHjhtM>Db1%lLA%kL2W^1Z*yFJhF2GTEtN@O+~mDPCu>;O|) zQ{oxssKR%0dft!jAE#cGs2qF+NA&_Tj;UHs)#Pr#$WJRl=*5)2le3jBk^1+_b^nqS455RHO^wG2K`9CZ7wi}v=(XzC ze`Ou^malE`6Y)!Lw-Ch20Fk*q`=4i0-!=&SPsx*i*s3PVNg>QnBalrar=QxS&Ixs? z6x*Akt0QJR_hPSb;2H->r<$+9R=QhJppYXsl_~E-((jZO8A$$UFVsd7M+TxGpMO!foD&*6T`?oFRrZ()f6=ADcI36Ji z<8on6N>}9wCiYXP^*{fW`YtLF)K;_n8$W4L6_9v@ZfUbc;LjX~1NY_@H}BQ$;_L0|@o;tgviog^UQhq?E(@_>Bk*WZwHrlj z;+kcHD_rP@@x0~GKMvVOP>u%~5`J0dH1;`q{ zM*Q?HbQJRm1c&0ART)+(91}&9_2?g0DjR-As3WbYr#*5RD(Og9`=v>Z9g%Di zL2>12g-WDxeL$0qif}x25jXS|uWJE58eyh^d{SXF@c7?Z_Mxs#?3SQZ11V4)no`*@ zFzvAYO^AN?+!L~cc}(x+m);L8K`~``1}f964xFuROBGjA=A+6f3nDkuX{JV_(D!^2 zoSlwn%!o?bpZU~{E$Y4Pfu@eN9?zKT={vwGl+a&5>j4cURT0F#AB~B2|}^4J*U>%`f)I(#jFG88;6{E*D_b_RzSk4HBluSPnvP8pLQlV{N99fr_QWV#9Sra~ZIYE4J=PDpDNBg^6Zy0c=EK(Sq%Fy8?vW;uHO%g6K)1vkR?c=X9t7eF7k-D&pCQx5%O( zY|}8c@V4TOgRU!j7bq2VfgGO8K8GE?-KX_?_ksm_(PZ2pxPXg9n^r?M4uJvJJXVx( z3bs!%Rnm%j#bk!6Q(s@~YV=H&!KMIf16m1oiI2@^oR2ZEB2rn{WB~BCJYji#f zNX;nZzi5o$G74|lCcS>k-L*Kka?~?R%N+DjcDt9pnL++D0ot?UX_tkb3_)LO(UfU( zop!dykevQ_v!zoIQ!HQG`)FZk9lLuzB*Rr{p}r&Be#<{&h-0!_qTm zi=lf?rS0lH71WN*nwg@-aeynw6GJ++5X2^`nyeKD?_nFc%|gOiw#KkMYT_cbEJjci zHAJDoO7`TMWG>h~3?}3fsBeKJMq7M~R_uHt5F}tEuq)p#WNuesgK1A3t?LgVnMs@J zzw-{u(m^stdBzwucUT=UMxpFV6a$h;tFdX)^L~OqKhmVVF0f$7FA^OCuM|d)3kibAQBKqqZrJ9N5LziZ# z?chZgkdl@mwwdECgI;okq87a@5*@%_l_N!E!#-7fO#;-!6fps+`Nz#R*rSOwnQL!g zA26AkW&ync+n@N3#EW%)*{U8aa|WqY{*~WQy{88g4TZ=+GO53qoS4(Ri5|W}B&eIs z%&G*}AOT*PAM?ySRGwzrkdr~I?BA2r!BrM2Y=zZ!tyQLw?1&IL5lAJf@nVh*5$F9e zl`mSXRQwYCK#}ktR6itmVREU_1z1cm1K5BrO;(WPQwSFR;)%hJI7&-9g4INsq@KR~ zwnP)mi3Bu(RmB=F48_e+^C;8Qr{O<{f5xIJjiGiNT?+NP$4~Ds3QL5bu;TlKHA_h<@_)7LAJ!iU}VfJYq17&9a41jGty^SHn&(j&p%<&6B=S z`c0l&0spRWB*glN4&wrsZ{t0HpMVhR>WVE5NZw6w{%oCACYsC8V7jYzC8<~e3B1ehFoVqsdgDjVpCDO0Fk={P*Jg`k{XWx03#s(iIu zspXsUz}Nyf{7pZPTI@lHH$54|1~2xu9p-5=NAgW>)zq8bqLnjk5w}unD#NCFJ3^W6 zD2|>z_Z^g-(~>B++C3Nv_3v2D#=q*r~v5!mo>|49vnngmtVYvLH52HirwF5;}nl9=u=@y z<4^WZkj?cR1urn%f)FS&mg)SX-Kv7Cy9fpeZ2!to_pjgP3 z5+gwv7b4q%z5$pYdjHi9@=d)-raDv4f~9PA3wrd&VL))t+d$q$!0lh2R+IO2^4uIE zMaz)vgJ-QMvN(b}F{GO0mSqv&&MI5}i-nBbZcjJYNucwb>@}97NefN-9XII1&8d5S zc_e8&v0JL7Y9eBfx|X*rvXWh)`>n>QZHmOqx{QJoka+QGxj|gB?2i^lYfUx<`=;V0 z$CPUD0?sWMO^@iwvVCBolM9joH=uJ2A!ZhKS7&`8-nlI!tg`mx0FH{T2dA1WMzNE7 zt10hW?L56e#EmP5NvjA6SRq*mF}cUlG()L@ zt!}r3Dk~|yYp>o@jyt24#V?h9)G-__YJl=79Gx7kA^UK5)OO>WlQinIBDYx5myxEM z0@iqeJ=~b_B5y>PIOD>vS1H#3laQd47A!S$TWqOgs9jvyV6(k06;Sy{$+2 z2zSMLOz{Ug!s;vg_n3_cAgcJ`Q{5>70*x)wd^g!m=rKrtY%hRsVnUKn5xTu4nW7DA z5m_mY?=x6LdWC?pccHVILH;rz*xuz;N9zE5AD#Y>K(_hyf(>oS4P1PMmK_fqZ%ZFi z32~+o8S%zM$Tdj)60klut|(f$ccab5_s5@)+kst2TwYn_fZqwykj2;Guw;pdTW_V1 zy-KVEBVZEiw9WCqwu`5~{-1Hbcy@uAM}>=(V}*);p0a^IeX&bKRq6xItpAPbLkyVI zX}sVN2RjB%Rb>(yY78f9SNlcK1pkRRv^XCjrry~uFN+(=T|7Atq)Rh~U)9a1CZRN+ zwS)au`;E#4gFs)V&nB==jmpX}G{od%>{w$jGBAt^Xg_9KtT@Ay3aHPb-!Q{%TKhCa z*;+N*@H%iboqSWY4X7m|KATg;0(97LS}jSa=4s57A9ww+)@~H{#Jyf4vq0JJU4<=p zF~v1|)LtX;3Tw8Tv>5Z*3UaKN_S>@OpV6qtflvWUJ^zLpG&=_ z(++QIIh7`EiIJ4c!$hF3(Mj~S7gb%mX*Eq1Q-r@^3 zlE-J))%TIPFK=s`8F(M z?gDtT_Cw!ocTG+qldkBxrMV{sdOTuYUQx+%RA=o~hj>U0{|mps1SR{NUcRaNV!axH zwmbo2lBTHwx#;_gjD~O2`<1xfF+l2s(qGgoD#>iWn}!u$Rm z4fG+*qb3Jf@%SpEJD*o)0zvJ2cpGV3>a#byASl^Ql<6HR_!-Rl4!e)=AL0sZ?gAn4 z=5L`BdZ{yHp)7QV+fw8uvn`q#FP`u+05Z@sPT zbQai7=f2@TvcOiW%@!-?f8T(zV{g6<|F5R>)%TWB$>dafYFqhv8g*9DqC4<8JoN>- zJ4CZ4YS7tH1$?`I5uM^{^Es!_|eIsmgG{YWC*5xVJ9Dz4I z?_e2GqH#&lihf2c&|%3p^z}qg`qodu*3T0ZwAhQM_*Kx%T4bnIg;KzmS~eB(4=s&E z8GR?B5G+gpPMA%qDe6NvY1!;xuruWwlBO%J|Cg%6$AU*Ky8Qy903Q&;Lc;(L`o)HN)s`W!|Ee3?##pISi!!ZF&9>m+4xw5LHTkOLLQUYwS)QVqxU6xzm zj1;D*t^)z!^!SWI^1nEhg-ov4*=km~vWyTX(B#6$(#VnssM!!tc-J%4)wF#=OrXTCD%mcuXkv_(j0_ENZ6|Hgk5KJipN|-Te zI$9%Lt^wH^zxYCS_`R`Be6yPwJeq?GuqPsYyh=WhzTtQ|%0Fy6CD5;>`RJ*f-VE)^ zN6Yy`SUpW%DeK(K@)R}mK{7a%Z~6|vf3cL7KV5rn{3XE6Y^7`4nN1?7!KDQdp4^Ja z8Y}cIk_@pyj&kfG)VLc&V{L@X8RoBr`n9`$A~Y`&R2DQ0FOhFdK{R1R0W@KQQC0FC(~12n<7)!2skt^ zM9ccaq^J%_MH5lDL#S^??@e+5AM*hWNM?2IrTc_P!TO}WghK@eIdOQS;7}+>AxvvOpJ<89_X<&?rzSXR=m;rgTZ`uC4V)2vEHxRSl~m z&c3oU8@5I{3##P^d2NTXZmhX62iNnv95*G?IqP^Mks?LZ?(3bo}J){kdZR2+YQP7_ItdE{LHl$U@S>VFW?@9( z4fB*wY5DWES>V`5^4qKu&7p}{%ASA0VLPllBPft$BR}vAjZXW8!z4=tvYoU9dzP#! z08y|wV8cA#TugtR{~o_!M%Q<&Nf!9V*?X6%J3h?@RJ+|{rH1gk!Sgt99U`{hdBk5Q zmAZeRkkko1XhJ!_O{qIpw;?>8Lo!4x$y{YD>b-^7BL!VhwMjlnGgM0vGs0}IDywY8 zT_qA|y$@^L<1N1+Y{UnQ^TwUV7au_9@U7BPl^r$mWE}&f9P?75O8`3utG-= zLt&bDAb#)SfWyibfPq0g9{2X(OAde>y$*mtdS&B$^uVD*LjLRE$%U7aOUpykjq^gf z{BQdN1A+K0CIdnu(B8iLn%GkH?%A9Qi)!(b$%*ds9@6n9U`i^}QIs;o@@R zw@6SD?VBBG%w474X%tNtocxZj7k?)nt5@Gv_7uC!NMq-^#z?=ZQjQ=x7a;lMfwH;P zAJ(!6ea&e??T(S90Ntb-H1c~0qQoB1fPmG?!(>jv?)W}d+ls$TVRgM86LrmzYx_zb+hVF6)3|yI)l)vjH6vfG>Iydv;4SXT}-g-+ORWd0KTy!A|MQ zc{bP0NH-GSKZgpun)8j@m&TEo>LK4O(>pi-mbmH>u_zLV(YyLuI8sn;p1xyXF0*vS zqDp+6g3~({{#HgFdLUkwav2`PG~vUtS3}oK)4;kH7;Y54} zk-g==5(6s=f=s5({BW-r=IEgZ)Yr?)?nQ=G;$D*a&rESxJ;n=H{N#+&jHs9%3f1E; z2-cE!R${md#a>U_Qj8Kr(9nm2HxPTA%h0&0m4l7ym+Fi6Bi zrXoU1`yFB|P}s|S4RQaD3k(zPE{)>z{KOGtx|%@5u*Wuv0C%SZatZ5r3ck(fa%Zn! z&Fc{Z=+mJY@r*2lDUM%VRFhyvX!MxArWN78(+LsD*gY{iv`2*&$UvC7#LDPwLulb& z!%Et)uANzfCdR!*5r_Xo@$pJXWvP--9GHnN)vcoI;J&(=@XUPZVL?Y>>UzM?kTUR) zQvWLR#X3F_am`SJ!Hk7{A>gx?i{$Nt?Qt<7;y}JXe;kHq zSdJKau4aRsc_}#}D54l_*e?lZx)kv~7Di}G{&q#lGiw!AQy=nmfc&sp8uH-nQG`ig zgUEuHdcf4`m^t8-=Se>FpW{&8|z^fGp4No` zu%RHx;QcEbxt7RxJ^qnT&<;Pe3bQ12Qk59$!2VPGv3|p2q1PQ?R3Mhphqw&8+a2Ol zn+9rK`D1+B+GDwnOG#uIC@~6+hW{XDx}4A4%od=%Rhf*ei@NHB*G1M{jkv#sRi;p* zWUAcSL+p&!oW(1+ueBt5B>5~7Z#Wb8DNq%4Wuj>osh9RdWw8R$ICXT<7bm8hh<_|d z%-@g+2Zb{TEVG%duqu&9&)P>+qiP1HCHdC9Srp15?Q zaPq?colBk=_tS;eOR6$0>65hsz>BLgMZ0D2v>kfp<|a*pg{JoX{Zjg`)AcXs=PmIM zrlGw?!z0yxSswMQUI$w$toHWT9zS131_qU1vK}gASvP2by7C3c`=7{z@Ook`_7HF` z;^q1Tef3y=#NVnwcs4sm`AG!MVoS zI_C)UiNlLAlx%G5-HP1J)f`r)beM%ELItl$@!D{m2+An8L#47yZZRS`R>a)SPR(rq z$6Er9v;&h9@;6xkFiQyK{8(g!$)4%yEaj&!Sz>-{Gp!bAyY&UOWg7v#0`Zb%8MWeq z`Ms)K;-uad{#S1I+kM11jFAfQ)7D6iz>Cheec*+P69hBpI=W9lYtfAjCX{iD7CmZt zK)%7Yr#DKYmF@YOAoWnaj%*L>M+TCMOgyTzR?@z4z05~=$9_K)@jqVr>#=y{1VB5^ znV*%AgVd`1Wl325luZS~IP050MtKoff;loMzCEmJuGF=wdCB?)1?CM)4ZnDRC^UT$ zgfk?ZMI?>+K=g;bd%U^4)2(v%$4S_4wQi8c3W>BaGnuWD9rxM~8k2{H5n9Ty>vOHmdh2`iW5Y*}-h!A` zhY1bVy$l#JPTOaF3?&=1OiDRRqu0AdF)HYzx+h&%Pm@_}gNzMQZEow%qdub!%HErF6?mp zExeVl$m~s#oSi*E`^ib$1!ye|9+lk7u`pvz=&CW= z2AquwyZHEm)R2V;Z&nyjFskc&U%US_Gt-LtxUlcb|JLBK#r0$E8q%W)%B%L;Y|G zB_^eBLW+IEGk(&_<85Ee@hQjXzA^WMydnI$$bQK#fU_)3+tcG+8rPr_x-#<(!kN3f zJv7N3(sAz#mlT{c>bK-2HZRre55mfgGJ_P2yVL3)_JCcFJ{| zfp@`tCPy#!l@h=8!XRj1_m*#+(UhDcT8oK$g0Vg8r4V?-uoX`TMY(Fbi$sT3N9s zLoWt1cYE8{fb8x_)WsSV6MNjly`}k3QkR3IZvi@LMWJ5AhtI=JB-GNl3@zY%}RBw(^L z77k(z#Q{{Q1CfF~Ax63c1xM5Yv$h3A&Rf|U1Ohhfk6>*c-3HgEl*5e#Lg*D}bO^~G zA0^2@|HclAzbVv?L@L!^w8xCz3-DoXChyu}6GM2vHAZc>9!sSe_g;Jz74AiW+v)7^ zFFXCSr`o$gMfEx8)Z8r?s=a)W{EO7XIJm=sY0WZu_r6X_o!Ou{wCUu9%Or0r`iNDJa>8Lu3&;Xt zDXWGLghxAj&ZM@3P8J`!9~KZB+E5*-qW+8OF&)C_ zH8qtluw*~Q(Ja1RIN9vrAlN#1!hZ8X_KxNu>M_h_!_7RBoaa4@aWAe&jw2BemBK*v zW;??s(pKHt?Vx1!DQsMvKGEU4*Y(Ll1hyVagzsfQ^Zl!PiH|m1Sur7WLyesoV>fr)hYpXwXzs z2E?eU0atjV9R|u(XKGeSvK6@=WVc>cC&vsZvi^8L6^7)Ggi)bC&j@dx2P-rt?E;j+ z$Hqh4$<1fG=1%}Vk^r@#q%hHN3;L2NRUMNTB+P1OV>g=s=El|MCGFITPXbqzfyPTb z>i~rAkBm^PC3LHBJS&XPMYXZc{|71MvQzH~H=9DtnwQJnBI>YLy!8>wT~w96y-xe? zi(LQx=%Lr7PF{jt^+)7#jf|XYc5iJ*^-h@SasXvP60AqF$!#z6yk)Y=`WCTu-giN> zC-f`y`C`?|JV7ahksw*lin>iiktY>EY_0GT!E^uQcIclbH9_IZoUzC)PVWZFMPu0e z@fEEpz59rq1e-BX>$DFuQfUT;U|*~#?U7{?J91d>l`FU__p)RNx=;N*cGVM?a&|`O zOGww-HS2b&Hn(;2;14Dc!{7Rc-^~%)e!J#1mC7@xXH82ycP}i&=(8XoV(YPXn}-|A z+n%a<GMautey~fRcrwTVJ`x;OkI;cy z^6fU96)XgY=gV3c*n0`=^G1FBW0?o^0VIiL=NkBgE;hg|o=H}-wMW=&+uO`c#ALV5Z-%G*3$bR z#s5HRqVtz|-P;n}od%IIoJ$Wa4CoMnOB&pHhNNebM^3C#&s}G|l4qdspv^1R>;x7P zslMxV`#Rklk@w~9x5MHNzKLmnbpd&pegUdMT&Be`LBk$#GD*gSq?3GxE?A1rEy9FM zw}!V z)MF;g2Q8*$lFosTiX4$2f%IOicH8Uffh$2N?O&*AW1=dOQl43#-lNTar2{br{@f3J zyfdc75mP@{G@14;5(M9eJ^eaGxLdV?ZX%r30959U+gXMPHy#(4`$LTZ;tH-U(mYx5 zh%o*Qdeq#|Y-A*rx~X8#+9fT9(IopFI*fV+cjP=RctGw^_2QfT*x;I<3_z%Q<6K~G z31?(cYT{M;ePZw(bTvYE^L1(t2h6Pcc$s+c7V$fZ4+};N)DCcU3;r}o*kD%?)O_N@;o(IC` zmrQj}y}YJ9M0zIN4~yt-VzTe7NOQwH{hq&~=P$`p685`|#kSl-g29qNFOi~x(*c)5 z`2JHvu*Af6MJ|}s+{6avGWP$Q62E&`LUTDy zZaqnhWVF#Gz%mkI8%GJY z)y45^U_v4q%BT5s$JjOxiiFV*#~LxsO7bhEJR)Mtx^E!X^$BLuo@vmcbvgmvS*9{MI((Way&Zh)UN0#`^nvS`lB z!W41a&g_SVH(c_0gkF@cssn+5_7q()?8q=`#IbJEI9FP5N-H_8br!FtFr9QiO%tYBvts@VfIcHU|s4h8@ zNu<3q&MeJhuV733Q;v7)R1p6?6A=@g3mdDG+n~EvpokGo1d6@E3Vc67ZosHaldEf9 za}a%If>_bnLS;G#o*of&&f6BTzmaNR=}EBERFzoB9Eu&7pEGW2Kb?^SninBHM}VuTD4`H6;T6}RxpP5ZPkVDk5_r^vuZhAW}b^q zHwinRW9C;~8S%L_U7_veK(qFEnej0e9_niI~a{!g8J- zH4&qKJdBLvSxwg-5fO>rf^?t42iey#rV%`5C0vxUygRYdnLoA{1%E%mBiYVgm|8N2Tm_9#{KYR9BrmWnm%4G zQn0EC@m^7^!b68;L%v9cMhtGJGNI4~C&Bq1Xe0aP3L}VL!`ueup7Hlau$z`DRwoKz zyvc+6vX>45ltt&p6|g~~@E-m%!LPZ59(>(H8=c7W(Rrfp1Js|D zl*$e19I5=hAr}K_rml2c*nFhlOB3-y$+pOJ&p<7eXxs#q{ z9T}S6!eIRpjk=}Dj8amnFqdvi+-7E9(8x8SksJkb9Z5^SeNBj14njw2{#LiSlQHLU zB4n6)f2qpE#qj&Pg8iB?vB~6Qf#bI08)Hn}cw&?coToIu+v$I2pqF!rR{P*jbZrSl zE7zs!`kyjY%LC<^GiViQ!mf=vH}0k3F!7T&b}z`1$30&6z^ve zlrD~tPz#!N1|3E{CHIp6>1<78RcF# z12ZkFparK124lv12;;gULG&Y*U2ez*i6O8fv7$v%2#p!)95oXiLJ-1h>PVR#FeajT z(rcYSd`b8=ikijQ*;-dV$HL%Z(u7bwVX+kAp>)IE3$S$MLA&_&A+zg#59@Rooy{oge@W#8(>UrRRU&uDPiMR5Yhv!1u}#P7QwGHt59g$&fwiUgcQCjX9y+u_#1g z%IJ!r{gE4se}1%c&`V_qC6vPRh0@(a*K9dO)O8==KgOWrL>>- zE?YiTCo7j^;;Xl%Vr6;7b^6PTa(CLOMwa2R)bU#SkZu05$!T7QBE`5_fRiBtYYA9V zfH$7DG1tR*FE|w`I_9bWg;4_$M&TU!%Y3a6J@LIU75Q5e+ge4H`w^YKN{BNFg5&A^ zmOA@^+T+m8i|DDrH{jp8;<+>gPYtJSk`*9k&GW8RUE-1lmSPjEEjrV#8J!7XKn-DzrMOaXCtaPWRV-tI4zoFj^h0mq%)x;weL!+e0m+=(_w@joU+q zcZNY-Q}(KL9@Ha81N3Y9)qg2Dct@LG!-2Qz$6Ml{o7s;|JPwN6>l>7+gEKb7Mfd#W zgV?l|cD{TvkYlB05Ey^!<3hf9rp;g{Sk*^jbxc>Qpy5Q>_^&0hz7&zvT zQdA9hrJ^1?V2;5Bd;;}Dz1bRZs5--fVKYD1qVoBn;*(pph~47Kq0j4Gy@EOj2*nCG zEY^~&ruWsOVdY#PItJgXaegYFI888ebS}J2AY{SJ>x<|}Ljp>*D{%dWWmFaqU@2CT zUpT)maUic)C)S7%LbGijiwJZ`Hj2{7L&>~n;v=@MSak^>uy`;f>+iB0NCY>#uWDfK zCoigWmJFk_syWh1`U$n94!vUb^Y=hP?vB|a2B2v?fNLPS$Yg=FSV4aUQUJESx8Z1X z-}jS$6>?<9yr;s4togLi%RXUq5KtjzR{e(Tei%vD&Dl!zpMdN$%)duxxwg2Gyga$1 z(WW@{f@ZKxB>j>PMk^x*H|A$Osep4I`HbUEi;!W5T z2-+JyVOr|M26d#I*j!2+AIdG$cK*tNb{|yN27RQ2sgB&`HDz~yW5GB}qb5D!9mg?2 zw-jYgVg>p&8pZc&ey(v9Ny&2toa>>Q_gy!F*ZagQ2}FSN$THW9@mOhE0#GSUu>SJC>dQTx&O7{ zd0TVjvG}yZZj@hU_WcQKk9JXEG=rcmYv_2S=CWB=o4pc!wSYQw9QGK?Y>~ioCB<@W zQqbt#a9G{BtpPx|HZ_cgEk2jPA>j_OSF}>2LthgTAPB+!p5dUkcK{ceLTn4sU-ve< zLffCB<2+5eGoDJ-qTXG9T9~KF2Jl6lAibUvnUN&53Mm6U_E#2W$6R#PSC^ne7cNt( zJrk?zom1vV37X8Cso3%c6Et+?l13P+#tf98IhxD7eUsWy4D5iCqO(rSnvQAs>4eyg z={23{Zs2F3oIxT2-AXIi_5dC<#Ry*@mU$gdnoceso_VUYoq;xfx3_J^@+ z5G=?aO)m^LqOCAJQsxr&k({)@P9fY0065a5tfx<4O`rI{C|5#GDoLd`lClX7IwMFW z6XlE4I5hp9Ui}IBgAFLxl%}BpS>Pioo1B%QwYznkCS?lzFL9!KC+vOv#e<9TLMG_0K@Wtu*Wb{X%0+`o7IS-)s zvf3qXrG~pXJwzEQeL86Lo{yR4P7YZ!g@)5e%q?1Ud}`Vg@%A^0Z#2>J?a#w#v}ke= z>E2&nQjZ0Y&NV%B={={p@w@Br3M&_rla3R*Nzlj4Pgxxtt}5YuGOs+{Anb!-mB}PE zU+dHyMLU#ch5=at_c%k67n7#Erwb4TF*^DQdUG8<*NEcxadDAY3P6vA>Z%T~rjhig zFw8H6Ki!xjl*X$o{O&i5Z_=tjX7%rgwW83EVpj*{Gh=rYGpWcG7Svl^ z;f51;pPV#G#p9p3d+4-*;X5t6TVq%Q{Wc~^jHRR?m#lGS5Y45|Y;?$v-6K zOveoDr?xjhCFvyg1q3@3New+A6{WMjkshv{el6%yu@s)%^ z_Rzotpw`r43{w7@rfdYfEh>B1G{m78YL1WL?9ul=Ik#A=XVh+{8_py|A*bpx5;*F2 zF>DSY>fp&^70|dF4yjkGdaN5xe>LU!t0UpZV3K6ppXpLVInb)N=;olHtmYg$LDy&n z?nacn;jpo0kKQt4{7*XIy5byuj`&fii#eUw+{CFQgM#cQi>yV?{M1YDUAMsv?jGm_2G_@Cau&r zRNvC9(~MgwA7y$9zi5u&;Ee;%6SJmEtw^cM$C}@K4`Y?EB^U5SDsp}G5-z>aLel9m;Nq0rgK_2uur7gjYC8_|$x} zNu&$rm%eMoW5a0$zZJxNrQB|JVTMJrlAHT>RsJZF&>cUvu!6%4iT0bv%cJn5hs5V< z)epZ*&;ytj1?^=H6hEo}+>6@{Y0BbYqQTD7JB|kO(j_f0`^YC#h}*Q(?q)2dZ_yv~|%ZA%zO@0%>Vnv1tokvJinf-~5pkVUP?MqLKs|H_u6^>Qczpto{7L2D0!Y{)t9#NHV zSrj(%&uy}dERS8O4yNq_Qwd;O^e;KoBJm?_)}6y#NVaoUT1~@z%t-_%+m3gFM#{=J z5Y7K(jtDa^TnDy`;b*D`y%kQ8ydF_Rb1?PDXajp**>bd$P4U7T)K3Fo29|*cs)HUw zgjq-~W`c?h0rQ%iHAB|%@zigL{+XEUeo#SV9jD(&YH{^zZZXCzh=Q8HY6^xg1O8hpi$e;(=P*c> zw@%%4XN_+2t%?w}X35a8BS4d^aanet!q~l82#wziB(-R2dgxL-j=o9jb!2*(5FCg6 zZ=D^_p+NWbtM^ovn|SPbrzj8TiSWNZWGv?O^Ibprk}$)#wWqOP`*6*H*n3?6^)cbw z!v9T=@jX4vd4DtE|No1B3*Od*lx~HWy7SfMi;^ZQHsh5hL=68#Uqe~Zt2$D1q%Qph zJv<68eK7q&Tfy=1@A{}VrGTc)jPW^jDaj|8NX6KB>@{A$W;-fMfk5QlDXK?`iT}$7rQy09gneE+NTu%Ngx|A81070x~qDv+^FwFYofj z*BqKY|4Y|gY#j;3S?iWoMm~XgH24!6FP#9pO3zH8Rm%vyNcEYwlg)Y$(S4m?u@Z3! z19h1X$`xVSEzO+#dpxa72pDwVJJyxfZ{$M2mA7E%slBOwWmxO{hFT3z2R~xRJ=fphX^z_$H3S0BgA zUYa|G#D|Vk)nul-_=&E{6WuM7U+!Uix<=6w!y!D|KO!5q0($AqmoG0gM+RZg)& zF$udg{mK_PvQV8z_}i-h;hDCBF=mhoVp`1iEG^@wONUkt3wi&}r$1Ibc3lXuqHguOi=*+wLB@<}>;e(8iTHLw$q!lEK;6O>enEzcyza}k`Y8@vr4 zl`+#yZ;FaNf1>~tryrQR2~uU`vi;R}TaRvRv3*jzTi$NLGs_VU2qP1b8c!gaP)rsn zr$pf%pberb?LzNFDvu^~v~Q*=M6+M&4qMRS=mA9I7S8mK35Z66hP*0X9&f~DcwriM z_{=Ic(T!@;@DuM&PJ3hcV@hN^BzYs(d`9^#$BUZ zw8D5|1lM#j{G!D*@?B)2^p^OzIrT@C^IxPso}y~p_l@*FRW(9vk<0Z6OM2d@H+))1 zkS{EOL{&Id+{R-cJ-(|E|GjCynSEc2qGy7nR>1;1orV#5Uep_@ZdRno^tQY!&|xww zdp~x#j`$^G&hJhCn^JfRJ4Lf3FQK#>PgsPZ2J8KBHFJmMgwJE2OebdbUsy z`uVOzkm`u093mHU0=Mw(adX%(l|myALw0!P!xqjP+p$w>GWG4i%I9k|_E0q4#E(t! zOO?~Lnz*snME&b$h&JFZa)ae|1E|@<^*A;#E*sQmJB6@z%n!578Ce1-`2v=}(vpR* zVU*Z-H?F|!drarv&arKlK)Bx!UrNxk%9}Bs%1X@Uh3KeW1_&SAt=*4I+gty#iY90} z*X|1*yNXqtV?w4zK9{WWCD4xVswGMB-=1s@vwAFUg{8pc;nk7&lxaI? z$TG-okRq{Jte%;(r`P%cY!3pS;#df0;Qe@*m`TQbV=d4k<(RA}`yX@Eqj zn$?w5f!nc>F-0Du!(VC6w&E!O^`UkqB20I&urszFW3&CWo)&nKV)fjT>ZmO=u7MTe zq`*(5a0B_bFLACU?a#8(Ddw6abLS!7~J%Pa1X zvr{m^=xti$xoI5MM4v+OFNHBxfz^||>NOmO#DVv_g15BaHjp0cu1`#Pt>;Sx+iKb* z0u60l3;zIKuggDGt5r$_j#?pv>#=v9l+P_;SB(Zzp7Kc|MmM#OPpO@*|GmGa$Rm`c zO$tX=rV3$SDUgnR<{XTpW=*Z!w}s|)_Ux7>a2WoRM$|}=cDH0h-|jk4)eP7QBoU!xNB5RNN!2Hu!P@@@Z2mOwZvM5>x4Tl z5L2QoYS2EdiQ{ID<>-(;qt%32S-e4pHrsWv5x~jb@+JvD^$=3j$&(7KSch=VL4&h- zW#Ss@JuVdY9S7pZ604(T-Z}MoqS=iC=%~Do<#<_%G0MJ(W#U6A-PrKxiB1ekf61c8 z)C9`1B~ZRlW)$pc{y;PQ&~n60jgp^%k-)&d05_v*Q>fhmX>;XtdtQ~)jc}qcXdAAZ zZaco)KYs06MyJK{6*Ks_VU2*y3Bhy~^2PAS7k(LW-1AwyFrwbch=!t1nziE+mtnQ%!qs-iaQ?cckch>^ME{#$13qBYMy83z_!mQqD4@QMwchDN2+E}9~3$^zhyWLLRtlz7IVPAe#S#GI<2P^jOj z<H~WJ!AR~*nalGb#W!!f&JWV^! zxUy2u7lmp+&Ppct+&Z?vXJYVQCW;zt zpGO8zg7*00d1|RI#G#&toWPDR64FD&cBo zG~-r(t&|dfj7} zzY*}6Yk#PJSA=23Z(F`q%AR<=@ajwV#o8eVOERXtXtDe|_FQvVeHjv~^wxPw8{7?o zJeT~^r8^Nu^^rM;fv|)n#9{^QAs=s90rf(#xLNh1K{eZETSAV2LZ?J2& zW7fQlac zHtCWQ^!g7ggcyK9b$#{r&LQ}T#0qC9PLz37XuXCac5)GV}2 zSBakeGeG{#<{Z%I7fnd`tr>AHF^Ve}4l0q^xokdCeaN(E%UaFf_U-X_clwU1iWGm; zll9Wf@bo5k`#$i?b@(vrgS1j5%}*`rQ?4HQND=RE4cGQCJhStTSgBf}5}TSVZoiWa z@yL-WCs4wSa-XS_k8U7ugN@6@o^3j;Ta8yld^|Oi-u&;CK3Y z4CfLjphw-h%x-d~J9cM`)XN^-`{^+)zcrrf8_qD6$+mW^53jStJmw3pFX$)J_ICc> z*|Hk9o;}RP_nMv8#LIn=>FNdt#JR4aAKzI$l@}|;Mugc;%fj}x;TP?OH%-R^n?sI| zCuIlILExJBFX;|n{=Lyp&0O=vZGnEmaQb7Th`a;{EJC~=nJ$+$Gw6!GVWZ`_qmH-6 z;BkT9OCFLE6`dJ~95R`ko#tzb*B{$|khBOZDW%?|MPN+OvjB!?vP!NDkA3cVD5xS@ zB=CqOnyl3HkVagGkKd`3vwNr1=Pa5#?#1X^oYYIs_xjUjS{NbfqNpOUvIW*q?6+n! zd@3#&7?WpTWC1e6m}J{luiUT$a6NVtGkbrP7AC-4i&=&Bz(J8d@I%4YYY^)`(2g?b zcNzA=%DWd8a8yOZNTvftv+*9&Ri_L-C4KHBTL|Q3e}1ifD*U_-oLxUMOwG72&y+ts z48L==_@zDH9Gw}+|7^Tgp8QQ@Txc==LYoO9uKcoxGz?9Zg^h8e;uJ&qSuQYnsE8P19Agv-G21fOCa8(K6Tb<2M++T#g1TKyU!tB(ffau-L8p zPszB)!f7X7l(sbaWJW%zz?vY(pr%xv(=I%@n5y8f#<=cwlz!jch*nq9^m>#77jSUe zLA-T{fkRS-X$*HVlA5J;3X=+xq|;*=_8IajjQrjP37bH#6x2|`G`q&NC4B*BC7}m9)H~Q-s>qSn<(Mk#0Iy=hOAtgl9 zT}3|#{=l8JcM+J9-1GGKS%sq_A$8UF4mIoUk0)oZIZ5~D7g`+OJ|ognH`j6$9t(O; zUAs|#XM9I>&d#VUgY|vhX&lXQ&O0c$9uUBMM+X@4h1BV81G~?~Y7+*|1wnUhbsMNm zpt6r%VIz}wVAuc{YawfGw_z$Msj>61l+yH_jB@57D{nq<3K0EcN1soX^)?~NwN2qe zNS6^HT2?wY%L6E9y96yN$*!X|vM$j-B6P^4(=EuGNg?8@7%UZZ2Asv>zP+K$NkSXs|Ygj2mWnCkf=f%Q)bO>O*IT?JtHiSD+-3Gi`~F zJY%<~ zq`Lj0YO!LLg_zb3+ii@bT2j{NiyV_doQGiG4P@JNC1lH?lgwgL9|1d>ur9!w5?ipo zEc=I1P3}mUTUy{)7eX;Cr?04d#fxTXcf0A!5HS6(ZO?ThT)bf|Hv+9i6YXljUhJRc(YQh8WF%_MW$Bqe!y^s7EvZ}o;jef~v%rme~(n=?Ol zvK#hiCb&GXW&RvZem!sBogek4%$cuRQ9rf^7Q>cZa<&OHoD!^cc(m%Ubd+=xidEY@ zszF$iNi^%ID2)mra_@0v*ZQ%8cfydg5m%puPUN=Lz1#{<(ABh|Zi?T&2w(YdJI)%) z6XRJ_pg{pb(_ZGhw-stjD`o*HpS^{e63+oZdQPB1Ts%2+IS(Sj5}J;x z9vhiXx+EqnEVW8Q6S1?-;jUjy>H2wi0dOMoCUn&D-AnlMhPC%GkUV4M0+xM3jQ*om zO~SBM_)VhZ36qVEL{N+-(gYNFq8JT;OhlXP9)^^AT}m9tU2S2&8Q`;r!Y*bbz^U?p zB|+&@eh?*heBmjP`vRw_{FrIAw9cRe)CGsqAbfx%_y`k8Y&=7YN)nWAck{Q#zp>>v zt)b=bD8Nft_EOQC!1A2reruQ1atB#|%bj{SvVZlsN+>23dXNuelQ>H#wJLxgT?8iLH5i zSoVM>sU>A@4b0|EEcVkCY$->Ijz<$BdjYd9Z0Uc6aYZ<)-E`69Q}onm!uHCLK+YWK z0FfdF!GN#nC26A^u+=D1IS|?mu+?u%6s;cBoN?9P3^0w)_t=F}uAr2Zl>rjlMSw5F zb0uY0xEVV<}+>j3~Bq~0WJiYHm|TQCv^hSKPm(;$y!KKas)Ac3CUYT@;VWtJOZ>H z-sktC!SjJY5H6pT{y50BcF;`AjT0b#m*DxWfLe0Hy+7XkS0)$ORD{#;MS?5jyt0&9 zJ<=>0WPc&+_Q*20J~NG;uUFEIo(0&eb1m!={qPL_+#GE)`|!KmGX7#sb-XUN>{M^h z3?xnnGBxQg-&^lovJC_$YF>7Q#r27oznZ>HiT6xE4C~ZHcyA@T5`hoMke^509Q%nXF_b1UPJ3$x1RXMIG!iG6O-I2Np+%r zLT%#h4tB_{W(RlP(t6E*xj>YR9Dg)x%mHK09YlL^rY=GKUMN!+M&E}~yu~_h9@seZ z0MyAopl8A)dAQr|5n`?3aNn)&+nmEIrplSL-yP6pVS(ohYa(g`!>rDtQhd^pL2?)w z9m}Z0_+xk7T;r}8oLXB)fcBfw-<(+JsNzIwVzZbhoUAcv0d)q-jk&XBP5mj$7mOQw zVXq)jT{Z_;inM?8tQ~$mbH@)ivv+;PQ)vkR^e3RxVnN#s)Suw&z8}V^oK3a*KEbTt zsM!FXtlzx%L&5;36UU6RwA7nI6vBJ)rgy}@XaOPp_pv=4@gl2M)eaSJ4R#Bt<6l2$ zl%P*$fiKkuW`W_BAQ_(gs683yGk2^DxR!>f<6mGbb+Q5oK+~TI@CbQz28M`DK5Bv9 z8uR9(KOXMx$Pqbq1CvK(fqz zJRl^;`e%Oy&XdfLp3VnEa81CV~ z;aERemGGL8cEyIcG|Ed|4b-Z;pIu)EtnG*KwFi_LD(%iPa{_EkUn0Ib(?yOGO)v=H z-s0Jgo5a5&g5GHi!OCaB9uVWlW>*-9IxLSM@`3 zM6f6x*ut3lA90K1QSbVm<$mJl(x&Bq;`jdknu7dz#`}W&5PI2z{ES(G{Jb(CX;DbF zlm2La3KVEl+>%vXigXc$!#B=;+gcBaZX?R-kY~wi6xU(Pvl1~m^|Ma|Sg~UI^xkw!vtX8FTeb7fq^cL%8#!Mvmln62{QC<2?$lSqOg9XW)4|r2H znUg2t>Xe#IXvp+-n2KpMS4-&tZZu$X6@(%^g1gJtH({{71Q#cj}u9S32}d*ZAz zQ|}<9o+{UriJ4aJ`t7e|>F4|eUCMicTy+THQDiyZ9Q0P=TN9(YO{8i--X5){3;XO$ zV5SX-vOEk3=w4*{WF2wueDzHM0AVJ%$S|(4V`k+kFn)^(cca>sR-@TR71oUs}l%;?z9 zc=28Hx74P;^3}Q5ec@!FZFs3T8;GUOCvLRX$wbbS7IuJfKFeiE5p%GnzCncacyTg| zutd})?(O|%agb(Msfx8Aw$p#Sj+io)W_a%mG!+5TQkC@O9s4yWYaU^GOV+)JjA(Td z(&Ebc(|it|NhUYx_(iepa6|tXX54HksF&0f?ONZR^oSmq&j!r;$01tCrL7w+kU`kN zdO|p~N8t0|VXR5BZdcFX=WQnI_cxyBqAqos7EFQSYQpTh{y9r-b+%U^al%IzutJq! z?AO;|IG~nxow5I9JyU`=qSBdXl9{xT~j~ZEK>7 zx|O-E5g^g$-O8Tgl_S$lAb`spx7{m>OE));9s4y6v-+@ zsXKU=BR3FaWMGA^J~;E7M5YK@`9l%3!oDtTlrz3WRzLWk2CGJ`RSaMCv!;0Ebo)1` zeqM7O9^ZWOdw-Uq)FJuXm_y-b%Rn%f5^71L(TT1wGQhn|s)x1=w+4@B-9wWpc?`+% z4zV4SUexAuD@skM{n{&JS;9I8CK=Cn>^@=_VAHPr$~QM^>+2Nz0CG9pzXbk(43vWo zJ8d}~hGVt%`D88tCN8Zl4}5+kQLOK2HtVXWSt7m{z~KwuJLnzC~2#++!Nm_3l7x%lkTw+C8#KMwqi$ zDG3XAJB{sdy-u?3XsxM5>bep78Vg+h!V2nU{q0lk@W%9mbx=?c)$diW>w)css1cV# z$&5~?ZpLhQnl~8!usn4L-Hi6D+x1NRS5d*@-a-<$nHBlEtTEiYrsv$ZiA4Ev zLQ>Pjlh5fEb;$b#=EmW(1hsIP&zTuqKGjdpE(f#eaadlx0`_(OL9=W<1*x}Ez`cR`m%~Z zNzhCcK({>D^!m#tKE1R z*8rj-X*~jiBY=|T1RS&op+W8;pbb8Oe+I_p2j&QeWO2yN4)(Ep@+yLA^bwaMC zkxx}|T`imSxpwLVj#;JcInF;KyPUn{pnCcQo@qB%8t_cIujxYzqkCsC1dbaGAg?A? zSLZQSx){aHSi(mJJlN=>b9ec<|50GkP_IsA&AF*=j;=>OiZjP9NLm)3l*x!>@uRHq zG1H2<9t$`p%$@mtIugv~Q_*ka^20H;4-qF2ec1;cLDs$0XVJFIpS`f1CSTyX3!)Y- z0gnF3{1VQ;6PGC%%vSlTWv72Gem+=Y8qKJ2nYQycTU|j1%$HnP;goCB{ow||)t{Sb z*%SDj&t47*;uk%T*o3xw+#3jvvG$_;K!%S<$_@(L`%TzpmJKB}Z@6E7H=aeAOD6CZ}JPb86Cw z!Y?{A@g^Xt5GQ$A7byxhQ6?!_uA%NWlYh^UK-TU1FwFEmd}Jkwm=uA2&ldJi^?NBe z2auRg?xPHj@o)RiI{*dJWQB;{L^xSm0xI|{AG&1L2&NFhZdi|eUdv_Cy< zZ@?Vfwy=Rl>bdx<;nruxFIP54%W`P(3{W$zxRtWs`Sc2u!Jt4viJAsQ_;(=$1KO$3 zj;S3GOGLJnuGe*&!6JGnxo7U+Pa5}ep%~*Q%wgLr#Z+4z(vSr&lXU#_V>)t2=E`N# zXfgGLG@gJp{m3U3djGQIAjO{C;clI+${JdGNHWGcfY~(&e@VfCZpd^?T z^K1TKHh~s??jGaL%^2&QEr%gFXht$c?1En3Re|_my)$HnA#FKe>gppoWlS7C8cw^j zb%!~tm#o3r2zV>0rEp&C5IgDUoi>H#V((;OV}d4m46W%g!7o0?AOi&fhW!Z2vH^yA7AA&&bPWe#h=;TFVe=urKP@hu!mWf$HL5LmpE20 z>B_~Rv9;%kUh&@GKD|Z;o>cgPWXG$?LxtL!aTez?hIOerhc(m0rQq6)?hCtJl<~5m zpS6B+rfN4H+rCrnPb09*UF#(*L*KF)X?lBT=>>VIUk9DDq)0t#|4uGv*;0KT+`5=C z7`f^Xg}@uwtH>r_@;?tWTiV#CSnN(?IT}|oRMMemTX1eNTwAWBs6p%&3voOHI^5R+ z#&j#9P7Nt@7cSmET8#koq1nM&rgu(f$^0a|WK^(dDMnl367(CWOHj~nucyPQnK9B9 zM+xisZ`(1w*R-@RN)LD!igFduq_sK_gj4m=>H^5sryYX+0~bwfzM~TdC;uH-r-y*N z@0g=M53eJF3crOF;}e{MeV^}Fd3AsOJU)(4|4v;P{_wbn6xObbPiXk{C-fP@!x>QO zdhHSTO|0r^R3~>T5YsQ(VTJpU>pTm?hE(y0uIOUMi6yht)CRmB+&se}Ad0W=8n<-A zlLi9!bCDN*+0KT9tN&% z(vNEfwm~>U3X$4Q)B}!M3Q46?4(uYbd3tZESxd!yJ4&Ftz=|FdnExoE2fhY96!PRt zEIa&*;4vjTLCIJ4gmvL4&Msgd9=0-?8x=1IMNv~M$OGpl70)s6X@kUJ-wWZ!0V1nv z#aFr+y z#2nGe;rVMJrQ%Cxp~Wt`HxyhBURsI_g{2)Y2c)uKnS?PdS3w3fQwy1gfvMn%^u9ECfIQjMWir<$y^=k!7TMp^%l zv_d5Qf6|)EW^b(mU?*BCq+gYLPBKRwguv#k$l#p%v4-R5 zNAfc`V7|0-8hP()FYM zsGDh=V&uMtWup=&F=7+1J3GE3Ayqo`+X8)e9)1&Nk%N2KN=}%ADOvNh+K~*O2SWSr z|6yw?nBW$_+64|KUX56*lynVc;zi_VDNehMsO@&8^5C)y*Q4$%pnkm67mKkDz3|;5 zX^!H3x&`mAAH*D49(SI!0O?;p4iCS-ejF2)@icj{Q3MwnkDRqmQ)-o)ycE@1k0wB4 zYjY;VYt$56!vxB@A5+|TV-V-UN%7#^v>wH~r8pqvn@k(l?KJN`@D%e&!tXS%z@fxZ z%jj0fsmvOMU!BF!T}I@Xxq4#;Kz%RHB^I`Ffy~tvaY~6PcapONbzi>a;;3(ifqPUVTMr;Tb+fr9^@<< zizN5Mu}}IqTwLy8`?~d3oDtGORh<+uILDU1#_%i!UISb0T6}WzTy&G!#+l?-99m!Y( zB4Wmv0fTf?LH8WmH8Ou9?k34YI zbz}onq8TxOC?9BwR3I)svwr>WPlw*SIRDkn?k2VAG%VXD8^PLaZG3AuFUln7q3KEA zijs9RP283O%k4d5#@oP(!Lwm&vP@Frhf_IRdc57%EGNDopc)iI>Dpbmav9dl1a&8M zEOk6>#U_fjBmoZFZ7#^id3hM}oA75PnDN%l^rM2^1wy=pRqZ7*y#@0z;u*;t1@Y#J z6SYZQkmDnhYK$*gFm!Y&on%wl(2Ja^5gRe(<~LDQ4#2=)c!NQ*5@2pDMQGgMJw#}- z$CpCYpynK7IOOLy58mbGH}f#lGQd{v9(hlo{`T7l{_VfSxhAVuIJQR7I(Uy=D23MX z4PGViqOiJw*5UQA$-|e-&TpZ#y7@kP3|)OWQsJ&=KmaSniD*4I=O-{1jld3sdDqIw zNYlvL>6U}z#n-BiI5y90i+JguoXUP+h0LS=+3Uem{SxF;{px$nIQ*SJ`xW%?Q~mly z`~9`L|7``rq;A9K_cjf461W(*4Y|mqGDDPe5o*WHspFo%-W}8ErRcxK`64J$y9@0Z za?)V^7CzShGNN7VNt?-&X|N+R#6`5S3af+kQ5qALx8>M>Zc$-qCvy)PSfeIjuj?o~ z077b%o()$v|j^8s9}>HuW+G%TAi8p&~PUh|!FNE>4x1LzijnJgGpP zN!?9+R`lyKukhWC6fVN)X@yz6w#ubn6@;%jgJ*@anc#RzCB^hhqA{i@K$p$yr^ryc zO*%Qm$QMJVPufA07byEhD~Y<`Xp*zhj_MAA7hS=lx)TxQOQS>fo&~rm8L#~i@G;Qe z-)>jg4#H|P^CaUkSK$4fWUjz_VXeTMwN&9AFLsH+-z#>JqZuQu!aEZmncO1Alu_OK z7j60Zcy2>dirkeQXC|$Gue)mY?@0#3?_LgJh3aj)p3C_Ov!q;%B)oP{8Tida%uo`K7DQUjrkdWF!!2y@*EyzMKpF)X& zOs_IVS^EM}CN&qQq?&=O%EAM(MUBq!j_XR*#K@r`QwDUf&+y_-G1?Neu<38VXHQ;Q z&dj+Cr9}qL@lxPtk2IZXR#H>tag<)dlton5)vQ4hGPpem}mq77t zLQq;BV=jHJewAReX|VLuA4{B{2cL1#%gQG%${r2KJ#3tS>1d`Gp}Lv&oF7)mH0(3M zCHWicA+NkzuURpC8F7cJG^x}`k`#Fq4}CR0TZb}W{V=f`;jmr;Bz{5OL-5fMqy-UF zQN=F1KT&u(i`m)&Z38~){IU>@n|Y_cjE1GbEF4uUQn!XZTlu}$1zEy8~?fAj&HiJtRS;xShL z*g!F=l($){>r1AVhl~HV0bOZX+R-k8cc5`EK_a(Gn%sb;UvoYunImN#+aSB%3hVW# z!qb-MT6Oy@QEBUYllhSzxFzDLu4pq;=q4z=D8eqkJzrip$6WtSP2nDsNtUhschp80ryAP8@bC0w z$4?;7Qw%e0(6ojtrW6T%LfocMWfNV4Bw(w7%;W^+I65bHPvqiXtaKZ-Skc0dd#51T zcA&TNcv#soY}#=gW<|T}z*=ZkbvTUL`PMxQ!g|=a*I~VbJ9&N39FA1{XfLgXm(xyV zyKO}*wPqeRd`zR+Wj@Jm8116W+GPjQ#UR5f*e9Lby{=UM#kRPSLak|RW4<7 zcd@68f^cC30_W3eOG_A$U5g1cP;UKA_(^rKwx?2~0!Qd%T4umpGqgaNZkdR7u~oXX zw{^8$4UlYHFTDVia?cS;@-eB`?>Svp5wG_#OLunNez2B1(=?{NH~CzKXY*wz^h?I8 zdYRKE?PMocbm<$Qk!nyA`Wb(9gSOJ84PEc(k!UpaRC`N>Y&!PueU3`J47t!LJ?K#; z4*fW9Cn?+C!RZ`K|DR5!svF|kZX#J_`Z1Tnn0M}MV^7pg#PwH$0DCRpWZ*#o}o7Bh04kb>?Z z0?v1n{g|(RZ$!s!AOjP_=;yNjs{31iAZms1qM2;SC04oA7Qb>kEOx-ojR@Uo(ipJ_o}^$yVtlfTz}I&1 zcRJV>ECBwCAzKK%mCZ}@JfA6%g@m(kR;2}_9AGnt+g)ag&Sy_Exeao-d7}}$9ra)V z{2rA4TSkE{X0fwq!@9J<+y>E`zMbD5KxhA@@2c?R#2=fk)H-rpH#gKde*1xVWe!Us?UGPmA<$Ad#=#0`+s;@p z+mz_(O~xbJmIg~aw=1F2_5w?5*+z$`0?R0ftCKIjm(EvhAZW=bkLJhvA6N@>0-(d| z_^^SGR!C*dsYwKiEG*)e>U;SheS^(LTB5}VlHMp%< zIQ1wQC^Wdi=$N2j<>i-f%BIKW4@GT@BJTOIya)pD4gwY8DIIRvf%%xSRS7GcJ+cY! zNQaD*MhkwpTJcDQDyE=C5@w)|;2b2_M6o9vq1?0$`M}8Rptdz*Jty9Szd@Y&hL3p!XLx5 zUJCO}{cyY~(}U>V8xX?Jqthi|JeOJmzSV{Q4unvYqC1dNh|iun-e8ic^HD|A9MgV& z*v<^Hrjgs?n~qh=7vi>n=8S8n5-0W{T{MK1TuuoC`Tr4h)W(fojr7l1AhqWj@Z)Nm zm&hMirnpVe5t&Lp{xXQ2c9?725b{P+o~{x6d}(>Va}%D`eKsl8Sh*F(&u9#fLcfc6 zt?ooUk+)*E(`{GLp5*be`MP6`jkXzP2hvRW5nC`G%7W0>%*F0d&sgiZtFMKfa%6kp zZ42jvVHUQ$amBKPHG{w(+!07bU)KDDWc8R!5?dB$_cQSU4*Nz7k2HU+2|~7wu5|CM z4pfaqcb^_aCa^uj7sPQuXtiNUHs|c|F;W=_y(03M6)HOQ#pSgeoOVWZmodsg(R$6J z4qADagRVA{F~aBVr4DkJG4k&QTNPeUR|-EhKchudiX(4>mwPbE&&DdSr&Ef9#IVdy z6Ag6ftmguaB>+oa3LC*ji5&wuc`|Z_$i#z9J_;Pcw(^Ekngybpz0(>4I=LP=gtJyM z4RSd&at?|VZ+D&rUSxgU z{G2xs`>ut@j5>|yY7giYaatTLMa!x}S(!gq$y@$`r;00_n$A<=ATYcx7EdV(U{M_S zCt_3I_$gtQ1rod>_9+p1COU44N{T1z>F zt?BTuAjYxEjJ+3eyPYbKyjV>f3RA?##~7h?CQatkQT&<&_?))*8H1m8Nu^=r>RjQu zqG11F@r>&MMVqPYTQ8?%IruRxMG|Uv1Rc!*f#fD+v&2Dl!`G>S+w48jUf^++y8Q zp-ni)m1}$yDGzV4e_nm1>iWdoF&WdeCkVo0C^aSl$%mlW28XFB$DPs_YZT~m*8wTY zxrPmXRZsP{mL@0dECL^Ld?O6UO98A18hnG_~D6r5P$DGw;kz4#AP6a@xJMWi{539LGAX3j^Ht3jV*WAO7a1o0`m#&F)k ziEm<@&XrI{ys1M_gsqlA^rX%YoyUR=ZG30BJciBd{#`D=-UP_evQn|MjI z2_65u4bCz*qeS|StX217ESx+k$U()t41`><&W(XkgJ3e;AM{ftC_?K6NnNIRASVbU z5nUtByBzS!6`k$w06v8lo zE+0+rV}uHdiV8nJGhKCc47*<;c|TvTg|m)#hJtxtIX&-^W~60h0UMBJtcsFj`{L}F zCwPprvXN^zb&ClY%oKpnj5Na8pMC)F<#m$!a(NVZ$NRq|KErM=1j_p>*5|6T=(7O5 zggP;47UwY>*JihoClH!2ly+i5AY8v6%KZlTI5OIE7ScJq{kjX{7FAZu(0*yPQvOy+m@bkoFMxsjEiiGLZjouPt8BHWN&=^ z&hsQp7tJ!jzOiON7q{~h52&E2jM!ut8%Cx;rDkk_(fIrz)#F8V?L2ob{TcMkf+{l(%g=MgkMHG6J6hDI`tjle8E(#YID`q9&z+Ab32*JIw z?OE7K&=5;e0N4kIbJ5SORXtJGnq3C0kWoz$i-hKJ@MH*^5Y8TpkycP$vq3@G32926 zH13j@QK~D!D`4Gxis4h&TJR{bkl^>)(Hj`tg{UQcKO=l;w#D@`g5;>%o?JfN{5p|l9&=H zyD`~4?qIQ^XC?(_dq}0Qa z{_IRd>cvuW_!=(u4EBMV1*ozR)n#1DFox`EuF`)ss0fPEP$sfN+Eqiho8#90w0i`-TwQ z4PQ%oGR`J)#oSoDI2BCwcHh_@xO5IHYz!;f(bPXw%&=*V3)~>wr$-&_+ECUXnIWuJ zKZs%;l}AgvLt2!_zZ`(SNU3*`eIpnwuyjmov|A1 zsso11We5B!_8?3R38=JGf8JZN zXNl#`j%)SiUFY@B8_HZ{r1~9a#C_Y}i*RH8Y*MWV4rvmPu!C(17KTdw!uu#(WZbm1 z+k)sK+zp%8Q@^3;W@ix&FC22!@BMSpq%iN^OOszauZ-C?w8k4xT?)&WandZd@Qrrj zm_q@u`XGdLV=X3Ar&WO{ibG@%Iv*DcL&1@25h?harudEdsr=7t3rql#O&<5P%djm$ zdH4@eNU(r_DT(lhV#E(V!z0z9wF8es1%^^`K0yQhCKKhjhTYZlJ~U_h=L!S_&rs+s z1rI}RhiC>wLZUlY*6Mx`pqZ4Mxw_|~mt60-I1mQ8B!iP{9dA-x40BzIlz*K9cq+^m z43hxF!&ajsC@IZ7iD@-`6VCkt<$C;R4dPqY7=*jAnqBvT9-RoZv@1AVk^BNC{?qG? z2MSrZ>XKwYh>O6T?1}Fc%%dLP>fEPyFy@>}Jt;l2>(po(sluA*%y2FJV-g!+At;-< zy)FDEPx3%t<5v6&XHKnqM;#YmRi5qgcvr~yuDwJ~xw?5R9m?DpnkXi36JYrfx|%X> zJ%U=Fc8`uM9+MsXnOa0{pf7mDBgPCAiiXMo%SoB!c_SMSI<@q}Dx)<5kDc2?4lX|Y zcIX<`t0F~4jZMm@d5J<#lk|{c5zy#75{GlW3qd>c7y%EY+d2jmo1jf`YvLg4PO0NY zSPez2K|yS_irQJEt3Iq_-ym*jc{#e9xIJ}>TG$0wB-U_Q zBWXEZCqrxwI&s%Nl=rkM?U?hldi!Xo?bt$=1zNT)C9Y4J#l8VrSB`#(;6jHlE(N^_ zLQfwF($iJcB_7nS+O364`v$pmwMPCt$hi@}3OVpyS@ir=v-Y=KA|C4=uE30-rB@(z zv#sVQm&{IA9A`Dk-w)$BsQ|E$!yqlL0xIH61Tpjn4PQZ9+?zr-*OPtPoHXm{v86%6 zDR|2IzICr;WlV2yL16e)l=pfg?$XX6(rpxu8);7S7!>pvc>9*~Rab{f|A%&x)l;qK zJKc1Fho7iL-^1^+=*tE^@wQmpbSFINLaEP26qY<=&*h-JbKBny&aP(37HQslVm!Q# zv$P}B!FrGu4xV~R3mpG1g_Zi0h9B4lX!c$Fd9l66S3`+d%weplkm(}!YwvSJxM$wF z(EUMO;XC6yAE{5*obY|mcbTl=uE{aSk4uF*E^1&L@MD9Y7q~}Vr%elpwZqT((VlRt z7r=<%v2?>5*Z7Cc4|-e+l%Kh!y{zl5s}q*+srJ<|NWlhM_- zDEw>ux(_yR@!cvl8JSjss!{ptrr;Q&7c9jFWFKtZkw8%%7J)^qTwX-(Jt@Owy=jP^ zx-|xk>zvQKOD!3ACze}nT%Ieelsk^Wj|K8g$rBkD6Z#Tmm-Iv&=%*xjZx2BfM3Nuy zG1m?nt_Xxv{yQuR9&43+JpcoqAM!B={Kbo)Q~oIU;%-+a1QT)jApCU77SIPlUp>XGPUZ> z-6KA8kz;D{KAf?BdoxOV*@HLwe9B7%Y^}dT7-xOViMsVH@N!|CQtPx`1qy>3<&ogR z2a3WE3W;;zLt?D-i-^-wFYGHP(#MTBT?^*P6nSZl7qp7?PEvJ(kLuKa?ja@{M{4EB zR;8>im`=9xw??rm?+lam$Fwi5zi5{PRUZ$)UfPXMy2c=v)h}by?Z1l3>tKKCU9!hn zwMOfLCZeRBk$;|AU$=G`DJ!I5qX^$oaoj9cPB&soO^I8~57gfUy$!V>Suh=(mwFZM0AU-+QACO)LZ5p*?gJF?1U$6KL49<9(jrq0n_9XX?Y1;{oA z4vE3l5{_};337U!N_g2L!U*HeeC|fM(KXo$+Vx-Xq#G`9M(~z|m%^ND@|n6&(ne~> zS+htMMjq_+p!7)r0oy;^V?F>?6nxK*8*bKaL6i<}yhcb-g!Xctv8t#<=DH^r<7()f z)#J_zN*@057)DAgUD!?8!aCm{ern1~j}jOpwZQ-hqalNq7vHSL$jq9F-=5C~E-90W z6eO;x;k&bKz<)X}@iJ-5H?Ok_EyKgqUz`40O7!ch6T zxYi8a0vXwT&}5_JIOxjCi@c~sOo_y~AR0if+7w_q%}Q(*#kR=`+s)0v2gt92e~(WV zxPl>Iz~Ii&+XIMcamDzjK{N{cMo%iD%UfeBoC3#cyv;=(P!iAF2dZ{d6Zp}5kxTqC z8+B9pQ~dbL%o`8TWzh(|W!n4@Q0e;jyT+H&tx}+5_7UxGZmXg!OZ21KH^@+tehy|% zQan{#n1ICOC9zve?>tOb79UN*hzEO37xS4LNWwr;GU1&|G{=3Q6R{|s0@X}}4Z^mI z018B32x#Aa;;yMZg)UfVhs87Hx&*Ba#&{XCr@ag52LI$@XGV>d)OP5!M;(yY z#+#g)tu{aMIX7n-5AhT+5@PeX2~9*}PE+}DEQ7A{JquQiBwLa~9oA=c9k%(ncY-Ol z^7%H}LO6hAbvl^KkqfAy6M?NmQiA{~)im5)=Ao&w1}SKP%+7{ERbg_S>8qLrN4P{B z>QxZ`QrNlqbgV>oi0Rzfu_b$C*{czjlTsetXH;L7c~Jg%^DJhjiEC8`3@kk((9dU1 z@7CuucX!4nBTE!8<{g9TC7McU%^vgjzLhKsg+eD+q@BLMFpapY$99Cw;ZDH&{XO@)PEzv&EQ*mESQD; z-{(9pQf2FGZlfP9jZ*IZxW}dQhIp+3c&Pm&+qB>H#i7!&|A;pAFT4la zX+AiE->~(QXxcOIjxO;Qkr6qr&SJ3%c^bMRk}y~G@>NLC2Xa1C)%Q;112a8`|07;` zORjk=8j8umumE`S`WX}{#tAf`H7%5PfT{720JzzD9S|l52F>)S3zW13Gn4TS*o;uL zaAZ9(P`yyrAFq0ZGDEGKqL8Zvtb;cv*LXj8i^}5zk?*Ba}2r3i^PghEef9;$+Ga zWJ(fab$B>kEFd&js{LmED3R3-cUlYZ0Vr10ia^oN)I<%VL&om?wIDT;qQ*C%`e9~q zKya%_=5vz1o@Q%h*%}0`~+Mp1IogGbJ;Jn|K_6eu1d9+DS|$; zft*^isrd2@tNNKJqIIicwo@0b05vZGS*eN@322oG_m@U;;3p{Lp@rV{HR}_jBwU~d zG5eDcl_nC&H9V2v1XhG;0v1@0`qMh+Wt3I6>fpN37{NzC81OUQu;3Y33A8NO%BL;i zsUD4(G8BO;pEf{fVh4Mo!BsM6-yE-E1TnN0sq)D{gV0FV#e&GR?cz7PR(@EU-xsq_Li;IHLJ+R!!hy z62TSG7FeJ~ux^A=#alj@-U=J$#Yv5#4~e5k=}CcEceL6I7qBo*jR7Xa0G@i$1yqkg z)EiL*@wRuad9fl)4?HquHoOI?X$F*Fl13)3!}t>1@bW!Pw=4vxLSwChbB&1o2ngb^ zLR~+kr+KOM0#Kre(Sd(4K{$XgY#eZAH=tvg>1~Mt82rljn9ZC9B@rAlDif?kCXr7d zVP;i05vb>ZphLL{fwz2Nqx?vVh=LFTG6nutHh(Xs6)vRw1t1}OuRJc><2tw>RPT2L zuy1alJId<$7>G~kqK{axk0_^nKb(ZTF~~P;#M_W)94imF`gqh<%K0&-az&sKJf6ftNAyTyw1-5qaP0nFZ; zX?nKE+=WK3nH@d6@E(gY#Yrjj>aGii3qGE{f05a zh(&U}L#*8MedLbn$}S1=`;s50V*b|X(d#|c{Ov|+D#d^mwF9o`1Rk}1r zi14g*guvy2sO%{En93qhdnm8~XTNm{pqf=O1<*xA7(LL153W`SIyEAb*96gnqA*Z( z3iBs0#Lqx4*+icOyVw?J4`Pv1irov*Zjypl2ZSs6SkPJnwM^;}V&ANH#%cfq^ymxl zAjfQrB_4`AKv1-pt9jRRX%nyyNDZ&WtkBAM47NzO&JTYxD29V9V^2)$VTZXY>Iv!> z@*X&WP*iS;gU3XasHwR`k+Oemp$GmO+|Vp7MHwcwq64X8S7x1)@?)*ECbHno2g;24 zZ??KCH)n>RCA)Iv)*~y?TCRMT17fqA#+n%T65K#@)2;wD?W1VqiB-N;#51Z6s%uXr zxt$uy43Kv$LttA7d)8`}IY0lEHa+=+)BU-%Wfr98c(5NAz!^S&6aeP)nRk4Yn3d|s z_W|BHhzkm+E!eKRQcI35A zm*NzF&0JyA2X1L=IyBop3h%t-dgA)xtPY!70rhkwq|Jtc=*k{oU|KLB-WW%38*1bc>{Ts0W{k&vi0#27c^Q) z-8?idy-g)Aij`Y&c|~W;%0Q90JgHo`C#}x1UD$+B5vlmhn3S&{p7K)lEg9JlM3`9g zt;d5ayMFWu?1#yU_~fJ=WCJG^GyfzoDtK-VbMDj&>5;0!CCXrmK}KJ>a@(Iet}7?h zGxNC?`{721;1G(*^UOk9*u#O$hG!7DxCIjzZDn#8{LeM^$+8XYngOCT52n#a+^5zH zF@JK`Rdk$BqZCI`cGR(AUo$lg%o43eT7nS6g)E!1D+mwKZ$C(`+Ntts)~prAu;UJd0YMm^TZx;l@nt&B&Dm1cPh`)Esm`&EZaUvY|$^QW!K;gfx z5+Ko>h{4LhKtGH1%bApPJ$}*jcV?wFbxw8bz-Hm) zvFFIoey(=g$m^rd;caU(Qmn_{^dhqD5lax8p)-$*`9Og+Dk>#9?I9GOq$Q=npv;k> zeW9j$!dYORb^^q3II@W-0~19s|CpfXsV=}8f+hm6kMvu|#&!I6XfuQ_aClh9i!eCs zYk=f}#dkL3gX6eZYDZf~TynZx@OL1nF-Ja5uMJP||@P0pK|x#s^` z#xfp5XO=`n<4r+GyeLo}p@vZ9y2x48$zq#DtpL?!`Hk?Vu&DexDc&d-~Mr6 zImVQT#mix)zkqI5jG~Fo`myOIhS{2d7+dg$s*WIp>Ytc1LDj&K(=TIE4ZFEk3vva4 z0wUA6%@9vBYlMeGivt&orBbINShXGAi;BG@@7bNcCoD2{j-c7BP2gJ9aPidzlXY&F z$lS2xCc+|0AYTN{zRg$^Wm-|5*#)Aq-Q$)su4uVthYA#=PkjQdX{r>nVbSu0mQR8n z2Vx;bCZb4q_G^?Ih+wRdBI9;LXN~7{&5n-&CZEw1)0_1#hjyrj7A2sCD#G|}k?l0e<%LzU*;EuBTAvq- zFaNlB21Ki1cSDvVV<+N8_3vLgmS@;mP;F=-2E>YPScP4cty{lN7%&ZVbdW1(k4T=f zgbv!cjMGQ5#5k7O{g#y+;s`QGkIm`MIGZG27ktGNswfe7mc(ETZmQ3sYO)X&PZ}|* zWErD!$n)cu7+d$vS-EydUR{Wy&Ibl>T(SzRTWxcMmTN$Jb&)IMTGWf#fNSARYQ6^7 z?v_dk*JaKFOh}))b;Sqy$74*wP)et|bmftD;1|I#%#PqrQ$K)Sc;54RwbT zO$h-m2B+X?R~@m8Clxtii}k|(K#y7J8d=F`y0emSLy#W4k}O+6sS*f=fchL509-Vp zmFZ}vr7>h^+*jxkE;&jQh#Q4a_~!0sV!d{$P+qS^_iUL3=+VXAnaez2K{5JK2p)SM z7+a5{T{qaWBP5WEluK~g?&c-S1Wo5qmi(4xM?Th<^WrTl*45@EdG2{Fia&x?EP6`K zGV7}8Fqto&Nhr?Q0QCgLPPL44EL+fARy@n|xvA1u$#l6W#5@zp&5CEthwvOR(~z9x zMMaJjB3wqaC6(+vclr@RLPK9(7#70=Yp_)7c*%cqjsY_IUy>KY3Tc<${Z%#g+VD6X znxU0c2(@03tIOAa#ZINVS!wMCe_!q_MvpYy01PzsF)8Fi`Tmf&FU zDj{g*hqDjw&)&pxj}#Avw!MiKuxEvw{%`H_Fg?g;6qz$aAdC zTM2T**cL64xWEl$L0)GS-)69b&tE)qT?d{sNp_+xNhXT5(#MgWDc(`mpxYxE>m;4b z6s2%9`9yU`{wI09aYUe4ffvtyHt`atyEb;>^v?P@wSL}MKj+rZ(*C_X{mbcah(+LD zty%R7B))S!<%m1}H*rVAs$vBeXx8Q0L9cAjZAXqA-rdiBO|E#(L_MHc>@CSV4eX3Q zWkK#JZ*t^deVW|xW`#V$(QkF#%5Cx%l;F1_i8@I~Xx=W-%gTD{1&BKg(6li8REX`8 zCO0ktP@Uiv6stf^V<_2$NXhxdu~ag;oiRh!A1+=IR-{|O3wIs)kAr3^Zl!ia*1JTMCh}!8|jya+j>UW#v^S@$AK{-Yb1pwCZ;C_EHItG-nkn zd$WY_WtUXbipmCui=ADZ1IIy2-z#G)!nj5C)^$R^;u*Vkfw^2KtgP@2p!nA}YzP0e zE7LWqg^OwiW!i*BLQ=d^?X19Js|$tmYKK9;a8GYp$yX5jz!0Y~w z2C5)+r((V_${s{l(29bcUb4uHFm1|sEU!xxTy57RqhWL_p;2icwQSJ=rGSoYmB*Bp z84cs16rx+s?w;zhH*+QY%ynI|r;ru#^xtBVO4M`p?dxyB^q|9XYspr1rti>#&@KNS zb?_NfPKiOJt$F&FM>T~T>(I#6zSI>+?lw%pSV<51(dQ%pgtNX`_;mBJj!&yAuu(IH$2oj?8v?#}vthjmx9 z{|M7%g{&YB`|)U4V7}`3|E4^zf4jQ6_$)lHJTk5c&@quSwXw=D9b7p8P_RnjI58e8 zdc!2ymMmc@AVon%Qt+rkbTvloE}EFD>sj{?6xL|SEDSAB46XB? z4&!vS%cZWutL`5Uf9RCrj*uAA?_r`b2^&Uy+X7Vp1vmg#sooV;H>i% zDG&**J8sws*Qg6ox|S4t&zhL^Du6*)XYDNggD6-PlT{PKH82l)OL+zYnoj%iS`IJ593!R=S)iPIKtH+~k(I&#Alblzc+N-(c?Ov1p+Wq-Rh_!)dMvGY z7$sT*IX!M74-F~=LVU@JI-}lUFuWiizIye=7hn9zoLnd;0e>?AoVsf5Ew~N*S>)=w zSp%^+f1ax5UTnVnV{VWpn?~z`>y(0A&WLmwSn?U!%t$&TcQfLU^#gl`LW(Ko7ahxF zZ{Xe*bdyplLF#QLs7nyFV|OcI(ydKkwqhs)nuB2};%UXV`d_9Z%OX#ox|Ly^uaMtUVZRTUC^JrIYtkR&}IDk-{-3PJ+j^xu})!q??7G8zuO@_FRbJF^*gt2 z7u4&<>U56F`N$M$#hpmu4YB&A$##Jjgu;$0ADNaV%Fj5;vpV zhtbzlr{QhTRN6|w(yj+3XA$-$UXTvjFE%@~(4addZJSI>Pm^O&OV!|C?+Tjp#DMvp zA-b<0BiJtXxaeJ2D1eyPmC_*_|xb#2y_0Y|XM9B7k}tw|8_Nnx5(@8mrK@_ENCw30;~dw z$hWn(x{ZZ;jm57=>l+>M@M%s9x@JwO137OA^wW@j;JrMC)lX;gb3KzEj4;+amArVs zsWh}1oDNM7ToRGx~iypP0YJvwVRXbg23ye1Q=ekK#?WFq}#Or(34^^u646(jfi zp?o2$tBbeUdvKT)R)-+wE7XU3 zp&?LhcvWYYtRjEt9u`OI>Ilgu0z_1{0NM$epil49_MK7>Vobr`Tdx~vx;-dA3!~0{-?jQya8sxGR{ClH5VzX~oCgTMqsdM) zIxhGaZ}i;wRzjAncu+N~H%tAZ1avN4yPN8_0HMfZ4WwZMTZ}0ay1t;*Cfe`{k<~aj zgU_hr(5QjSi$pkXV-;W&>YAfiqi&XNmK?wIR^o}$=y8&z*wQr|z3932KO`x`dg4q1 zq*Shtcc2>BR<~g4vzVAnh^J_muRuUU6jbg4MHp!*>LS&6>`k@7cfA3>w`vJ4fERch zZH)a(R#9~yo-dw1d+_p)0{H`e)iaYDanjt42al*^H2t>7yix2upnW)?@F4>2@s$o% zgW@Q{n>{f-=DpE~o!RN$6AfNYVO zLaG7$WLp=y5UD)2zg3j0Mb6)A$SKY287a0RMREJ8X&iK$s-npTMK9egAmBTcN6L#z zS9VhLzR}tJX1m8cKpZ;U$)PzJN>@`Zbd!y36$OVv#_BU`R z--|c*EU7E!6$@T3kD;rxT;H%=Q}+($2RI2P(55}Fx+0K!UX{@6=~%ZLw!>tIN*mU= zL{we36QVghCIj}(J(rdIA0}ucU-AM6r>l{PwpVAPx;i*uHfI?Q>(E6u1B_bfCIiAM z89-J4$5Kc3ib8JqR;Nq)Ct$*O%?L1p7uN^o_U2zT&AJ>Of8}gfe~k5<`7O;9Q>%p1 z<oY4>hVC?%I>t*@b%TDOZgYBSQS#=-F34YZ z##QIZFCv|C=z5I$+GMR=1~2PMltPc}7m)Gxs?2oTXTr<m$lI(qd+Cr-g5{*Mx>zb#Chp68Xz4J}=yKS;1UlZaZ;&L$XB z3aIG3NE;bc9T&L(VcSDlX=~Kr633#wdZGkAnrJgaC0a|9mnXLsV$=Y%1(TN~5hWJ# zM-W|eic^tO?!;k;vKOlHD;17O87ddvHKp}F#7d$Wm*6Mwu7y_%v@5=$U%>tiEaTbx z*Zv}c5!Ji9Wg7aMWp#-Yi5j{-1E$os=2bJ4#%89RM?;Zy2xb;h)6K}XstIiM*TXZq zX(fAHwT@&TxlroSlE30s!M8`D>iC6c$FxMG7#%S&C=DPPoP51ogJSi0F(dCqbza1v z4_Pw!Yr*7uQNeE>b?I=hBNt9fWIB9Zv1_Pkg$1W$4l6zb^2&KZM5$M(6 zu+0eihtK%B)DesSSm;mlPz5G?SV54en9&0h1Bt*NvpO=yHF}-*IVuhYc~DNb>Sq1F z#%T7RWJ|fblnFj(MDk*tvCene_HB2eTJPxSAsa9)2*W^2HL5=USE`_pOrsZ}y6J84 z=!Wf%W_!2u=)5?ZHNLaXZ8Ln<+blS9j^MkaL&b@R6BhBxcyyR{d%p1`9&|%3-gUk) z!b_z^$d+TvHss7}fB?A@qU2DuM@0!DP_zo3&@8Ey5TnC5RO4PNrID~RA#Uoe+0({z z$_osKX~zpHtMC{#ZrE;)3N60hZYvf7>gqOatesEvUYDYV`({c6k6H?6vnD{vo+YlE zq9C7b3mtsL1Ypx3ceaePhVzV<|Z?KP(%2+o>~;F2LF&7)_WFx-yl#r0V|i zhTTQcsN;9|axX<8W$Th*F(bHMaNTkbw_Y+*7wnd)ZBMz>RfkQ?#I-uP-xZQpT&}=@ z`afTb_o6cX5N9D7NKxpe32&9I4w()qUuiF_Iw*IH*`-|F=Cf7Tlwhuc##3w*c*Ez` znOM@yWNo;4)79R)&B*mXTMI&G0KPj4jGeEc=r~#w)HVExl|t9@v>NTb=xJAyj6pSM zT99|Vc#ZG(3Z6lX;}`RvM8Qas)w1&Cl9*!z#b3dt@Ej!tSYaUim7%L0Lf|%wp%suy z^5W@>r_Wz14Fr8cRBGI^{ndWHAU``>R5N)_eun?Q(EtB}{7il(#;STg~UQE+Mrtc8Szy9M~*&2_z>G9(RM9@{9jrzJ)sf6%mDQ;rR(n;gsBgLmJ7uw%}h_8pDsF6ha!gU6rJswsdr`#qo&0Ux^&Y6 zLkiA{-5uQp34)PZ*8G-{4PS3qX-C_FKOF8cH~F>LnVW*~YC+Dc<6{X1b<~=4B7*U- zwIv;&!`XhO-7RhjOJ@+eg4`{i#uqOR_G#qJ4UQB`oA)gbIvt==hM+j-mm0$H!-neRgFTULoBt}h*7i7 zHqEurr8GfGD$+J$_yDThS)blIuz%GwlZy1O!O2$ct%%>rhzGX|rYH)FzAVTY9()NN zMl}zwD&pB^=bjmw;HLR^Ah??MNG>3=UNxf$(S_nB;5%3PY;*k1-5b?h*@9;?6L6(H zCuFJP!YFu{?0cb?+;dD7uR%j%qX;u|O=WkRp6Lq$FNR#VFwP(P^szB#b>t?HqQCMm zQ7&N#p;$@>qLJ2T$i8Pukg@Ld1h15^l3p zLp%U`wDou|8UaP}ss;iwBL%wyuBaBrd!GgtO((_L%yg#Y2B_z*GltznPpXYIn%oyB zUywO@cB)Q!f!7E&(H2bf&J4=CcGs!*0uGy0TrLLpA z`L5@oO5Nk|Q!CImX$e&;k8WhBFXBMrA9ro z7_(I3={Z;@ULM+dW826wJ>GDqAKg3=d>IX+wb>2u7>)+iZqOBAA**Hc2-NBXEPD58 zuCYM%MKJZotwZj+#Oj2d7ZBYm3Or)IYqrX$D<>_&!l7373PL!>UIP0o-xYu^+;jlL zN4&><@L2l}k9FMfPq)EBQz6Wh*S(w{feddqSp}F#{73wNhPyUE8b|vgtPg&_pVFm`OsUQd^W{L~)L)!vjG}&P2 zICk(K?u(@&VBm@)R))iUOj24NyDE%m#Y8Z1^JZbjh@&re2g%MU&ojkmW#_Fg+VXW0D(wPz{tqy$}!%_2PcSK5@Jepyz zKj{=Yt5=UO+8NDxHDtLt^gi#|r{aw1`)XU4b3v>b)%Vox^-y%5uK=A#Fj*WQH`Xh> zj{W#ZB@T7;LTAe-M*Jle%QB6aZPap5KP%Dd-yB-*TISsQG0>phn08>AM0*30I3vs2 zP2JY~Rx!dT10NbgH!RzdlBKmXuu&^q!0pd$YXE{oYRX8CIvaU}uuASr-eKB&1@z4% zFS1)!?#Nb138Vu^Je88KA=4}1o}?9OkT5%=Y#nKB>j4V5^F0OM*{H4fU%-E@Ro?vnd^KPv@kBQ0IdyJIq~YG{Q+NS(^e;|ZyFU5i>5He& zPt=@WoT}MdmiZU+=cmL24%%r^&y2~AZT`G4iOZr)bqu-QL}#;VW?0BW@%7yBTeyDK zjlB~o`#_c%ErON?zf_>8mJj27v!?us^5ju@RXzrbxW>^D6y`|_4rB|ssYEJRExvf zfd6bKA~&H~G3(L3jR9PWtTroPwFTWe1JD4}MNu}{T|ISVuGc^D?E!0}K09OqOzo$L z*}u`RVxW-r$m2nP%$mz4d170NTb?>q8eW*98-G?Z@PEAgvs#70L{%{IzGcP#$KLyQ zH*Q;b2!j_pbPbbfd(C+VJ~n}JA3LQN710otQHdEWc? z;9h_PB@zTBdB)Qr{%9u_2?7_uy%+b_XRa16$g^!GxL`D9ZD}NZA{ILp)%8426M6-) zh~^RpCy7csD6M&jh(j0kAGxYSf1b(i(4Pfi449`*KENM7NBsR0C#U zftetFPWPobWX7U)q|2_F4N~htJe$AFY1Ce(#8VJ8q!emE5!5y4Nt=|Iiz%J*gsT;y z;N!-!daShQ!}!~cHKt-6p=6c_iiPaR2~mq&6!Qgn)>#(+G1Ow+q6(`D1|oP}2OVHW z(6Dt(0kknt)mn=J-}|(122>)0OI+T(niwE;i{=g1RLzXRPmMykdHUVB0WC#QH%4*h zpxn|zEkv%*2Cz054k9q>0U?NEYp=uw%-tECHL?ivJrbNMRC+&Jy0^yweA)0>ke#s9 z%MjiH`Fx{&X2|E&*2a380n>3RZhpSt=fo6_?E~$*0aM+;LLLv3*d36XYzInB2DbAr z&vrIy_y)}9v*$Sh-AA;&A~0#JXJ?`s&Q3wg+C=}as-1ae)R%_^Y&t!fP)!rBoQ{?V zK1xhOMrAV=Q1KWZ9BFM7eXQ-bQ8Fl$0rMCX%C6CUcAR}sCy&X1B;U4>ZGqgjTc$l`dA4Vmq8@Y$ zlTcei7Ip825XlrKmAWO8Um68TqHUxQI)C1T!)JYG!c5SARH8)ySn&RMn0LPI#2l58 z5*t7HYM0A++N1bpUS!|{%~>L74034$JH{g7NR~I3ht^kxDV2;I-b~KFX+Rb5vr18@ zx3jl-%<>zavLARBivoYX?x6Omw1ze0_~Y$SAl$L2q#qR3if7}1$-Q`aH< zA!l-F>WBk`)#UselG67;1xRr_-Tpg(nlxupL7tKO1u6-oj!;>VT+wvd9tUmWE={+8 z`7d06@oV~CQ<97tN&6IL@U}JXR?G-Y$_&#Mn{N&*Lx?6-rtr1&>DU1O@?Yfc`S^?f z7@xF3{5?vyx#!5W!f&-gZ#m@H27c?|ptp8_-6{dL8oIq}oO&4SR`c%f1G(i{S3lI& z+WIx%Y?+MmHmt3NfxEo`p=;f?qH1-ac3DKNWl3HKP3y>%a?e@0O%OiSWc)Xic%np3 z=S=G$X5Y&O;)bDE>p1BQpa6F}JqF+j20qS|`^B zYpaH*)!JhZoUJ;RR%?$P7+c-RtRA&;Z@5~Wqv*oZdJrtFopH3f?-d>RSr3GrwQt<4 z&gBch%i0nv%WKWp5+}=-`m!ZPmKVWq8+Y}E2P9!qb=Tm^D7h`BEMbr}21-LpqXlMJn{r(2{ZdGO zj=H5yUt^-geQ|7d((mcLWH{Zap056xwVC`vtiChZp6%2`ZhYRB0)RUhfZ@QowicKG zm{*|mN)+UtV$KY3nlZWI%NOJ#lLZ#kQfr^-GzmgVXqh-Ptqf)63I>7K2;W04FvqmR z_xpssS6@CkB2V7Wr2fB5&E%6YxlES{kGRsBN;YG8j=N>b%nRN=`OH;M9%NH^zmJw> ztvttq+-tp=-D!!)@?7L)ol$3Xwea@#27I>;CeRS|mO^B$D1*lp_k zp{9~}>s9F1ut6_pL||3dBU6ADi+yleKIH9ky0Tx_9N5`}P}9`>bR4t0V;R$DM=%83 zDIGr3+B~TwPo6y)lZn-jj@)^d&v|>CiM$*L)I~UXLJma^BZ_E75=QTs%>vSlW`$Lk zpSGsblEA7}B&c!~*#@)sz`kKIvhu>@D$PV5G2{8cN4(ULOevG@TDOyYKn72O{UF%; z<#{-r>>F@_&G;_`BbVov!7o`Zxm22E#9ZEb31x+@rj(Y%ho>q1Rb=Gin~53L(~m^7 z9r0i-g}i#V$8|)1Epn#yk|0H=e0p8E(SIlBR8iD%fCKr`_Dg}N)t4#YRlzHclCpz7 z5dqtuzZP*1rNVj1vh2U=5uV}SNveAzZ$!rY@jBOx#zddoBmCzNIakcfje&sq~`0o1B&O!fE%r=G(L-aGs{S3dS zqFU*xFV}R`N|c-2Q}5{MhK?JkU3z=Hct8FSdgZP(zvY=XmaBe>_0!;Zi>itTaOs^a zsAi-jEfU3-o}xd*Zh5`Ipk%S#J^JDaT#B4CtWy-fun;w!I}o;OV`r}s_|S0I5(iOl z^6nlF(M*|@uNfy1W147Sy$W8j&cde4I8 z8w!(su~jMBQGe-{#g^lGANcP2awl2si&NW_)%xj1Sq(o-SboPN)^TAlHmZjc*V}KV z_*30XHsZXOM6X`4@(`|XfLUh(-&_z!@1$`T7=s@< zT4)l!wLy~@t7k%^QtQ|hLiukmIW>Wk)R^CjBtcrMoUtR<(scacwTQ}aQk!arPSJT? zgC6%S2FQomlzpU)KgNBX*3~<8YENo^@47uMO|bkeTM>W;orzrU&Ah&sn2LZ1$Ax71 z7(e}3h}$LTV6fM$rSwrdX|LI=iL&O;r9arua8LuQr`1|@ppn#w@cQv-8=CeOk`nfq z1Lz+lgubVw>C9rZbqo$6~QO?bAhK$^c9Crs&M_Epce+Jnff7rO3)ZaiD-RX^?F4H{os?=4|Z_)+HHe~946PS z3KYWw^f6E4NA}xxR=%lQlFGs$bzPwRGh%*Xi0e`;0SUa71c+(l3$F?}yM~ANbJ995 z)mc`m|1#yOWH^3zAX5m8jEMhQvxy8i9SC&?`fOZPhwZRQMatsFl=#XgCnqO}efrf` zUlEZLN@5oAbp1hZ$JoQaKYKPN|8x5K6@))7l|<}x{HlJS*0BfSQn&vzdHbe9k*mK6 zx5kp3wZLKjF#q)Bmt%6ybHFm-2I$E~WZID7OejJDQ4`c8|J`xhR$;(y0lM_4a9!A#o7ZUX)CI zz&NB%`(*7b!P9FRp#he;tyRYM;rA00%K@Pum{(F4tPzWyvP3prYnh9G!WQs5l5Mu9 zy{t&1bX$xd$5mhvoU{_+LkS@WuLHg68A2@GTGC#^A=wn5WVak~6{)RvzQny$WE(vr zWToGLhWr`^N@1S776}f?3GaLPSoaqPaQq`tLp+Ai!Y1R^@uBw-}0c zTU?FSa!IJ6)W<97bXx~OgKB%QH}_hIDeVx;4Ffw!jZ}WSrBue&Ae$9tttHGA#;(8( z=ytCn8QTL0W^8@LGv*zK(6!Fw%z<|mD$DC2RWP8(J`%Y*%o*FB=#8 zEf*|hxd$KK21DAMw*}2&gUTGS92g0EuUM|v;%stB;+)^HT$00Ye5vcE)=I1AKgc`* zUkVRYhQ}>y&YUAx4I5{PQ|!8;mKK(=-fxhg39N7>iro0TY{8{S_#NKox^mu3yrQ^F746C-Z`1FD;CZQbsKPTSAbnhDvdP}T;*MWPS~7f!z&hO zM`hKATG~};YM>x=xlFvy_aItlB4$^`G~seF>236HdKrVS$}4*p4mByg4Gp$`$uh}7 zFIlU&qnDwJKI;cYKu%x6Or0M6)0=Ai-*=~YYF$yk@m;ob-ysoA61iRv!?Jz9j;_0e zE9}(nG_E(Hr3i#>pjI(P1aQ{hi6TiVF!?kiBC{PF5t=0SJ5{{^|7de&4uEgl4bGNN z-r2H!6t%q>A`h^oPr6E9Rm}<_E3mc?$Lo_vc&b#+r-kxw**@@r?if1PvPR1F5rJN# zukOIcMuxzVbIbxP)aAk5WQ>(k0Qvy_(Nnht1}vfiYS95duH#~&X`y>%yjwRQK?26P zty&w4`|O_P@#$4-{#acltDbk2Qu}A3|4yf(P&S9b zc0KLO(Zstn8DS1>wTf$%!I#2-mG8*|0)2H{U>fFpP^FL{em*(}9S`JEbeQ$9fwDuh zOrT_xM+nLauS7HDWT>CcA8H+<0b$?vd>(xiicoIX-YF=|69vqCv2N?KKT^I^A z8~vHEED~|3ms*c8mkQLX$`s`~Pg6VIUT#So-(FFfY@MQ)TPj7BDpXcHG7~O~BnIn5 zwg*WPo^mK25Ch|3Qb}@%e;h~4B7|p{r&E@Se05ZIqhGJ}Yo2%-!Y^}1q9V^(rjpeI z>Rct|x{}$eR$r9LT?XdUQlAcVUrEzR?ARB)A<8Px1 zZsG8!G5q5XPrrWpoBfYJK7D$4_~YMRe|vLv@els=w;!`2y~Y3f?eIr-@sGexo<9A$ z8OUM1)igeysgcM>NctC0t1h4uOQ3DeOC_XACRS?c53tqysMwvwTk9iua1A>&e&xeO zr*w7%F0*qUn8IkEt|u-imM%q3^A(AWX>8S_{)JNjW9OoOFus`QyJRt@EapYpqmTwX z?^5I*iqx9ow>G8%AM-nLO;K|i-Qq2&Bqsu^IC-{Do)|xP%4nA@5KYk#2q9vo>1{j0 zG1bh!=t2!RtB0nOr&>>Z4Qj%JFCPgm0>FP|ue6Tibw+hb$#B^iED_fd{_HSqq>2)7 z&ot#^W+V|2O#;g0k-8QGo9DuwRqF`UTS$KqwLdV6Cem;4rpsl*BPfUuH&s~Xa@*WM zP2cxCEmBfQy~#ypp7Hi-PE0w#dt1vNilguhI;`K{yL5s{Ny*9-1R16FdMOLIY?v9K z{_E{g%)q^0s++Q`h`wMq9wTT!bPsfoLD(_&iEoXG;yITe#VXw;<{EqBmmV~su5!#8C! z3|MJDHM%djUz>E8m?ca>1`8|8_G9!+D(>{={3#>fNp!TfyJu!E2z^@14wZO)%@~U@ zKPF*i_j!@YBQkwzLj>D97BiCR3qpzBfL5fOX#ztXn_Q$sW^^eR0^lVhx}*_T9vsJ< zMz{J^@Y%uej!5*>gg3j(R!{L!i(C}*cKwZ(Z$}B0mb#7onp?6Y@hYR;sk<0gC%G`* z9~cLr#Q*obDJOoCBdFRg4xcO(sS)EmJm6kh%rMWUoBqUg0iD~9Rsch&y6 zVd*lVDm<3s$q$az-0{BZGflI2L=(O3uD(C(iWDeMYAqD&#n_3D-d%}!vSiWN;Pbr0i!ouD$~!7-U5VHM zFUK+G?4LGz0+y__+eTEetTSNQX2WJe864{wxTiEt4#8T8#Se~%5(4-R@Ir5&rx{`I zxr9n6NLyGMioxckf}%gEcY-Z^CAz0Xsf;MkJ14>pVY6z_b z#;=pHh)GHF)e&Htab;toF{0K#R=ol#%@92-XOYOH%8LkAijrwAZdq2QCTMAJ<5L2O zjS9e(u@S5gx*6nbxqmf@PU%g{1U&{gjn8O^><( z#w`hX(kggJ4?ji96pg8(y&|(;(-Jw$M7#jOoJg3koXzqkAeP*PXa;_)K~EH@zRU>Q1lrN&s%d@;a9 zl)}g*Eq%Kt-^WFVcyO;p9OlnkgKn!{u97b>xkrL|3m*t_JEYKyTF1C03wpP`rMc2x zM3~2xC~?s>ey)ZPQ6Hz&Q`_4x^}9t5C*j|Dz_3&<4gh~I{LZB(`nJG&`s&CIO+{Z}bk)!&X1Te*@eO<9I6}+HfLw@TJ}{Aze^HZmX>~kj{F{jGMKLO@P%4 zo@cen4>ik>98KG5vvi?1ti|P#&N4h^1FCm$PjzGW!Oj$Vcd&uM9lHfklRB^5MQ_rq zR@MWdLT5l~!0EzGf-7}hus`Up9TDDcp>-Q2naD;aS-*7N7`@v@5Ti|jW?`D0QQBa% zZuh5Gmxf6nW+FqjhtGy7ZfHqsyxd?nrS@U-$wMWqQh74Im5eqy@ z#%$;!Ay*>6kSC=oS};kZ2-@z1@JuS2Bo$V-*#H7GPUA~*711yj?quH^3+OVT#zZwM zvIs*RxmxuJ(%ZNwfK@}Hm5$HQlTcX~!dR-^NMM;;p%h(M;_|IG~CnV%rgFZLWfq9UO!m(Q27_wytNolRm6|3p=LkI_lF>_m!-~0IP@))@h+c zN)?a50?22McP%YZDr_u8*bxO)1e7JSSmsPNYpa3#ew~oc?!u*UXb`qaC7rMoqS439 zfNwB_1U%_E3p|{uzSL2G!JXNf5T(o18SutJhG_|ZAU}W)-RTL>&&!zhe@Dfo#%`51zqobC$83DsX(73Z8sM=*CcMM`3G1xZ41o z%yvn8a%yMUO%GsB)H}c)-oWL8!cb4icLoB!lrLpoc8h%=ObJ380@?@Ab7M>dV{&+! zEEmp~0|iUTo66MHI#t|p2laTC3n|~&ZrjWtb_$^e20P8+CO+K6o2ED1#GBgZa1(cU z*w#}mwX?c$(V6tt2bjDO$^HGaz?IMB<@&_ z8{p#ol4g>N>?P$%s7J_dmC7juP<8eAgq~i=lA)XBP7MJ}?JwP@2@GB4?2d~fj1DG^ zp-`w{NM-Ko3qCKAU2Ui`gx=(F_-z^YV}y#U{w3sU zZ--~bk}b1i$#xmCc_)hZ(;^Eyh1T2W!~_F z-bbQVb@4c*8J)9H+3l!G4)UTeyRE0Qf|RNF!}<7p&&Rbb^RRY8P%?)vZkboOc|vMM za_#r}=;R{hN--PeT%HVwPl*HT>$%PF1aP2WTr+!LVr@DX0Jv%2b59n=def?QMtHhJ zChe!bpFy*MF|B6!P{WOOxF3>K~F zu%T`>-A><}hu@=mgfWs0OQBi!rbz_%NRWJGAHXTfu+h?5Pj1;N+&D-~;I)AYw%uaJ zK@HxlN0;3tee137=EN2RU*)zoC@WkWz@iBQ3UV8kST{=o8RVL5B7E{>8@m@ayJkC( zpUJ)ed?OUR|3MZBr@zAz8UG#!zZzzFL8XnJSc3kT^W+ zUZ}j`J$q~zpILB`*H&4f_H6DMXNzX(0gSM-`g~1X|Tw%JMl>hl_&X zud3ElMq%4>U-t{Yx2=zP5nDF^R{d$aX~H$+zL(VgGY3WJ6(l-+n49euYr3hq;eb)= zNa<2@o&T$q3phakug5)>=W74q_Ufz;A)&i;2n9 zT33PlskRS!Z`w5k)!1pz5{mAW{ zfIteVNm~x;@w8r}%{Z8#@Rac_i)KOu^=?B+@!D2m-QZ9$vHaiw<8F*iJ#?q)tCVtM zz?7&@1k&NVk*#f$3_-XZy!29*^eZOgHmJun6-8$Dy{+=KJ4N;EZB2o4c;+-ziGg8* zoM}3liUIa!;P@|REYET6rmJSo74(@{FkET)(h;s*a|~yn>^KvlOUV_(V=EO^>=3S!)rCIv{IQH$T4Q)0~mB1kZv` zP-1u1X0bi+CL!0hN60_|9dah}$cA`Ys89rs(<}eIk&Q*Bq*rK%fqEkOfB4}|IM~`j z^-kMr-ithm?|Ix^@MA9`_&yJYFY)z4~e*5KRTbW0om> zxO2arb_l&aB^Mk^@6&Xc!cI5NbgrPm4s+jEO=f zEAMvtkc-eQTod*l4?>ubEg!E7TS(jDkzKIPYl96?sWNWk<5KdLMf2Eu7MWrz%cPj| z*3wuX6c4|G_AVq8`SvfRgM`1Uh}aII{?k+gZW`g~Nz9fk>!{}?GV-$8r$~^UX>V}6 zK0Z|QfkhOnNet(PyAVk%345<{>iu{m(o`$5Wq&h5Ge7GJx;pGj8Hhh&WdI@WJ2I5l zJPh`gIu&Yv4!dF6gm#44ujBH=$z+-2fE^YJtEvPbd6%U)ff<{VCF_MQ&_Ky&B=ZL@ z)<;*%=C$^@#tHb3eu_2RX$I-|*ufDbBICbh11dSr#y?myQ!h+F;F%A}sjKgdH;(YJ z`Bc5FY~qgPcbwfHn~bl~Jy(m7Nmf1v1@id!@PD1w2E0_A#<9uS#CoZwI>Xq3cs+vD zo+DE5`0FmKSTB{Q(|ECsyYQjm-)B&;Hw-q&JiBJmFn=^kmSd@Cw`R0hFwB`X($to+ zvmY?Vbq{yT@3>9mBzci#?TZ$qno&h1Q(fk`x_(x@qq&EBM$7;SK=!{+;D2HG|tD4NrVX!Ui4P z9V`SP_j$hzSi9?Q`W!w6!Tk1w?wbkjsI7G>uCCd#HQaDblfsAB?7v*;KM2VI%9JWX zk(oqALVbQCwEm9C`Nh@s#o6i2#rX^Jt@#3XmFF?J5tX^vbKxmd{tTcNFH*qO{Wxkt0~1ftnwW55|y_aWFoxidbTUCB2;**;Eb2 z#c!%_yfGnfZE~w$W8)j~nq`<}1!39ng)ZI#cB^9Q4%UQi&iS3G<+!sgx*AUK5<7>} zUIVXR?LH1XjXg$TAim5PB9W$uMzbhOo`n^-EIG$+bLi%*olD=%T@eLn)_P>u_^W+< z?S@+Z*`%cVLl@v$1N^ENa=txs|_iK$>^tsWfg(XNevr?N^ zi{Na6k3R7|D~0RS=<;t&86GUa2UH6Og=@+3lwQ=N@RJ&+SSZ< z6#h;&s5#P=XLHxr);?Wlg3FwZO7xi-PZ(71wz9U}cbiqF9&4Gic07DLS*?ZgIl5SD zw_rl&($uU?l2ru+Xsjc2j>s^%@suoce#a9w_Zb2gJexaJwA|WUre0m$kpEi$S5@bt zwPQFRZ{FSja4-0acfRQ8fq1qAA*0aJ>AIsmT8Ch>XmfwH%0YdS2lr(Un|5bcfJmZQ}CGuW=gW6lA z38j!qqjLAFeR=fdY-sNnw!zwQn?m$5E%S_)&x(YYfMn#{Nbak zy=DhmEyJ7AUQ)=8XfK+XnW>y+QmxP@Y%S?R?-RI8=II=VT#f?g!~^Pf&Yll$WNpH94s|6%h^dRvk{sCRK^b*6u8 zaV>@zYEx(}Yl7jmnwOeG4x6uoA;gu`2-nkw*w{}zHrA%MA!7>OcPQQ_*56zrZ)+&<9lZL1b3;{3S`+w%F-hqB&E)4- zr{7$>0=i*zi^n3V7Dg-IChjr()V7_k&SlF&op#xpyDlSdUQTMBx($z9y9H(4Q@88Y zDVw!Y;DPL&PW|cHB~=pzz~S=oL)zv%6DA+Y9i?ZE)addY^>;*dD;7Hk*9B3uTXD(z zWMCfP^Hyik=LeZjuGzn4{7lM_+H32(?xXIkoeaJsGmmZA{IaPOQ5c77eASc?vQ?BS z{^vPXCWFluG-|4iWaDUP>Y!=0@9+K98(J?j1?5+EVxG`Lzv$5NmG$J~7IJl6NH$4H zsI*eogs-_-$`8SrccwLYFuq-~?H@X`M7>05y!Y+r&ezQ|%006+XIKa&vMn-enqI6s z?XTd8v+jaz)sDdU<Mo@n&gkcqR=-b^{;o%L8TUwJ(+^vkU!EHc z(`KScv&l%nE+d;e?ZG8_e0je0!4$c=7kQr|9rah%B52HS>q(^Tw~@zn_9k%l*6{Y) z&Y)i-E-$S;kD%WVQu^K2j^N`udfQ9Mj;ai$*!0e3tEvn;-#>86_ux791!Kt23s-R* z3PvmEEIl+Av7_0VljP!`P%1QxiP)#nc#j}eNupOpWG)PrDXh>QZjTU6DP1BAJc-49 zc2D!z0~Q^W=71Zqt1Magy2*E{_NS9{LkrYUr0YOaJdpI zR^f1zzjJMUR4qCYmY`hnZ03}<<z}9aMi< zuhF|PRK`_RfA1 ztMArQ4ykqfQ`XR5IeLBkjH763QME#cn$eUUX>Mx^9M`J_Nvd!*IdK)x~Rw0SA>>t73{}nA{jv6I^e=N(~+p zav_C+ihjChvp}*e3=Y90qiG@{e>nkckw^`uNEQ=#i7Xh6E&8-piCniaz~cpyfa?)Xxyot2LgmG-on}^uPY_69 z7!{&5LiUwk-6y>P?O>lxdf-d0@h5y0RJ_f@CHsKd0bmXa?Dfr0&_6;YVIXq@d*t@$ zm2qlJjr9#q6*Ng!BlcdiXMV>*goSOG(rWM!m^#8uz08@yx;;;D9+`lQj3@ADVAlX# zP$;Z=(AF`#WBJPT9n^1sfWvL^H~S!P26st3@Rbc1oIU#O4OO%S#@Rz4aRy>>_UiGL z8Ajzpi zb5|tKEm!%$&^#Xu&+~y1Js$$oGaS{kPnQP&a}T(lTbzsDFYJcy85TE=t?|xlH{n)R zGlHfgQ>EJ6DqF6m&NVi$XT2+GSNpJK>v`OTEjQkuhFWUp1v?_K$PSb-?<_@(#h;30 z_!Zv?;n%`+b1@F(Q@wq&gHY~F?>6$!cgB4L zP)ttMu3U&<*lhs@Qgr@JVWtmqrmDqyEY>*3d_CTe@$AMFO2H~8GgD~a08%y#Jg|40 z?pb6NNNLT^yZ`kHS4zDBy6ayLl7Z+Te@&M?T}tJ9E{+4CQ&A|J9gDvQkP9lHD`CBF zdp1t@xSG^cSt0bG9{X}c7>yQM1sj2q6S$|e!f7?jb3NJ8SGF#9dQgoxo7D-JFn*UZ z#!8wXqtG5f$IP4vU~7XjXU+Z@<{zrZZ6Z`BTo#8z`UkMyA#UAJ_;@BP&LU^vBcp*i2*%ZCM5zg zla%JSEG{*x&{ew9#4F7cxfOa4g-2Jl0-NE6+=KD>0Oe*wQO=+~df66gNBA{eO017aViJ!s*ID-h#`Opelo;di5C;P}Qr)GWe<^z-kLia!W9l6L8h5QztmkzgrEs3Gl^)1bSokb)9XZL|iOul)mP#^Ou$1=Jvvhl@SLVv7msKP)Q7lg-5wrfm zA8y9lle=eu(Uw))IzVD(r?L}iubcwXXtRVXcGCCqL!4>=r3Fnh`#W5_4@8#jqCZMf z_Ewa8hI8!hi^jH_MW(?LW34N_R>^0HIkIX`-^-QH5@W3^{rcQPYTHJiL8)%Ny6m#V zz*_a{vD*^kT332??XkoJ=gV#O8I(WQ+al$+#E>Y+tz5Ow%M#3?Eyetjmmm1wmY*)Mlim@758tok2NWox~akR(Q5t}@0Go^q27rr9Zz zJ^c)a*ugkj7DuMDamrGWua3$N^^ZLTTQwTf8$=tCsK|4csbuwmHS>~w;UOedt1oDY zY={=JQU~j;Kh8=dT6t8NWAtBHc1P}L?ycX{)V5ZxV}8eF|I7%Qt={%J8U$->ZpGN^ zZ*LW1U5^SfDgxTwMWL33GP0`oQW{og@i)rTc~st)dUvVsSI@fh8tZLIm124Jg8c8{ zPk;Pv^z`e)!yivZ|M8DM9{x0jfBfO;*H3@5|MADCPY(}&{M+kqZ>}!>!Jq#2V^*ZM z_+P&r{>U!=5!lJor(genJqKF57w}AtL_WeLeE|`Wy}GmH-tAmhqQ8P?>32aOa7obY z{sSG{?XG{FsQhi?`zW`+{_d8?;zH@sIhK9tX=BEhTFlGl$C#>+l*dPBS^gNYYFNTO zu-73_?Gp}E!y^#+UBX%vYfQ#az41q7h1 z*%CH-0Enq|+d_yf0jv+3-92mw8!ck4ukl;#G5e|u?wG|aE`JEzVuqV2J=-xkB{_@u zlAAIK6#tI9&J;cl<+of(mdsEqqFGfoId?9_=5n~hkWn21j&jc>BXS{%Bqllgr}0OJ zGJKr>jWe2B)DV!koI<~W(j_<8GY`S1*!v~VVY5$IB(iuL!_`I~mC6L}uwWBQT;ye{ z`W;Pp%+;!l2*cy0?;f3%*{fzZrmWVS(B(20G+Hp}jQps_YMT=FWpUf2nDY!T1<?c>X0>h$5Hc^L{=hCCD>)6HQ9?eyvEts8r>}O9`&rO5nn*_uEo%TuS%0VnkQ}t&)Y7x%))Y>;oU{ zR^>Z8`lS%ll?sFEFwfRY^ZmE^;|6?S>$n|QK?5IHp$E3S6r!ulb5VGU64d4bPum-^ z^(a6^YyC$Hhy>GolEp!02fWz6m~^IK>x)S2!7rk_XHc9vpJV4??VxMp}sjy zsU(q;v`7?RCM9*eleo5>87b*{N2p_Zr-&?rkgiM0{ml2U90SV~;0!~cD%!W8f|eqXE%nrOD-zmHjDlC_Ou3f(_Aps_tV ztq*wFX{Vgcr(sAo2l+Iv^RR0g3rW6#Qf221ly+Cvw$f!C!gmv@w-I2Jy{G9iVRC%? zXRx)IGj*I#S@4A!f^1&g(0qAqx=A9c)LQ&=2QPY8%?MSaZh4F?jhvhPT$?Yt1qoJ z#aYZ*F8rj9M7qpH%B2-62F`6~3{IZXd5`S~kC#RVIGgwiS>!rkk%{eB_tM@oze z$wu58`xZ3eTegBgz~CvH7ixDuCBVoFf7i;!^*86=$u7H*qgEd?6&*)xxi~hxZ_kYO zOrBI^T!}m!j_AO?fl@FMNn)^K#EhJ=<)ZAeH|q7C=N}Mq5_mxlaUv~+@VX?{Ae)6q z>c2Uk4oYGyAL=;Mc)l|6pY5##xVYraDqU)}hwTD{wz| z9F4S43zjJ!p=DQg2OPSri(Jm|RuQupuy1l6Gf7JRRBb|Uml;n!1a(8*Hz5w(ey-aO z?Ue27jtz3O_TqD$hPdsVi@olP2XJAmF-Z8!27{z=$~*F-9|eo-zLJ4n)%vRys@vYk zWJD8o`8McQW}dBI%qHfK4nA|oHKYwA{BPyvA(@EV9R&kEc>PA??0SSP-c@MNsbcpu zTxOf+E%{(}G>&s7rRAVzmw2amdUZ*@#jbY8DbIyM_48b$lzRge)-iNzFWO1D0ybna zS7krMbTkvWQ{uPK_f~VC{D{!d8!FMK0}b6@WvVnL$1Z163@~y>FH*?zg3u`=8Hi+= zp@Z`7a5%b%|N%g zML*eK4ERnJ8l*QaHcezgzKv)7maCUN^Vw&EZQpaGJZ4o=T?x$tANY1m!uDym4FSI@T2+lEJKfvKniIABvsJW>kF?#1 zI`~fa;XB=DH0j6l;>Dk*7f;q|Yg%I*Fb`1E#54pU1GsN_{!XthNt83l9HxmxQ0TRK zHiq2ZUX8YBbvmjPOP3uuxPue#lXv3n#UZdLQksqQ2?jTwBUQZ=SQjE=V$LdxRLalXqK*-)#*R?Z${uh>{cw#tU7Q0I?JLOM zta_+dtHO&KK)B1!VY-4`ue+X~NhhB&lJYzkx$)+>)A05Qkth0|$a1_IF)EG{Z!rE?a>O429`iU-6VJ=p7eDjy)?uo|vDWXj!=u{_@_|GF@T*12tvCnw7gD zz{ng6YnU4W&i_Vt<#O3_`46ly-pQqwwcmSF%<4zLeWR{9-c8`AZS_V|j7HW0!^Yv(_6+-+#iR)#Z8rm;rpxN*`Iq3+TU-Z>a2GgRf~u=V)X&oLqhL`l;S2 zU_H7#U-v+}+7^D`&F*Q9Yb59;eK+2G05a?$bJNPNE@-L-$$7P2umXbvY;7EASxAWV zyi4QT-6ql<6i|(NI|l3f!U)!);RkP@?KYu%nw*JLv*Ttsw%T$$IkPb+67%@*EvuKq zWo<+kjK(Apb8`5@>8qy=Lq+j8$>?&q8k3(g^26z?T7O)cK(N^Hpo0={m5Y?A1uL4w z=3oAcJRd(l8K3ArudVgFdWGCQA3u9GMyorWiaR#mWOKet-#wdlQ0N@2S+W);Tb3sR zMXMUkJ)4rYDouV#rSCep6wRBkaY%o?xw#@sn$wgimZKZMH-zqCz)%>CnJ>w%HkMZg z{nEUJ2iX|cX&j>nf`R5tE=4AN#dg9jo5qj8iZn&Vc3p7Ya-|KuB^G;*0wgWhhNllmUTG}Cwi%1-g{a4zfUGr>d#hd z|M$sce3k*eSa+u3zM>HMrntL9@9( zScQ^YB#FKX6w8>Z%IK(aKA*GPXISWV&2%hLO11dPa6`1PxK_5) zq&(B}dO=P+ByA5(S<*SX;VNmdUDU{VgQ*$rT}Z!9lzzW?{2eD#C9?OONH6BitI7Jc z$fK2YUpgX|d#bF}hG8HQt8wUe`3w?;%52N7LcYP*pc$XPrryemo5RC*9H{=XJhgRE zb{7m)G@moSr}be29&f5hM}(c*YoKFTW&OwP&3uqr`C)bPdr&&OK7Ip$Wj$D_+S*v@ zK43cA)WQ$1gTJphvf$FR+t$$!f#6%y3n8k(&q5v#lc)!Ol`I1an~g~*4mCsqK{;~U z(=GGX1i&`1oquh%vw^M=FrUw!=LB>g@#TW$lY%RMs-SxqtS!2_GGQ-7LN~yfv`%)1 z8_>6#u>!Sz_}Tb5MAWl%sa8kiTTyy1>YdZPJ4*iaj`9R9hMVukA}UgrIr+15QT~y3 z-Q4XlpNoWMbGSb*Z|BERzFI1AtP*+r`{nI?6p2hKnkf&yc4vZ2kL$EAWI0MW%hb;i z-A1!MJKa#Mu|N0EzKd$T99da!W6;apHt6Mpfb8mY(8~wCe9+4Ww$t7Qz5J8mIU#yE zs9C3lTKI!1>&IaNniW}MfQ6&3)fKTZq+9!RPeIPg0bm#$XL3wlLINzf4i)7IlSC=3H>u+72fBdV-En7`GWN(Muwbp+v=Jsp# z?-aKWBx<)1vw@7iTf9DI(fR;cLxFS)6PEo2C5XZ9-4G%qLlLFAgri?D1_{M&1Y%0x zo5c%c1h0)3*)cLzWzWFttC?z5Ims{SZeC5EebMDG_imGvXFVr??&3X8;Q3AyK+%NS z!&!fe;F-u&E|Rqhji`$t2Q+cLAleUn(@BL?W;pqpVeyF6IQ)TJd)Q|ya=O9Nta-tz z9Imp%N<<$K3V@|Klm6g;BZ)zxYj$aBE&#%dYg|3jYwI^$b{}K^5gvh6uH~mv*f#zi z5Ch-vW6KXxQB|z%ow{s!trsq52xdA3^2g@2hH&pFDWuZ+(zQkATc=@zRGM?OD5m2` zq>g|(HYHKj>Uf%n>2XSR$68#eoKFiSa*0t=$CA%SG>;Zsu}Bp;JEluMibQs&wP=ya zaT@<#E?G39SvyGwsW1QQyibl6z|KA*sIa7E6)g^`C=hOAAOoBk!8N@~-`2DeaC)R^-|9 zBQh=Uv+u^ZgT@t)`0*d(P1<<7{*R-zDR4_7xC^e~Q5Ric@B6b!mbZ*%Qj_21V7C6W@)*AA^1^{ZtX6-@^x4iCx~R7t zg+HNilb<#|hBR#aNq@dcIB%DdxOV+YxKwZlyJf3}C2aZnW*7Lul&+&ZFR9jNJB0al z#g^>tfMzQ~G0N9w`xIHs@&v&1QCv0&Sz|T;D;citlI8l)v3N{OF`R@cR(4*N4=o>S zA!f7Aoo}a~Wg;6XmZv;})MrrA*0rsM*)=LRoke2|Q#k z(`jYhN>-J9%gK3y$M-3@gU_5bf$9HqsBOuK0THnHx@dC{8_y?@KPwmv5noy|P`;ABq9|NlhV?@M1wPVdL) zPbnnJOlkr#2$0rtKx9OQtl*h*g(w1lQaU^mkr2fj!O*`** zqtWPhmD~RXp-ax*E0#gZK?&mf?02_3i(inlLMoA78%_dQtYEbHoi$);c0#jEU?7cQ zH4!>C6PAzWEE{W`VOsDcW;t|jKYe#He)h!}Q-AGr#?_!}&mtjyXW}u)`qUZ;@H@?- z^oZuUD3&is^F!FB9XVBO4k$qQ-y_mMa`*-5QwhE6!j!SsYSG^setgBvy>XcoIZZa^ zfNXmqa;Ohp`+jcwmOPsm3C-7E{vE-po-e90k6NDDOcLGK$WZR?8TJ${SXw4Gh$YKT zuP(p;Vp99_nxasjj|s7mT4fV?%(FR`xk~f}J)0Al6yBPfbs%dOF8|bgyLUEgO_tVO zqk8m4sfe9iYeNBj!GM|Jb&<=;?B8VJ7c)TX0D(Iu=--zXt_-}HIcTb4p7CGHzDr=# zgetaSBhU&@?gqY_mnXZkCo$ejML*hpj4(3~rKNGqr8mX6p`K{#koQZ{bOu;D!` zp!YlkVp!kJX>)NsL~wj1^M5@liv;xe>@re$3On~boKP>^56H)zhmd$ z3+Lad^Ka??T|WK#>HoK_eJyErk|zCwdU0tJX{wwSNQt1d3$1B zw&jY@xiVe3aU<<#Ba2Ii7oTTPDtx+fSIFuqtpeEwOey%>hzs*C8!W{$8{V5PMW!X% zu_m*Y0EQ4FONdrr(&})2&7U69smRrD_cYI*e!caf0tOOzsE~1%$43*belS`SwmPEX z>kDi2O8uuPtDjhruYIw#OF|~B9`sa?_5acK`Hw@pi1+vRYZnh#u$kvKV0@?zWn?04eRY9#Uze%>*~Hn&+*yg=(1PmKzbL#U4AG0Oq-87^8P zL+qgy7}`5ld&;!#$W)Q*TQSB!aY+)9%{y{#Kc0SwEpT;j8C(90l=-L2KNaI;AuMb)kdG`a;OFg4rdaGsV`T%}n&5Ej>thS60h{N)jcEdg~ODFbxgc zr?R9N#`9QM?<)){GF(^WP*@pC(eH(CqCneHxI#d(xw|MqM4^_2La@wUNpvYwIgb#A zp;O*=bh}?^9FmTuS$XTv0n6ae!TgYD34I;+jb9J-+J6}D#2i$w9I0emvpbv!eEG&h&gHGhl;kKy1ktWkgP7=8$kp~DC!OBR`>TlXWnaelR9 zyGCD(3_ym_cD{)}Be+ltk@H_$^8e}v6!%#NA)=6sh>Wm1mapt!%!>B~$#O7|n=dD8 zRar2>pV|s2)YiH%EK>Yv{s3tQUJ- zE!h~3QVAdxQOQO;lMFH@?^?}FKV{^nC*M%Xqo1Bw)D`OZrj7gLr_2^ed7+h(w^-LMHvq3OrfanqI3P9pXo`$et}h5;hWBNbjS6`T z(>|t=J_Dne=MbX4IyRZFBgALba~{Eyqf%Qf8FCwY^dV=Ba5}BOuFcVFQuMH2)IkID z-`*NdZR-?Xo*OHz<%CsT4d385Dox`$Zkw*f`g_N!9#*A(+Sr_0S>0hpnWp_Lm zS<137&Y;f|%1`Ibtr?S6fPz1(ABK~QVjf*#(wyMSnL(3=l>K6wcD%h1+# zS)<7Pf=9LJM~J)wQBC7ou^7<(KE+T~t8_rzaX%qgnz+DbZJe(VV=BE$uNj%Jc1pLG z?JL+6({hY~S_u@ma9#toiE9X*WtKsB3zUEy0)AUk6g*GcJ@K%R9~_!`k7vXi5L4L z#8lep`)-(1xCh*8Gpl*WxAudFPnCqBiai*n*j9*Q!IADEXkw35{$qOp$dexQ3&jX~ z7~qp&=#y}Cun=spE@ZI4%!eIN!9saX56`=oeobEaK>xe9-#)&1XZrq~*-3tn0jPn7 z2d#D>SN>Qz)4pyS@~4SuG-x>(ip#V4x;|rL`dZ1be@#QysVS5_9aBXkmMPYn zINxC8BIo5>a3TQYMr*$Y5+EkLBWu2pSRgnN5lw8br*oDmr|+$O*({^eg#GpA=IVLS ztbVW?OPE&)yz8+VwN4Ct_xrIL%|w!j`)h`gk71R2&TOnkd06yJs+<<=6aVZ7W4_7ggoLpYXBXV|meobt~yvr;w8^l=7fXLlk$?m*$}gJPA?at1XXue9*KUmwBKrPkQL{Bv7X(0eU?N z*X_yH_5=bISbP~6wyini){F*Ctmd(?l6}|*$hI~DGO(x5j6H4G7=q?C@TR_>2uUL& zZOS3O=2Z~8lr^tTXj0S$v#9UE!7REP{RXpW>$e`r4P$-BS2U--dez$$nhad_HFj1X z(Yy7uD=^tD^>0kE+`plfAauE8S#0YKDRIj(^4;|-FCPmcIMksE5BfTIhcCRV6?LOlPF@7tYhRrnbO{Jp>ISF4Se{D$fj2_o_d3gHh!8?5A`qjode|P-~3`@-5 z1a~a7CJ4v@A2k9C#0F5LJI~abY&stBO6E~X(QVCSP-j0p+A>agMy~8T4{ucl8|<0L zG8U=7LU8AP?3)c&yuY!P#r!jRLxYCYt_XzbHl3Qy1L-weXf=VyzFVVt%=*j&XfuIy z3)5sCMvn>9V#0Kokh;S@{Uu14Zlk-@T~1dUoTOX=&A!S@5r&Yr?;ALMs4!^eq+f{( za95qZee>qx?56Y~j@LXstzB;83EH#g1NiLAT%;F0%MmB_BV zK<;SH=`>-I9BO^;=d0_tfB&B&(CsGnKWG+{H*fL3Jjo7y+_De3V!m?(6!YjsYu;0Q zTkvH+ckM0k7QMAKy&Ov`Sj}-$H(J8?oc0~e+7mcTiG391h9&N*@75f zfzDd3rkMVH7y)lHlF)n(71@g=;fkbuzA$Jkb0K2FV+OWgo<%vM5{qtjghfwTMD5Xt zsG^COV|j;YL9;m{G)c-5_03TPtzGwvE_lqL9)b&J=TmO$9kTw@_cscWy6dM5J6P79YWBDtpxLW7&n)w>vJt!C3^pi*Fmw zfJY!JI-g)XOgmD`W{nZtJ=2vTqq&DEI!99cQ)n3sb)?%T4Bo6cLW}G%sL5BJgf^wI zHq7zV+0%CSz~sO%vP+Z+WLm(M0k4D&>;cxSKVTB?e6@ET>qbf>dXK|)(W?nLYs+~h zq46uNK}Z|lJJWs5OkND8aw-E;c)ws7xnnt>t)R9fFjK#WqUy}+i54e-w^)X`mF~F ze@3^%n?7}DzBA4!^j!eCOaE;Tjcp(G%OE=Rn$7x@tT;V;MQ+S{t@SGOzHfJREg1At zVv24SuNcsW$bdf7n_dI@(0=3oI#4y*Jg!@!X~cZa6;0kojAo(btIwO>X((dtKG!hf zj*`+_m$GgeUKcE4rX2E5{JX&k)R4`2B^6KtO3%Eq4 zD;u?Or2lPetmr+C@o0$e(%5H(!nEgt+h%JBJ@cL68y`ghX?HDg0V{-T$rPdZnAAL7 z@zm3)B=4{;$-AKr^>Bq6u24JDZ@5CWeru>h{aEWz@2gIq_xq2(#NO7!ABxj`+~Rcm zEUZ>nq}uhSH3bV4shWDYn?5zvX!Ff>h>{rGke`_wa)^>}+YlwuHrOFbVqi~0lmxU- zj5iH6+MG66*d8yN8!T*}mxXPp(bhKZp+;N7V?&KLY@bYxHqo9}vms-_+^5AVI-OId zavn(}t9JlAQ*7RnVLg;#8_KW^W!Q!?Y(p8gKJ*&OusLlg!`3>DPreM>HNFEXgnmiS zIA^1r(HKa=GpT46`RnIA=IUhqi|Evk9bGOHsA6NUFqj7eTg6_LcJf9J6?X<_<6zDI ze60C{v(asXv$1WkgR^m9PlK}&+9$@FhKf6bdgqRP;Gdrn{~Xjiw+-rD+h7OvZeUM? zdI#+j<4r@wouT3mwjr3!--cke&nuYiQ?9tPZDov11{cJ5Gx65Jcr)27645ZwJ(a(x z27^^Er19W`|A<1N12ZT$J+YCth*&~%MwTq+BIXf91pHF4oa5@+`8d<{E~;kImL4pZ zx{E18!4oMK8*mvKV?yOIQ_*qGq)6_@nmWyW?z!wmCU>fK0n`kua#7nM_4HF9i=p$w zft#c}yM#_U@P*og6Z?n3Vb&`;H#tf)JXk;iQ^ zt2>Tv2j{wdXFbGn9saC+jk;jb$R*bLY6{Y5S5f~Jz{h+Q3hFT*063>Nk2y(ZI|vGx5^irEzPQRp87I3&1r3b@a{T$P-%f zeEj8qcp^*uuxahbW<*Zge>pvE|MB(TTYq}9B?z0p!d06z+ER?O=y|9Zzc_pD6XO?W z&p)6TcLvi{yb5LI=?~4zp@(f_eI!=uSmyjrUpgcLaK5R?=0sC_TuqzBjH*T|)Vj!=kE3<*3oGX@dIwq%glqX<+G`-oM z?009P$l4cT!E#m|1-<#l^j?l>N`DpEI1=gc=?{~Ov*(s!o?cyE<6(-J{*RwG*!Os$ z(zg5?LXarH*J2SBDa)`>&rV~&PT|9Cho<9JwJxO1FbyZJN0{mx3ewtd(W;B+)(683 zr`^WIKsy^~uN^kpfGoHkcD;~HxR8_zO#;u)II)gTY)WTl+7)4NDIKx9Ag-8x^3Yi}JnRmOhhT(F=__PfRaI|{s)(;_{Xvni)9W{3n(7Y|axD@@POsk_5tAZz{<47bro8AQyB*>x^*gp7CmUud)A`2wfI1B%`=fMwJ2<4jvHxu z8JW>UvZDrB0SSww$Rg7ri%HJpQe=%_3njdQLAW;thxP2ahJ3RB7@z2;8v1O{(o=cC z9Ft9h)zG)>GoG+e+t^F@*#_en1PFXM>zF$nXEY9df(NuVq&-`P^e}cZ4e1U7uaU2u zwXiNETloU%QPN>du303~lx4BEDm#GyJ35sjDHPL+&_a&M4-4IyRvtl+!(WN!*hki( zVMsL6bEC>rcw?CheY{CZZSAoGIeG*e8k=TsC}Fw{r`f;DGF4lFbGgPp8h#q@}| zBvOf-#bek&vW$L8MUg2MqdwBwYqh(^Wh|)FoZ9&T*|%&hmeJ>SIXQocbT;AOdWXY2 z%0&&^-=wC({`kLt_{Sd(fBM6Z|9kw8KRo^Vr}5!GrO19;rO9tAnkK*fpUK-dzx~ha zSHJzw>FZamdg%vpFFw-yLn6%eArhHlnX-BhoV-u(7U6fuz@+vkp65}p(E0Z4!Y6aS zJ-hgTBIk)S8h1}M`C`63yV9d;T#c=LF(hp{BwP7n_ABP*kzdL|KE0l9p;+LY-cb9@%&f!vw3Nqt~)*&NGHYRD}|`9{I7Gwv)|AN{pb7(K4ZGCi1k6 zWT#{a){0*WF1fpsW{cfxmW{h`J`3^5)P^3bSyV-Zwxp<8W1v0*+?2{50P~_}wGos`F#D9$R)$%XL z|LIU;r1zEn^H__?zcf1L19UjmUY)7Q7x19205Vx+ioQpo#ii|$OEZ+I=OSqghvpTG zv?y!F5y_&uiGJgx=v!2w%=ySm+q~g1C2u^|BX858IyH~M{Avk-+I283YAoT(sOFdF z?dn}u(YJ@JY%&;#gD9-%e1bR`B8&w=I2D}}Y2PXp5-?06@#P_qruj4#g9^^NX5wyiz3ZQHhO+qP}nwr$(Ct?$gq|971{_nS((t9o^( zQ&~xUsC<>%;i$5taJLMS5U~CEyy4*#t=T?d6{}yg;E6||CLVnYls8!+$sd8K-0gN9{O1GnVNRgOX)AxX1=dym4(ye=+-;thVV4P@~Y>RpK*Q5}HR6 zm&26zg4Ss2S%*5UhIbIk1!b4($Ya99f{gnlDk7;G%wIuDVoW5XKI6^it}0^BXj0&^ zGi9bvlH$A)YSa%)X<6|}b1;_DeWy{SUD*~=yGON{UE}F&f8}pJpAEsLQ)a+raR{mR zkIhp|X!G19MV8eD{aoeGq;Dc0TbSwJaLoc}tTq;$`pPC?M<0d`f-<$VuUbtd1-~}s zd6K+8Z_ssvP${AW{R3#{5yGS$%9+9l)n*(9R7c1iT6<=s%q0!Currb$XQKRqVBUkK zH{E$9QZ1FB8TJz8$s&=_oOFye+mG$b%!S4AktezHs80A&IH8u8iSvRH$&^8qB2Dd) zeTWYNU9k?XnoH3yQA*==gp%8gO4l)zj^1KVN8RI-jC3wLW7#lRB8B725340+6^hK7 z-KLSF$}OGGR$2QsI(f_8pd4!RaZ#EIt8vc|{C4kC=)b(Ye>FaBDGuG_yU@p16lTTP zmIl;XgBa1TaG&J?Zi7UjI&EpZ(DLU@h+$Q5W&D)o7#K}hh+pxtZ0H$cCn7x{=D0z} zJJ}U(I_X5MmhDgm_-wT)ONnZpknSEfJLQu;2orHsjBB zSHXOl?z2u=@3U6ze6qRSf_*-&W@c8N@LT34K8Mr@>drd$pw5YiGWF{)1C@gfT(dW< z>l~%{9TR5yn0*fa^|}A{rj&L7E?LB=mNW<1vu_A z7~FO6%8@3_bxy=jgL&l9a+SBDJ6+1h9v!6~^+&n#p}nD11J8|-xrkk~`)=6f$k!w0 zGaBC+cZKFxPw~3``HIw|d+TSvSjD)nBZE8kvK@7?|Bo;wKM~Lwj7wnv`Gs|ZFdmaw zw-V3|aIFOJ)U*-M4cO#i*g;8gfr3_>?f0p`J+RaTu9sAMxCjDw^I+Kcz{81d?VPh= z*l6{=@XHcnlemjGU9xe!RQXB_ruoF=tcX3^_Uxmj;x~o6{=n9N6ik&+lz@BU1n2p> z0Vb!$0wHLL8mX`}d3>IZUkS+YZ6UG8O0hOm$fv~#hXjFTweCts>! zTz+mpQS&JnLq>;VLqh8fX~4agA;$0U4Nxvb58FtyV!^u;J(LxlwX9C%w_V9?xlr10 zBe&p2Y`*5L$ZfBh3)*jn+y`HEw^fM^*sIqFDM-ST!p0d#THbY>7+b6Um=MZg2dJwS z-#{&5BeuEl1~wP0{v)fwOnUe75+(WZCE09gEIDgq>?(VKT>S{XF-m+FRiE0i^toir zI+Jeoi}Lcj+m>H6cQSY&ExA>zP-88tJb8)jlHhT9nM3{xfKuAq=ssDS+e{R02(ie> z@4Y@*1*w?xx|ne5Zm_{~q-7|oU-m;>Nil=F_?E1`gD`r+GyV!cW8V_{4%_5C;oE$z zcm=l2Gv_;fji?>6&AZ}!yaIf{{fOrMo%_u2_nDl7st2_hMzB%2>(E(Vu~ib15s+$E z&M(<%RJm$+jkRpz!V#2Ai?O@bZ1e8gPAM?-I7<$lPZ(hP^7t?fn@4PeEg-T9!W9$) zdu0xe<^LLXUy^U|mcQF>fd77CI?N05JZlx`vBJL2--W%lkiX0KG3>S|SLaox9%BtA z{n|pv{PqYT^_xVU0)+HvX)9={zJmRGqRM>)+tn(PRT8+XOlNZ@x?<_HD3)_LN(~C+ z6<&1CsMlOk&g}@GcXKBu@!<9SS$PKujAeA)Q%-dMqTHc(?opucbkou#zHb$RUT32f zz&h8LXeJTJ^z^1j@{vn2++z;xuZt9ieR6nRAR<$oRGuxmYSi~pt9e_yeQI6jBE=;( zUk!+mWHezy$#Npida0?0x|OrsYh2knQV~_cOUT|i+g2qHW5oL05CtR65DX2Uz7L-d zKKcB0=s9vc&Yb0t8u^{u?Q;{e#r_(!|9LIzPi)eai0CYNAdx-2ZUR`&McdHKiHIfU zkj33F2_~J}3S0YQ|Iu3F;N1ggkpJmsf%LCgbtfflRIN@JC#+Qosnk=U2xN9R{F$HO z^)uD^92i0#*uny01@%TRq)i9reJB%a<)8}M52zE*#RW(rTvE1iIXXS8H4>!JzGcG=0WH7z25fAjZo|HR+N0-3v)!JE6E(K~&$I{JC`8!aoc7#|(0z?q`;c~q)L8VB9O+LPD;TTeKhmOu zJbn>%_a{rJOtFjXQUoN~#ely|zy9MQHA^el!y?WZ zCZH(14eVv%GW9ndspH2WVOI+|XDg)!p>tHCfUIMfc|Kj|NlKat8abM*)$S^Lxz|_9 zI@Vw+yoC(FQGrwq7BH-_TEFj41Hydq22>$N06(_GW;(0lIWg)y^G z_J5bKfuzhJqBS;$ceIZ;XBO!eY7TG3=-k$#CEVmR!u!8Uxq&ie%ttHgEu9;%PKQ9b zcAdTU3}O$nggIJF%+}@UXy-?hiv)8!Ku*dcq@L#*XZ+X=C=wbEhQRx(i)eobll^k$ zefQbHuOyU)ma)uqfLXB0)G%%9D|;G? zac`(jRmtfb)OtT-zwo&)4wq4vp&Q%Gd1g{Ko2NN9SjKL|se0aH#HpNxOQR?_Tb4No z+qj(;vJUSFCSZ3gZZKwQEnu}Spt&gE#m(3M)vgF^VE3P7Xyqc4qRQSHUXAojKz7-7 z&nwW@dY|*CMr~cDGq63cMFyQ*a%ZT#={>QCYEl}cny(o5y&O(X_^EfaQ+E--G>xMS z_|IBQQpt-6I2_8~cjqXkX4G5>2bY`^@ryQ)}BY{qgV=<{{Ho@Z#2>GptQ0%Jm z*V}pk*>^666;r3b_+JmdvKEvbYiPh#@Qs-?;i;T;&AF{yadsU#5m~4GUfm)b7JkXq z`MCUv@q;2xe}XbE*&%mNU+L1SA`U^h)0={d6DvK>tiv~Umr*L;01tB>f~Ijp5|IAe z`>E!6!P>j~4u!OS>mXBEP1xuq(p#(X{1w-RHX$@L%@N$UfqADEXQ*1pam3SXs6GAn z?Xjpm6}&5@NV{+dO3GWj_PD1jq3D!q)aM(c_K3XAA$p2E`=NAC<(r%W{o2LfRC&bY zinp$3pS38e8N}2gQ44>>4cfeq+Ek;C%PY|FH*cmDbf_U^(r0L`vbZt)vpp6#`xyf)8n|lgq|qh%gz5bEkl$uL zE>CU&+lTnBf|x8^9^lkhjaJP4#T6xG*>__YUUYg=_@YQ7CY5nhxcMV>Es&?;h;ogn zy`90a;5_NU(bsKX26d*10Trq8>#c=;H|vB{S-Sdcyf&4~;t4o1t5@*-)_&8IU0|9u zPI#FEjrJOXR;bht)9+9UG$S9IPPd-4oV+il(Z8lh&kmAbeyXbD;tomHP*6ZGT3z5u zR=C1wi3N~(9s&YXwZI{#dU3J7oLS#DX?*|eV*6FczPc;EsV{p|M|E*-bB7{m4JO+!Xm!+8 zYWIeF@kESyw{!#QO-8i93Ua;;;Ae6s6#bmUEqtqZZ28Nnd1V_m-CKt?nT4>|{3n4A&D!OMYQM zOUU`QVL)}0fqk90hb^Bgfv$Qn6PEMtmL)z!Dh`}aZ4zbSGjHc;h!nIh_<*X)9MBZC zq`dmIC9rLrm6ha0jA}}OX3=Ase>AQ6?jV4!T!o5O`6b7Dbz7$tFl7v4F2fN0_+!@V^kju(q9UWjoEn zcAAOpH1ne6^?wDT*iJJoKium;cDCFM6B~6^WK&&kw=(y3by~PNF6i~M+rMAZ^|VPO z7mZi0{w&Z2G-Mt7k!0LWbXn{~JrR^mTO-{@@}wQRs+2tR!_%y{Sgkyrw_QUZy4N2s zF9^s{8lh&GsRYQ-l3q#C{3h1Boa^}Yqejv zVfSni12D3~B|BCHDIDz;Hqz?oqG;L*)!|HVMp8(NZ4`+nI_Nq<8Wy)s5ylB;+T=!> zH8*;b2{Ckz zE=CN^!VC~jG&fSNf4uV<1ah)9ouJHAs}t>X)0C`VD5%Gkp2?iQNXe#4AG!(dmN%ZQ zKaX3J?q7%GO3jv`aW5DKDnti>XLb71? z@^Oi7Bfdxt~e z&KP7jViIBo#cl%PVTPM{M42}XakoQ8t4&hk5RnW@~2xV7DhyLJ@=X-;6%q`F+L&50Xi@%4+Ge28T?w{C! z)j_Pdr%YReG7Ch3=6b(}lIVIr51X3E^t^x4=6>y8H#XGt04MP0 z*9HG#EkcylzYngdN2t#xbncuK4?ebq&Ze6m`~FFMHKUjV?a84l(?n?yMAWPAS`4bO zpuYxhsdvoo)y$RZ`LObEk=*^cTYH%%M!cxl(N3l*g+vdQ0;II*aNQ+M7(YsoT;y6- zi0x7gO0!IK^#$JR@O=nN^_QJ%5TNSmAOc(a)}BDMckyW;hOpSpgTpbYk*qRuXQ2`j z88+fuS91JR=W9rUAh_k}=E>xF+cHqwvM-em^?9#K50k0B@ZS`;$o7+E0-xjkm42qU z^c|0P<{68mZYNJ^gSb=vs#K%3uXK!C{pykS*>WEOV^AZfw%!;u2kJNkMf?rB=$<(18J{e7Bt!CChX^wh601UFQFAz$# z=#0>YlX)Lhh{8oyla zqHHi%!(*L*r;QnBeI^%m9(z;&jiz-wxI7v8##UO5uGTL8i(yY_sz1sS@;O+6 zl??6kfS=|i(d<@om5?%OL(*ytPA?iIuh6VN^?9UwT&Y?dio!Ja0CghQpr(xb5$HT4 z6`7~OK}!Ct{3fg|S$1s0b9G9aSepQocg?SaRCG9*i}g(mFh)7sRvECJxeI38R8je& z=h~piuF-pmv#ja|Z<`gTmhg_+TbYfI{aD}95+7icLG@=@kM1ftSz?CV2qC*gBf zis#5;`sMgeYS3Tc@&|H;%rY-b3)lP#A+s015_>q(awQK}`0SNME~2|-YuKypqU+g% zmhJXC#FhzeScNyvky?2oISa{HE!#i?yI~u@+7JQXWxbH0=9F=nhX@DCK?5PjEN#mJ z9y9?vdakdYFSWbl=IU-+H9%doKWFXN5;^x-_p<79Y*Ax#AWKcKqH2o{Cn%BcRdHDh z6}y${1yxfI7cuqRHIzig`OH=h0sTDn%D{W|9}JetOzGv1e{&*FMw+$^%PBKqpMt7h za?Qn@w6_oK8EM=|Zc}001R0+&bb*D@KP7q-44Gd>K8LXQV!U%(cQN~ z1QPW`Fk=PXUSzFI{Ola~Z%N119QY^XOuhq6=lJlW`vvQh?Kk>BVE9inwz(W8y;KAO z#TzB$VyXwdXT0~yurCB-9Yw_(*_)tBLL2Gb{c4Y+)>LRK4G@){D*t;zBfL)I?AV&9 z0UK0T*vuaJwP)Xp_#kbvAcoBo_S@v{yG%yD+c8^d%(v+=n-0HIA}PKBo=7K5Dz$>PLptp}fkaY|RATu@WCDI10Z$>ZSoAW4 z1WaDDdosVn;IqZAY{hp|`}adz3!AM%ASU|<^($42|KgpkEiE6g75}eJpOF0fQrDNM zt1Oequ*v{)SJ_2JdetYPXa;LRlp#OsyhU0m-G>?R#O$E&4JX^Xch^3A;x&+V* z4Sr}Y4)BEXJvUj!lsmF4ru+wp7KgdB-%PkoDMJ&fg`BJP64)c;3?5^wl1HVP=(%wGLyh$@foWv9kFpa>tjZol5WIC!}3y+8YyN6c;_SOl8&F zUXReAf?t0qNcgTlNNCHByO!76>L1XNO|0{1u^Oh8Rtwgz0x9fH0mwPVOr)gX;#6I2 zutt1QxCTK2QQCbp*M-BBPWv=~Yu9LTa=9dz?IZ0O>(+v9I`qxV+X};A5aR*5>xR&k zhq`oN9j9`$TRaI^k*QEm`WuPm5( zk2txjfB2ZPpLUx2ulk#@pnn2snVks1lm9{};V6>!H}6D&rlZ}|!=S>|%Y{sVr@xYm zfJko@B=cA6b4SpTBM@#V*zN-u)*+re%zI#usTTvxA1rK`hO(RhRz8Sp&^Pf;Jw304 zm!Cltyji}lDr&}cRxa$j^y@mHc-|O?ID!$hq4;;(RKCQ~WkAr3pe=1qS}Us~-pBXy zg;8f-0Yk#XdzTnC)=;(hUNPi@yvEGraQjUxxd9Sg)_X=D%#k2x)-G2;juoA(%Pm+U z%wXd|p|c*;`QSH+K-8A%4oR28(;*V(Bl! zX7(VGZUL)AAtvn@uTs?Yd#$Y1hcJ7uoY(EXDigL z%r+@G-8JC376*A*_|q)5#bI#`!}Q~2D)M5`=C_HDK)TX0rW~!`8%%jEFS;96zhrXF zm-N!4x&ZUoEAu7RDDW-(v0u2UDDm>@_F@;G{X%+a8`Dse)IP-nxLVYKZ`a890%=O| z{ZrlEutM!6LOKZ~CMKaK5U-*+U)`>u>yQVSuH3JK__aOQ=eiOzaqkYe=!ej~=oz?u z9L#}hvN6%uxnU>KKVO*1{k)zGH0+{4(>Q|Ap-tIHx&B&@{9)}m*$XQGhPM?#p`G9V z<&L_x4{%Ag$zY%OAoMfMpXxHl3Wgq0Q(piv)96zku%NsGZo=Flo|?`=%FC9V;}Wzz4; zawCW&x0nsjuHV&Y{l=-kemc|bXEaLwmW1KEylyz!Nmsq?%tn~y20Z=!s@9gqeZx1` zS~wU@O(Atv3zRel3Bdsq4_s0lA64LIg()Kn?Li1vQebdn-Gd(!x|hYsa$smkTBK@h zaV3?(%sN{x1leI5D*N07pzvj zB8l(mQiuq^H7EM?3RYX)4FADf*weBR_VWtkY%FudUf6qiO?W6}e`b3nDHORW=^VDa z(2tp$74UL`RLEKrDU$i3!3z4&S$19aiCi<0nYr>LupODwY{;K2)pjxu%F4R#W1cOq zkA=D-M-&Y~je)vy#YR=VlNWnkwk;aewEKr?7pbHkA76Tols+Bd^L1x7j1n5SS1o^#Y$VdA}|s4t#84J-kbIp?NBzEs0TQ7?~kYS zPw1pj{b(LU(-RsMd4B5d(PAMCd3z2}9eVose98s50X_qGevg~* zj#aGy+d*xrNa9fu=?T@6v2i8Gkf8DgsK3h#8(LTCZXugAlZUyc8X5z`sxpiJdR_Jo zfDWjsgs`~Cf2Zf!wDBL|0v+{BDsz=CB4_tzIoJ;HKCy>BC20)j4Z;0am6(SP(INxt zKL-z)ZwfEKUitiZ(+wV;03Ne{a*JQqWtO>vMt3`tgd=zd0gtg6U?Z!W$y_E^8a_HR;nk{hMG;KKZB>$aWFIZMJngZP?Tq zhp0)6x1K<9{u&f5cC!S3oz$;X*w|F`)O-fDx^nn+fp5x{(lrA4HE{<2n#wTMmOWV% zEMtY+Az>QTJ;`QNqJof$FT*g=%!MfQa4x()X*0qn7fcDT2lA2ja-Je>*Yiu8vpPXL zD0@w5Ns4Dn#)m0=&M)EFRG+QMzl?&hFatB^-IHEZR84m*SxoMio{cz%JGiH^1k=4( z#+j|A-&+)It;_F0zQ`H=fn?Su2xY?EUCpgI1uNR7I_W|!cSUe7Q@_Iaxz!%`e$WW* zx>eXR5HRJA@a>+qukUH;kuc$=G91@~s&yX>;(FR|eerfOJH1`&EQ+RXU)-uO+O+}Y zwdUgs8O9-=DrU)PnV2A)>yR%kPJgQv$2=rtiIlX3<%JXwvxidMwy%vgM>GAQ7Wdq) zzFiY&iBT$(WMwLT577V@so4kAD0RQ&R-IzD+fsud4euHq;I!|dF&a+FH^JF@{I=7Q z?TqT^3*;8^6>PC_7}QNGu^s1dX64G{Iq!wE{qtqy$;#ciA12S|StOP_p}M#U$&7Sy|?< z@24>OF%tgvN<|t>*Cfbi^}3VWp0fMtb6TH8V!;ir`1Jkq)Qh2C7dwA`*-^$-y0t<~ z=TI-%vQl;uA(7<)vwz*8V)&7uc_NsIJ5o@h2Uz>s6e^0Gr z>&1qO(2Yl*O7)}f7Ms;_ew((my}0O^%m2Qccv_y+pbQ%{5ivJa4)`}fa5$a? za2t@Fx2k>i)9o(mrK}jQJ#msfwyNE~0GErs(0pE2N6c?yX3~aACPQw#2=zde99OD| zD0MC{X~_y&oF$?0ID<0Fmg%oOc_Y+N(#Yw}PzA(%POtEt=s!n?zOH{jTlR!`*Q@$fQXYs~N;F(eO4m&LK?jr#~Z{ zJkeNq1m31;BVGBlh8_fQHFB_9WXJbP^$LTdU+)()C$qtsP?X`L{QUWGrQboS=V=Bu zWE;#`cp4+<AtaF?~HK6je2usYgdSgd-?@3|s;XMW|6NCa zv>j!mXi1T4ooIGFU=4vB&h%aULURvU(ljnt*4_tg==&HN+xmaZtDMACJCcy-|CrYS zwwalknV!!_$;X}EwBH=>7tqhw@l@Sx@8{H+pZnv8nx41o>v7cF@8ie6SzfNM?+J82 zelxFk*LqY%gNLxDCX}XZGPj;NiI6iV_#CFC$zNaMnJGEw{5#n-c*$N{pQdy+h2(&M1yS6voa->NUWT^e86oVWjcPIw< ztYvO~pvQBc6J&15C2|W*f|R|TgrszHPBY{DWADQxC@W*QlI|H^0UUtVbYIb7V}W($ zs1gl!$jfx=_o$3)?DOH2Taa&6z1cpy-xHFS_TH>UhDyxO^if zt86f&y0i#Km2Q7$lwy3fSN&3;p(L;=_Xb+KX1iNi;rVlO7==LQAi4Uw-yfc?M(f(L zF-eTdD?(=WPBHiRDa#Ut+d?WwN3bt7lu%cE8p*n9JGP#r6uF|2M3pps^gXpGiSvR8 z;GEkj{EfroGXqdHDm#8MAwZ#<^`G@fZjf+|9DiEPft(E2P2IQ~JXW3~g1PtyET6Mu zN`N~%^Cu0>fvaMnbq{CNho75@|`*iKQyV3!wyL9c0PoT}+XbY8K^o zbd$nxJ?~a~{ssq_fi{cRo6cvJlxLcQr}ug~_$b*p^X(J)R^XnufDx*vB)y< zcI_Pe1n88>$2kUSa57xial`;f?<8&5(RCuOvO<4HhiJt*XehOaLh98~Mla@dluB5V zRGf#FPaBt{1541jfn6}{3(v8J!cv)KxD>pfVCXJaA$!WOb(NbMDL_JtYq|x&NOw&} zHV{mzc~b3vLIt7w6!0f~To8lKmT-LP8`>j7~Fttvf%O{evRiuDwN+iz2xzydIvwp~E2WQrswRyMyi;`j)U zbsj~WoR6WH_FR{o3}*)eL=f1SS4;`TM;$mFHg@4YVXhHP+yhGu5zHK71=&(mmxa#w z_>%DLC4Ygk9e8MvXT4jo`$#09{f|krB}nLRoj>Y?rz3a&2q8XaTD$h zIf6aS@|7jCegb@Dw`t4~x7MM_|eD9$35v%geSdu7L4tWf6xATv-Q<&2C*6CK15St0&(wG?bW z4zuBXSRayAWT|z_oyNr4BJq+zIFx7GLW`b#FJ%Psf8t+1Ma7HJw5b3cUV`XSBytEj zi4}-f>=$!l;g&iYh=u%HMa+629yHYvOr_l4Hlo@z=`N4IM zf)^?aM7cK2?TG7^U!@p`(jKNRp8BN^1D05y!>s3lr&DEIZcC)( z2Se}~&tS}3c`Sa5h~0?ercR#YwHHSgYnr5Vx3)O|ySw>yZ4|QUs2C(CpR)48)r6*h zGGdGwCg^!P8y%EPJr^)J17##v3W9}L4?Y_95dC7+6&_*b#t@E9+75_xEAy=+@K(as z**Bp6v6JIHZeQ0!tqTob7T>N#ty7e;wB=cA%oA!5mmzv|u%G1#`0|#Nf(*(x>V>)3K}P`aJ$4 zI->tosGu`b1x!xUV8A|R4pf1ZlRa~V&`MVi0TJW|7GGoV%aFjk25%&adC8UCx`Bkd z5-kgDygB`t|8_!vOrKwrfZ0AdAHoNigOwKqcii*~!zEt<9P5AK(z*za`I5f~E?5+o z{M&t+d{2r}!)O4@So*80fvL5|tXBQGHHDR(O7p=X0^7Eq*{p~L#1uL(S2{O4C&bkX zR_f=&Oo0ho>5G|EMZ+ZNnb*{G(`+YJ?C8g@)%2X{FHq>}9lwf;U7 zRRN+Zv?HTd{DO-4dRgC#n>iTq-76HGy zoUwQACkdr2vrX&>{W_TCQij{Xjg=SWg54qT5$SL<>q_h{VAG;s-Sg1)fq?5+7hmT&h$3xId`e3awcz2w&H_r?V{U&C<8Z;auLFN@)nKlT^J9dG^r|NJA* zhrv1j-AEncQqH}wTx(3u-zjf>^&xN+{H!PEGW_)HvRn)dqik7x@(M0oYDaoMnQ>1m zVqMIsFUU`HvttlLXy&XxR2>w#Nl(w>cyIaxV;GsY9C$|NqZpa(7?~um7Si=WI`_2^ z=j(746%Hsf_djC&F7vT19&v~)o?V+Vpalafuk=MyHC36#X)_ESBw-zhLTe70i71N6 zSvBJe*_$X{Gl(4;_cK1$!BS0CLD{M}{W{nTJ&DsiRU&7BjjTN9kvfcINhwPkkBy`&j?y0@y)E7_pXodUX+2e)`GfM*w6 zenWt|Tj-!$Fu^2Bn6xl638A`GpJgjVmqyQEs{br|AC+U9*5q<%GKn{;SL2j{WRJ&9 zYi_!5G;2i95Ak?5yUC<-zmG z)+6%QYH%cXv2+SgZB5})h*vc2Y(-s_%@PN=8dtH&MW9(3)9QuOTC1v9 z5kyw2>q(%jw}EQgJ6%MD)5%odi{VU!%R#Ic8Mw3=e+!HT&{ zKw>@VMZ2%yf{_Iy+M;5>>?29-hM>+!D|uq@0Vak^7NAF;MF}nwxf)7f9;k|65qoRS zy~HM8{azjt_UW_t1kQN>R&I6 zrCma{$D7+%f~TAC!u2hhV|E7B1=EDn_7cj6Fa$Iert(w=&3c0UH;3;zPc~@PDGMDc z@0&=3se&wf954i}M@Xh{EZ6pENEokc^o&Z7faIg3Qimd~THaujM~X3G~aSvZR_ zZJb4K?jFK8iYl#U_oW;GqZSLBrL<_O%n^MvD|E}JWWO;Sx1rnJD&V%$3dC1!2Vk%5 z-E?cWI}UC~tvQCBVN(tsBfEvSeQbWrrvxTWUEcrP_$&kpZNx)s9NCCeP{bl*vl&*v zm1{F=c9!w`QF4-%Dbwv|m)d1k$B|b5{H7`po9GL-a2;aY0TRPU-7A_^KnT=jP@*sB zoJ@~M2_r0JZL7OqwKKl(qH$?`Ry;@K0*rv<7bi>652sar4YL^M2k<)7n>fO?xE3iq z|CUv+3INo4O69+)V+Y{ZFB&E{jQg`iprddZH+JrLl60VZeiKLIF9 zr%ly3)`2W;xKXNt-5M-z=J&d!UOP(px&6sn$5CdQC{d=I>6p+0J7SKacs@GA*4<^V z#>P?wl!e9j_gQaY%`zl(&I-<>p-98yZ|Mv#Jv4^f9_OW;i$QH#8+xtA0NvGQTAtWX zJDKb<%gIjEBJT;@uT;3IB(X|*tt!ZLH?%D$#ELriNVF;u5G)6LmW+3NJ<3EVAnC=5 zBJw}-gES-kPuA5GKJweEM#I)TvfZn+4Vrf>u!p_chQQkf53T+?N+2&?ZKF5>T%Ryr zxXKD0{8g{`XUku5sK*ZCKK&d)`eZk!nx1=*{;$Wfn|)LigE7W^twB-D@lfXX{a-s` zB|_$ZWa6&Or=yXdj>8f#&FYKH+{otoUp!IHW{KEc@n$NJZ{`_dGtH%3iFT>4 zS@qTW(qLqV;xB@T!v8?w8ASP^uzwcuFMtosAA}+kbC4U7OTvkYL9dvyi8F(_?M6_* zE8Y%edp#TrRPwkx(WP>G6Wk7Fj*0z%TmX-I-zo47#UGJSa*Mm^l|P7!nESKGsDb18 zBmKkKcp_KeiNjoYYG|6TAS@&(aurotiSh*iPFnkPw}?Ny z^IwCD9ref-&lXT%UzI{8#=;Iznz#)29TnMQCbBAQtwq;vj)8+(d=}l|CN4B}#%yQp zgaNW~#thy%XT;C}E1J@G#&m98FqyY3n#x}hO&jQonZoB?HPX)(JEBB$vsrgi4dxdO zu3e8Qr&3b04kNQA5^wstM2*~lNZg?*J!)ejhY3G$&wh^yXux(Tms^hbejc&Gh&Kto zU|=;Zf@_A66C~Fyf^CUrXGCnk(sIBMXoQbh3f~@ZF!AHZpo_XlEt;cV%Jp*ue0ie$8d{*f1^^ zJuwDkBh72N)#vuu(10{MrlM(39sJZ)QAPhhDN1FV>fk4a#;XRT*}Dj%xDVw<}uFxf2;BZOjPUtENF$3{eS+WEd3RqqwxQos19cO?}^bV z_S|#da8w7(hCZHC2k&4S$jQKy2kzvYdeY@MvUY1c&25(7Fy#?P1Y6~)Oy#7pVdg3{ zX7VyPchjAg970xy?%}LMMqn$$5ArlsC($epqnYeRGTHptR)^7$x1QiEr0v+q+cA*0 zo?Gerw4E`Kw}+eQ`?UA(>HD@-F_5>TAZ`Gg>WHTUG-r$|4sYiboc{!H^obw@!g-;)0oDm9m`G6$>*1DG5C5 zIA~^*nr`n7@~4ou6MHu<-lB*Q6Rt zdA39Pgg9LU?;RKe_2mZEI4g!@u6jn|$~>LoRlnE+hypt~j0kbZK@Mwcg+;Y-Q#h5SCle4NfavB@oXWNs>2gb+Uw zHmQV{LYzH|5kBiYf$S)x=H>^M>eeU&XMj06`mp*qvhoBqL8~+~hjgxY**JSyelC^>I7{|C zWXGL-yZOuxbzT?nobYM)G~1J;YJxVSw5}ZdghE%PJ4;Wz8t^jQ^;z2IY`);3a7`8y zvgZ&!XI9PL=r}dDC9V;2-hG4ASa9i(%?uerCF}sPZR|#T^*X6===&8^l-@Qlgy4dR~mclUyLo|^} z%?*V`Xy3bH9>7-Xq7Zr@QN@x-OrvV85_D5BH4a%$20;Qmxwu7IJ|t{R)@y~J1?o(Z z1etAOCSVh#)D{&EXEW| z;Yp3hA#D!Gs=+~F=3U!VonU#Ed3wi22?!%?--B-9Oi1xNtVV_IL8&`!V{vhLxzVEr z-^r<$2&E%njj8Be(>8l`4nR4c|9I#2c-E@z-0ya1Kv_9Vf$A+K7b`szp`kRhRPhoE z7xLi&+VUPlGC9kaF^bSVw@GMf zwd)qzn__Ap|K)Y*vL+Lc2Su|GEoO$vUrY{!7Lm;3JF<+-O}R34G$?M+EFXN7&BY>M zlV})B+SK4-vD_d&s6sWp0NGk%plI8;C|mvJ9yn1-mbn+El}L5DN*73z2rl3-eroZF z;CiN;WU?ifZ3#4fid6?;wf~2q)R)qo#gmU-XIJ~<@Ns(hA`_4OoCs$hfnht}m*m+Y zoR0mBbaRBnu$}Lx_v{owZacrXg-44x2kR+e=M-VEo$tqJscneBc3$%Sb%YdLQ^Y?#8ei2|a zc{X6#hEfNR$B8W8uM@G*)P_E%*WKIketI~Pve$umu?Z4(?_Yoecd!+3psdYeH(*Wt+%vZhvwHUkQmM)(@Dk`PKHO$|HaxY~;CpC*j2xzZX& ztIFQwJ>CD0kbi9Qt?Pn*!Ln`Jwr$&X?Xs<1wryjVZQHhOyZ)t)=ekcvM|Yoy^J>kR znIqQ+n71cN3>k(?k75rv(EV_K7D(3&L}F156@jT3AA^P(tOI zeQeibfdP<*B10_YO)kR0{%F#5G%bQn0bTHJpkhfk9vIDA{p^ptsGIcXdQ@*-a6$9U z#0C-WPd?h?&$m}4mqDSuv(L{Sauy_da` z?t`d3(f4~pa~L|*Fw;*2UGfa{Bo=D;89|HC;VsV5!`Ba_HaTVA>>9~v;ROAQIs3`u zpiDVqWn396RFG8clT#l*B4K+tqDBDHX4pZ2a#puIT|;Pkt43NDYnr=PRQzi#FhZ{W zUV*C?7KFy_qV0=(-spR{4$hH$eO^rQ&MDiMsp&--`ar5u5!|P*7wFnf4ctz@WFJY@&+0}b=$M|u3X_P#C=B_WCAhiUWrWF-hKB9cnDkp* zvkHqnPi4;H26q)3{5I|Hd1*d`3{Wx+&_Tx^YW&NJ^zoVg*nyt1H~~z!r#lEhqI%j;^O??fR~$@lW4>9hdRLq z6F1ABm#YQE;}OP0vAFoYA5*nAoSek}c$2apY84U^rb3g1uAFpQN?e*#Etw|C^-EE) z#H;ym(nz>%TSHEFSc4{rll8^bheFf4r!2z2(`ddIR**wJ94&s=aDNKER%pD@zlignsZfLkbM@O7kgk=zhg4@?iN-5O^cQbjZMmW9mE31b|BN6d z0JR_nb!mODxt7>P(@b(i*LajcSorwHN2Ge6Ie~Aj71H<{M2TuzR8*kgPNEX-+LUbO z<`>BZm8H8HVVMr(DfAW0EoIn7g zA(v}gq!HU(PP{_PqtP3{uOCR1Rj9G3RSzWWI$t+50B7OTM8g5@2&IB!I5$1>T^%WWFQZY@3MEa~lg*M{&E?NCzMh7~ zpi-riD*ddg#5$3q?+D4i^_{4HG$jm43k?}Nq}XUW^G*Q9+^}`Xenbc+yCstMKpgU=eH(5rrdR$+FkonJV( zICw}4q?`cFmdqN9{@E&*dO)j4QG zBBO&D;wr8n#Q*X342qD6Ti`$5SzF>3=hU;QSsCN#7K$0xk$PD?#g?y_(y&PxpzVx(ec|rVp+-@$F69Oh z%IrCmi8lk`oL3Y*MGb&tbe*&GOtYJ8*N{fzna}5iGJU-Hn=;u)$`v|pjN3-ogGkaQ ze}g-O?Lv%hSXg>v!U{H8>z3o7aEBesroI_A^S1BU0Vn zKGk9ZUDPF7eBC2jTEzD(p_BvK=e_}ajYS}c{n1;-&5(LL+k>C=ck@ckMgV8;Nm5QQ zR}zG!?I0scQ~-<`nR6OCuAIySzs(PVUVQdDoqeq+P-n9)n|9ELCzcm*UZqnd>U#3(kgci5ZgM% z=H|6rL>sLJ0cwL5lvjYA=^{KI(~pvR8tZ3WmqeCRI+R*(^OV4xT)>{t%AR<4vo?sAVu}_78?S>Kz0Mv#~JJ5d^O#0(gEKjJ0~} zbVL_MUlU*hZwYR?7@XBA7>F+ExHOX~g`j$u;Jer3eYB(6yUssxEh6e>wbq}RsckQl z!x%d$a65ieO!1Tc*ml{pgJ>OVIhLr0ZtukD(@4G0Vl znPdS^`bE-XM}b;jj|4MYEcBL8;2Lr*cARWdD zLCjSSM~5v!AlNu1%}Dt*Qi;^ZylGIWM+u-OJ=)22==`5USrJpPhFsd+fAF)W|Z` zFAW1i-n)?>OvhJXh+hy2WZL!*^{UUxAKBd`;68?w4`;p@Y!&iOx4#U{kL$GGV$loY znTMIW6SM;YD898`ajhl;;8=fzQ2k3*F|B&*vQJHq?Z;jBSf;|+L;p1S&5#k6;FvZ# zWE~B$;&80B$v-e#C(Wr5I)ueHV<04WJ6Xg~wc6?*bIJ-ICt2>Yu5uaHQ59F2l(tnZ zOGehp26WN6Nf}A_%VS+go%2%8sNbA_i zf7hsVpL_pm+OG~<&I}zt@;YzROrNvnLt`Biq1JtZvjEBtZ)Vzs6pdAW%5tZ#5n`uj zgFSDVh`dfI-lw=z$-9hF{ws7%QYefm%o`z$>g;G&W!)a_D8J^B7Q-UX9ve6TUTUJ}j;U-}qx9K;N;QSyj2NkA5u7<-T$%{$5P zxiBI$OCdnL3#k75Hc_j^+B0qL#@lznyLAY8LAXNc`oJM&40?G#|mW7{E2_ z!M%nQ1D2w)#uR?ixq+a{J2%GYg4Dh`{PGv|^B>K}#Cuf=*n=$ZQFiZm~mUb zSdk%bJK1Gg6;1oKJl++N?RYAW=b3H9?vSnGR=!@ z8{+$?q9G9YlhQ@lgtp-ryL#+n)w>rO>AcLbQnAR4+jzds_hg9oYJQFFD;(pfljbo} z&qZ)t|HY}{K!TUtC;D>wO%2jApX~syFn*$-GL9b$bFB(p>|D$$0-}ZNM>B~;1qGIc zsol!_HYahP6*=)&@(Z#khSXXp(YmKfEnUx%l(-wsF7{4m2SrJp%oyfCf-JcNmY+PU zIQ&|IeOjW2RLE;qT*Pt%g5F8vG`mEJ(Qp8hhcSQHkp0?Rs4&ph665CSs7iAkmx!1v$)8JHiRHb-&D|VLr!59V!Sz zD{(+4I@9NuqoA=`T8~M(8%BtZjX`ClkaiHZrHwb~U%x&u8Jpkl&%QXU5nwp!T5b0C zhEYcN-)U3sYubLDu)gm_}p)p{pr+=a`k1&lGjt|^(1f>pou)^rM z5B3U?m&VF_TXBKgJva*!tq@=B9elT53zPxctafj)-m_1^xXjV?*=zm;=!&1T+bQOQuW=i&j-Hz-#m>4 zcA|DoF)F_VbZJvH2XYLRkoEJT--s$NC{awmg*9258VGG;0kjTs0}<62jdl_GZb1mF z@-czBd^{X2pZK0*pvG;^RdMM`?3KO8AGiK1K^1CP62<(S@0TVH3 z!jXn-&8A)ekyl1&iEZM_n^~VZc`K6KN)7M`0gb|4E&c2Iy^cArE3$L=M(&>zH?On$ zmiHrVQwtGjl!x|@SPr@eK(EsHs1LJdx@&5cCr zEf3@}+(dYAtG3dhwd>ywwe@>V4bGTPKiUrZtHBb9%M<51m$5nJs-(MaV~bJr7kyw< zuTlZ9&J1=5^+(Up(n0Ev7OjraOhWw|qr%o8Q)ph!&I9O*o(0R1=rxML?ww`9bBke4 zXBJl*p@4&rlSm2(b-ddAP(Nxn_4obL4f6sK@8xq-nP>K6(Y^$K&(xH(6Db=`rP7l) zTW@eJy*G*z63KvdSNG7a8<3#=lv`x4TP727{dnuf(hBItLz^TV`Wa;sx*6`3O>zA6 zUD=g*O+u#&QU<@_()jjgM28AC;Y?_zB|w5{8%gjB0unN(F0YWU^R;r7}a_A9%KfG;!-|@xLysrDUg=W-eDY@Pl2anPmX{ zY~|a2Sv?6r#AwC@|3**N=5m#{8Im%Un_n8RhA6`MO4|$-K)_`fS!l9Z9!S8L1*W@3HSK;%n8wgj!-}7QqUUkj{c|)nWtxRTym!Ag^ zet(@AQjQrC;(xnxd~7iW2mrib1ivocuZ|3N1b=*eeO;fI-s*aLI(#2s`F~z0HL7o? z`~3LYXQ&hSHP={WhMU5RqDZB=d9g>cl%`_n%zc|e-DHUAQi$;UxCy= z~JOKJ0l}D$JE>wNO&7DQ>H`ucgmb8vb>UqM-W?ZDGkg*}OyUlok>1HO&){hLtWG%U0j`7wp z;H19Lb`E8mv>Hokw-gARs*sA}-Gco|jVZNOI{VcG)#o4&Wo~xvp_~QXI^ZqoCA)Sh z3*OI1k*txpAZaZc64k*^*8~nFl)Kls4E-QmBr)xd63?BhJKX|rhGJb5Rgo(-snD3Q zumOz9oQp6Rg_?vhkCXcBSvKKS(Dw?J{!QHergY&-Qr~0TYp7L%BBw4%73KafEzxOC z{F8_$)mP?kzU0!r?9me?p}%4Y5UUJd-!mr;eBa#cgQh;NG!B(-4P-XL{tZJ?eq zIN{WME%w``;5)Z~#rsGVo_w22q@U4tE#s$?pDEWBQru88Alw%s)$u^5g48dt=wK}N zUp`e?1G;MCnig-Bz&jyCJsw5%r$mEJ6;{18BvZ@7;hcOQxs}$^h4#J~94)-~xz~yg z7kKNDao&~76*PUUj3?=*$ouWI;2Lg~92*B8f4;MRdnc%5b}*(bE-45u;dzuP92BxG zwa@xBDC}i#GHDp$l~ZiXsO~GbL(YyLx2IBwJr6MCIeEIQImixmXh#Vm5`}ysB{DK) z3+caAgDdSX4+fjF;FjI7zxwm+vrww+GU@y0|4@DYY>f6Ku{i%~1V|Tog<%@kcd*;EOy#IEob*aV zf1tL#~_OcojlzAG*yo33wZ6(#lVQGIMM<9%43r90I9I(@%>eN;uSowI8Wr*6`< z#FjMQ0bEITWXAearl{9zfQsi-{26;{;fAE2gOC&{g2B$PWGEmQJax6+0MSGli_dK6 ze4P01`{7yCi@cL(HG_c+8ZfK=yI<=fMi8D#eJiycd94UXZ1s5lmfXreC-)!wCUpxJ z^;|bkY~)s^{vr&;_(3Ll+>h*#WSzb)ehbXQdxBD<{LAQv|TAbjYM4VfJNu(v{QuE!N<6gQ_R;O%}(Hhq`OW}R387? zBS`D`f#_gD$)h;2b-TWR!nlARcq>jo z?dPIO#VZ9F%DWN}zvYwNKktMJ&YQjmW8g_XpBD6TGvd;t*bD)KNm4cp!E$C>Iu7@t z5nJ>XEI?ets(f%fPunfbmCPqW*21aIj_G_g8dXqmFwClGWQ(pR2@5`>W5LYCziFfv;CA9?G_o z2oe^$HG^BPeA?R2a9XO&GQlE~wpt;m-=cF6HmIJ)qv9}vNgJw5$qhI8<`5Bcu7_-H zKJoU5`I^3BthT*juKCW}MN%L`9SH&UaF8hvb(kwjys;WH!6QrKYCd5Z9Ln(m4wclSvC)|BHaiYmwCM36dcAT9I5GZ|bHL+oFlTn#~a&v0-B`FIDC-sqruuS^?VvqQf7!u4s4$sk&OYsBAV18G5wi z&vj}tfg8p73g&fj7KEm+SD0ooYm>=QL2I8Jyg3XkoPO$9g6}EY=>wFt$oU8h^#tGiU+h!WoUCJs`3`Kcn!-YSLT~RVSzb{- zHVq}BJV{!2OR^|0yVSzP@%59rRYLn1tiTdpl(1|gQM=+0eaZxClbY*Nk4iO)Wxfmg?_=d=89WB>>e4Xx1*DbBRW?HIts~g}iR>hdp=N+n_ zM_7TbXL|On4`xKBM(Hat0w!%_J~Tg^xRZPC=TdTU&2wj6?Ob-a_qcp52tifkJ`cd=Du@JE9SCB-p+nM@7eTu>!&P(HCj!`xbcK|U)8QCH8Ykn z2p4I0sB&~7lT#-mD2Y7~hxW*FRTg}p0*?{@#Yt>LosF!Y9ksjH)}?#LYv{30N+Oqu7DWRXoL z9BzpjtxN7NVm(mNSPMuIgY3Q!D-#&uKhSVe;U2dN<{kHI{YO{Wz-?Zr?6U?rawXL( zthV$@ok4NkUs8CW&zkR{zj@jG|B`^!KeCP^ui7Pl-CE3LzFq_a{Zfcj(~0jI$z*NC zWEdB~nppIk4rh-|fWI(+s_wS6_ERx9P&TOZ62ZX>Oc^g!Oz>7kKow#*q|kL1Mi!~E zp+rZ?k8|>|$G)3@C^KMP<40s<`Euw6KBKT@V1d~Q(FQIoNYFKOs3|FQ{sr`bHj`sl zU0<)_vT1B!S6zcyx>h$Xy)A2*TUIwVeQ#@-m#>b}@Xn9Y@ao%DH!kfBHB8QGnOj!H zs=JTU@b+^2H?{u1&$A$_#>NBk_n&0Rcy;&It^bv*jMDJty*~WoG)>K|{a-NVzhTqV z-2ckTkFXhQsY`pfZ%tG}weADZAEGQqd~Y{Mvxo;Z+xlVDG3r|pBP&qHB^LW$f&SbP z+lV?7r{@r%px7`yAI@-|C|vY?bm39#e!LiC6)HMt_1V;0CUt?ZUh7d}r~jnDTaYA? zTg0?aVX6ahW+M+G%|*IZ7?V-!nt>UYIyPQgq6a|!?athXp|%bnO(pU)r_f12FFQqr zq73pOs;weXWn&bKBLhz8OiVl(21Y3jO3lmYdrzB2h;MQ?SU{C>>zP~`B|5UDkt0Tz zVUgi`^;!~5mPPC}v=Fp#A6`VrEG>vUlOb^c1XhBcE>V>(?#3NM4^A=*YI$Csen>6n%DVr92Ofo&+qYb|(_L)UH^J$e@9(GfJCw3qGcfX_>Z zN1XB0Jsz|2j@#2@JPIS-dgRJ8m+T;ce}VDVpvAK$Zr^aes#D>mB}nW1w&CEIHsk2* zF=Y4{>N{irH6Y$}hrO!jr(Mr$Sou|{i*2Y?R})O*Azu;6-6}FX6`mbTareg!-#p^{ zCyQ3;yX#wS-QI}5CO_BjLDNMJ3G=+p@~>nnWf(EA;tGgMXp`JtV~~W#i19%e$Ls8; z&WPQ?pVY_GQ$ihw>$51CBwK$I60*tcO=74*(WxCGGIL(;o!Lk8nI;qQykz!RdVw|1 zA#H6I8e-T*x}ynvnH$1X&N*3iKD#+}`YV20AwdnUIci<3?(Y7#vaa1v{Q3D$e3SGy zjg>{D*T}~>Cw0aStGNS9&}o8{U>J2|yc4Y0m<1OwN>{`aTvH+XSY#GOZ1xyu60+2+ z)L$rCxX=ZVp=vpIL_!5Q_DSnlK18npJRd4yD+HhtxLDq$biNkGfkJjG4c?SyE)BsLGzJ=qX;tG<=YdRS5~LM%k!&fE!oz0TY6Y0bQi z5p?lTdAJYvoJD=mnJ02J?L3al=Zd#fjmFRZozV{h99J8glny!2-xRb_<}->#^%v+> z8liIn2PBMcql;xNC>fWTDvfeO7l}!oDB#K&>ujTlQaJuh0q#$FE!yX@5go(y>Xn@A zp698PNBsFVO^c&dl^2!tYgGSI#=sJq74(cZQl-dd4ee-t{@cj{eCBOOqdGbZM85CS zjO-ab{2!b(c)0c6y#c68Q2{O?$!2OBLKiLE@iCG8*@2-Y-wh z*v^Q$X+35rrRiBE5Vbi*0!&e18M%Y?3;V8QV~AW!8w5uF z{OscRTy67M>AqU&Qwf>rkdmq#$(VK$F*wxLl$oM9yYQV@NJ{Vm5t5$fH%fwYcg>zd z#&i|dPQ6026w62vrY!=KpPWodkRfIUPG2=y6kXwZz1+>ire+i=6Me0;q^SNHjuB$U z`;5)=y4K?=>VpQ%fUAmCJDrgYLik4I`F6}i>QL~DidONxh15fk)dAGPy_z-5UamDY zabG-DSETv>Z?>?uP5lF$inI2nl~ zcud#dTQgRQ8W7B#rqz)w5s#M8IyRAxHqA2C|E<)UvKsuPgPZABm*35m<#ko>lT&*L zZXszEusOA}G_?0IAg(^APiMwvt1du$+u`LI!~&Fy?bXTU%ImJI9x+%m3n>{qv!4Q= z?j-5tu->?3e`<*0Wb_8ry+9Mam1`(x{CV*(!5*~QsQH=frJXbg>N={44=^ATQb|-}I7bevQ z6EZ3#QZj5(ys;U?L{h(qQ!#(p@RN7#*&0lj+s1QZ#BOpUCtPN`if8=%_0kdT@?@JB*no|Po!Lz=dHFB!gF+HOS^s0jmz zolU#bc_Wr%84VGU5|v*Wm9->t^7vRWJp6oP3N~bK zM*39TLU9W1Am>Laq$DR-sP_a_ZT{INKuGeq+X-?bGQZ14iJr5Tu8#ZVB<%>q`g%7p zA#toQ$Fwm{5@REh>E#`Ojy^RH)vk5mYhqq>NSpQCO)gF1>m)aG5gIOdUaU)@8<>{{ zi5(K_1_6#?2E-PRgXxBV_@%j)%9G?)X0^7 zo^ji)rBQ!*x~4}MZ(*CPKF}4>1QK5NOJfkR*a^$Fj zCl>Fi_$;INlC4sq_>zq!NZD;u-K{aLEO1g)g_;Z_Z?{hT17U?J41#NEqS#y>(%6zTRL3HQD`1@l~A5_o#>V)fg$RmzG=W+T}&}o4J z0m8)zx)+m^cn1=^pKo?TVna|q4*|6kirv5(89p2(0$b^ap&+7>R@q?xZhPrOEy~Y@zm>}vjRt{+`xW@!_7r$Ezb0Z#U_HS{8S`!o;c}RYe*22 z!vL3$)bCYHvhpEa&3Da1Tu~Z#jzTnac>yxG& zA(4W0t|)5Wgrp5=)MzwF&0~p+TDaTt+!~ZR5eBhuHVec(oJGpfsU=ZQt#`!|=c?4q z`;jErUXY3^ub|`VdkwkKz){QN28_xWNvjIa?J!LbaoRifSojfD6l8`w46EJ{5_0CK zaHl_f53^)p4HJ!jnV8oU>qahZKZ%n$%+jLL+8QH4JP+eAFNfqCUcYF1!6TBu8DB}x zNHm&AQ3C3WHfAw5)h+Cmr@g1Kqh0P(;d3p+9Z~ezsv(hj+KrlkW0Ivscj$UQb3 z=@cZDSEDbU*65+YUqHv0Vm|yM0;QBqq;1mrH-X#WeBtLIo(V*Fk)%PqYRBNCIJ%2D zrZJrXEOZ>~%>KH?O4NA>&HoK;fjYA&Sa9{nb^p$eq5U>VUKhN0s)7W>^1ML7Jj&qQ zK~$|&x80zqMNXp0dPAYvIYc~b<5Oja21SqRG48U88q-_QZ3I)CMLP5(*CJ9!L{6pp zzQkQaF}Bd$q^d{0o*y6mjqiJz;x1?Zk-E~2wIX_za)j`A5Qpk`jqiV4F$hRAO}w@< zq3TtQ;lhO(;`zu`c2$60Z-=KJ>M=x538)@P81D03i2(!hDMXLhXS*02=lLGyw;o9p z?(z0^YzWh6K0(QLvM$t zQw!#}kl09ZnHlzA8|cu#c5jA=)l>6U>gZiJ2RTK1ao%eSd$hE{?@@L1Ycib4NoJ+L z>#C?ko&S#{FRt`~$b+vu+ghpm%Hgzp!Qghw>+Q>Sk?l|$R=BDP%#aYbS4X1CrIx^!3-{_* zOtvhE#28udUQVhE8&UY0M6;oO<~4{~WxRyx7UA*5@xm4)Nab~wHIpaIOpX!^V=QFe zis5~GDYZ4DH@Gv=aCA}@>dr|PwhL9uxtNVL+9D<}^UGkz#V^WtC53sr4_T>oq5vMg zulPH|uOmT3#DI?2jiZ&%{uBAf7zkulg~0zmhIpl{CHMaaA>KoW-n1XW@*W{}=Sq5= zL77aYo0vq~d6#eia_B9zC~lOIogwP0U6spH4@O*7l64{`97K(8|}=3w62_0j@jAJg3S6?id&>H~ zhhkcDj{tMAVe}55~Wh+?s6Y>)zkWabnp`zH3}IqwJ~&N)?(3$-Yka_ zPfU&ezqMwjTbo0E))P+LX2L1IshXBwz_=@oKwnK}antTK2Fl z>v2sg7`_Ma0Kf4)ZwGN(qmSTyFmc|qGMBY9Thedc1bzNXl6JgSc4Mup&PlnkpKeNI zl`y0j{S;CYsW1L-{Oy=P!Whth$*G+zqPQs;)DzR+eVl%OZb79ANhkb0W3gG7HxP}q z${E&r%sLik-KMU&6x3zm(ui?;SD&=%bv3XO18x5m*z`LS60%-cGmKBJq}+|o1+iHz zaqf@FV1VO_lTeiGbH;R=nG<{HqJpOBSvjN)_?;B;0GLWjLNj|AKz%Lw$g`-7(Fk?d9c zJVdW>Dy_(B2$6NM+cpRIJD8NMX5KiiuYQXpo_{6r7gF!eQc{8gI5)dRx0oDL>1PA z;TZ3od$5d@LeH>h3v=kv6H^R*E#G2s(V(=$SfP$=S_q@doUF9(N_mO0bkM_U_0@L| zfjW&ZI#hjhxv{J{Xef4z{N9V7>nWi7txKkj#2p$$Ql)V#1X@$f$kozbhi6#1MWJ&i z#W8P=W4`Cqj9HBf(d#BeF^iKcWWw9X^F4)eQ^l)xKi6Vy917&edqd< z%?2rRadz(bFY#b(Qb!N~zEN%rAgO=9WnD@tk=?=M2&Q7Fq%WgRw{y<#>4g?F;aO(M zE-lOR6)ZzmHn=;I51Ydd^b+1O@NW%Tli`u+RzxbxTj@MUuS_6)W5yw5EBh~!CBkbw7@D{{nOyg=k0h(vuTRww;4-GX{@jUpv6L42%>T)ofm z+~{K!MqAy!y+XnuHqeJke!HvdwQ!5C`6I(I3AC)EQDcTq5%bl$k$(XBCJ?ObH&ff7H#iC=m3SQm3ztx`{p z9lJV2rkWo67IzWTq1>{a6l-O|2|haQmgS&OhmGrD*nrYjR`}VUZS7EFa>x5(&)R1i z8An9N5?Xiw!iz>1vDj1QWjaCjONQOpXS5fi5WY!DZr!b0c5{JZ~(JN4i61QgH2 zj}K}_Fbx*${@T$BzS_6go_Ut;Gmg!d^t^J-KtEP!0lC?PMx#j?z|v;dQ|dy!K`zR- z6UjYxPWz^Pr#Z~1@(TuEUYifA8v6R&Fd)BseKA85V9na&Qvt4NdD1d%Pf^Lk_~JYz z*Rg@ADJ48NoZWV}#dLm-rA+VQZ+asdz`n{}6S-U8=spoGo3*pOKm%EYUh|QJ$(QCV zg_Kk@=o}~Ftu-;eo~utNPg4dD-I>ZC1*P3K%^%)Z63rv(eWm zt~4ydS!mQ(e36BA`E;O=#|jG}ev`B^IR=o=Zj@|IM(MAycw)V6Bfj@XJPqWOpmrnP zILSUjywad2pqST4FTTNuk!(DdX4#tF_y0d7WLr-~Hmrn!d~yUG6C6*75(cyP^&y4&kKN50cOES8EKmXPW3bbfqR}R6wP3gy z+K33E31xy}$JNO5hdJo3;mAR>4E;&U|DGh>xi2n4_6CC{!eSMeJ#n!@VP@VO=Lr1N zZrq-g@%_{!30D>*|99-UIxpq{RJ?k7{CEBNpPHYjesNy>SXUfxz0U)vh!dzRj=wxF zzBKrqDTyqOztp@gj@mWB=hxI(lb>4e1Oerpi&2!S<-rz|vR^1Fj(0Bz)vfnp7=2m- zWI?co4PH<|YVy;V@8W#$UkT^{k79|!3`J+Y5YCq5Px_z+9{(C&h;!pII9jV+6L&I0u&J_emoG&zYn z6r$)Po5u+I(N_hGM)-F)(z~wT&WW&|TwojeBjc}Ra5dFVx)K0hx~eb<5K9Lel^E>J zfk}lPDEzPl!9?YepS;ZrsyZzTwMDCtQ8!>D7sLg~_&$P^Lxe8L5~ZUGE{02hxjQ)2 z*{eyerL$Z2C4y-5Fq&%yAV>k4Ml_5ZbX{JSUz(Paw3%I}FJ;;cEFFGDnAkrK+T$n5 z^y=(Y&C5oft9(mp?OKejA6{QTA4E*F?eu0R7vfite1J*X%AbaI3z zb_yAU_ybeE?LMxVHcujsGtdt-!lC^4{odio(*$UN;qXU70UQ>`q&cP)L#_eMj7v(n z2V{Zj04A>rRk3a|+Fdr-U9zhkGeM%rkdP#QN8;6M}xCt9h7oy7~%IM?6Nw7AJ_lLpQo~5`}+%* zhqR308a^|P;S$wVUFDho68 z<-amH!|F3(M4~Tr4VDTClgw9AN8w-Q=N_|9PyE###2q|Ab`weLUTrCo)t2TX4nmWU zb*Qt4j-ZK(%$irANItaTC(9}xyOuDR4%0W`pK$9K-k^z44-qN%Z+dhKNik#E$_qC$$)T)~^@8pglR!Ry zXW-FIDL`D%k3|J}hux6ype9K{nOlR^c`%c9ZnCjYoaAzGt`gQlk;`hRGR}`PVT)%J zOp%PX6SwYv@0%r}7rUu4H1f;qQSxd{VQ~eEJE00*;qy_BQ*xU z{1L-yqb-*qxLH^2*eF)Fx&H{c@Wf`?sdki9JPKN30N2OR5qJbEH@j*$$ zOQ&{(@}Qavrh^=aXWZ-U3@ey!49c;Lk^Z}P0FSiJE3=$3q?-WSFcj`c>m&|SYYh?J z*np>#zCE!7{>6D~Q0!%3g}h{L*821(o&ZhcqwyX>q#lQaUE-oQtpgjFE2iHfin6Ve zR;P)5ZahuhMnXo%61TCGH)*2Mdh%~YnU91`Cz0VqlyMIrD%00D+u9+SY{ zm40nLNxG(y@&OeR>T82VDu(@ZJHJ#1!gmp5o{-!_yPh~jJt{^5S1584*MLs6)zaaFtwoeG#83zRK^%-_cc4 zq0c3|rQP4xiG4BuGk{xPzG zqossLi-K+KWu|%nPuTzYJy-vEi2DC!J3He3Qy+ZdoGyrq`ZgV-P4xgjZvPG6{hv(# z%N43aoYRHq+W&X$|Jnk5K+1JS(0(KJCxR@yN~AJ9%E;1L;*(JF+`L8D9$yLywml43 zzxVZYiKf{o$SxFYEh=qob?+^7688TQp51vX%eEbN$W-}Dcs~hEHSWvNkGFthkAm0b zC0E;8m`$}dZi#&VP8k=h!u-=}$GTA7t*9@sPddZ zdy`&yNa%fKl01Kbqx9C`PdPz+as~RHX6o2`ag0(pu9b@~^(1UH{?iWE-j^!G8c?5v z=nm-Cr5K5-i^cl{qn(`5H#{|9Z~X-8G6z2~Ce&TK`31QR2TNRiKpYUSwPL+`Mvw(- z*Tr~cCgU5rC#Q!G#$qxahVH!2MIwWupOjaE3qt7LhVDGc%)N~+O3?16GSWn9yc_Yj zfUs?omrn=RO^}Tcqfik$Y)^yxsv6a@JLSM)OxP#D=le0LV1CSm?&&Wz3W@C8ywL7Q z`e@0^nB76vwCkK>o!RTS!?HSh;60$IJ%RS;VIcbQHCowdx|TW3iWgO%Dj~75?^U+x zSeEYwMh{uNnUHdzW_&i0vNjX6qp|Oj;+QDuAVA%YOHODvwI9A{*pRNt!SRv@di9wa zQ~iXZK+mL_uNr30Hww)*3C=bOq_xs-lHsHL{k$H!6PTI%^>u$8TKeew{`RmW#JE3b zI|~r_HT!+v`S{q_Kn(Ete)+7@3^+T>_eYv}h4=-yzJmI^yor&Vodx`)8#Ub;(_iH+ z@Acby854LTI$TK3r<`kW6pb2ra?O+S4iUT}RX#@Jt&zEzX0B%+@qDLNtD=szx2eP` z1f0vXY~!&(z9G~`Y(MH;Dd?o!BP7*R*Oc32^ng;a|)>>Ot6@}bjuSQtXI6m2F z3n`nele*yIK-26Fz7R>Vh&E*TO=C^TlM3$DWSI;^&=!7=9P^uy*_pFD&CA(&(j*>1 zr|BJkrJcdkf0>DAN3`9SZqV-p-dlq$D<^rkI1PMXk7cq)k^&Fu=#s8Zh&xGo&?Gp# zrdIcZY>~>Dna7^@F7K%4kD)IV4qT32dYcbc#SIF|EWb9xN6$AQN_=cq;_P%3-vEI@ zsta!12;xi;qpSS3CwUva$gW$DGy=S|eDRAyXYaCL>9TiM2@*kI#w91?mWqhbbveOb zQZbhXTWhhrXq&nCJfEoSh)iS)&R9LW`|$cz7dPY{3jbdKFhI}0wf%M$bhGN6;gjTi zwbIlf{Y}VfOYzO)%5>WzlZAu<8-9{0U|^hRAwQ$lNs5?%QPdmRtc_uW^}eMM-QPF|CXOd$d(_<73$+sLW8G`jf?Q&l^Q?PE zd~983Jx7Xl?d^|GU>}$z4r#JTfQp?B$p9w^*-+(0f)u-U9gZ3yqf(DzSyS)}iii{O9D^ zIjn7}h@&4iN`7$;TmHqx)jzKP&yRp%A1K4ib?EpDv4TSY z&_c-+nbf#>`>L4H|MbsGS#OdRKVs}(`!r`>G-ewd!^I@hHTYD^ie)TkB8$p^Zb^iN z!RgE8VFM0fS-p`$Z_MO{b!j6_UGaZ!Q-YJ+XgndHM*=ogI0qXAwwM*DQ#f0HYnH6g zBgl)w7FK7`z-GRsYes18R%k5=6RN4qMEN!*)5LEtUj|7&C1<*g=diY*9(yg(KGWCr zR+fAeRS-4{krgtY84;4GCfoQ01l_N_e=_Ev1$_et;Vt%u>3 z-zrv3VwT@NFthYUI=#lb+y@uymen+;b$F*BgLQ@eCro!#6&+!-q3I6NjI-GhRvJ;m zV{G=9XQhgL!$tEElT)UdL9SoHqa?E<1T%|~sJXRfBnIPLcGk*lM4ScwLNUyYMfh|@ zMJ~5{jnslL>wsi0I*ce~=^X`t$1?&I?3P7fKl@1G`U%|%JJmpf6m-c76BCKS(Qy;O zq$N8gdQ2A6vhd%@f?+<2?UW|*S@j8SoYE()CU*M-O7!Q{rzUdt?)6C@*4;kAiTU~T zDalBsSeD6cc~wcRYEC)>%%iOX4RIqT1|I|O>=L4tmxM-ZWU4k3?f*!WX@C>-4-@-_ zj?Ya~H9v%#n#ex;EHZ4yc|i6lGg<)dcziNRe?t2Yx5;_vIfL&~BGd^=%oI!))?g@P z&OFyC#{Is>Brknml%mZm)Tf#CTIL<cH zMzsdIdDtQwC2AYl-!R__ScbaYbFG4R?~9W4#`0&yQ;fK1l8!0ECJ#|nzQ;$c9G&TH zQ**IazPdsq{&?SNKcuhNRp~0T?923#W5Cbj4(k(RIz!Yge2Av|vmc0s7Ns8II9zWP zeNpLd)SMRAWf-2E?7{24zP{)_4%&{%b7Am1q4}086$`u2*Bx7Kq`V<#!O}Q$#n%cE;R=lnYmuTL|_exIJek23f*DAx)FM+AMB*rpQ?CnZFIWc z7zMS5vm5RB;FzIo`*7f*C}?F&O#>JGYx^9@8ODe*h%=&lF={F2fwg**Q-JI-q)&YV z$wg7`C3A0&(wojjs4Z5l|C-m#*!1&NY#qpjEFV#|M)g-RMn8BI&M(4@f z4UEs`L4A(cLw3*@9qr#<@w-8Tw7#Wllul7Zx<>1mVLFB4kO}Y}7C`d2RzNaj3H;d% z4AEQsz~RK8UO&{B?>qLcWWE=HJlY9^^XQ-iJh4tlqf~i-i>CfZ9trF>_74nt?zW@_ z18VV_)XX+KAphFhGA{o5KY@L?AXAHwcjd@3laVq?G^4`7HP_ObTucDJ34ql)H?8Pa zA2y1ZAzrdIhd{?s((c4K^-=?7C=K$OZ25YF!X}}BxJ0b;vygOr6krF-irmp^t4}m5Q{vSXE;tg^Fr9jojFu3w*cbF8v6bRDbg9INactLz-B>>R7? z9INactLzNi6O3!h`6$L+3RH**{%=n@5N@_)Ob}Tz<4~hSaj`BKp><8O%{Z(09s=ZXfZIq>^Np%2Mz=W7728sd zuOKw5WFW+K+Y~h~-OGt$5TuT0q#HPhL}|lHXZTBK%1rm9=;Mmc0P1>M);o-hGfA_3lU2X-tW+lh3T;}T^2b=;$^K!5PqO-0KS^!Tp8_1%?Pb6^Mi@E;|^0l z9(RHV6}AR%^ z+TD@vg$Hju?~HuSl%i{P6|l6ADX(B`ZzltmOVKHZm4L>2da63I?iSnQnx}BRl}lzPMR-MTaLjyM5x#+q{tL#X#++gTaCamS{Rwyu}J48+nI#Zv!hV zo*`JuvMJDN{FPqWFk+2j`*}Kp`zbV{mUNH%m{>a7L}_BtjkQqouyIyo_1UjzQ7mb8 z^Im=<*Xpf!RaLU;w!NU5t8W@{6N1Wbn^nlQ@!Hx@<)Hz^oMUr{1o?2c)9fQ})los1 zZZE1p{(Hxnuvx%*SigAs`#<6~A+jQWd3D?68;c83VH?VNj7agTwU^&lo*MutOi9d}B&bOQ(k-P)3cCl2wW; z&C0)n8HU1Wk#4poyZd)1r5a^BM}eDy!5j}zf)Y0XK!&F6=_{F=hhWnp%<6!UmKKMf z%z~$E>hIbMgW zJ)g$M;xBANZyBV6@q$*-T&h}6^-Jz55XT&WGHu{ z{(>Tu$!UDaM_2W5A^lUepAI=tHhsM5LniZ6 zPHhg&@93-r=WBeX0Q_%2HcPE58th6J`4ZmLj-&CS*Q^rM0$`SuV{XLz4HZ(oYl>mn z^1{t!#ma(b)I9%V4bT0f(hvT_{PtrD%!QsG3Z+=cV4S6;1m>C6_w52~@>7M{6ePyB zYbmd!*7*h~n${*#6LW$F{xGRi!-L(Yu^2oBdrm2RHY?*LSqq z@ADijlvBoH%f&lJ^YH-b!Jt#I=lNer>-VQbu)AVMGR(FfF-3Tl?)2+n1Z7hr+TaT6 zq_sRdtEk#Q45|c==wM_>H;UEOckc>?Psv#+bHBdk-UCC7MiC_?fEi&nJ+%3}4LcEp z7UjkUo`5jd1|5)Ri!X*E0?^0zZ1KhK&B*NqtH2Jvu4&CyO>xcY*WJk)pxKF1+50^l zqY#Qr5He@itbp_V=Qw~~Lh`5?mp9S|H(kjb1IARgVR=)4yLb|x#xJjHSk|Yy#o$p`zzJ$WKnEy)#S4Z$>O$T@&}a@0TP`bJ58w@iO*S`d zL@V7(|GHtjHitQ@m`UW+6AsJS+E|w3;OJ1T!9IP%M!*pw;?%g3-wQqRB593MQcuQJ zk+-b6kznVV3C(dr)f33(d-5CjXQa#T-@bec)#416PowlkHMKPtgqHj}Rw*vU z3~)+kq~ZCWhy9L`ago-w9Uq%(5W07^Y$fs#oE6}n3MShI`Gr7t!xn*Y$#<^bFlAHr z;)u7joEs^)mRp|1xA0ekfW)zF44%_&wl=8eW?hZyQ+BU+YL*5IiC z`E=m@LqgqHGvgpzffZCz-J-|$Q_sfxH41#x{4a?7xq!_q@i<2>Zc1f zT)f3*KSGLZu>)&1?z(A$YAN&A26`>hxs$8*R-b#LN`>J+S|Oy^uQ5xE!49|PSmQ77 zhpd!(=YJ5uRof!qVEG@h^;QfLKrvvoMT;_Fy*U%l$#2Pr6IzywE;rMx>BA7L0`zKT z!>>M^pr$nk<>JH1VuTzP-OuOCT@ulw={(4Ro_ z(Ps&&-CJ4$XB>^!F+u$mQ?jWt1MRz|I}1R$+_ff#TFSyi?ddapJi4S}b;a2&BeYub zx}w$2p)o?B#t!zZ-ctW;ShWKyCgh)mP)rdD-t0<9S=h>)uR4sXFKi6U8jWJreBJPz zJ@Fo;kYj^a2KLT+g80eriT&p>^){hA6pZ?GLag0k>pS);F=QwQP}X zHjJ#9Fp!AgWD|szny*!y)`s)arj}b;^Nbe7D0-TORxT^Pr4=uBE)o*+*`X&=5T3Jb zDQhha;iB@zu~8%T3KFiR7%~A*-qRdloL8Jx3KfL|g!{mHDP>g;gXfyF=sCG#%dq5w z_+5HPVSl>BV9=yRPS#q^fNU>^=oN6l)RO#o$Cf`%6F-gm`l6spjrGTGvbmpR`T;_f z;%m{0TUTu3iY90ISC!np5-V8^1^IXP;#HIanY)m%_U0j3Pd{%f z)fzHrFJeH*26p67H^gmo!~BPK%=faoQN%itkqXP-*eAh~-@$h#ujvgVPy}+M=%H?; zBY@HA<{Og+=oLFFDX+*&rWF&tg`VqGHb9c?4;iWz2JvZs-;Bn}^p=W0GiG9pkp?RP zL+k=NDxBOuxJlh~bZHt?9}`}|t9DZCJo2>|m3rg;H$&(@JG(T( zJ9=9hyQ{3l^lvSHt|~0a2I=E0WYvSe?G?_Aq5TAX6xJD-wdM%eS*|H z#7>PK7#oi_9@v=YdUxzncLwdB59vp9N>-$G@0>DbCki*XF`bwtw}mVpNX>Y!SW@+Fj@X(~dS$ zmT$LBjjrlTG^T(wA1oPS4t-lSY#3WN7;Zke++Z;MC_gDosTEtAL=S!E?$NBjlxXr; zOz*zXI+85WX=f?(#lRW~yvG%Zxsti7!sXAAshn23h#knAMe0Git&C&RR+1v|lY$Fl zp@qSq!Dz@}EMTy6Q0WZ~r_~MHjZAl-HT$ISXCHp|X@36av$G$b&j0Q2zdQS|fDgYt|MU4L`^WFj&(F?&_}lBR-(S7@J3s&Q zgJ`xl_}8biAK0tEClWb7|MS0$H0U7Armoca4IUY7UW(p3Tlx`#gD9rc$Q2Z+X==~3 zI5rhnaci>N+EIjz|7z}D);em`lY>mP_JlmHwO?|r87%z0)n-D`_Ey^{x3m(K<(nWD zIqF;yx#e|jY@{BW9t&RPbq6=y>$dBtXR#&aITfJdVlfzeY<5>m4}MB)Kiz%Wo&PvM z`fSUKg4+;E`|p;EriLW3rcxxQJ4NmoyHRAvXr%_SSiD{T^poAJ=eu^}p5J+&bMN!U z`@Ho&EB||S{^#@k{T<*&m27HuRk0QS7*TWH>mk&4lM-(4%v8nJTtQ(lK*`=vkr%^N z1Xh@p&y|JSH!9t?m&QbET)9#ZN>sC)mvkszH`FEH^fw*|Ic(2Vt`!Bs&}jw(o&-dX zxn(A2!Y}+gWc$!+fdi6nP03X4#WR?wL_iT4&)*A+=&;IPxrLpg;PoFtOg8HoDU2@y zN3WyAj`6Jbo1U_(HN1;AFURsV(p8>|Ky$F!?ibC^M@4nR2MrjXnh{EF*luP*iii!3 zu_{Ah$_cVT(~!Md5Bq(zt5K`xz;N8Kox?VOanWD6oOw4CSkyX(aCuL~hPY8nZoV7l zib3QCEEU~{Q)TMzOQANrjDZSL@NUVK9er089?9eSyA(6>M%MbDS0A~GWUA@8d?}fF zBWt+(-h+cpOCA~%sbLHpTA%acIczsCZrmkDULi@Q`^@hA`0(>RM9$ zmPut$Lj@_~c7SnKGk3%OICcU!5We|EBA2{ZBR}6_B-8(^*)uGn{||H67UW{weyT73 z36mk_6W4ULwExeB-_pXw#ZW;Q3h*ODxGhZe#TWv{){z4hXu2cs9#;ZQ(^rAzeD z1D^n^a3-sqi}g{WI!aWd!*P_T;5tfFqt{WQB1eg8YWt27)ls5)Fo{aBtbxc}DQfmH zmhXSvy_;-gkt+g_>5wLGsS#S&6<>l8b2xJ&liO_<=b6=7yi$g#^y-m11c@tv%l8{G zRRmk9uPfP|+_gRyQ#hS9;5NK{qhZz_s-b-s#QK*^40{ zF3k&MRokH=OR8AjHvO!(O3oo_1YWqIw_{T)Bw?$S)=s4$tlY3It7y?}l)?0VZX!!( zCJcBx9E3}ZoD8B`&MPZ-yAzC&leglXl=W+_;MhEY`%f+acjshG6x%~~a!<5jGJ!Qe zLUn8>_-p>4aM7v}7s(u7a~93`YN?;Y5Oh&jJBWzf!<%)bfkyiDtzyd6fUpOcEA9fh zZ2)|1`p!lFd|l|#4lq}4Ah`R-vYy8MgVGcA$7tF)9cas5{<%pcqIEo+=@1s^Py8eC zI#?QSBqV2P|KFEGQFj+R*KL8q-O;e|U^wMJ=5ObCfQ z%Cvtx9&!&4AbB?gA|emo4eh!grySc2AsK!gQ4?0+w~CDzW%hW}w-3gfd`Kj{9J_lT z*PKnGHayGTFdAKwK7(U(6*$HcXK>)HV=Fs>pFPq+_DK9pv2VC&K4KA+6w|U^QrEBG zAuJj3u{LT0>NUC1Yy3dtajC$MT^K2ZWGXtj`T!Lb`CyWPzWOb58He8d91qd^9inlQ zcQK5UXRj_^9vA_*a#M#^0W^O1^yQFwE~Pv|T7a}1LIoOZ`#XsRLa1xJe)tz`JKU1i z_V>H8*LMBzFWB~G^hn!lyFUH8ZHMFNmuh>&Hmg{c$!&R6$<-j>)L!?{Rss*1%r$%D z?iB)2Xc-wbr-0e$KrS#Uhux1i$cz>nsp{Z(?2SqPv9^RLq57@xT__t5FR-P-y#`DQ zic__)lBkNl$0C$)pYdJQ z%iTgiAN>s%xxpAs92o~5i~>ySY{accEB2gx5X21+nYU{$=>4-9X(?;&9f~`#B9#!* zKPV?zV}V_`y%z*1ek))>>9)?bKDJN6j?AzHktAjI2fbsMr*IH=^{nh*vPltQFI+4H z*yWWkyS8N_jXI=9cwEWrN|!U3ZR1NL1C?PAo@PhBzI++7En7R`8DF)+;W@gL(3P-d z+a($gRXq1`0Li=*96DK;Ev=jq~}0%V_Go-#0`G9-4t;()W@;j1o|H8BchYk zng^gYmEW=%VYf^WKGyl#hX~`s08W^r7}nO$ss;wV+$mL`HtLTf00iGN!u1l6Biq2oZzP{)_2z<%purv6b(A@YVL$Whl zZlt^+XEr%-&Ff9GTx4?V(f2rz)*&Sms&|!wH)%C(BTU z9{X&!)4;A^s_vz;>`y}oQ!6?Z7Rs;>J*)2wbg@4wXfn80$Be?Q#iYdE{Iz|;FI~r1 zom%2S994p%kaLWkYMR6lJ@qZQ7e#?g{tBwLv|K7c8T9!Ly=84qoD=&xtw-K)hNTHR{XVakTD$t{GRpwnQX);aA_x?u>Zgiv+UkA&|+gz_12d`wZdFj zfIu4Q$KWU!eE(SEO%(vkp_PFf+sHdtApn!jM-4Y&u^{J)E(2_B#OkBXlinoPq#Kww9vl|qRy^4SDy(nF-$jOPV z)yJp^9f@}D8e{6ISNadzWc3|mc)VQOaSCk~ zcUXfeW-|y>vxP(jY`uZr(~@sTH;?(fN}2C%Il&8m&$VG{MWm%%K_M0}%Ki-7#l zIuNBOV&-wjOVa6QZ@&NW=W5bn1#Xk|BFN1jp%C3OSczX>n9|&TSdgz|Wr~g6B1Wqk zv$H^?AQx-2(IUZT{|0C5Fi_&xOt6Y)z21#kw>v0_2QEW5>T`G2uCWpGy`K&}dXtCqwy3t|yF)80%naCe z$-vp;CxAeJyF*F*?!}Do1zXs^;MTAa0IESjuh$62>#DW_GlS;)QhyrNAOEDgte^Y$ z`JYUeF@gQq`xXzj08waztF!;3#k|DfQ$L#AFHm5?V-XK`CD!^`F#zSM?L;~K zR!i#wvSX|*QOFtz2WTQpnI5Mz2#)(mw`IX*q?VEtQmoGbYxF`8_K_>tw>YMOI@8AP z39YawX1Xc;JRiZ)CNlMPV>-jo2PGd@UIfEMrR1YBz%d z@Z1$=GHBjd2#E?Pp$;GjrVvzE7^h@e$r~m}Ry3;ika*jS%!<)`j`=uXPcs>AAu-7E zrY+53#r&-;w`$9TeD{LTRn4GctRC_u8+GduG`eCDm=6N*^|bstL$jK({0m+L;wDl& zaKss&!kNt7qPuOb=~%rw*sBq4Nogad3H?C?JGzkC27<0Ao4TO=}Ge z|A~NSCMCROl_{0=gZ1o*5wRzXBWN7PKh~B`fHBa5kE)rdziZF-a1-&sVxs1ib8Qd+ zoQC0T1TNw3Ku0KIyacP;WJz-lY1U>W4Wl?YZ%1@$OxEdO7tO>eX=)be6EocHv_%dt z&NMf9E%W%TeQDTxv`wV|{F_6-2KLe%2t~oEbJITAI!1ko)>uIky2iL>Fwe_7Yj29i<>{f40Uf0xgzcL+dYZsY;bR!ryzLX<>YZF*bH_mTgR|1L`y{j1(OnYbDI;Xc6ZL-xP=K`vNp!CV){MU`j}*4udllC*c)Eu^O8aY z)UIS|=C?eNq{F^Wx_{E?Lm*!JfEk7Xvj%75lW>(zV8)U=XXo`XdKZhHYp?w{x}I0_ zv+G?f8m_(OFv|(I1;Xe0jf@KVcufJe>0cfGceu=>nP<0Pyb(}FswMR=S3RkeWmDK3 zZ-<3aZ=j(w46<=xU+ITCK<6j~w+9v5Xz;*A+Ln%%F7q$<`j&B#GXbbQmF+h;*hZ9# zgjbkCepY($O#+?^*9tB?mTcoj*};3LCIeu6V`{V9rOU7rm!E;&VhYRULZPUlXw6n; zN+rmEIhF{TCOqzAlo(7kOeAPThGF?UX-SxV?g{K*J1}q}e1%aeNvb3+U|FY9qXZ%B1KNYdE25TUT;6^Iz%x5gcOSc2s5(g2zI@gbx|6ADX04~#JeB&UIHp4 zUT1kF%hH(Qx99zS+e-QXab_`H6DQ166bBqR!m3JEcr^B#nI-^xT2*-JG0q`a9n;wR`}Qv=D;pGsisz}?vrl7O$JsS=U?_Q~a;=@lR zpTFeujOguN2gUdHugMi~Q_tanI=R1CMcn>1$v_{gnA^XOd*Q#vUP!9Yd!OAfR7-?Rt?0*iWD-QPsA-Ri)dj(>8yXHS3a9Sle{5rYW?e=!H!p6>tD&0*d`53w`$vHWwuE)>v$EnaZtW- zk_lDa9;TUlRo*6%4OfIqt@^e{bqdcn{X1k`P6?*ASGqY3uTT}a{d*&&%KFzt#cgVT zqgCko*L`%o$ul^$YF)B!*Qsq6sbZH%G*atsu&P}m>9pE+uF74ql9F5tFOlNZlC=Hn zu{4VNg&Z17o?={w=LOR0gbz!Zwpu&ggoI%PT!A`d zUAWIgRdqCJ62U~4vJqwt=TQ`Ndxb|*_uJ-DyBFVv@FY6J+NNP)5i6T1ZRCB=XSQNh zmC6Q|yJ7Cs7T;pF%5subVR%S;QKpn!Vc7xjnz5u%I?ei51?J+KQN$9LCV$Rzzs)dl zK7pz!NJwZqe0EqL?nJ?VTb2bUOhV)Oacsl!bAxMcg4gz-5VXl>-)osqJ?z>C9jJ5% z^DBAh@KLG3Av1r)3#N9ehM46Pl3Qeo7~b83T}K88(7F#lkOxQ&?SoeSGIrfIFeS~t zKg2R2^xdt`L`fmJ*GZCYTxp%dY>fV#h;}`CLo%99XIfsGMQsrbEC^!{FX^h1TUKw_ zXpDmgsATNcj4IAlP44Ky772QUWH;=;P4B>1K})e`fTo2MYsK?1q7HV$ zA(8Z!(!o`HXvx1r+^1AH!HcU)^MvCqAZFrsEGH`4uq{O$CCCw+ffV5wG2J{--zmSy;q8?@eYti4}@-!lewlx(FoN;Sj z+mO6WZnur#_3jDC9{PBZm3or1TUI<#d_AXCw&69)^f~f`mV6E!6kzaOZ1a?8l>|-@ z4sGqutBq%+U5VIDk_`xd_v-q6a8ivh08Gt98MdbIc4GC=bFsn@;m|U1XiQAxSnm># zA+Gfb)hr_;5bNhM0U>_IR8Ju7lo% z#(*6`t*#rk3zmSe*Or!`$vTuLe|Jy^uw;}Qiy9H@l9YwQ%DvJcOIEC|IJ;$pR!d%2 z6o4^tUg1=WnRIOd9B_OC#HOa&R*kYbJVt0k-61Y@xs0RUrW#fU8pW#ly5Tu{;w_1= zJe_FGD-tZ#KjZ3pjTUIbW^dR|d0t&7WyDb^Ea-}?)>I!`KqaF!TgyuSd8RU1nz;l? z-szg*H~4Rq8h|qbUn5vf&d}V;H)u59rqWai!iY z4AhSRMOe@8Mph4LvFV%VK&=f7Yod^l3Mcj$P-34Wj98K3ibt1UJ1+K!GOLdpgkrhNDbEw^;AyGO6Vsfhr1(x~ zYg*#+#H8g%X=_^KS3(kHLIP!S{K}!RD-)oy9s?%pQ6RFW!eb>9eiUe|M}Wn83`nf$ za9AC%s|fXED6D;;tC-9Yfx#LAxrz-%2m)&q;3~Qkg78-(;8rP}k3nDcLJnd|MjZBP z57a6qPDCKDMkJ(|ycK}E8UeG4@s$y%t27|12=RTGt27X+7}5y0_`*a+LS;v1nV>i*pxtX z1k&kIK?wg#+Z7T4nH~v@>5+h#CW0`f@{42urWRf)MbAtFU+O|DB}r|+Fz`~k6qE|M zbZF9HGPuliV42CFGM_b|%)P;69s^YAvjtR|gnO3?q;y>4zpgd@P+&=ufRZc(Qi@=c z1d`;zA0^O?10;<@A0?0<21go)JxU~fhREOY)1|fx#Dc%cuXd>*;0gyw};CH4#4W+^iO@bIY0CFXzZX{O{5e9#~;Py*Ru=%9OvQx7doA0A9K4FqU1 z08j_sC(Ro93xoe8ZoUISe+~fri7Pad!G7YXeiMOy5*#Rx1M>6m06){geQ;1CWTyc8 zjKcOr-Z2d76M^iBZptp8PXb&|gklfOCk?76!R=yfK*=yY5t98Zp=5}j=zt=C&lGr` zXz%$*+|npCPi*@GaL+VYp6ImKy!d<}c^(gr=is25NheVpg69DEow%Ac33|r@>%`dD zePDMc4cf%>It^+kfw$ZnX6M1OJYfT;=XsVD%k&xcychK?m=+B*|Nq?5C}~zVv@m2b zPcq$w-%bmH_-yEd@~O`~n^Fr2Fx?hN1NnVyW)4w%JG1#c#wmFRZpAXjZ<(47t%rQW zNU0!v!ok?>k?|=dM_Hj`scw&Hhs&cd2TuQJ;y*C{H;trerk08dR-5QRAI4xqMP6ue zE2d%|hWl_bI^ziCq0)jihGOj$|H-7_LwF~@WkPSM1^F8@@w~BPR6_ihs;S6nm6N}@ zJ!hQ^J5ke*)~cIic+?tuo3{I{n3ok>@sEg~qMnnvRRqW!Pd5|867PS@L>A-v?g|kuYFqfihC5)SssXm;rk5toJWODmtyPIe5TIccma1x<=QPRpL7%k6~}Vo09@h#pBp%7wTf&qN41K(ARolw-cAB4F@fcB zW!9NPuY^_b!fnBKu#hjsRVCLIQ&GMK@NFP{gH(h2z~^kLrWJrgDgKi=YSI*G?f_2a zWkCg747Bqr$_ss1Ln{)D2!0#2_Pm1U^{e(9u6b!}dy$$|TP|pUnFhLvZ)DD}aYhq) z-W#L1KH6&tG?WXjHn@u1IJ#3j8nw1+DqywlXbDlt&kWm& z?Ipe<>|;q$;-(;w>GNc(gu2WtpD{d?qX z|BHM$8RsP*PJV{h;kZA_tM<0`BfsU5N3OqCKb#EZyFxk%t7n8qLt{JkJ;TS)Nx`K6 z^;l>n33_BDr3g(=j`|?Th0hjEX(l5#>&=pc$qqZ-hW!>h(uf-&?}U?GK>L-|&`BFR zV0?KPwwn=mtx#y}vw#gkF%fz#;@%XGgOGVwZGbTVuMK!+RjI{|Wf6_Gmflg|I7>&W zoAtVL_1%k@{R=v4!PCHoYy|vU5LmrPP^DzXX*?PZ=E_b!n7oZ+zbH4<%OZ9rEU-Jp zt$)Etb!*RBgr&**IQ z>X_A|x}2qA?A+hIF!n>ei8%ya;zmBIoJW=OAyv*cucefgUVo5Tzm0|#EEy?iwYK>& zp0Uy$IVNGxs3OdA;@Dd7WeSI=hoZJ!T>orjO=l*Yu!=J+bW!3w?Qd6o$mnwj%=4)m zmMb%em#ZO~zHBv_HZ8PNZ{7gsIdqt0w8$Ea9>n)f9a}Gp2_Y=G!h9)~&qjKo<&Owe zL%~3v7+eM++g7!6+L02YhnGW9F)PcuGHov~Y=;(FqJu`HILB)f_<}%E%jV58>C;Eh$>lY?GGbv(O_Y7^sx=9BQPM1Vygi@<`yK&J9ry-{0MJ?uP z0TRuL7_1Bj=w~s0Tc#9UPaib%oE@o6v!!BQF`7d?1@mjtxN(sK$i7(jc9NbKZ!A-W zYf{%MMpdW|&`8hmF^q^ulzgms*A|dS>NeH)SM`i+X}00QoFix+&_ieS0XL{z;fbc>DyhXT*a~c`QWfK(cW2 z)O8d-e%|lO#O5 z0~{U!tJ@{4gXntRk4c^|^{@8>ntMt{TF?{1^U9SMh2fKKO*kHIZi0zyf6j-g8aAv| z!EIT`8Sx|~{vL7950zP;dJgnclwd-+KaqB44$o5pm-S}y(xrt2B8U=5&Rxx1`>&C) ztjBOPOCo`|D+q}j1Nhwc6XUbA8pO?2Kz_nMT$);7f2f;UvZ2tug-6LTi0ZaALw%al~ZZm!jWyoW#m zSs2`Ah^Lu3;o;EYz=E+{nN$R?w!<4)v*+XuyEFTQCC1JXG@G>vT&o5)zTRM}+4?0i zH!QnJu!s`K7eTYHYm4SgE6NMMKvc1N*mA}-t=8;Nfr9k8PlTG`N+}bTEKeBuB z9#Ui`MaGL?qTE2j!I~&CmK&<3$a`>q|D3m$H~eu-DE~P!GlDk+eyn9-{Y3kN(HVft zRX@kGl=-RhDeQN2)L_>k{T_*-bO1X*#J~C8(F0+lu4%S0sfqXt?Cjy?T2m~g3Uw$E z${$QT6)y`${^YXKXKb}%S^e)QO1JsxV^iQV!~bMI{rk|Q#$*$G>)4{}RyT!PJBJuE z-9svRK%mVF@+t^FkE%OXKBPs_A)<_A=}l}R1$BR6p75er~wQMC#yolyLqZGABV_8qT5=?=@l2$d( znu1ogO6l(4d1#mEwLW6=Q9Yo`Jo``e52=z>Tp|=|$hO?uC}&u;#;#epvV6yFY7g!% zK-3R?MSpaD?s0OrV;AHvJDZUmjE(ifJWLzSwlP!GGJ%z>AkFYBmn2j?o89uPK3|ak zhgA}monUKP^ESB1`mAbGs-7zHl$^n1MhRrWTC3s@KyW;|@ovl4b88Wsv)1n!_TBz3i;evqh=SaCyhbyO zVa)R6KD_fL4cDc#d>&fdD4UgfK%kKF9TdkDg%iw`MRh^|zI*u(T_<}jpWAkKXpK-> z{*4(}-+vCd!kN_2<5-0An0jnE)5otja}Yuny-540Rc<-88F>QxuFtrSuc_VJA9F4u zB&Ym*=Pz4bv90X{QCk_r>jdqBN%(J1w_IdnTxyE(+5!8wTpbL?shRCeQtq11KX-fT zPEO}mL*}i;tP%zXX4lZ%f^eXFkONKATIltd7k1;pJ7chrx(Cz~*YvxGKLqIrIlTnm zT)tAU;>;yVY&7GQQVb;FZmMu+Ia9HW!-%sPZ&y$#@ye|1G}V(>^Zo9QH(YCj3va?0 zcnCA`Kd&tj@2yjduO~R~;lDly|9w!u{SrZUk*JIMuK$Ykqi_`-FSjjP@Zr6)RpF1LmJ*E>G1$!cXOO=Ga_S;|xA5s1qDkM)Y_2y~ zl#rE?P9kHPIYYT{_wn&^UqR}$i@G&Et zL6+XH{k{$;C&NDpzL6)*t~iMG^8)#7eSJ}qz|&>oa>|Gv<_Gi6@D2IECGI-GVK^v5 zb_S~Oehm=fx9||H5tnl^^Uh1M>v1^dp3@6AYNmYLU&?XND$PAH;I_vbWB4x&|Duju zzV#_dgAk@fNW2dAQ}X)5;E@l(8z6m6@8O^bliJx5^YNm5L2@dO!(oXF#K!I3HJAvo zv_@A5qFMEy6)fSs|Gmvl1M4=;MKzw+bm~B5`8$7jmo8XGG_wOx%K^=RX9YUTpT@_7WnJ7O|Y;-=0S#${>3+edN>ad>9t1#9w)ksIK&KA z0t?*pwpt%S!{aM2SD&lWZbSS3!Z&{Wyo7i!9HeFdADK+&=jOoEVi5~IlV27p31=;9 zTh~m2TuwJ!8;nMTrq}~v!If;l{H(cy{ub$Ver(BS%DIeKS&db{dW@Ycr;*cDPCAcY zOlX2n%@59?l(51!O+~k12HeWBYNjA+`~`(!30}BA7Aw1yZ^Tn~JH`!dir;$c1Hl0! zL@{x#6JoOufYTn&+sOUV_c(X)byH-Pr`<(?a~$GzX${0q0PPV1Wn8tPcROb7%EGeD zv!v89C$q8-E0Q%5?e&C1Yzv<^#}Mete80UL1QWKPXv^}Kd}^lX!+TC7%|d=pYpatM zcQ{+6S!t|?B>Tqfs!*SUT2U$QY4UjSi2|;bZ+QVxioQVSE@QcdPTnS8V@mw91^^6n zE!=LMj-ML?6EDbgHr|Ai4iXD}U<&K%IRym?Lfaha|ms45#i5!!!PrRvArM!xqoIa(GB1K-m; zxf;FDrq~}tDR}uteg&S7E@}sgq5nFbqMgeZ+ETlfihhsz#{Rdadvev60R$J6A!{0V ztZG)nKzBEAl zxKh;Yh;HRc;gjVFb#Y&9?rJ?_wi9(6+m$iq-B;b`e!J4tyaXM^?k?cB<|SnHQZn&j zfw(}Y{R4X!`_5c@mRFFT!~~p%3vsbeh?Nzj(U^fZdvbR00io8h(L2YV5eQX6jb2tO z4GGjhKCo5+9sC+jz0T&!fQU1mGMx)-Mm&wzPp}EV2ptwzX9;A|-htQZ0^VxRwkYMT zX`zkg!70}O8Bo>%H>{gSa;~ui;~{5eiPw$_p0;Uu*lN__#VPOzu$KM9@adRC={{Wp zl)EF`k&;Go*m0Wgs&9P(UwA6=rU2*m`ni?iSdbDc#LHBuVz+|TzOfuKpBy;G01f*W zc@0B%=s^_Kad5nS9OLK$WAu~&0|hJ!hoCI-4}qnGZ5a0jX@j--++ zVvtK@m`V>LUz&njS^5Ig@)OtsH)bMYM%QGP0{B2gd3)1tnFS|NuIq8@0wR8~Ik1nN z+PSg=arHFHP<~5zN`3&Muv5XH+xhYm8>`0s&!yv2T5HAiW6^Nh0J}NL|NCK zgKFaZL4|Fqaxpru+FWL>K|bYuHLy~_;zydRZJRi>Ricv?9mOI{fJ14>!#W`YCJKN` zsgtrLIi8K&5d3HO#HQbl?H@u^pRFi97J%Eth4WP=;=SG7<@vynP{8gDJpK0m!InIS zWDT0}+Ur2=yw8H3N@*6NQI0A9{&~C2+D2y{^M&Wtaais_UPX+$>gpM+Qo;EtFae-z zLrJD<>tIy==&c`XJ4P{o)|c4;GXm-^)M!MsZdhB#oQKS6KS_fELUoj3+bq{v)lz)B zJnxKj1$K9MeO~w-BMruG?6yf022Ay=JZr&^$uxnPHiR6w7AP%>J*qs}4k)Ld8yZ?w+>xN0h@I7})d7u_ZmgCOafyr_6ZljZVymwReaMkF6YjkH zT6&mI$;(G@x%=eyo>)=n4r0csyE?p8*1NiSR4?)}KCn0VNA#gBi|xU%(a7H-)C7$7 ztHAT{tm;ApSdPgn(8ISgK8b`pZW23m@4a7Z+P}Hz;R~aCqe%MD61qq&=phIJ#CIR= z`Ms&tL(!ewZ@5je%K_E&3RU<=7~wjcm0_m#&h+DqTnJK&r;6)i?=Q5RF89u5kD|qf zRVmON+v(7AUbO&>MpKP&AYft|Qas!51d$@4i7wVIb7N)9 zEzy()1fT|O|I7?gzz8Xjla+?zwUA~c^R-4*rTd^m^aPr+`XD+$4hBiNQ0vhv8JaVHrjl85J{lqs^H63-#?jU5s(88E6I z{k2kKp2lu=JggT|Qlf1@=-TSo_o{<0B(o?Vl~T-NRGyIH4@Ng;8K&cdn$Gs1?}3Gb8XF4#Pa|Z3=(v%3CfDFPIoF7ihy!ev(WjKDM)i z>+tw;OkGPq$=4tJBY?+~%;pXN55{>LMO(W4Gk-kjs@`bIFGTr3*j(q?$Q{vW|HT|w zmBg#sz$QH+DCt`4d5t6*V04!a-J!CGUYg;BwA->o{tjePc+(7K=+Et#Bo@k@DF_fG{97h-6b<=m#s?4zSne~eErSU!H5FPK#yE!zgHRCuxPBZ?W7$%LlHs+tcaw z_P%18^KC(SbK08A8#CigEP#*MKa6out4h{B8AO?Yl$mo8PcjJNaftE*&gY_fu>BoO zZ9=hS#pY)JwVWPJhd}hV)70)%FFXI=KhwB$H3maaZXZ~Xs84) z6Wl-KJzMGp_r_YJv@=`7@VNT#^(JKNXR5o>wetqZgsicEIIkCT04Fn8VyX;9H8`evNg_NVn=tc#Cu2O=S=?Cc973H1QZ-h-!p6&1 zz6t`?#hHmq^nVDcLUWRcly#rTKQQAKAoE5|Wonx9 zYX=-tkdi*-%V1t#Atas(9)&rJoGN$bq>+?182+h4^>K21iVW;h`(`?QefG`*9euy? z{qBuLVlFH9yMBbj2#HCx6ZPVY4UtA2Y=xXc{E|OrI!o3CMJe}DuN5wD!Vio8mi#8 z?uqPVvIX|(+;6hyHV7uJM48*jx=D+3gwjAqc1CKnSUv1h@Kn{biJ_vPJ)JLAD52Nf z2L4O<7B`omo6s2reG-O-B@L413+p9*w|jjehl!@Q{%A1s1NIjEBp*Dm`1*Sj*hj|d4LvcoP1d{~B~M7iWUfFbw%=&gp)ve07`jeHZ};Y- zM(Bjpr;-CY8={N@GhiT&HDY_bpW1ofkK$>A*(5IJi>{h#7&pmcsndGI4 zWL!rBIPx1eSE{bI8|yXUA`X&YQ3o$zbi33o(T?Y7~xXQFa0p zS`LqGSa#AoQ#DE!PIl{CA+%Fll9_j*Nrj#(L!a=i`l#)`9S=1!;wkCqS8$Hlvv`7e zH_3tVsO{vPUk5@6neWjiz{O(d10wCjJrOHY%y}}J2-S7;Z}1$T8$~GZPFA?&PFAq$ zF2?)y_9A_l$3f(>*Q6>y@xzAiz^^_FV{*Gi-dx;}uskj&mL-iTn;^P^DPqkae^=JO zjSGd%)|1fb4yuW{7n$P(Z;E%v_YcLNI|PLjCmP*81yt2MdW-v4zZqgwhYRWKbGFRU z{>%+nyTzZddL`JF=!TT*CTLHp4ZQ!Z!2;;KWvC0_U>1ootJZC5;cY+_{KmDW4 zqTN?G51w@D0?Z6r9e_iksBvGE>M@WTBKtiA<`G@EA0GH$ulrv?C*r6ecLUD;4sZ=L&yV_eqegS*;MSuWutc zo!U1;A+~O{f5bInBZ!govqHi#&l_iPgQ>&))1K2ItVd&2%x!SP#Goq7i;QXQ=*IMH znPjb#d>HGb(DChKP2H@I{>oK`v==bll;T3 z%keAWqqKm>I`ZnjZUy9t2mYe*n*RE_1;mwJ?R27A&SGr2lY0D$O*O#76?un{L|CCh zFJ-i#N>p$r^_d;OI#UVxm``C!&TXc@{;AX}ZHg&_AuTm2P4)rwAoYnnO}nGI?!v%? z82~vB@lo=Hp}*iVbJXwJjHYBZX#szK+&;QB@{RN-Xqs4Eto%y|iIpF)#Sk+QD>83mRar8pcd6Tl9Y2(%~{8eWSJ?6f~YPKd~BL=QX;&3-G{u( z5v&~5Isq7r9yY&g&85u?2U*V5?w^V7xxsn9XNLdBDskzlJCz8_MMupk-{TxOkR>>Z zxbWw*oUb%L&1vbQl=g+%uS+^FTM%>ti%G?3{Xf42dX*ph9KI8nYIs*WJHz4G_HsH4 zHc~ET1X~LIeYaq2*U0a%S5w~Z1x%yNQ{K&aV5Q(MACxYzWUVm&R!7zM?M-Owf>|pz zkZoif_;AU1y}jFLX-83e0bCY{ztp&(evIdy-?fD`C=freb}C~&+vtfXX&!S&!6%6L zQ{h7eeGm68a462>V+&F*&5a&oHfG#f=2%l|*ueXg2dC(W1NtezSrXjJ@U(^MFAZCN zi}i#(5~{pIHd_@9!Z_-tL(hS|lW=KdT7c|!Ah{jhm zMd&nWSC$(h1!Xm^g<+VY+jhfxy0!M6zAs)6iA6ny?F-jTo?)AOEMuRwC9gf4tC?o3 zTL74jaN42~Gft-wF^X~h?>PeBnn=e9O}qw874WD?z&1oPX z=K!>0z3|_w&M(s!{6%MZm&$;Pea>{VfH~P0bRt?LjbLD{77)Bf8hZ&6 ziH|338-v=aiU)(C7SK|PUk@pv%a|<=5WU?Hf3TnuN3Dh8ga4_(iZBT}mS>kV+@Ox{(~lehv`A|1 zXtSRqNWc$6JY>2|pE-cNKEWaJ+;1}j2MWeAenLBIx3z&`c03*mu+*`n)z9QiHois} zpCQt(itK83{d{sJ`wD#S{pSC@HTe**+Xqpmk6jhUnd;I(1(c(Iy}Xb$M0rc8|IzdqS#%3OJx~!8r$<;aAkgZ0j#w`ZdgZQ> zXs#Er<}~B*K&-4VnHX`xAmUN6)l-oh^Y}}9rRDE%3tXw^9 zf|!9*LoxOOF2MopUi{t4MTa6I#Hzq6P%k0{RP;}bce0M34IE5jK?4Ei2$u|VtL7tV zXkZb!Wd;vO2J)Via=uxZ2)6@T!>! z)MW`r9C}+y@isJ0OG2fzeN7U~dD96;StM4Yel#>tLfh{KL#fx;FQe%q))+}O#L-It zxr?>+rp<_^yGim@(cU!G$rQCm&|rIKjmzk=H16}^3$l$JLc7Sx6Wc4n^zm^A5Q+#F z2ibV5wi&aBNZxYRz(#BE9k~B&ORwt>HBHBf9SR$Y!Wk`L^-K?0t|*eTwiyah4Jb95 zW}m7rm5yJ6mMJYaqHr@C$S`PFESGE{nvN4SG34=TIH%E5nF!a;}s;ib9J>Zq^V5Io)y2 z(Z7CSb*BHZj-AkQpSi8vvH~er+$RLCHTe*N4k$w@_N(;Gwi^nyj@OvM#}I}LOUA-- z)hZ5^u0q;3Rj$U&h*nD<#1p6ATh(*;u+HS&lKYZ|8QH43Qe~Qp7H@AD3-@V@2qLlo}Y_Uuj zH(;MA>;?2NQ3*Gz4u9Z1O4bj`^7T+gNBxnGKM#`Dje?E!?Hr0{ai}{0RN%7glU}za zcwlt#m_-nCVW$`u4>?jr{xPvFIOqd+h;;AvW_ymwVwFeHfvL$1UR!Ahz98Jd-Z@w~(pQ`qcwA?=uG&|N_Ziy;QL7B8J*gP)!1 zG!A4l5~TF&3g@v)g`=n8(b*Gptl7Szh*~HYlgKr<4Cnb{Fa33Ry2){_Mh@L1?8l?% z^ixi^-X+w$`|;p@!pw2KkSPT-p?!SDA8d^Rk3&5Vz;sWi(`*tIwyh8D;iIW;DOVnq ztg@Ad#*$htv0&`Jg-j-X*wS07j23TpApYFb8k7#*N>lC4%z@;7?30I43M!%t()d}F z_Iqw2WnLTiP`i3d+Bi$^h`G~kyt$BW%O=FpT>sZmoW)B*%Oo}yUtu<-=@!1MeNhm# zqT=CitLABax07k7`HGpn!{63glK$5KqY)t(ZBBUSU$)Fz>JLavL4G7&YnmiwgCnZL z#42bKV%klyfI0KN7J`nOE=>GO7X55vltaylIn4AQE%%vMx5N>NL9{)lNs+FAQ~?CG zc2_OT!7lkw6h|iW+|*m!d*au5NLedGW`wuOsDRYni||FGZI>BlUR6tC_x?rh!!n&o zJ0z)XH7FO$(;CsHVzzJM!*$B?o8X0q@czqyx?Jl;nzh{1G^QzS@rlc)vz>*Xc+yY( zSWu81r3K_(5e>|l#@H+SuPxX@Z@KgFb#1=~XED(oAl*2+s1=Tv2b%N{j1{>A`-ykQ zPeKNj=O%b@Aa?gyMOb}f{mB&Jeh@-z!p^&zDL|>Y_^6~#?N24)#-1sNxVH*3n)H>> z_12ZD-VklrTcGiVX+wq*o~F=zl}5iH-^gIz*!Dpw2<@8)qOQnU+8>#+{mDaR^QLV@ zRtKBNPUa${+n8UZyPcKl%tt7<7iKua^t$Q&07yv1Bu36u0|o|M^#?SjH;dME57Rb> zZTmlHCne!B`+kl(V_-gT3Q~inO-{o?9bkhcmUxH+Ewk@`aC@E`e18%va2>G7XRYpw z{2S2&`BCV10;(rikwXfGg8aV4T866Uu6%@PZ**u#wZjSbFMi5|AF6g2bw@qHb<{5M zg@aYs5f>!>34cV|VR@NU466MESZ(&tH<_a9uR1>%FX7q5V?-=!Y8l;!OL@2#O``dz zo!o7*MPR8J06l8ckRS+9{v1Br1|mgcrm&mV10OM5YC%VwW)r_R8qGmSI`^&$vsNHT zs*+^n-3M%omZJntAWr8rg+N^7p1lhSPq%I}8DRH5?3i+)y3?z-ID}<=I3oYCnq1yfU zyN^k;5I7MX-rBT?SIvwVIGDmm5~{h0TMl|F^Z^i~2yNd^$qpsom56Fi3??Z%%zqqj z%Kl7L)Fb7QFC8ojD6_CRcP*ipl`yL<9#E~*N+ZeQrbFJyE|i58bH?L2;`7&(D_ET! zdfQElG|mobrOsoqzU$ZK2)qeHF(|iSRM~XIcp)8$@UJH4)!`1Hn`B$*TmJfZH(&`;7AR=*Zd?caW)g7QCq4kXDIM$ z$am(!o9Jjgpyr{%_e0&Iegxl}w)Jfe+dPr#T5gGjTJ5GZ;ze|l;^6j0o8s@@(Z1FY z6KiEl(t1KUt4R7HtBBUIE}G89#?ArUx}xbzu5Zy@1g+}jd4YkGPOIjvvB8tbZIXJy zOoUs100KId2s2dwze)R)lDxj;|Hp;=CaTZ zj2nSv)=cgW+YhOv?ImSoYTm=-hGe`SxT@v}6MM1@gEkq;$m?M*W<<( z+cx+?qC-dS2}u<$006U`cQ6I~8z^p#KXzoRi&?J1t|5!GaFrJZ1#bbLxXNOul+siu zTg7y(_A))}ta!~x(DkT1t@{iMr7CT|>4G&9>k-fM*EnQQZ571yK}!+SiAXKcI%BqD zti|5=?-=UHp+-U2qk=#~Hj>>~w+$BJ{Pt*kwT?c5Y#ZFLvrld@W+Dvv!g1GlKZ1sxtuDotQ>o1~XbB18`y^KbyKcvBXRf1IBty-62QITfqLXW1H?rh=Y2I_Z!DL(Ec0=WzU2HMiw9Pm=vLk{$hI&85f9kUn zIed5%K@>*hOT(q$Rq(hvDrb12VNA=n)p#O(L#fwP@m%!+`12=Suk*#)FneQOimQTPUCdLQzw zZ`x6~ZrtQEq^qAHWM#j>4W1P1r}1iv*Q~JS=Uox*#1f?srwfkf;*L&zdInqbfoXB=-55@Ao#l z8gX=y4L6+&^Z7pa{9jl|?5;|^Ew^k$CacFuy+sU<6_V6`{&}01rjYNFk!~0oof?zM zrLj%)JIg?SxT2+ld9Ca~Lo+;&KOH8P7> z*+l!y+~>^S*waQ-rsfgN$zsFX+OkFw29f~CD+5Nld*SX>aEz~BtzieWr-BMRr!_<+ zB^ON&cGWpKg!}T13O;h*xt@``_ebR!S`?KUcQYu0YgL>Mip$WtKl@{x9lH49t4leU z3=uk7sTWRkCS>1n;oxEIq!Y^5UU=J#S2|XoL=`5TVIE-{jz{-H6Xrjh`%z-UHC8YZ zyZz}EjYpr&t2~Mjxs>nu(T9R#uXbCTyz~6)Ji&&f0>inttAgD8sGMM;$6vpzVbh4D zIJb2K7sWG!#yLq-)Yf+KZ$hTVPSmLNHjW{IBz(R?Abg_~sk#izoEkqAh~u~VebJIt z81-tF`|VRZ(;jubJQD)3?tF2ZIs7W`uMBpecJeryv=(3HQ3l4)o)pE^ZUk6o#yavE?#J34@j{9uG@h`7h&s^4f=`tH1Z4HAAo z&G|dEY|U}QCyw^jC#-=^BI@*TTX3Na9g6WC;KEegM)7?U!I}AGdAF4-@PiL zYt-9I@w_HIQ?{qbXF<2EsL6wU^fKI{V_xH2cC5d119{cjb>0YM$D*emfxn=Nq#Tuj$^N8Yt`ola7gnlgl%xEp)86#I^g}&PVH&2 z#Ux(h_-)3@Y+WY46SJ55g*}E4N;=%z_26@j;jUHjx_cvDKw{VN=U|yf8Z>Xk(-NkN1tc58ol*+YE4V+K3)TG*&3%m z6SsG1?R99TXt%tuGxP(kWDa!?oY)|6>Fk!C#%?{ItuAZ-_ohzy;kz&+D1y*~ir%l? z?E7T%9~m|*omw@#ghlcSH1R!Q(UYevOTELcQBQAsSErvuZ;R1>(X9}*XA?UvDxHwX z$s9z?Yi}MoCXG55UrdmY7<31YzHjPZlMPnJ(uM=di0W2NR=%(gHWN|PukH6qFcnD$ zwB;2~e_#>0Y>o?B<(+KESfP7(^!(fMzu#Jml||apjuqrhKN^G(xkdBx<;iJ}%zX}_ z8V8}}P%v6O)t0i*`l@>F%ESDwlJz2~d3w?7&xe=@;imFgZ{zMu(erQV1$my+CVii~EO406k*IpD|;_%qSxhw)-%}X{4L_)jxn3 z$&w6*AK6LX)`qMl&F~^`{isY>9+qa)ee7ml4Pd0otdLlB$r_BmJ4BhSowcPl8DvNt zLtd~cKGC;WEF7c`NCBY{lsiav*Uy`s6$xCrrAL{&{iY0WDn>$>SUTKlmny7~fwt?P!5 zyZz5Ey@&ep*%8e_Cy`x$f7t$>eb!yQ1$JWmb&N218ca6&ysx40Y45vSzp={~@*Z(5 zp`~MIdSQ8nID0>dgG!3s9RH7PWwv`-!vto5?d?Fgn#kRwp5wp7d@apwzEy1NNlSQ@ zRAXoKDt|QwM*Ja7xiZ?MDJV}-9lkZ=P|k4O93y)?`ggaDXmzp%3hib$jEHj@21trH zzAy3P9DbR84^t*6U%l*i=mTQL*!8_yM(m+Gdj0)Z0(;uYTX>o)nB|Z1lf^kl z^`@N_Ns9a(6YlsC#_)DbhR*Z?YS5$9EU=$w%ardxs z^;osF-Cgn%)$-Cpzj{K?`mtK! zMzAl>h*`$8L|`o{ zZn)-vT3)bowNx7MG;xA-Fbv(eAL{jTvF>g!?=Y2g_hWl~U^3T%pU*F>Fu31Gr+HL* z$s*OB&W_GB!fox?KapDL<5JyY4_YHKHh-ezIEIlQ&(xx`RtS%b2AnoFvh*P1t38e) z8ilysD9}V^k-UBnNijzEK?~U>UYNXYiDUVHEw$AtP6I$sjt5jXHgtFbURRK~5!LH0 zG`EOm$@3>&TY=vfhq1tV<6X8+KtDxQh9<)v^)>s~vV)*e$rO!go=5Rf9p>>)&Q^Mb z!w@BCCuH$y`3fiy{kDqFm)yfyDY~PoI#UY9r_U{1Ztq~+e~j5tGCl6hGb$u?{=3q7 z!1QLnvl5>eSHC(HX&TdScY^8kAK60{(g$x`0X?>wZ*A9Kk@MNjseQr5qxH0z4F9sI zLq7c*U|U!suz2~lFW3ygxvno>(z_|m@;jmqf~)rG>1t8QkQAJT01i+)PUo&8)}AK?rJac-(%&kdWv9^c%puZ87o zL#l2JF^BEDp`GBB<9_to5%g9zaq!*LZWOPTQOA6EL>uSA+p{{)*X zTB@e3$emmxBVG6|TiPIKO$`EJMo_x&aA?@VWVThT3qh2T2jO9u3K zwZ#VXA&yHeh^|aBj-?i-`E+?v4jv|jUava{`vlgn+elF~Ft{n5Q7r=ssDk%BRwKoU z?;bBftVJaz{kp*zjde=x#KZ;x%I~x1zn%(t>K|+3gKZc+8a0@e z>4p0Dek7@FCzH-FT25iC=_odCOK2%F&sGlWRZVK=WiP!Ud&2my2B`rwbR zPAt*DB7JtiuK>Ls0@3A5mJyg(G~!i!{9x;?XS9u>%lUN~IAo(FLEAkNtP5WU{iXmgl^be+8cmBi1=&`s-Hi3`2>a?$79B*_LzBcYn%X(fq zW4n5V87HSzm8ZxVf=|Bi$@b2o7c)rfuD3?G>Qy&~SxoG(Rb9nGq#ne!{A5z7;Lb@) zi4)9{zw;6)x9G-9`g!w~<*SkURA6trZAtBRI_> zVHy;!7!S5BV}$Prj+PFy=_;)-7MviKd$eZRLAIEXZz;7Qr-|{I6}>2M*rfF+(`ckk2emq&8eG3wT; zcsbizW?~;={+eZbL|7mo|FF^Crkx1#Xqa=ryrmZW2X6$8?)czTg{3SAHa&pY)%6Dd zE>%f^biFMdvrQ~aS~2>#Y~|EhF436ZfEpqE0E^?Q(;h)zi-Q_`&bhq|7~xvI@{Tw{ z!{Fe^A1FyG1#d50)qXN2G5p8~Kr&ZTJCK zTBJFHQ)5`7IsYUfPyLgUB5xApxrgzn{IUMWIH@Oja_c)$)S$l|LmkoqCZcY0?d{HX zJp%PxVS+j{aM{1@(rkbl4k`+e+&sxm8`#V%A`TlE30-a{qf2M=+nX9Qv(hXh(QT)B zs;Aojs#uOR{*i_4>6hokI#=iCTrnz|c5wO^b{94ep1+}8gz<<-i+BuZ_>;z7f$p&h z!k`QxpXdisd2C6hto+V=t`cr)-Wq#Y7e9by+$;ZJf#wAZv;biIn_&(Qj^cL9s)#RY z_E@MpDL9szaw{;IgTO;CBp53%kQ47*WmdgeTNKw;-?%rQWYo*U60yuUEv19G#Sxs| zSAaxjr_6V?7@XdI!yC2F@b1UCagf8yUlmU>f$_aXj)k)0Jw#_BxV#D7u1oEIzQbDRke0_C}tnqf@VS z zs(A`ibMl5^SvnM8#vKa39T$WYqg~f}0jB-j*5#~6?lV`Jwu^`AvR2Z1f6tCcFkTwY zpTq=R84fu6=z>8gzF4?Ki)8|o!h3O58LkqD>SVh2eP9w8keF#gFQhRf2uM{d?zH+u zlqutgzD<>FBA|I(r;WM5jnSI+%Y$wZh&ht>%aiPo5e8|ybseoC-b5li z%w4aWB%+p>m|&T>RySKGS&B~^V_tG9$Xjtj&-Z}ZfyBn#m^xPX1EW%KFL&M8cs3|1 zt%i8wlt+Q2tq0?X)#StG9XiRzS*fv2s`}@;M+?=c?wThTmvD83q2RcN`6@#=~SfrVmjzUCE{<+hdk-^v0eX`rVV8X14(V$5$pQeI}}@J zpd1|7=GIz!kp9+GQB`74za z#huBpiJ=F!c>(pr@2*i@?w27$amMtB9O^XYjTo9Ko$?-bMpv_uDVpRgE6rN0q=(1- z1U9p)jiIS=tEuuUA~JT*pBuI|jv)|>R?Se@UD6r>(#pvFdhW24)^BNQRRkkY$5{+b)`1CsnV*x{pato@}>WuNjag!|N7?y_;)Y zUe!aWvR1rR!#PF>FkgXK7vRrQR8rL(GsD$xtzCVXAce=wnl?b0&((^~D)VI&Ta9*ZX{SqCb z$H=q*_*+CPXxUK%pR#DJzB0WUxqBLy$u^_R-8I;YVxOear$7IVL?&|8_7swApfQ^p z!pNnQo{>ar=5e3mJuqUT9tUfNKIOb`&JMcIlD5mv#TA-%yRCSb{wp^@6(8{AB2?eFBU{Z49A>=#Rt$Wd5v85` zcJX=tjGgHF`sA$;l|Kj!`25E@_kI3#mvc6!q}2DlwZ(elQ1>-k@crPcQh$AC^eyz6 zZ}2IcJpOPN_*M{2Y<~`||0lYJroFVD41yy%H|P3u*4_1q_m;^T0Ro(Qpy4CI&B;^k zTT^mLPA6?lG~UJ#2&9n6>RFVn()B&Rn$^_;-J!cLzcDmZmt5v4JNah;@I0I?!+ zB*v|Oo^B0xIH&qffL<1HXPNZNtYo==gt+Iapm=$7OD%QtKe`Ar6?CotC-WJr>=~=f zBZ!V$ebdKug#y;{t|MEl^;^-|X40-9&cHmYCcHQ0g^2>J%uH^*5DZPtVCzR5Gk6VZ z&hc-;LCL?lgxmp3c4< z+U!zQQ8DFqF4%MSK_#(f*S^g+6+Q5~sqmtK_ zv>fC3p5pn%_|thq=X|CERFAvx>UgZUC@|ZHiWQi=4c^qBV0CYE|F$N;4IKg85*8*e z6$YXZOn5$OQ;9M-mNhR|(t0(1*=2RjQO$_l4>+p?qyc{mfb`BQwdU5k6RAh~ zw;5#L>|G`h-UjP^5cebKzV^ciJ59;4H%%5NKN+iptZMIDSC9)PK&ZFyhzf{Nu{omF zVa?+k6qkVOKbokuV@HF3W(bnKBC86a**T5i=$|tHByYo?Pg}~8%_TJ5i@bGSH}Hvu zeWj*%lZ!mVV9GsX#yQ~5byY>C{gnJkGv|Xi!O$OTO0l1o3MN)!s?Yv}k#iAtgfsDE zci=MWay6q3_g;4Kb7QI_y(@0fb>8wNB|sEy;_2MN(g<`V|H_Q(E!+^s}b>EO5oUXBnKkVF91>s&uKrB(o#OUK4&S1&)w%;84aAWPhL> znxJ)Bo8dX|hpL$%zUua|YQiuh>EW-A5pl>^maPAHk%Hy&kh1&>8G7mVm|tt*EX_l^ z!sPhS@WPc5;|E?g$$}4ebzsotZrF*|o1@&*16FbgX^;l3w)7zyQah7h^1F-7n`aJct<+akpJ z|B!Ft>UI{S8y|=7#W&o2#ZEacXYSCU0q3%2XE`-l=9r&$vKE3XEUBT_nqnd;1)LrC znKLJf8?@P76b+kQ&Shvy3`xEIp_9`lA-Ke0R;9YkQ!fPDr8ilTGQL79X*4)jZ%9k@ z6io04uu=KqIyq%-#0CWIfdzh+C@k^K`e?EXCqpcT>w@Zo0!a7>tZOsmt@4WiTJf+% z5jC;;b1J}b9h7xm?71w8d{OQkr^M3zk+IAj?H7eONZuLukH);Fs_S(+DO-#JLhj+K zQ+@eGTqQ9RJ9g<;K&bel--y(GNgxiBi#|1Ehs|H3ow81#WI2a!!{7Rn7LE@)J@&3v zQJB|hN}F+>xZA$BnV(_PAVGxu+kq_tm4?6QZC-nFzT(`zrFWif{<{jKalcQ!M^asg zmamfs$PKskQ&^LhzxYnlmgxcvC$UdLgAojnc2;8@lnOfNm*p`5F z{qHe<{B(@^Re;Ln);Yc`Egx(!TZU(R{9C6q^CVnuOo{_-B3dvN5DGXIhApk5! zV<>&2{y0RCjIG(<$&{Y7M(TqK*8kZhQFSD(u-SBdX>)h}3;Vvj5OHvy^o(EYwD0)& znen!CdGG?_f7pTZe zzp;D5YqVWcYFMd1jILt)cmPVu=64V2{BxQH5yxv=GTj9PyPty5kItUq%+0uzfUuN6 znw4?=mZEOAs04i`H)iWgThW~lPa12pUu!n`KNE{=XT;(N2U$z=)id$M#q?5^s>knY zz2^gb>#blUHG|Q^<|{GgCJLB&egEK=ee4RBLj!0 z_tyvCFiU%!c|EPIh3OrmYC30F8z=l-;{vU|k<@jX9-G}M zqUXEZ&4dTF{H1nSff7faRY#8j!CVCcoZ~Bo?OD^f1#jgdactp|oKPdgZN0XHt+XG` z@i{|b%7pL0XulUC7k)iDWo%P9#%yLQ6N{T4*=@klkx8v-wtNt53Ir_vyp-=~lg6_+ z15HNXz)cA_eq` z9i};FFXj9$E~vr5uej5DZ7U#;WBz=ku;>Y~O|U|1vDledd6N}zyh#1)k$yhz(#22c zD_Y}{HhxcxirPcsRqVPc7o?78iKm4I}v5T%7#G{nju-S6j>Es9s&06ImsYn z9%PQa;+8{w{v#)$o5bP#8GN^A{thGZ2W`R(26`GJdhr{`SXJ?>M!~UP$^2>>1w;Obg_# zQ@q$g`_b1zs*%{H%Bc+e@i9E*%CoGTm+A(BUh=!G)^$@KS=xC>N+LYu5d3?ufq-zG zqD9`7L`UJ}9Cx`WjJ(f<YXG>AR2bQA$$`MtU#JLkbe^pfUZQgHAhjJ4&?-6sgvc zj-YgBr7jPX+moEa@M!Ue<-wuS3F$0%NsBzl&aB2Zh;zehu*;7>ksW}-7LzdJ>qh(Jv z(@-9EEV~Y{1dPI6m$mx8dVC)!>c>?Zor%3JMqjKWG%Oc1rcFw=;hR|4D)NHyXsvgI zl(z;g)Hk386TLmFW;yOeHQY0S^mNBeXeh~FXiP}HAXRrr(od91+j*Qm$5S26)rx;( zRRV|2PI1b^!PU0H|)7IsgCw literal 0 HcmV?d00001 diff --git a/assets/rancher-monitoring/rancher-monitoring-103.2.2+up57.0.3.tgz b/assets/rancher-monitoring/rancher-monitoring-103.2.2+up57.0.3.tgz new file mode 100644 index 0000000000000000000000000000000000000000..96a72b6d55cb3884f6447e031cf64ed7da071af5 GIT binary patch literal 480143 zcmV(=K-s?^iwG0|00000|0w_~VMtOiV@ORlOnEsqVl!4SWK%V1T2nbTPgYhoO;>Dc zVQyr3R8em|NM&qo0PMYMcN;ghAbP%zeg%f?du*pwMN+a7o83A0D3+aQ+7BJcPMcJ{OaM}c=B-n%l&^D zPY%Z8hyQ}cAE^J0e`=|)_?Pi}_tl=A0^FyO2|Ky@L=!31YY;Qv^ta&Uh^{0tE=wqL`wkQk9&JMhgnfGhU7= z!Aqi+q?Sp>%d%!vtwu{ylu4$6$O!J6N-9BciB1Inkz{In*t{YFD=tPC^^7DDICY&W)PSF)*Ilq$qRjI5Byh_YEz3>C+tqf~$Nra@FP0_u7`x;Hsy)V;oGDd$! zlXQQg|9P*ssfI5roRKLqAHzw@l~kncZC4RdQYB#?1!eWy-A%Kp1iz#?k;&pxOi|NA znsGt6)SH_&dpNqB^g?ml63uikePk!Pmrfq9pQs=O!IIbo?e6fJMnBa=Dw$+eJw*=> z#^e6g@byYka`9eY)J+!By4#9H(Dt3XqeEu$-`r5;@&TP{vtK6;xRWJjV>DgyC61@*QN4 zt^K}(kRTP8dSt69n$v7XdmwHADqB@?1q9Ec0W_Zyz`85@{7}1($ zH!aO!UOrV>zHTJCY`e~F%`N&n5&Z3HU58M(U5~En7G0iYOOn^c`W5#Tw(B~qzN!1* z3dcFF`p#Ro$_+#4ZQuv-Fx6Bp8U)*P+yiL3UP87z@*A23L)q6vIU0+&AbU3-gWg&< zjA&h(P1~c}^kTm&*rLxZ@LpFtW(z^2e7T_PZEtsBg)O>mRllkG{xj!Gw&-x1)28dK zKWb6w7Cmlz$ixNN3Z>owquv3b*5T83=(HU+-GofH!=>Ay(ppT~5|OsVqm5{E3oP0e zi7Y?AmQJ?HZ|JyNJLr2qo$Ta9W`b7DB^3yDnzzUjNl9n%upm!99hu? z5iD0o@g0PWKuHi`d7^kiSRM|@GWG$-ZaN9ZP)q$)ElKck2ccJ1P)&E19Qe+66M$)^ zSJ19MmXatrD)@rSQCH|mE4J9VAge1ba$x=))SV=_2X~V8hV1Cy-v?sY#LHeKrc7bV zhyZ}3#I%^|MHS@g-+!uco^erWi7%}$JT95y#Ra))nk*=0qMlLy?+g7w&%Y5sTv=Eh zR%vm8g<8@J+1JXIHLZFP?cXJpil&wFEx&z%g`$l7BRSJW?Oy-=#|yzpbwSdc_%`la zQ(0zQsecz=&Q{T|;Ip8M*{Z2pQo&_`FViK?|FNXxvS&K~spw)Qf+hH=CQMP5&Zv}} z^|bg8Q{fvZ-&rdOOVIPc_PvXO=cZO`G1o}OxT*@8!CBn#G-5K|PxsQjKh~8MY=3P{ z+voYGNB?_?i<-y}r~uxm|2^2>-#>WR*8lDu9Ng)DKgFLRIwEsi7YdneM{_RvXCVm&of<3*w3g@zXLGu!g*nzc!(sdd#@Qpr4TngO`8(;EM@kSf(znqDF>yHD z>3*H=AY_@HDN>@=B)zFM1s(MeJy~M5AV|R)&h;{&%zYxrt5siJ5+OAIzy*A4e$P=4 zP27pp-_udgogHjtx=v-A%F7HuX>MzZ9*-Z7VbM?j@)Vuaa;aI)P^r9Oo$UZd;i~53 zGP&GKACD@WUEl@jV=errs+P-2@V9!g`viz+x#zf$dcCb0=>bSfo@g@!1%xUD#n zB&!91=^-rjVqQFw$SA(|b`T0wr-1&M?=>mC+>#ZPFlBetOx+qF+S{-HP-Z$kyP(zg zM9}#vDy}6tBjS=~f)^yc@a!fSAI#jU0y7dAIS^93AfmAbA(S98pOZ{Y(cmS2`j%w1vQ-fZ%p_0W zQmN#i+r)_u8`}h|p+k1)9)t_2gys*YXz*v)Dg((7D>u*1-tWbG?>55hK4q}=e|9^s z|3n+x+1zM@9m2UkeY@PQvw5pc?VV84+787Z_u#DzgtcE6UH9(Lb#H$wbe(LAuKS-C zx`G~JD^z{hhpLrTV??uQ@MnJ${`?wSzhjk4H_v!)|Esq>kHu)m0z%Z--T#!|#IJqFJph||H#6U6}luGiPvij|-D%b!7 zOwPCTl1QhkLtyij`lJ9OCw*MG!#F!7Ms@V&^tqg(zYRuq0Y(|lFRA2W@OO_7mqg5<(t;@eIYGFh-wIw= zP!v=b#A&6_-v$~%qyxSg=C8w4NVDb^Ax+1G_TSFWEv)kjtL6E5vn~pY1yNAZcLj6@etQD+A~_ipRCzZPHG_+3TwRhWF^Zl^=cc{(%CqVA4^|_xLa(_-7$0x*r9p{Al zxw9QNYa&1Os@ex*NH2O|kU5>2`<^+8T2dwS+ifHL1BGDCdZAg%}~ZLq>2<)F+b?sb_sf zP=*=GsUVpuRwxq$D`M8fsiBZs>WPEeLc#+0l>yg8{bfNf2}8K5z)Qh;j71&{?hf-Y zVd3fMAVKA5PfwX0mMN93!-A`-bxayx>75L($&YF+OW3|BTO2PK_n*xSC}(k7 zJy1};waH}o2SP2MzOD3bx0fjdbU{{AH0a`tCpUxMhS5qKvvrkXJn8Fy(9onV5XCEA z@WtvcFvxg-8Y?v7bz6CK?Cm%)fb21BTE8a|MOX%)Fgn7dqe|Pq~D{*)ByJtUS zp^p4fzL5|Fr*2zH5FHGNA1P>t zd3c!a-;swq^3Yw0JM!?mCl5PdnLG4OPrs^tZfDreh62hT&WaV9k!DNi95f?AHkGeo z&O>xOcgwJgsDd@q_yW6q7C9|7&}J(rb}DcS8F6qIH5-f3&d{$+GQ*PO$R$89+NYco z3Cb)>04$~$CAx_!;uT?FRmlXdh*Kk`JFO|do2Q*m2lFmYl6P))w|2Apq&A;Bce{U1 zce`P4Zl<*~u`lh*2PBLk_)ka{!>Qj)8y|o5_{*3ywzJi&A&_5gB9OnT;c7TUZ>ojB zIgvZH`G+3FaEMN8hU^!g(qTsCT(5Ie!UfW7%?xK3=#pZD(f{BxwB-DvClGWvM8_pw z5WOR{gug^>_$S-L4nkDdFbfgp#BjWTT#~{BcPPAoL~MKM#KdQIB4^>+w+)-G)6U4u z&<-0O(DBG%vWB~8zKTOrb;+Ev+%SK(=Z5D3h@l}U=DLB;eyPhGfwjS?)H{H(GE69NPUwmj1yW*V+%B-lmJl=C56E$}(TIYHW0oVz=ESu3BXlv-f|rOk zqgm63Ke|LC;x1ymxjoHr1M-u))ol37pMSA&(Do#n15Vij8Ft$--woTXae;2O(3(NN zZf|q2Vi2#rwuknfcdfN;`<;z2er){#47HAJzgjSe&3r>+=<7o7yUB+`bY^HDWr~PP zT%;W&39_p=axCLETF9uS*$F#K0us_zc#P$c0!y8dd(VNrfh|du=pA6BKOI{bb z-d+t5R@LoY1igVWXh8orLyq00h%$(ShkzxMbYrp%~`fj*}#D4R-kVHL5 z|9H)hbFoxZtkjsN!EQxt0^*T*~H^F~ZQuXuj0#|}gKdoyBEVS9c2sl6!w>wfyh zgFpDM5!azXS+lrn1MC5arsxlo2B+<++j7mJ1eoa({Pbi)II&rVFdRKizW@NZl=bF9 z(n$A%K0-ejm59_(UcP$$bgIurl#PH&L)muWb6HD;>MF;|WGd)M$QL9Ss70ZvPMEo1 ziUwVA)m+0qxq)GeJgM{u_0b2}*2b+%kT7Vi`o$yZwGkRkYsH&9QiztokC_V|k+*=m zMyPxZc+&Dgn40{%U%G&bHI)Wt6o*B@uR>P_$M~6wIbrB(Neg0pJ6!50OF3N^>wvcq zOP{=YCKXk+BK`)Q*F|AANh6%U#WZYP)Y^Rm*qpxTOyi=68)v*MIdkXq=U;Y)NM_{{ z%jFc|SvJ|*-x;EEWltH#7Z+s?z>iHP=KjNTdSrk@%BsFNnyi+Jmk0!BJudFhaxmv& z0Jr}@l1n0}S`GABvsVT*0MG!y2;O?w;F?nqp{#H6GfWGTpOS)Oa5B?Jf3B&@e%Yz} zf|N5NFHrd$zuUuSEwm2i|o0TH1zD{`l>WW1x z1yK$aHBp?0mzWlKMhn=+{yN{(f{Ac9TIJtSsrW+Ra$SY9g3DG}*JnR2FNkV=F|MJV zZ}zZ#+uhZuT#KBiB;z92vPfLZ+`xxb)gGiNIBgV6J*8)g3oSbJmI2{z6?y+{IC9<5 z`0{nb4HF@UZk$Gg-E!<~{d7`yPv2p2U)MoCp}vo=yqN0?Z34we+&d+Ct;HGZwP%GJ zJsKo4Ug?Ios$P1tfVqdJXpnv}uprKe0v_>vPBkSt*Mvp?r>QSM((LUZ~gG)0uYT%t48l$G<`LQs3}}C%vfg{Tss|LxHX#X zARgXgjItsbozturG+Kb}wbKEpWbL?n$TO^^BhL0mhcw&oo-o_rHTMi_7 zZK)y1J;e<%OfguYENd;@Vdlkk-CpLs)~79V(9!^En`;oia~IyCs&41_eDtw6C;(h%o`R1)hAm-#|QrA6?SfQ`qe=B%7U24bj}@ z$ZtHLa^nd-j9*!~JEnKX^zN9x^D%7>*x_O!$N~p5zlj=JYL`Tv4+b=NIVB z>2sN)H`1yPLO~CNS}LSM6&6Yhwfc)~sM{Gu9)(QMrIq=$^s2d$6htIb6_j_*tZzTc zv?p<6nv+Xk!SP2&RDx`}mcPht1utbn3W$^ABYwpilLG?GJ3Kx*B4DuyW!)oE36fz& za&(1PVZv%F1q-)32)H`4k5-B_dWObC#uN`l$yCdd!(fBrX_)O9Hm^CAt3<<1=CmOC zJF--CYJP+j&!NP9_adqPo!D_hUGvYT5Fzrm5@41>Uj!9Gea8oy{o1*|u;g-R#%`V! z>Y4PqDaRAuINi0A-B}cThKs3r5=a$JBQL9tuY$J@AY>F^Hc#v+Sx~9O$~&$Nhv=_H z(ug0Of|7S|aydv0_3Qe&Sb-)qJ$l8jLF|IBWE6|Q=b=W(HI&F$R(Wq$vEDqfw(oNX z!fBn+(@P>&eg(i33%bZ9z9a~1A*|68@;Mr(lXUz?Zwu69$e63{JNd?h1v~T6UV4x| zdf(`!ZFBQBnjEH^18Xpc&m}u=lgFKTPC(I5$V{ST-lE?J~`Z)P%eG7 zWU8>O`;48ohN)oeo#5zYgJr9_C~Pz#I-bAe>O>G}Y=MnCOrr%m{pRpVU}n?TLF{;A z0_18*gbZgHX~5u%u5!u)*P5=XO;LmGpaw>PNqW=Z1 zuVglfjO3+KJNaB`j6wTtK zE(%K&tc7#vS%!fHpkbMz51<@Qi>}sJVaL+**jFUDPa*ubb@c*mvKLO`x2SnX13YH7tK0Z?F8MNacIfvn{gKbCk<9&T2j1I(CoOGBpcmG>lJ8azrn^UaY|a~tocr#+Mi!S$kxA8lq4oyvZ0&G~f z&Y0~nNfzXlO|Min2w5^*S}Zo6emV8R$nsl&2EW zvaA&xV9hew70Tfd0>(H;R`AL?`o>Gy1<}>aFXQ!*n{Ijte2_*u_$QUJCUW%fVE^%> zfc6~u!G8mTUn8j?R8nSDlLc`>A|-zNhT-O`H*H0D7%zg!wLvd?S%pn!J_0iuMn8?x z1mMA+et1`8rZ_JGn1a{(??hjD3CZxF<2WlK7v z>RiMz2S(gF2bWJGT9Ie>oZ|xe8fAhin&IM}uVMVCZf^SjAnWvSP1&3`?a5DdGi=++ z+dA*X-(gX7;l10B-1dI^fum12Uf%$HjgHxzADivs;?oQ}xdHkO!_%nhBdtIgc0OB8 z(cglO{vMTHVXCHRe=MV~jnC#m>Cw3&SOq1My`?Y82)!g?ite$x@Q4Xx2{3$3HS6Zw znQA!G!C!TioG;9!;2?RoXme!*VEtoBuEsscogZ-o=ID;TNL5)C_ijUz(0weC7NXb! zGUo51N@#&db5dF$Gb&DKjFwm~QHd)fS=mXyT5D~VZjHJ$2d;?6&kH{9A;}Bxkz*9@I zx>pn??~{FTOYn+_Vugsza7A(?xy|adBrD*z7*|FaRSCFhXDb6tkGW@)w78VYrP-dG zT#|xUlr7G!z%+z6DigINiK>cZUNi7)y>WLo?zYC|bF(A21jM+~BB|lEM(@%1+&XUp z5Rc5U_k8c(*oQ6@N9LKd#)dN-qHk7CI@$FNoHJX`JBG>=MpT*d5X@6JI<-cBZO!C(jFzvy#u?oi*TMJu9ietSVRKO;@lWWqLN+R zht%w;X2QV}*TC!V%cw%Lq6Xs|m!#?9mK5USZ1r%cH#gcF8=qs;)aOq0g9Z1}qW<&q z&xw7w1Px??vy0TWNfmjk1}OREm!NKgFoUq#d07{VRs|WL6h#dZL|6_$dV5?0B<%Ey zzkPf9>dgsC&=FCX7V_`S+bgdE^%pL%D5Z&wq1MB};F%J1u^{?}z$je8G#(S3qlH5E z3&KzuxGJUJgVgr`jl-FJ0X3|P*z}9w`qZEVxP3=Z&bUw{KfzSU5Dr5B=YRiy=!>6! zM(Im^;r{Z=^b7QZ{t4ZpUw-+a1zWFY&l0yj(K$+=zEwmpT)a7b9u56DWfvNwff6{U zf-m&$u4@u>vePrqL*rDk6{cvyyB1I08Fd`gq-PoI1~g1fgj|QupPs%(3CzByW`Jxk zqC)nyZ`bPI4-4LNsUe&*=N@f$zgzc*TXf-GX%2g^Hk9cXD4c6g%?;oK6-+AYvc#e_ z3-iq)(+#z^?58AB;zhjbFU+@|etK$e*-!97chuKW@N##zzcsD4?9+NwoY$g=`aErX zK&D@yKes!Ln_W+3MFlR-8CT@I!uj;yZW;$Dxhx9wmz|PQ-+T8BKXX>ljG%!(7=tjv zU|?}*@Stb9o}#wtXW+ZGnTUR3!nxfivq+&va{E=ZM`3^l7Fh>q5TMZj4TAlB=S2-I zpTDJ*4QRfN0qWIw!M7oL#zknJfHW(UM-XIX!x*hHFiQ9mq@O1p`)K@VvMuN)UoT;y&12U-GYk=|^SW>k4qE}yfroq-2BtUSQ6?2;CX!G?45bfr zpDg;r3zM)~GJX(G%q;CT4gI2i$EA|qWN2N~ASr`QIjtvWF*tHh0rwXl{ApubL*qlv zOH3I&bMHDk!D=a;4Q|ebUv6VBO28;wVYPHe0F*qPc=R^@vQbr_N@+-eE&{Ys~Y&25&1J~WhJ$o~v6cLFYL;^kQQ>baT`J`4A zai`DCWQvZp+@-sL<923o(J#>J=VvXwNXPPX9G>f;^_%hdQ({)*!e_hAUA%u2XBupk zDq6wA0L$QTgss*WJ5hz3kwZWKvxR$B!gR>jYaxuLW%_O`PWsn$9d#O_*{XSXpnY~t zV|{EGSD@Lu5>zR|tRn_41i|^rWG9eLH3{o=JU6*BVzozi=sw;QGmnj+bOIeji7S1^ z2+o%AByA$DfG7@!@wzzzNyMqG>vq9#Q&ABMu?cj3D z!S=(>R>6i+PpTb+pMUOdQL2GhckN$(Nq+tr{lNlwS(h`Aq|&wRj)0mwmZ$Ls4cxzh zX!eY%!A#I;{E)=m{@_62sv;~mnpF*pu-xA?rkl_i)()rv9%g-5GZYWbG>+0gY`&ZB zu9y#W#n=l2aM>!|U7R%BK15h87wNchtliOfy@ymG&w@KAK!!7ukAmcmn)e<%yGtp0 zf|*A@#TNt=wjf98`2-JVg?FFnx=lgXq+{;?YHlEn2r{>Vw0?XNA6(7uce-*eEQ+H7g8LB|Nt#1!vg3>&?|bGaa6U9S z!c1lv@>U7#qpqM~w39(zoeSRo2Oj zaeaxq>k7Ee%)R}b`VIS(@j?CY=5!1XlPFzhFRTaZilcYM+~ zIM_kiu+u4;j3151!JTC(-l~elEV!1RING^|P~p-sdAHi;iV0Fq^~x44(td+{uvYtX zBUulxx-CfjyVCd$aeq(h9^AUOq4N)Yzt1_&XO?hSpCnE6K`kn8+4JCeK!twebm;>k zf6JBJqu`6!Uw2JHyyH1|-|Th)IU&Wdq)tu=fqO0>rV_>6brZx0eVXihgT5==CN}H6 zqMvug*)veaJf|gfN?NH8aAJp2Kmlin8J?5R?I3fXj4qP$FkA|)A%nrVmKmOZUD?$Ka;I z$W<$Z5PI5bbOm7xwG2jQl9GL*A!_;SIgH@l25YY7Pc>SwC9N6mN6b$tvnA17>UXbS zpV%Lvy21mBq*1u#eo%!pB<}}vlN;@*Cg9*rhNx7Oqw@9hGcyd=RnWV+xxZ7Y+MA-k zE{%w*FF+GBq6f8?_NgWM!-HIOe($$g!Dl8)=!j>v*CmW_Mbk1Lx_ctvDD1c*;^-$m z?a^!7g@Y(FpWf9fod9?Ws+=2FO5e6~d&~69IOBqF`7V=Gg)}2}>H;?xM^|SA3nCzV z;TbK+)X)1xn11xhh76y0u1JG7yJ%LcwOK#63Hf@TJ+%o%gDL+;lBsH0`NoyDtLm2e z*Iv`#e)iTbOY*djd2F~6UmQ)?NC<1h{wSY*6ThdqM16MVTF})DW6Lnv<5l_Lw>Iol zx=wdcXE05kW`$XfMUMFFN0O;~QXedwV}(=n!kqXy?;UXv=hMV=+Todm2AnBDXSL$O zIxLSQT_jj!OR7kwYC-g{lZ>-V!mO`TK6L#DWD9<~+;#c_4!u->BdY|j@B*twcMwU3 z70+dSk@DQZjW|-1DW1>Uaa;`t+$4(c&z?|HA-EuUXlTc6U_l=FuTuAXj$GpCX0;MS z<68h(1soqEh^#m>L3}|D8SIq5ouI$?qVxOD&0Xhua)AbwPvJfzh?=LK(0$`D3|C_4 znEL=WMYraRIF-5|M`MBs7qXTFY?_d|6RfI{ZP01%!OZ}J9Iv;?^%egA=IC&o}(s7pX%;%D}Vo998Pl{~P4`3DUMUD>lV2K(Bf~{63 zS{N4wV9%OgnZJ_R>*r@SLdXfi&giCZ?~*{m9xWbdUjFP8@hIVZuE@;ZWe8o7SwyhI zr3^T&nR_I{TnHr*3*rLTZ4hs?37T5Qhp7YiEPMyT_Wsn00T%V4hHNsiM(u+{+qN~3 zXcO^ojAmwV15@B4v)M{x>A*@HiL1j!`&+VSb;z3r@7b9CSNBZhhJ9-<7*kLfKIwAI zqJr=Zy(afpg5q)BP@GtE%KNQxw77y7WB_>|6z{C$ZPzT!oM2|5!B5+hbi;3Q`L^-0 zt;2SQkjji-60tH`J6i0Tr;)NH$u1DTVhu&Q0b^fw-htT)b(=>C*CMLzW*dC<&|;MJ zpEqG`cLKLY-)N*C28&xhYny5$dpoyT?;97|qp&fTy!*;^k+|F4)cbaY#5P9OATf`= z9eITc&FJ^?nrpDsR_et~y&Pt#5VisF>$i11Xg^J3&NYBy$Jf>aAIMoWnhQ}XffZR; z1IldGcXkaoAdLl7WtIV#DI$yM=2duMqL$BWCu>)umIS?GetZ3>)~-Pta3gVpZYqG) znl*aP7w8#;PTDqaBtZpV%*!TQ-UEH8f0Uf*f`wCI6ntSCPHp|bOtr8W(w919MwqaZDX~TN&ai^~XLBYL_b#II6&v5ZG>_1ySQ51Qu-A_Xd*Z zeQE~INiNa|V)4CqS~!H%>Q$PDPCd^{OM-I8A;z8#OY<&=W+f4bzH=@-dkD6{$=-N* z&t#|r!*k;gEyx0l>crDV*W=&D$>YEO@qQ_<<#lHdl8ssg1}(RLq8aQOT#fmZ<&SSmp942IeoriTN*Qp^9({Y%gu2z z@eYjWDPfak&Nyf*eb9u^RNQ0$?$)LAR)^BsE5)Velmsu4;M~Z==;-AcQ1aAeO$fXO zTGrt=qL@bWGzMtv+17GYSWpSF4j59FnB|3)e3}uPXpY@R?;l2&>eJ}VI#eP-t^@_C z9RQjH7G?%^-*&y;1ZSG;M9`>2QkggOYO?F*Fsqf1U$JIkB0V0M&4?mY!U@N=3r&H+ z)}ikeoAlF+K0fqKJjFC{XkBl0`bdVRt<5&p*(mT9{%sg;j7rF|c%Y91A;-E+S)(*; zaIw-OH;9{iro-iwSa49#nJg4U%(*B@zKh7)j8w`#Fudza zO|Fa=0nbgh(xx)c(K^e}W?w;Y{xHNzuUI%BT@=YLtuZwcpFafpP=QU@z8e66)Un3t z7GukJS=EpZ(sne8uf&KjBinJ~n`QFSX-YhPLH*aPsv$x2xJif;Qu46jq_<%ZI|b=p z6;0f${_vV^C4iT)&z#ys^M;UC1qnjO*- zi+sb7s6=15B!%xZpuRo_=1#PU;cfQaa64S$7sgu!Cb-SJ|9H-~ZK*h9iZ&jMv4tRp z*o5S1I${o~;uSCWV)YlY3f%&thQYk31pH}oz9GeTL!!Nl=9m_>Ac+m+>q$}j56;Mj zNxW``G(~Ys7thkjoKn&SbID~}YiIWd_wg#*9(A8cIL>{8pJsCPa(^w@8PXAN~7 zzek?qa>#a6L$I{LG|Cpo#-L(8o2Ql|IW;DGuZbvI3XVPo zF34)&rx89@t!28V1Que72Jmhm2Fo_kFb87N7 z9`To4xi#|@LHE_wS*_Jt*++>Z1FM%xZ4T`xoGJ2Fp(QVJskb088VHwT8)0Sa)HXA% zePEFV8NxNRgh`7{TOo5%%aUudzW%)o+prqvLBt}NaMWX zH#HENHFu+$cMWVVFw79H)siq}N8;QIHPg{?hG^SNZ70FG4?cQIus)>5ctxO}YP`}& z3RU(@o#@B(@~iY94g7b+B8t^clDd*g5L|w((SJG`P4>P@$LTnoOq&=gTUFOcY*f?N zW2iNrMqxd=mi5!=tGE|bOw-i=f6{zjZQrH%OAw7imJ(RvMFL>M*sA(2^ zLolkgJQXymy7pw`AP2)E28NaO_N@CA5hXR8(=N)XKuQquKu^P2*4=VB4?Ts=Lqv_> z55VG~L=!p@l*9D`A>qjIlZ}6p7Dpo%*;Wtqg}pxZC!~LQKykq5;W~w<90()|0)3u;Oeq)~j=PWN$pys8hQbB(jBM;eOPmodsus@g6`3`&wBZeFXWCXXkN(BCG1NB66GR?zH0 z6CIhtcUz~qg&~7!N{O1MTr5UQRTd*L&vd1TmiEt}P5y1QZ{Y?xb`cjrthbNCm=SeF z2y>SWR*IBWz$b{Fhqzo)&>)kXQaFe3#*L(mj9ps{r^g+nx5#d z4#52f@Z?8|9v+l(7wwPbE}E3PXtLZJmy-keAe?$UrwEB{4UIytO~hI_WgEn1qHe5V zMI&tB1ONHWHkeEARBO4c>Y~_ciz{3CH?Ya=c|~)7#zjWVlM(hnzu*wXC2K)i&QG2{ z%L&dDyt@*<=!Qz&612L`=4Qk5tWwYFYy3*Rv5(Y%CPdFhR@k&_b3v{LoCz*v=mTa; zWPewh&Jdhg%jCRwuI=Ogf-+jxWf)Opjcpp)jaC&SMy`Uy1QpMNun?0OMh{3I5$j_? zl?``oc(wY{0THTHnlOJf%_36vFe32Rey-$e)80(hOCqkQB)jI_nK0Q-r%`);+pokK7|4P=+E3pq zfvuNg7G@I9gk+{|#~pq9yn6j~>IP34q>&29gywV%0#Qb7uI*N= z!*m3Ci`0%1I4Drl7DfEpiC5kMJLRfVMh%|88-7lOpb_KTaMkBrEY6J=-uIMTSra>K zx~*)r^XCO$P!3 za(_n;DEYSfMz~^5+=4tu?j5BHFWew)v-2e{$+>TPZUa|TS&f>(`1dAlqeM4Kf0Wj3 z#Hov#F`v3eo)kx?oXzih0>gH&6~YQ(TgDGjN^FR^bD;4KrDgxk59t zqzulY9Z=0INp2>oIUKVw=Ba`Bqyuc<>#h%DS@~JJH>9AGWp5{72s7Ro-I)hZmT1v~ zU{+VJ*c%5U#&Io%{ev^&6}M}!-LPIBg|Q%0G!y(v>W?DK#HuXBT|`roqNlZH3P;cD zx1^l$S}fKc9ygRJs#WF~ubi4kxaI|3+%(4C6KMf7M-t~HWhYq5D|m>y61?=@D4k@( zVG0U&XSi%~Ei4Tc8oh}dhpwepR>6c?V4J9Kk9ix@$hP8DHs(w0c{{Jwd!)YVe=@ga zAhjK)ps#bi^;cDeOmGG~s9w=V_O&sJMpVankhHL8SUm|Izq97=X7OHZlYzIjbe$Zi zj*|*f@1iJGuUF<_wS?7l7f_f^&oU49c_Z3aTx?vCJ*tlDp#ID0$_s~K6)4)kDftTn`;{yU(7XqHk0n0{_M-9Q~s|A4QYc$Q5O}<9QEca41 zcyY$6uo`P?{FvYx5d}W0EWfdur9}aJVBFu(H}LF`jOCt&HQs;*{#lDViL}tuV7I zKHZ3#4cIe%2N1R7xeqA?%@JWa`l%+yZM?HJdMjQ)t0_8pb@n;{*p=#hPP!AG>pc%P zC9q8XsR03uY%PgIOM>%4->fbN<_{f5Uy}deHHQs|faMOld*zrhU+;rNGmPhiJl-dh zy*&FW+aq|gH+eLlJjP`2fMj3J9zEWF_~rQE@px2SECMK_Ad^YD;KS#82M5XXy+>c# z;h8x!!?$VSWRGyWhIF@fZ5>G-qGyn=8}#d727%mRlwFty5v=>N7r#IiX*ajgc%%Ui zSSyDnOOkR%a>xwcC`0uW&Vt}ugveXa)r!{wXXH3PBD$*%F@AJ?SEljYqI^HP<7IW zC8hUIA^m9Ffcpd~qz}f8;I?%JP2=OWgKm{4r=T-|#VQHqZOD!@N%tP6;{+EKWlqUs zxI%;(IyS`If91f|EZ>?ju$pAcZ29zLb;NPlEw9%}{MmZG>hA_szqcM_p&y1hkb!Tp?4Xuy7QaQ^86qhwwK}3r zMu#>ch7)ZrPt51rO)lX@z&!!*_c zaB!~*zJfdUfdg#wsyg*`yyX2#&5w|b*xWFo?-2@OMm?E{V=PS-4fb^-cgwqMjb1(a zYh-I|g-^!8lP&^sFNlDI_{TjLH`$Jw?2DOY2bLx#HrY|7j{Mo)#D+P#4pO zivke!B5)GpWyKjh0SGuH$ewP&dE-Cn3Qto}~J}uMV#FuJ^C^(}U}W*ALT!>o2drx_)&1_W&lCH0wz2DDULEwE=UD}j$ zE$qh)Htn;${{PfK`V*|FpH{>=o}b&e8;evO?lCY_Tga)r%%%???!U+LBethG=TkJp z;wDzCAV#HV|D7zk)K^7zm|s%KJ$qIdZ|1e`xAy&IqoUba&ukqkg|nqk!)^Ic=bvu5 zjnw&J%II3!OCKH>`xpFeq@19a?E|1%XuU6ptVpIF zxEtO3#>`fr6BX_FwIyd@f~}7#~#1$n>p5w&`awzr8SIzNf~C7xWEg!!OJ$FER)1rT;79qpi0n-czr*wFde-JL12bCsF8T{mc!@>U$>XO&!S7U{a*O}s2Ehs(a7?1bU}vA4yz$y|FIKG;2b_GeN$$Gm zfuCnpeSSZi%X769T&aTOUi4C2;o`dBI6udiMBoKEpN!?TwY0^xYDq-=>A}5i<~BmO zxWX%m{y_vsb6OAs{^Lz}F3;~%exB7rXbFt+bIp~srsKX|5xQ8a&YI}cCC?$Xer`WW zP%oUDlIYWwpjxUh-|UaD>86E>mnS^zB^RrYw@3#5`TY`C70J)_zFf;ITy3|hmEf5! z_-MnsuLpFV6PXEGDK6wSKM@>KAEy3?hDr8%A^MVKAbuxVSi1S#@4ZO}EV9bKmPc{vINariX2k_uzKZ`F-AD zhQ;aLU>ogkcj)xt(CN3Q-y04c9fJs5lQyA=p*5M?UD_*~*AmjuLBMy!IrZ>NNyn<{ zNL`;9*pDJ4)R9Imth647K4XjWZxl-mVY%Dj-|YrFG4F1Rx7_027vr$`x!OCH;f`ha zuq=bPBQv*tD){4=55q@>Tu_fS89wnPFPiY6Zp8i2C1Vil0JcaA<3>u zifv;yns5iu-`W>ryP+E0&?(a|s|$#JteH1Q8e;}Hv)aUaIm%ZImo$Uu^BI;VKZWT# zOao_)7$^_Z^0?mk+$*7B_>_fBC+Xf^C+p=D#>{bYXw|WKeW9z?3;CTF*zAcTq(M5=vFaLOj>301bI#hVqSrdQJ285iV;$rMr)6KZp~QK_kpUXLRN z5hS)+pC{oBJg_12k0%HF9Ujwv?Bg*n9?1kAQ30BqGkM2R-f@(79OWHH`73agU0QU< zg50qncPz+dL(aVBZzQbuI+bl>cK-4DqW3t5ZC#PRFRADk6A3U^QLLQ{~E-zL& zligMaYg!$=H|ui8sN69scZ|w!icz^^CGJ>>J67V(ha%h-%WrJJ7nkYB-cC1_195?1IfwjqVdE%b04FBP6W`?lL{j-G#^ea*>hJsKAszxU+ff?m3OoF=({A*Aw;?g?J{6O?0xcY!Gq zG&c@Q;!^J$%}GergC5pAg7C2znqTMo4d%O^tyyywf9T72o9p;}*qdK{h5MMCcUMrY zQBkc^Qr)DcTCb>DtE$pcL%%+xc^mKu&;YvfuYVYokw+E#&M?#g4oLswsjb@=*=;t1 zvc(ys{q~O`vifa5%b~yISX@!ImL(Yu(K9YGV#a%f<#NWc$k7w?;%e?c_!rs{xg-qH zISPl18nj1CtnrXApNrpVz~((2$#BuS%w2$ry1T?LWys>KtF(#Ht9TxXoWSnF4R9}l zanDdmy;qJtH_wH;H)MUXHK{Pm&1#h;X9&{P>FcaXzScZ6?^YmWs~fR78pS#W6y5|r z^bbfUC`mG27!f;d62zzQ%5h@Uj|q936?IO&{$Hc}Ik~*91kbMJN?u=LF)HY6l=JL@ zh*A1S|HZz#FiQXU;Qk-3M-Mi4HkXN7RV3)>KCTs?%j+3s^HQCcRA$$+YJP3fQST5&~v41YFmB1O9OQ0cZ-hYpD z9`7a#^e9R0q=Y*u;Z92UwWWmb_!TNRTcDaLTIlZ04D8E`=4~#ty#Vo<%MbsYjQtHy zQ*6Oy0%r<$tLE$FIh)J#`FA}u+jx<~^k{}9F^`uCqA+DX80qzrcxvW7izee; zG&b~lfyE3jh~o!45CM)LmsFGJ@2F{%Y7*!XGQ6^TE;v-ea(XhH$BpGK~Jg|wERUuzc_Pv2NMfM-BpPx-!3a&Sj z2#Kzi9Gq^=YM6w=FHcb==%vOK8_Z^nc8r)b&^XW0oC=IazhALqjRZxZ z*%;TXrwApQuQ)X#UwSNUkv^8SU=j^9iVk))*MSD6{Qz;bB;txnf(E*8GS-5I83i-RA;P=T`{fqvS1>N)M#c@si*i$S^I5?s&y(bVcPdehrv0ZkT|`_=m7q@GMXDUzOlL zrj(ELDooLdAZII<-BdBjNkJ4zDvPq;aei@DXBiM}0o`xH$=DS6XH6|?lM3(L&>G-Snsm#Zlsv;s**ZW=h(f+9L>H03+`p7K6Hf(?yLb$w+sF?5;y~=wBV||r!ksF8OD=sdyY?|oBb#mg` zGZCBto+ti_4!&DmU1A3^>YPYZerSrD&BwLkAjq9X6<+fT!osLh)b$K7f0zq2ZRXU~ zW(P~c+e;F@XWJccFw!2fi=0*?{$zLg_#_;dIYZ#)<+(u45W(3J1f#GTup)3N)49(E zp;i?f?|UzKPBfrTPXM4;o7zGd!P z<$-SZjcqsR>$nIi*sbm>Iy28oVYVRM!5tCC6_JL?pwge`oJjDT(^Q|=iXigVkO8Cw zp3iBfOTbbAStTGtph3FiO7j=q=ecj-4%TY}%CV?V41j2SZajKb!tV|RcU`OFlU)?9 zSYN*y$h2t??-G?tUwg+VdP4<~QcEk!@~JKTp5-I$mg(K{7i|dhru)8@L?nxv=ENi7 zXy_Ivq1DF4J3fhaa|n~Lx@sf)*>orVz_<&f%eJ&sq%vY#hUl=NJhzAKAh*d^1AeFQ252a@R5k@D1azWMVNr9yVM_L8%&q{Nao29MV6{k_cCv;<;Kfukm0#_p zWB*YuaYS9Kx+omcI-bAe>O>H!QU15ItHJrBmxK{1%^Q>LJuGbLi7vc}X|_s-*Goa< zk{9_D?FWn`RTLLTq`)h$I@oVkxgvt{yscD#^{*j+QvkIQW;(ivNNtXDx_)ND6@Q$m z9emW88HJj$tV#5*&Op^8$y(K0z#tB7v*ulWgtDrhqP_9B48EE7&EfM4>c6|5I;|)D zttZer5Q6PPb(*gO-iO&&%5v1)k&U2oU)I?YVTlg^dWN3< zpc#L#>-(CK4C^lJLo!KKU`BjzvlYP1;AR5HjhdNgQ0PGu^Lv^fcyfGnYUHK1(o=#e zNw>DlV|(?NT&kCbwd|@PER8T6hGuA_;WP9lf<===B1h{jYn!L5XrbT8dc9|m6~!6I9>hms&BC#k&5QYRqgyu_5HHa}pia=A)#CQ}0| zmGyj1-v(89rMl~ySS19@c_?r-t20G|y+Lmik~2B>8VBPZ^@KA;-Ud37x%^h}x|*Ud z4-WR58H<`7ZYX)l*(v9$b&W#N8%adFjU-vd%j!h%IW2EW$FOadDV_Oc(!wD4#Bvv_18g6Zq^{E^Q(l4!5!XqUHF z8@RO@<~}4+UCl)iGtEsBl49K1BnkM7HJDvC3Je zfv(7mSH_V6OlZcP%$bh+Y-G;wqH$EX4;X<8D%FE@ry*ZdqOw*sxUYb(y`+~`KFbMP zHRDLpnW2E*0RS^Z%1W&epZ!QO>po!PHJtxT#3}*Z@pkvn;%@^Jd~cEeRAUu!q5X*s z*ac+mu9%jh-(}VA>luejw1ZlU;4YWp5Iv)XA|jZM?y}HW-HJK!`b{hJ`K#V_fdiL6 zaMEEJX>}P_GV(>IO4H|BC~Aor2T!VXJvF|1R>n925yM*Ky<;`_URnObR!l2Ggq14X zDu<)CSoc?|mYN9!Eb`tqu3ZERdRyX4mx(Ndme|+`$`(oVeVj%=&FYhU`gr{Kp;wa{ zfDf7xed^()9&dW&IT))?;~sCs6b;;)-=s%LF*ET zCa?hT6PoOep1_Z8!#RvKR1LUW$uw_Lgb-lzr+${kJHtj47^^k&2}f+JFxN=qMY^r~ zLbFsK*JUA0ts1#1bX*TuC=P5VYi;eB&{)*Fsb2f!j+O^P$NS($zk^$O99WDG{0zq4 z&mi_O7>E9JT`uzoN_7(VKegck|-RK|C^$gh3 zC&1;}>hlEn&|Ux^%mbjcEP>^J3#)(ZO>bZoZ92Gx#k}j4-(%6gaWux>4$V8(`|ZXN zL+G*8-_lCI!9u^rj=QxDca8n_7Pi}7YrGB0JqyXQQQ~PzzuR(j(EN`T!L@*)*%9#K z#2#_OurUYRA>IcJA6k;4zuD>2!=o2Z(=zX6Xrdd>${C>`VaNH7(KB|st>Bs1t54sX zb^+a*S;i5TE?n+SPtk;1l(pc!2;i-=B)Q~-K@NgIjoO?%4dW~V+qz)dJ~MYI9Pd&% z{_3H{zt0qoXHf>B_f9&rCQ{gk@w$LZtHjwDDcVUd2!DO&7 z@~m^>>38Ajy6^P3?sVOD+Acd=yX)NGs?&1Q>37lTanI><%;_~2_BiD9JL9ae{PsHG z^qYLwnS0;S@#dqL$Kjl8cDC8Zw0fgiwa=s)np3?g)vNI@iBL2Hr6VMk=DOn`NO6tT zX$Se1CpA?sJGFD1fqBJ{;yaVDuAg6_iFaJ76dT(w383<4}z1D!1Z)L>0= z8DlWK@127jt15ynO|+-873*4YPY>M55N@Xsd5a;IpEDAIO1*i}f0B(2(VPxLnUkPO zQV{`30hoVP!fW%I^aCIhz$-axHYUT&F#st#3cN(Jsy6x)V0sKXn0f`tPa8)d&rhCN zvy`;=w;O~dbHVa9aHhcQXr2oM6E6JkLzaVZ;jBJEqwj6nflgJo43>?sA^qkkkUUrw zhXgJdro?X#O{IXI#{A;2#r<`AqHLR@UgCPtVn1&ip{8+k^`!3fZ+QJA?0wx!}=C6OJ zP3(gH=P5CIslhJvKhT6{;9RFn&o6jRrbrib^RRfGql*Osx*WLg-8#yrpEUMF>4c7Z zo1$}~0h^(@mzrWytZ}H}{QFdV!=Sb$6yIP;j$yh%S)lSGuuauvGL;Aqu&>X zv^JDzO(4=i7>9Jb7^Z_pNAVi`X^^BjNiyt^4(PDwI=zj_INuy^NyJQT`&UG&_8+E` zoz|X#6;mN>H227fU>fb`#!=}?i(Vw}q%ZP64;O(!uIo$G2=ce>F*>Ome4gIDhrq3= z-^pn$=w}ehxdF7vl_MK#jQ$C&0+?E|`R`wM1dt0NviV}Sue9Z`ktlvan60@aMVZRw z$U4QPmlHSU59C~Lc3@jakHFvIk{5N^xEes_;P1`1z{{Z10GafLJj+o<#|A}_fDeH`T9nG>vPK_tqF%mkh3%YYR83f(8^ zA~mX-UG!5;#A??dE@Z6~q|{I}_1m>L_n=9k1m01aarHg-sntGU62+WIx?mxXkU-zY z7;3g$|2Aw_s2dC|<*^UTX7jxB0P1*ydDc7x)Z?S}&*f=m-BZpwkLEcaH22#^o;atp zR(aA$H@;%0q~dwEpZ+(p5Im68;Vd`1vl-pxXYI$<<6O2S$@6nJcx-NX=cGhs#xIFj zAw>GJZUsSSGP~t8<`hHHO{qUcWvGL8AK_!EN?=qg2yJ_$10s zAAL3*?FAiu=Sv2tG2;iK+E@8AJI4`su;ZEEA!As7_z>F>Q z1%=OzC4ap~=x*TEh@4lW^X?1xA@l)tYpG}SGWxz35s2XF25V|RvnQ%a=|4m-3CU%Q z;8tDa5|d(yu1Ha|kHMzrLC`vnnA< z?FXo8?HhXOYpHoqDi!6ZeEs~)IUks=f^mBgQ}eKO1>tVq%E(n+JlDb+IGB~EeT-^H32o^-ZP4f&)B5pkI?QPNTX5qlUJKVNwL+57n z#1li}5rEjf5Sv`Xa9PX1RWQsS4ELG8jcNriONF5&?W6J3_WiCm1$s{(=HEJX{_Wk6 zk@NKyg#TS>`wf8&A3C0)nMSLl;HD_3RD>bU!Z{htdOUs{cqHlN{>^&BZ;4*F%5K>gZ4zu>UvkIWCwBZDMP|Gc-@I*f zPMGR*L);8PaRjO<@dCFIIXSM?(mpTkAS@$6VS+2_b)8;u z@4sA5Mnk*7XBDwmW<4vYT)yP$R1fjcQH!>g=-8$d$4q%SLe|1M>t>uW%u)D!kWyt<^Z?3Y%4I41Kdw_@kW5&@xp0V<5pMCr2p!%=D6b%B;bm;WzRR< zNx#?2C(?&kJV&|DsgGumRKHibzNO}Lb0Yttahh8}@Oe`4yy@F?9prn61#2o1Ez4Sg z?vk_2MvFVI5emSL)JwlYXsCB_0Pa)no^_4Lhn*-Q?rr3^J{b|H;{ zF}h-O%_yE^)ELKZg=!koYrQ!4bE1eSAxdq@ulx+5tH6P*ajLkcEAlt+OQGm{9% z5kYF>_?!_Vx&w+<$Pd6W2O>s>L=_D`Z*a%|+=Mt#$KEYZH7j9 z*AiX<0=qLa?1vqffpQ<@NEIn*J%lBeOOm4*mNY|J775)39yq(b8G{c)r9l)L1ortT zx&NQNcU^De#u7#Qx1Iur_H0WT@urI%Yft(+YTHRPZ6|6=?(|G2qZC;nSy8c2sRAXd zZSC*8#(BN-B7S?rP1bhw0IeHwV+3@j?-+s6MkDF2YczpSK@a^|6k1xqT{_7{2D?f770)G0%e(*eu3e74O=1z#60B!lh3QkJwJiP@A_{b4%Zl(&b3$w-Fxm$7x3?BOeS`!& zWf11UO0P-?t@dlzZuVCRDo9iAw;)7yI0+RY=FON{7X?C=$FsbSX0e7VvcNU2}-8i4~M&v zl`+gb*rELXg$~T(B=%*y?DvwX`jtiWmm#1l(D*Vk}m~NVKPm6Di=RCDG`;~19 zy`IQDwv@gsV9&iq-C5qoXcMnz6R+nF60c`tPPeK@!>6~c+e9+@tRfk?htn&c>QA3f z>a*2huHP*&#t7)_2=(P_LUPMmnk`aC5`D9I?qP?dT5lmyugt7o-TIhvTUE?oFyhhtao%R2BH%v4^3^}{^4 zl%c%}_LRz;1;vD4~_q<|aOn8GwC{a+*P2ea^dlet%3CcFfGv> zlmwBn3?*q2BqucDvHF*!LD%Hv(O6t@Kygm{;QkpUQ?|MKj~JnZ#IZ@25ejR*tnRym zbH;8HGWkE5SnK%)#5}BZX;fx@Xp;(Ax6aGf*A3oKRLr_|YdGP)$`WJ>!I)j0{_K3~@ zt@Nli34nGJF=Y{QlS(IS9kQwI4waxTB&fczA0t0%`>@;{nT!)~&+r!5s(COn5D z(vk5T_Iihdo{aiqN8xCZJM}Iyr%K6&Cu`~>R7{!*nks+JRQTjcnF{aAGp6$8t&+0r z%6Lyt*eXg%Or|#|PZ2oNgmKBpG4fa?IrnvP&sC*Al&1M z-ruGE-(C-J!RMNRau`z7-TkRyG9{)%cnuPbeSiwKxAF#=#1`^+aDOG_7agt~1Eda}-P{&QQd|3|fiJf+DuA|5Ao0YddG4{vM~)Xts7%`FT7K(!6hV zbF5%kJ%yMlXDJ?AH_qRABo#8Bv$2iYW@AT!IFTUYCCq4nZNF>I1Q5%m51Z&3ytFnW zyRAj>uE;2d$aSLUsOKd81lCHdSX%-977Pk?{1<)}EMlF@v5++xXdYMRhzW-hc>!va zm?h#%llcCtj$(-vAB%9&KLh&FaET|FJ3n`GrBP3vh$Jme$x8n9{tIB(> z9pxfcI0z09!t&FZk~^oS^_%c~5RC!csOhXAKGYv>)$(K{ z!GTtWRoz1K!Ll-;YTD>oZtz|PoaI5=Izl_y1p}-OTNU=J?w+|OOO|3uD0*?Xb&W8@ zb*VJ&d}-8q6R+y8UE$`t{8(Cpn2{Lk-D(e6{pb`oDuZ|A^k?O5YMZ}2*bQsv>+d#SCW#uzeL#oE90lQ`()p~V#8eDlnEziy>WZW^-XfDIgYOpKr z#VRyD)jrI~=;3m-fSrH$nRt6Po@L5<%E7G6wvHUTd=rO8_oC5KtHx^P6K*etM`35N zm&;ueM|kD^7xMFEo4EvWYQ&Bh_k-QZImEs9BHiF(dX3{!QlbVEo zoW6Q{^5)eK=f|&)Omjls1dqN=vuL!SSb3C?gp#|&mZvxXGpBqUKp@tTQ7-{W6IeQ+#idzEB)KV$>j&!cHK$i}mX;xS!hxF4iZcTAEie)i;G zY%SVXhnM&*Q|Yr+XeQbKB9LYz4W%3cL z=yOU!6eKk2s{FL$AVFz{Sq}v#$~D&+-;-$^jBQ1mLAFf=q{w@sG=`QrB4b7-w1eh2 zNQ{Jvx-H?_{8b9ECBNd4-l!^o=rvKc5p1|P0kc{MLX!|QT?AQ>Pz-URWKa+bHGvY_ zlocpNIr>Vb$6X2`Uz;6H2ZaSAPb6tghcwepi1UHR7LnEpdRr?=%5g#?9gNa8%&?3zAoyJ!+b@gl=A748Vrpw#N^k*^cyrs14j zQ-(p%Oklg@%PQ57sJ_1R%i45olY+S>1+%$5_#u4(7M_d|uZ-YwU#fnN6vIjB`}wQ4 zI0!9Ez(p2wM2`A>US!y3^(3O&@#d`p0CpWls0$66uOCkp%WlFLHh!c&uyF&VoPh%) zz1opW7x}Dg~ypCb*leT(Bbd0U=kz&3WIz$BEZwa$-(vwJN;}4lg&~J zBXqZyFBK^F0wfdvie!Z7f2ov)_WeU9KNMF)J9VK>Qn)`D=nRJO1TCSZc`DlsujPO2F6-`tX;*Jn+NC@r-9kzvGZTy&w zapxDaq%+%!T-uYTsi|fFW(br_AX(6KK^eOae$8iC#9oN>-AV?{*pdf9qQfF}XJn4mQdIcpyq~hPv1)ic zCAEB|b(vo0sH6@L;NCU-iTEQ}R-QNSVE=1RugzqJt-EgKKonD>L7!O?(Lgw!s%TTD z@2TnV$HOF8R6p;@AR+c^MN)bCXDh~%T~dd0l*w|9CZ%wwhyIg{+=?JNfbdQbr;B8B z)oN}?*o2&%OeWDdvU?Po8(;Y`uTT8ON9=;X?V)3548gup;VedJS7+b^KLMR6--<^c zL8>c=G|@ApGF%`BE(X$jY#!X`oH*c+!ugJYYHo1YLAC1WyA0~#6WHj0CRrX=EWU5=2^0JqYGvEf+u z>ejWmG{^{$fi`W;_zx;*Z-{aV4$T&+;#`k!(W@91JfSDtlO|CTB7WKDx$bs#kVrbi zH+rE$3X>^a=7%l@sWpIewy?@^^pz)*rBG#cclqAt)3f#CC`im7u!>C{Z)1$&>=hIWd4&yjt$~&s1gL9z^Ew~C=2Nk z1HcZ89Ak)t2udOFx%qWE#s+!gk&}cH1tZokNyJLY=||adlbm+I=Z2li8I}1gU~KI6 z3~Kn{L&j62vPRL-H>k(e@V)6cg#pfQ^;+VQU4P>au;Q;_$7JhSjU&|g@B!f6-dZ0; zGLqq7V78Bw&}^b~p`u=|w`B3^6uStsuyw95 zuscDK=rC+#V*BO(Mn=kj2I5C9W>cX7H?Riya@6Bwxhd^Pwxk@f?Xly%1mZ z*m^tq`p_ifes6G|B4LaZ${>XJ%HnaR7e?5%C%aEfXd(-uEAAIYHD;g+b7hyD4%QZ3 z?u^)FmpeOy8w00qx-&di*}~!*Z3^I8(S3qr#xwi%>%(*8c+M92gGa_K23K;3tB6Th zE*~=2Hml$S3e>WpGSl!EjNs@izem69^ zgcxgS#huXPX2jSU1&Pre5M_HuI2n4%LsIHVzt9;6QKJBq`fgM~E6bMnAB0%ivr<2A z&4{H$KnC!BK@y7JQ#2!S2=+5wZcL@Zi3e1e;cL3iAX$#;xHmG4UirKaWe_c`JR%{ zaoFI+4P}}c8H-!=*_Q`#{>hhcK@xef0(Mx?ybvow#72ZICt%{_D#}P=*^{0ddj%|I z-lsT=U`}Fg9U0un_z|pIkfk7@@eQmIvY>1iW|$ilGKvX_trIYw6?C)^z|d?17FTT> zqhNe_9b~Ywf$(x-U((JU>-%|PGdCsze{1b*PFha6m^_2&HnZOd`FVDPrsLFnuObiH zxpGw8jE~l9E|=qiMX{QMye02HK7IS+f4+KgesS_r35eOr&;OQpW62M69JmO?9Rem9 z{RYANdN;v5HlqFg2!@J6Vg5o%%ec9g3RxoK2}X%nj5^>RDFQJ2{Rl`pejnu!0Ye@e z(Znzc+w}qI3Cgalbp;(aU`wbTs%+3^Bes6xOJDD+g+=NyrH z6W7hL(mX9zu%I)XP?1tsq-+Kw@KehOSJ6uq-rAepd69;JvhRmProbKWvg2YBXpQR; zQl!db&!E%FvnR-=hRIGB*(2CR-Kp_|hn;8Fq5o|b85=+P0D%A7W2;SlB9Q{rU$@#dX1pl+oRUiX)r}n9*%@C+!`D; z#7@qA{#bSC{ogtNW|M#!JQo2b{iEtBO}QmeUoXrToTbW6-AFwhpmO~H69thWZ5X9j9`6x zb_`VVe3p^LbawV8H_!0v83xxDn6Ury%xr9eQ@LIDBO~q263$I z!c^WWv5mANt^{XhATZWL+3-5fj<5JchpHef9XbEGVjYaGbD0jAAd*GXDADQGxX=eK zZ9EALkFuYOo#OcFozEmF?hqn;!LNfG@YeDUx41D#xFgJPa{<3B#=wV9*xOY*eYLQ$RB>2k+~Qs)K&2ZO)F?4=ol82_$uX%f<|&82h$l!EqVEgWS2Q#h0&x0UvqEt z&>JNv9T7USBhm3tjN%Os#c3tS3!+S~A%0J1*Clpn{ECY`XLsFSVS!4!{IqtoY(sND zOk+gF^n&9PPK1Yf`4-z1`%COJGqRl6^Mcp;E~8xvSz{vvK{~YvDx7C8IH!PO&A8hf_*=k1C(p>nx&dBU*${==Y`3 zd?;=gj47a$v4N5Y>Y?KpJSu5)h2tBpE>LXnat1=ag1br*o|t>%G3sDe5Qn%g=w_m!KBC7AvL$c=bIA6URcv) zBzds1AX+__$J}2rPG4puO>xL`Bj$E;#6nU;$pnPF5<63L)-XHg-X%_{&R!uqu00+N z?lHN6P1n0Dpab0=-4rREK?QBLW|2S+ixPMOizr7^I3_bA*#(4?RrxOf04^P;9vTP- z(S&dZoSt>67HgP^0p zc7UxGvs09Sy(gd(#7gOB^ojSSFXGEDKq$!9a~`p4gxCR^x<(VL1t7K8d?KmIJ29py zVD}$!M%(jqR<<|v!ONN0 zt(}Tpb|~YN8d5b85`tHXa@di(jC}BOcN_?#qJ$(}*sV5rNLtG-nZWlMdP@#R?osaG z%%|6059em?RSWLj{m`lYmx&HAAXK|3GQ8N8YOvabF zN=bEvGa8LSaUkQ(;%@ieM2V9^Or|holP|LMu)#C3-9vtL9#@0sw+OZVSj5(js3}w&XF)!b?#^(M z$mV&(i98B1J0cYS-?QW}1GLSWG(uDqDu68);R)`9lN3X1hY!U(O5U=cXS{2gl|pYt zj0BY$i*cM_7!#UC)no30o1<-+Kk)dcGxX}OXFDPfKUW~W_~AsC7zc4QCOhK3hDMl2 z1%%)*B<`khkYLr;=msE;xgCtoiz9M%r6kOspp#Sd;^gI9?j{xr**ChzqZuKWxlC4p zu1R*OvP$w>$_%3!$?20EC781B#vMJH6r>-N@bf#TtSMo;!|h+9IhZza)1zo&SrmyR zJd&3cl1bL^5295aAVdgbVNMw@874VTp3eZi;KTiGl;!lNh|a#}AW*QNgxxO3K^%)K zfdSpQq_9fIaTMgjn1^*^&bNZfo_TT#gDKr=oMV{lEu@tqxDh6dY-Kl6o|uyG|NDRb z-y8tsaM#yZLEG3#cou=eq&FP&*guA?u0p}GuI-{=*bCCM$FAbB{bgi+QDVRhqUsc_ z11SICkf8e!tLQ2M3o{wFiExP%fSZVm=lA3}C_%WaEkx6dAw@yBG9p*Fhqluo2m3>= zsz5Z2`ORMIqVBA53600B+Liky!Hjf;S3xpnh;h}Zh?d6g zv0TUL_O1CSoPl$!Cnb^GKs=x#B=u6Ujegb5|EkIUlbmbZ7AX(hD&fKQg6z0LFDC@g znykx{AAE965YqO8E;1EFF<%J%m$LEv>G(A_xrB&B>|l;DBaHCxiy&qs>*-*iPR{M&p@cV&k_E-NJcsz`o$)Tc6l=;pm5dN0 zix}tMl1xy;x8Pt4!~{r5kVc?^WfTT=I!JtQLOv*R0Y1TfX$kd9t=VZ#Y zraXqOjO!*KO)@e!bW@CV%BZQgHNCJrndL;Qg7G!j1h%2(qnL}le7cZ!3z;B zzq8xm4R2RlBi^tH2wheIKPB?cTx>#$|D}Es#FyOnM2pgaeQ=e8X&Ohi=ldgnpnox-9ZUFDPZ2#K&Vnd6-Yoq2$q( z;AXXflj=lQ_@T?p4C)JDAsc!cu+b*+jtw4@Mg(LBRLV_= zWC;fr_a!s!U}WEYuq@9r4e9*#nZr*Bc+OxN)9iz;GnY!(0{*G!Faz-77#MP^fcXQB z19v(cC_Iz{1eE|oX^QJ0;*6HSJiCli3(^(N_@6Q!v=?Fo+5I!js{ftUaEXT>{<&#PY&c(@bo- z|3}8LeauDhrySFr-oI;iOwQuH*3m6ky^D)qj9_rJV=c|%%zgYxM>Q)M&q8P2P#!4> zec?Q++_nB^yZ6=3WA(NW-NOo1S?9ndqlH-fvXN-V&nw#|YHQ<@PYXGE?wUtFCE@+@ z$0-Tb0nwUC_OO?dTQyiqV{)3am8UjWt61b*jU;SY`8e;?V%7 zmHMV;T41NIrnYlX!$hYlRq>a~K+DzQ;hyhOibg`gnpv!oxlH&eo7+j{w^RVTKt#XN zUo&_scQbjrh;g>%(tO(?WlZKNNpM2*ezu6gBW3H+qb&mfaADl;EhaY>Xn~&-8HE=& zBb+dl&WrVeVt^3iN*;{IWRXy}vfwFuTgB7N6M>lq`K1oOj3MT-5@8wd#TBP38tqFI z#rtFanlENf$_>Xz%e%?nsTayii4nIj%Ckkvt-lu`Xw-c})UbEiBIyDV8{7WpxSNKf z{v++|-wjy*b>qD4Eu?%3ZRS17Q8b?~z%C7uWUSK}1xYAnb)aD32h+>Hh;1X&TZ zJks}8*#>^E4w@z_8?9O1gB0yRy|Fjk;0N)d0E-S8Ext z;^1Q2C6jKD$f{fNQetB#+)B2}RyUFv2F#P|dr+B3>qZ^naDq-actcra9Y>&RF{Kjh zG7tNLk*BOD!orV<5pOu4v)sJ8iXI3{<1Qiz=9gtLSvr4~3E9XbVIssT+We}{GSO~b zFi7V&DMs5NrZ}6!aXus0YDl0dLofh%;&4Z?19!SWMwn8Zbr}*}Xqhb)#!&KC^d?mi z(qvDL&**ljp?KG;)JLYzPhes4u3Pm_ad2-GZiry|6u2q!`kZZWSzC_dr_9nd29 z2m#7Hww@Rfc=7`YF$40%Y%UQmkI??$`9LXLUJL&3MHS(hjY*m|%Q9MG2+axau_JfIBx8tUi^Gt?7m};%1m9b2?GTC*!7hx?Vt`a7gHL6u& z_i*VfxpMlAYMWWpHfWWaW}JidTX5DI|CLy*$<$hgv1B!N%Nt6r1I2<}VoPM3+a#_7 z+oA=aMi)-LcUgRd-o1UzBfkhQe@Kg`7%?S)lfqQnHCRD0_McRd)M!_F=!FX6DGuac zmHy*bGc%HoE)eB3B4~d8`b?$@#Yh?ki{%+{<#=<`yzBInz0JzTiBNCR1CR2vYI2^I!@}nsYzzYK7>j z4A8^JTl^;~KzN?3L2v({H}q){{7L9duV{Zt@ul-?JNJPi@VhmP)j79KWz-_qaTz&9 zGX2oce0ryv9QMZA?>YOR$ACZB?DdWxy0UEaC9*d=QkTHqT~({4uy>%9Ayt=?-3Lo| zSq)Z6a&~wxJ=eGsMn^DcT@Z72*s`!+b@yZ{01Hn=<*O1siZqX&G_j~Et7Bub#8lm$m&O@n9iOj!m<0K#Ve3$;x(Bfo+DY*dzWL8EE#Q!i~SCp zuHSNYJtEVo+=dB2F4>#I2 z6dSOuAq%ib`5jz}aIhMXV);a3BLPcWR~Ino30AF`O8q>VW0h#%mh8*|Zaf}D36Bmq zA(`?t0!icE!R*N{kGZnQsX>S`JYB>AyYk4*Alv)rp!@unf8UNspFBgOkq@^r<)WK$u}Q+k#`zXY(k+%{ zTl6Pe^qQ6&8%`m@sYB$gDXLG#BuOL#PAqp4eJ{7;Z2&PU0U{T6uu-|vM?oHq zFW5cO{emEv6`z%y%<%>LStyeRd45f@P?TXmtui?y6=OS31~M2GYXhi(vO&0+GUNvu z%`sVkO3OILU!A^v_2T&a)ytkd(%-#(EsTsHhaVQ29dp-mp;_rB^DcH*#ApShV3#KX zb|w(HA1nu)uxxBU`M{bcbyd52k;Nrj+gT{@viQi#AgvTc@7}&Pjt}IVA6mexXc4%c zYVYU-Guj(uATyFFA<&p)8RPTVxckaR6Q{vkXi~B(8L|q3u!BhVB77Pb0o z3xcKfC!8jiixD2vm}749@--o^pmL&wU@!b<5QykOCvyv-vR=}EqNtqTgMQv_3w?Rlgm6gR+b|{LvpWO zjH0+cJ#(bZ6U&YPpJ2VW1_i1e?9 zeeu!_3?UA|A4n)-M4T!Ikns$Mix^Y^emPOKK^@qWabXCX7;_rfC$A<%;G{*pijjr* zjL|$2>q{iZIyE$7ZE$0Bc0qAhHPp97dQuFeg;GTs5D#s6gboISH<9H`6(lt31{09Z z-9UKOGy@qb=hVfx3eWpA=Am7zylPx9suH&{a>B;UiWhCg{6w9cOlU{M+ESrCbX(#k z`c(??alhgQe^eDf^qRv2M(V^PWCK~E+QL~R1k2qxcN zl%ymk)0@BI8`ECMs%R&I65)M?n+2j|*EAd2MIxzpS6I%EFb#V4>i-u>a78E1HA+Xw zt|{XhY5EU2yCfInjGc6Y^GKST?$I$x*x{5rNfvpkF!1hS6L0O-K+T2PtgMWhBDG=V zT6+nJ!Or+Xi<0RH=ivD3SMNbcPZCquYtpN8oXs6oT9u+rsoLz$;yv6A{?)ZCMEFyx zd#ljC0kC1o`#p=PD0pH%Ve=N_%vMJ%J5lVAr`;e4yBer9TSi-!xe905FY%2|Z>~q8 z^Xm6S5bH1E6z;(P`ZWf_l_S*ofr#0~od%ufs~_cQh6A_=?(Ib=*c2g#>dk?kGw^6ifql zI7proot#=?aV7GA(DJ@3#k-STfX3cWNuEck5qZ%t@8nl3-3RwdbFZnz64$WH!gB0O zV`N$a%9KcgF(eC`E+{$^=ACDl8cJafS=1X*PWPSox?uA9HlES1J2^V0$7ULiR z6_K!~15-e5WiGhP90hz5=FwCp0>jES&x)oY^M52%ELgeH!5SyyYnSJ0=9f4ii6Y|% z_-&H})^G&d3X&!X3i#;J7T~HcmooD@SJ#NR34$QX1%~E95>4RRO^Go16Y*1nEDLV7 zUz2W67(ovUiSwXQ@!6~>R%Dki z8p#v>Y4a2wVddKuVK1U&+LxIf^ZpM%p1(Tl(f4$dNt@Y5086ICk4Dg%=bdf`z7jE0f_MFrVce^@Dibh)vZo63 zT%m}E{tI)|jZsI0KK}+Vd@JIh3miLJTYr!8h&GiLJI#xT0XfYUT7W(y*E(nu`d@}} z!8G}Bl4%)c4_w#Jh_|Gtz0ZK+RPO(a8IIXW)T3#Cz7)?}OUY`G^Hx4L`agWy&*TuJ@o~o8+Q9>uE^KW^- zvlIZFgoVE?%=J2Uutta#d(0NM`ht$WLEBLR>ENpJ81EPmqKONHf8$?i7R}EVlS%X* zb^0BDNx(hMu~pF-4DiZ%>#67OfB&m34<(%8Sp1_S>l{9UzuB!AC$xL;?AfztU++I3 zZoSL7#q9^A24oz>aW}y^#bF0+LmaLi>YTm%%Zu-i-=1ImaQx;~2X)S0y*Yh-eEzDl zgKlrPAlqy`kZBf7f+Xmuzj~8J9D`B}zS6KP@>b*VAL^Xl-^Tcg$KsbH_)D#C;2sltR1>Nr;>yxfA^3vcXBI< zkh^YaP=_PbQEha@S=r(EieISnB1nT#6h}0I((If0 zYR5rs>k)#`E?{xSZk{;Htr@}MEr=Cg(Q()liVcfOkW4LYx19QrUHad%3diBRtj?bG zfwkFTSF(wcql_R34GGB8s7+h{-v zg-ry@Kh_RFCEUH@*Ht=v3Z{(>I%^d)5%Pbi?PMvuJ^RZlor^JT(lw))dOr)Y5GOeN zYsvRw|JJ)7eF(Ki0Qu;D6jMIGst-UKdoQF&LBsPjg#|Ef5$$ETv;M?}eg z1@qV%AGu6Ky_d?_87>boA(x93b%Hd_`&Yvb>gWO;#5Mr$30DarThSmLp0xPq)+6*&l!W9uM`w!^ zBE|80hkPqNJh{_%7^*D<>VJ^9#8I+%&(qfIZm}XG9FuGD+AsouxXn^)j>bA?f;2t@ zO$*7f36WnJtaSum!8pXQJjnU-k=6y`))|pL0oyuhMgVo$3$Foz&k&*XN@}3tUBb@x z`;TF7c)<2vBih5@c+7i9e#8f-NJUEv`L33xW4)4V@tH?g(fo*~%#UJBZ&Lite33(r z@*^bfZ~d>>|x7;Q;>BH~wzNL)ZFWZM~2ohQOQWSyK(}+I`V+v;eBKdqYk5tHzC`a4E)mJz~f#Nj9 zMivvEYko;bG;62^%?jL@$ z_q10OIQUnLl2Z82Ao)=(6j`RUfLuv?Pb*fE+`MoMLH8{NrDm}jc&ZuZP;8;P!qs}) z8bK}L>;7BbaiH;b8^QDtqk_x}eJjS(B_EPF8)WF1+hIIs8)NUOxf;j>RnK)(Y9d>$ zULZN+6=If9dVqM4sgA4 z+m*c4_tnP{!uYituuj*E@6vqqv4vz&lglNzZ$3*CpfAnr^I3Ov19w!# z1jxA0-b#{4*j0qW-&0}FCP&YWYw;d0JG{|U|OpcnGDhR@$FW+^Q^dv$>b?Yvg zG<-zAF9?O6&Ttk{sf&AD7(1h+O9;U%n#&Bh%tMGXdEY%N1{jS2HBmU?t3%8UPzV8WB z>FR$H+3N3Q3Jn@3vgFqjdGEYdPHeGnI>zCh&HB=fwpkLXoho7ZbzrcQt>?Y%e6PS{ z`=)u_Gq|?NF`2~`r^jNR8%|I~owZMq#TV~1LSeaSY7DW>MA2e)i6!UBWH%d5Q+bsY z&b}g(-ShLrYnha$085RXAmo4uS}!#GuwR*+hb$$S*C zI}31Zc~fbiVZUk_h?#;o_-;vdT$8V|Ql2q&#B;naO{~M-2_FMtZDkA>k&WKjFmozZ z?W_uyZe{zI9!?$|NoDpJFER)QYb@lecB5?cPWbP@QDXVuXm1MbwPAcVK-j(;Mi~-^ zwx@>iEy363w|(=QJOYjH2lh(1jFRvqN1Z5v1auu;$KoAalPwGDR!cw(hkqSFXUvAJ zC190gLt>S*r{2GGROcEHf4yE9H&|jr&A39GfSm<+pF=1Uod!(WXSF&u{N~^iHczkF zW`QZJ%2rqLv$en$InH2Jmo2mA({Jw)?i!_>csNnn0~=h0vqVl~QAIAD`O-nw?r55{ zLHr+OdiIx{o-<4f`CiRPhFm5ld>M;#V1Ms%0Fdc2*SrrX+LSO;bmcr<#4(SLlJ|r(2}e*(2GBrcf5+MbgP%g|Ue9M46C-WJE>XiMaJGf-jgWQpqcbOUh*w^?;mE4GtpQ z+`#jzY!eFGt5~0Acu|T;jd1xD$1gX_UOL6=Sn1 z;*$A;U7=E*-MCu4SXQ{3l;7%aZ)IqT%J>ha^33|ty9wsaBSKWif7shU+~2q3KkV%f z4u_lg5BKqWsL%l$+szE(q?byM=X-`=16RW>9@NRS1l=+tIw`_gXEx=v{1ntyZHmTL zTPD>qLrMv!s#dmNmfm{H40BLzCCAPa#*HQv9e6CatPUH&&p66vG&GM@S83&9lVa=h zd})<97(XDVC>*l-ynAd$&`@LGjlIFTu7uCc+v=$nTv1IMAT*e^Dxc2+-X7123#NsS z4NUrmBVn^n$b7o!_V%b{WrqCi?f;$6g1y71Z7cJhbX+O)Lm7)t)+;`!4{C-x4vTp= zxOHq+6(MCmyA;wzwx`va%4b3AhE-)h+h!&YZ46n3)#rzU845(ixwYH0Dv=X$OEIs;MTea0&Y@~%^DHz)40j&i(Eutb9GRg0p+QC~FhL*IIRMJ+s%|6-1=lttMHJeOO%bR^z{yz{g3~qE1sghszSDss(jDV1gL8_ zCL+I|P)xa*@nmkIC6cZo)U~r(HNOFrX0}7hl**ge3hBI7g+y-h>7ox1N37^v&hBu# zR_3-{nXC={(?s%@TR%jjNGY6ElqtX5T0)2Ar<`Y*te$e7VseQ&zsAvY2HAfHYM59X zayD@`H)rH7mq^7Tsaqy@u~0t2QjxcCD`8)WWWkEKx0-Ol3=Oyh)ibNF=O8W3%O{7Y zEN$NsxRb`m|&}l*~mV8SOMbFY>bKuc#73PIxm;FA30FhGn6uA99Nr(*$SL~q226MT$-Zsy`R@iR2SKFX%VfnD1=JuZuSeL7N+L?dW<^LNV>^b@W z_V%9bA8zbF@8hYo|I}Gsdmu`aCd7~VGWbE*pv$0KX65d`iR_%MVoDg7>?*dh5oi>5GxqQN^!5staXWR zcERn6EwpBA$phYDfG3R8Rk(4xSe1 z|KREVhW_8jQ)gJ+$ePab%ju%WDttoOo|P3Z3PY<%-DdxcxsRJiTEPw4iK_8NOvprVe^j zO?vbV;^`ZgQPiq+kvCctxq+^Qgu*OHqZ1M6f3Y|IOI9fJQjOJlCWimJo zB(A=Y9BfcIV1;mjgor^3p)h^{uK z^WgYHD~yb`N*i~^^FwK@og#MK&1%}LCbMxbO)OhUb(E3DQUe|5%?!KDngv^~@}NMU z`qr{}*7dZ$|ED+;PIHYV0X6sk!>7*%&i(&jxWBi#|KG>6(*0jjGn~B@OQMOmz*D@f z;J^Uk6CvoUJdep<2CaB$mF}R77E)8*U4Q5HU!VoTyh8JCeS!B}CCGmr@S+t8D!vHN38pU#L!8zGQ_Ej9g{d3hjzT)o8!Lrz*6$XrP zBa!A#)ZvUbctC1 zUiDL{Mi}#vm7Vrc5+9hffbCY@Vrke&QBqJv6z}>5-u(WTlJI4eXN#1bLw{L>)5dl{ zb@o4=``^J}qyN2^XQlg}Z%LE{jjJb`Zj5$A!*yD0XSv976>&MfnAmcTBB=4i&7=1F>Y@FR7d}x4GvuUpW(s5hW_8j)0F;i8*b8etfw?jbtw~$D3*w`ms0wq;{SGaZ z(=57*Vm!s+5t;;X&d!BLA_eZ4WLQKpK=dZXdCv^Fz|{*+R3ObK@(rG+^yX!hsTXNR z*b#w4^@dC5)z6SzC)Yt19-p4*N^Gs<6eslW(Bg|Yi016;jO6qOp8ry#%wC?HzOY}M zp1jmAvPE*7zss=@BRxW=ID;HtbcS_Q6GRCs$=#GQLZ}{~90fVL#&O&O0PF^Kgr2-* zV2_iVCm^VL9VLtRvVrqil#7*)f;@+K6Rfd>gbY}Q(IStMDLPKm<7`eclm(E4iq3)r zoxK8B6sIYi<4RO%_65zoR24{PL;e&&TkMx-Z{wJ&$>EgaC$1*(>zL!kPh@T z58fZ4r-z68hg%3ua?WhrXPX%g!XK0P<}D%g+bG67vyPnMhj^a1eVU*DOSyi-Q*-|B z4-5LA;nSzjHs}9+JRi&)xD^b*4%$vgQ{S&ej$gfxavCKf1hY8U`(JSiHHl)3qMV(E zafCy3J;Mp?PaeB0A_@CC!|@zNNlt?}1_W6No`+EAT2V-F4&m}KYtG1#?s%wgm$<0c zyX5(W@fN2-GCnrjF(H?W6m_nK9n|3+uW!J-B&*oztFeAUSWhC4?z zTi_j+^J(NkZw}ZR+{8R~P8~L%y?oZ65pvl%LTQG(Vs>}o$0D6(L5PcmyF3VB=LmK7 zJN9QOV(aoluXBVhF-|Lew{ovHevSW@r_w!n-mk-#zHBIHYQ$El95aZ6V#Z;yKC?4H znLPQ)hzWsv@s;@E>u{7Raib63SU~DV_AXcS?rfW#Tx4BmoF zsECgF)p5WcM`2d$=2hL2G=m0Eu6GUexzWmh@Qj2{(aS5UF8j&*cown$~4i zdl{i*OyVPS{^Haa!wj=!gR|~69&H(R?XHE(H^biEpvuBUoHeJaof)O6FE|13`PYN5 z2Ze4~$TXt7I=V7j94+X+TT>AlH+-gRdfC{{a?MtjbwiQ!F#pF^au4Ta{lf7;Dbj|z zo#O)Oyoe*5P(>x2@B9kRM|UhrN{Kn;A$w~Bzm_!3LH8uoo(--r(-hp7N>lb&l)*{S z^QpP!FXw4}{|mTbM4M}1z5mCvg8kS2CjbAvJnQiP7q&7|$zOAYlO{S#-Q=2@>O8bN zUD-MI8H8kQa2(17u#T&l4_4{DWVuw>ZcFVFVOJznxvO+4dv|9%)|&o{g6pII!@=Od zmH+J@Y~+9U^Q?pZ`(#kgA)uF6Z%i^w^4^%t`|i;oDu3xhzSU@Pc{&4UI`pOMwIQeL zDk5AFNz?Knl&CrcARlKzieE?b2#8iaLT=WCW^Q&XpOJ`Nba9XiJ$LUd|J4(*dYxd8 zw(ymQvN4y>SrKLB-l4=kI_z&-{lA3|PQsM%ykQVnqEZ-9BZgtn*iQbQ(sZw}RN1J5 zjA(~7?0~EtS@~KJifU7|)Bx}VUTla8B=>)p#fE?eem6$N1+Y9S`FMu)tngpW#r#&~ z<&hZSAj283INwG(w1@M$FqhxACLT)%F1qfy*YV8neff=K- zqdgu9Q|MuZg(0|9WrSJ;&W&>fO-KK{xCnTTt&59aq{s;bl2GDE>P;@9P*LtQ4#r?} zYOTeZhT9`;63V5~W~6)>#LdQ+-!fToVHmGkZ=1}&G`x6f-AOyX32Tb0I##Mrs8+a^!(0tlarTrSV6Cw9Lt70)_ZVt(%(X(k zX+x`ZN1t4^=V2@Cy>LHD;?VSzgk2iY*^-?r9zf*>T=DGGXI`tGH)_;35>+Q+t8SZQ zwGqi~)xT2cjc4~u@M+jQ&7aotzxBs|-hcMgjsJ2m+}p_i?&VpB{I4RQ^Lpbia~O4| zIZhPi`-?$b*dgnSLELQ=gxKjSBks2KB>KY!6JA#sVQVEei12HK7cQ^bAjYsSp05qi zu|eea9i|;(efbY%agFVob*69ChIR;fjqO^uQ8ckD?SijQ33i+G8`**AwA9ef0a0pN zSAvT&vXdRs{#eTid95zu-~C-ISlNkPYueg56>aM}vzW*txhCtm<5a z0csbBZ(T-WRzT;R1{6;gagM2jM@qtO5tnW~Au4UuC9~Rq$~Vo9$y5n9U&;UB_iWN| zmiNab!`(2+m+}Isv;R3L#Q)eI9&F;j-pk|R|JYo=O!5NjC*LbOV&}VF{;2W5Dbo*JXfQx^2^JN1*LimWBPc#iga)5!NBP#m~#H^De9b7;rhQf+qfvWgSb> z|5jE&HT3`Bpb-D>;P7x0|LI|j|Be23TaR0VJnzTFQNLMue1V%fWCM4sLJWVr@ zypp23nF3ZdLRx2#0ecKAqoR_D_Sq9jDq3b!tuoG%$v0zy&p%8>!%} zGFqys+Dw^WhH(>9rJB0__6usy{#B6mvqd6eJLP@*25WnK^eL;@!j`23#|kr+Hk-hw zw=3HW8~c?{VZYLBIn?Mt%d*%ufL2%!8{KB{X~_QrF%HrQbR2D+0c!dGeb@i%>ELi9 z|GAgP!~e6nK2D=EnCZ`vy${leFVj0=@P$FfgJ$mC5wO^Mdww4tli!|Iuj)~eVRtym zwQ)txQmY*1MyBztAXM|3m7p4NoUI^ugAmLpO>@g;P`_!$QC8yMQxO(d4EPsA4q^(kr8>979`D(@;jxH4b7ef+^|@DqAj5 zAZWe7r)JOY{At7gk4Zu^6348#FbY&>Y1I{=hX3Ci>>s-P|KV^W|G$@~g#Uk`rut2Z zAz)c4>Cv&eR#^NfzJ8p8#Em39qzS)a8{`Q%&f13%{6o7@18m+ z;qKBw+i{dr)N9eFrnom6W*FZct9zERedyaX!@~JLJbb!2|L@~j z^ZEaxu;%=KxFv^i`C>pm?ItIo=Y zXpz{OmtJO88!M7qL%J*D!DV{cSVP?yQl-`$|6UumuN`9z`9dK(Ups6;L5_BIL?A+m zW>J5B7pld(8jK{0ml%qoy0I`UepMzb;+Ix0OGk?;)Cw)^Qt&*>()d|vEYYg%wzjxS z0$sah?#6ziHTKQzCu%2n1v`pXur~(0cmK4x|KFAW_tSkh{?GpKU?cy(m#5_Z@ALnA zKr;NbICxvGz>bvuA-H%m?yf6h$TEcHLH@fd^>*h>ZckBsd9Us=fW@dBUUc)z8|^Gs zi!Cp`K4b;FR-Whu7`39{FTxXjN&3G||L?yfUDE$gB)t8jdv-g6 zP!P|4My}l@=&nNqyRvI{32rUKtPjPt+d7FJj3l*`{b@E2jRm9a{kZQ^6uPWKcun0U z1W9$E#gx2RLWI-?+D3NtImwR7XV$Yq3PPhcE5*M?3SKu}YGT|@z6@g0tQRfft5?bJ zmJ7zvdflubW!wY?X#BLf|6w`~n|b`$Po4a4uNeRF>Hgv7{&ycw$^GvY9f#fvp!o2h zy?od@UDGv6fHK53?2U+m@ld>XoH<#y!w!J2@zL>$_{@sM8XKpRhw!wf|4S%<>gfOB zp)3E}-ydx9f8Wcq6#f4UygcsW3~+Qr@cvUPiCWX@wv@W18peWTD_}fqf8pe6X~}Q3 zR5Z4j8YuF==nSqgvT#HwDqecH9Sx8{+F~(lEaP8gA7x6zT!t zj5=6|H_h-UR9|*5F7b^GNW5r-MxnCi!TIk7fz14y)E~%f1$C*fnPqJ4o59(rNIxeP zsc%kK&~?>;+bFv>PsP*b{tq0$5(>at{oj7!{y%uOvH!o9r{w;B%BI?Tv6o*Sv<47X z>YDDy5|(Sa&EG!|6`(sS>(&8s_{%5(t(qGx;6r=b)BpAgz*_y^vx5D{;lU>U>%BZn z(*Msu19&%QfTIJvyF*|FC13+$-I`RlR0G4&}1N2V#LskNM5nhKD z&^pOe{fxpI}`uKuH&lve3;j?%jC?Zfh~Y-6oQ|VLa=mVS5OKzg4tFv*y{dA1Bzu3&HhyoM`1u42!U(v ze}lvQ!u@Z5FxcGx?&I;TT{g`;5U+Jl6Gan3(Kfq|o#dz!C9n)SRF0CI;vnpG$~V{N$XbzTGs1DIo4SN9v>P_Jh=DQo^4VB$Nd$yu0i7M8 zz7$`K0}g0&L_EkN;vo^*&-1uyx1UPWIJL_S)2fB_QG+JQh1z4>2G>?-rx_XHB{*4! zgmEw%0=6Mwe_#T}H^JSJuQK?n6E6;Vy`-xMbFG9c!-#NCm&@)lMnauW7>$(#>rLqg|@r+o<`Hxy?>aVjXziW0V!_v^~pHoq8| zkF{Yp0-nugzN^i=5tFP(OtQReq-v{g1Rn(%38!sZ6P z)*E>7GA@4@!4!CCo#sAY-3E4Pc@aXvy{%`;vF22lio8Y7_ja z$Mkc0YW6?Pf-V{ThQ}?^{S=hEm#zS{ZuXTb4shXY5Q#~=1? z0pl)mP(;Wy_=Dql6BRT|jvh;&G7m3|bQF5G=fAt6IVfC`*Furg=pu zhC0Iz=XFl zD(dA>5E6J<09C&J@Lr26tU$t8xCNh7ak4e)WR8PO2p-|&t2>fSe^y`cgQ~VbY7)?h zBtl|qyeRrxwgl*SXO?=|R-eThwsjacufb1Y=McT`6B9;2E;NlI5V#}su`7_$Yjs#Q zjUkRP#m*{cwF@6g1@7gb^a)>-uR|L)^e%eLbulMWJH`0y1{0$=J^AX)2J^A7W6Isi z^RlX){XyPoxgiROz0%Q1kP-X%RN)|4Fdo%4bzd){(97!j+FnHEA+0LwWd>uraI|@i zD}Hcd^9fKhO4Fl$zdcnnCak?v{*loG1CY=V$!K|iRTj-pGYt2Pey725$;g6YQKN#R z9*=RJA0adKRFsRj$BhSA)zIrnH^JEBmx_Z1jU%P$8%$@U`I^yHG+=~iG*=sfQ%h%) z7-d^5G-_*HWlOHswk3#ibz>zf@E%rFcAf=ed=kFIIgJwYB;!v-F+J{GgRZ@6v0*EK z_biAeMyMD)ec3iBe+L>y>5jYHXpS=+uw5Z{%3GYq(KyJRV@At2772}F#+o#t(3ya? zQ%*&@cAk(LYTD@V=j6028h08nXoQX7RmY!VeSWCkAmmVg5lv{{OoXq+2pgo!r@@_Y;Q0= zJjBDV_a?)jKbN)$eJ#op%kd?iL`lTYc((esUJ?jTI-UU=93@>))%{=SBtaw#afT?7 zAL%U0(Ikp7nnYPnQHFyMO-LM*>nNF`I7%*$wq#d%H{HIC$$Y*@B6`!G;dtI1&w`BR zeVSq1p9fLW7caa=QhyeVFSowxLt>c^GJDcFL;(_FCUo?T7PPzw(qI(D5sh%(`|Dza zMWNnzkt%PRKU$F7z^NkP$2Q3t3w*t5+#R%?#wU5nzkI5yvBK{`7p zB&Jc?IYL9@(<}2q$bKR1L#4i^5Z~!o!wAc5dRc~u;y**;w?@I4dRBNui%vG*aF7l zQ?)?iC;=%xGoQc<0a&nbx#`i7LfQ*`A2y6qjSW83VY<8gN(%R65NOciS`qYGr2|yt)=5+s`eK>uNsA!r2zXO%`M{F^-wQnwI!zme4 zCXOsKp@3yP72kW8BVH0>mC_4;dJs^^6#54V(DOI%DamAjpnGNYvhIaFH;&wC+qiuQ z)@*FOWo2T zQuH!hMjmJ(eHe)xcr(^4%vDs@TW zb`ODI#e{x&i=DL%meT!2iWyY zn*JuS_(MOfp)>B|_Id!)&y%DosO!w5XG!@^HGEO?vW&JaB7#yA|1izfig}8PG|xQk z*T)!<1|7+Ya6je<0APTzK1}T~#XCkVz8YNaar}797D~-d7Md6%kG|<9?o41t*O?`s z4!MJ=r*TE!@}N$s?#YXFx8C)LC!XUPQ+K2%4}6%{5{VuU*Ir{?(n!Lq^Pf_Xk4MIp z$*rmG{@Q%p1)(w115mAx!AS649uAe1+Ds{jvGP)~z2IfdnK7eBsXcnMv-8z49-rjk zcP7N+FUyAk)NK4v%C$j^KW0J2TPZX6T8msO)H{(&t%bhTTWpmtjUhVF(5KBJb_5+Z z$I~YfylJ&3Yt<96KJp0MlVuCwuq&D}6j>Y1c(b&IAatV8`ilaMPPL0CLaA45lXrsK zgkqH)VPizLewFYh{Eg#;>tmz@TmZfV=a)Cl5Bi-k#Xk4rYk=Hb*ae8x=$&On58`lTN6j*dW(-vpQ& z$hRW|#Jp?;XxuRA3x#M^b-ZUoFTHPhb}Dy&WD~myZ-)6{-uQPhJUs$f1`FynW`D3N z3TVLv>oT0Q6h^>TmoEdLinF(Tz79aB(yW58Vafc%)5apTU*~jNM?aPAaB~em1&-nn zMjsA-OV+U+93bfQYj7o}r(T@z%v}BU$TQxd-7+*-T7kJDSZU8`yvr4SMI#_SY3H%J z$RT~dbQeOrN~t}f8HbcGAuG{h&Pf-FRlrQ;M$@(4!GfZVAa-nD+=~IWd z2+-u<0{ksNeVOrT*MV2Zfob81K*WG^8*DIs@JogUFC*tYUlh+vn6cumPncbzCq_v- z`&o$+f09s`cas>O{aE(pC znmPSZvc-3q9i7$_iVh$fez_{Ic}>pKass;M-cx$sU#tXcr}b9n^ybbgqpk(-)~F3i z`Q@{QDbFg%g%|RM4MIKb>fFcL(lw{?{aP;-Gc|DMB=p2JzF!8)mcz5fe;{V#Hi zf7VZnXA;0CCAqLduS#ij0enFi9R=qkio{M@W)xPFKQ_g8-XZUgoNO!2u{NS-Wwl_? z%9W;AMv#n0jNBQ#fxEOQF@SO^hTw|kxZfEw`4V4E7`zI{-9pn{k*;kVDzZ2J_{*7E z7%mMUFzZ)nj4fm4wtc2q^X8kmv|;Q!1-70nORe~K#@gTF&&GMZ_n1@}V(wE}QAi%4 zmSxMIQ=3g3O_oNUht_}K=r}wroasa62bx@KxkEB^P1CGan>UWzj?GC>W94sEQR{cP z-1uI(A%2k3Ad@@RP(0h1HX%9+_;&a7JZIpwkH8Q)xfczIpJo!}b-IPmKdiW_q?X^P zBz$I#Vc* zm>^L8Or}d6#vG=5quC~qsR*U5E6zYxi(wy@mJkN*iemK-tk^WT40Q2Q!4IQ@|9ucyzjR??5T8%q3$BQnwGNxWaB}r<=Xs+NF8TR-V zi-a@!V#rE!d_rO-0i6dvk#wZ5w%J2*cshq+`s5iKOrC%e=Bm_cv?gLUTMOR4;JAlX zwv4YcWM=gJHr9Bg(+QZ^Z;mkkBcJ}MXCPr}Um1aLtmuZoX$fs7TKf zZ7XI=9G?fc2YOjV?Tcz>SO*@9NX4p3VY7cc8Brq+A?(qa2zWd2BN{#Dag)&`S3G` zN8SaOW{S_SyWnk4me2EM(z_w&@1HuIi0zLBfYYemHP_B7~g!8(SkpqjPq?t{J9n$eE8d;MAEc;%Mk>*Rrj;=~0}z*T1@lzve-)Ruf~&wh6JlHXXcekkn^az}*)(s8;PxFXJe694 z#fDSVPyHNe&eTnh<#5!MNe;^%b!J@tDM&5xE) z`jEd*yHqi*|k2QxL zSiJ{t0^8o8?`;a8X zAq%4@{fV-&xuI5ghh9b!cX4dW{fs=f!G>N?Bk!3Fm90;>38iLk~n&dO^j1PC01}r-f&y$s~O<~Z*E2nNT6E`I& z5aDxpJ$-P`3-J#~(u=t7!IG?g)tKxgT%+DVhlnd%%6Y z9&u7a-O-V3;N9WpP2635h_1g}c^|2(3w5s1{vhE`L-3q)bqOA`M~ZkFKHmFz7e;fR z!g5`m(CH>8FJm?zh1}t$N9b1@I{nFmVwjfbK+VyaT`(X zt0Yf#D zJtToRb8e6#l9AeVx8wfTN%hzMVNlbbxLby%Zn{??_OQWBP1-t>2!rC;75v1v0$3Kphi#VdTxixe#`0X+UH_(xX6sRtcGc*QZcH z)YG&Qt{w9rxbC%p=d27jX`S|$(V8pn6GeSRa85P}$5V8r$;_yKw>ANyPCEp2I3{M}um2UmLl|->1 z#H2zjaBuScAo1fUVDacW$Q9ZW@H3+c^K!h|Af#E}f_cD^VTn-%$i0VS zp?T$yIc)>chU~#GYuSI|7EHNafLy1=D}az3z=D#5G6)1G^sik2{l7jzFczjF1Ktl0 zVr-0orK0}c>k_1ZP(bL^MP4M%a@$C|w1G=4wOD+bz6ehm<4zOZOUP>IRYbUYWe83K z)wfzLrjjZD8;N!==4@$0c>O&zJ=IIzAyS|u*s<&=g^f>ph>B1O0H(rtPI{@_E&vBW zWL(X+Vh}G0?!9xxl1Gjw76O*{hJt*_hZq5iUM`pBlj+#`_`7(%S>85` zmI^*Ti-}@o#L{#bE`2n_1K&0r4HLn5T$;glxzYCC z#dyNJhU$AxIt^Ut)X!QX>PGve+0bG&1KGDU#TW6U6OQ#^H5VHm&tE(~*dH|DGnG~5 zrB_aJ9$MyoYEZpxsZUOF7)oJhAd+e>YAiS32doEb@-cBt)z&R|F`dsv*4b>M+B~l< zg&iK1U7Ct8yISKFJ{+1Clw4iZa76vFYJWG`KI3CA># z%tEarRnQ*e3M2ekDPz4I~U-k=crTh+{uc)}^t_ilX)suQ9)GE9qC@n3edguGBDlL8f zbXu6(96A10@n`k2RI|%@);-7RmCEnpTRz_)2n1i><~PTEXAFKTvC3#yZao4IVc?Dd z03LnR?ast1uq)h37RUZQOm?Wx`5e|{P9OcLe~usxC9T16=ww_$bIi8ykW-h+f1~0V ziNJ%s_pnGEulVIyU!+$8>64k&(P35PzQ^0#Y@;TS8?*5^`H9Y?Pjxg<%~;xUG=TiD z(~lN<1yT3~eGSdGc*9PnJgzJI61e9Ab&YHvOSBUIVm>}aQJ6UuEmrL_82O9Kc|Mak zZj}QIdZY-DrSZnTQB&Ssc-F~6QMfQ$v~1)Ru%3d@gmigTh9W*KZw!N>6F=M_B1aM3I4q=GrIcNyj0)+A>>sNGrM$ zww892h&xlZGWa*wqy30GIZu_+X;RjbM*ip4?St!UyCXwkCHlLy626b6nR(%)D%384!|5-jg-s0Z?9Mir+I<>#6l9cFj`D4q&463)qq|Du`EsaGNjzf7Z4?9szRW1KNM7h&5zrlkC$SNZq>0MGtg%`21x2Dv4Q_hA z2d2Z*r1~QEVR(ttC9zq0Ir#JC@q-1$xGu%fr+uQeS*^F3AJV3cxM0oz7en>%C&dTu z#Dodh$K-^16P_pTocWFSrYefl>>G@J<~QP1gLMyUG6cPe=$v2>uxg~jOO;z~;zQ0G zZ>^x7#fuGBV7^e|vdvp0_#}%Xj(}d(e;*aeX*Ie9p|$#*8Ge=H7T%ib6yECB!!#>- zZ44HwXSO)`nY~1LtzNQkYsDV!DyPp@p7Vj~$GqJ{iI*#r%etusS_3p5@%R0$MO4HRN$ zey7NZB2_IxbS-a!pQISeE)O)^XP!R!0p4?+sA3Rv-$zo`NBkS(_dfHrWig*}+&ljc zr(NkN^61I*NGeI(!Zi+?Fgco(8J8w%E$zvNcqsfIxc-?*m!hAXHkFSj^W|a zVDXS2e7wZ{*UQtyonA-KizNFRj;%&4(v~|2mc1AFm!?N38Pi&*7{(ziXChwVyXaz_ zFI`cRuhu*<3^gu#?RUYK$%4&wR1NA%Zogm`2NmMNW@2Z>_hP9>>h0Nj$?LXbz78MO zUfQw{(u!h~omCwjqw=Cc5-g&GC;MWspW!_)#qC;LcDYWY!5Owv5w zkHlhrR)#v22iuinY$OBNBypO#GHV87v{pa>Gndq5QZO`_!fhgblu^dv2EF*V9chT-WEV+%Xmz4Q@+Hf)*?j#DvvwAZ{JTD}M$@yw~>fO; zaAIf*oeGyvsNXiVW$pI|SGjNlE|Md$rW67q3#b~mD1Q54C&zFGzTYqD{juzDUuta? z+VXNY(7@_cE?^Cot)Mm$bCNV0?q@q$_BXzjXWt*qD-3}@oJ_Q6nD2aEVWCJ-qGlFg z0`Rm={;ew5kqn0|nr95I|MZbzp}<(Ir^yd*MDC!l2NX{Dyt5B`IikNf)@Bo^=gqF0 z#hdgX^IK(t?muMS!FKzl9SYK9t`QM>cuMi`kPt&OMS>Qndu8Du`~O|M%ASv}?1yeC-S2pTEXFr|MEsJJAc-2%lUzLeGkD+d$hz&kAp- zS-pICJsFX>+>_?#9N3jDME!-&@bxcA>p0e+J#_WC?GkqxuD1HwPf@FTul)JPs)mEg z{od~9yo9LhC+DA~8x%eSzp?SHZ_WWZ7E_G-VPb(&;(4>KH^rC9)m{ki0bj5;h=U!| zwhgbaRhZ>F8=O=#^#{Pf3ykzT765m*h71xEEI~h2Svw=d_OQJHxhV@ufJC7{8~@(b9V0SPI++i z40>4PytE~N#F`CSxRK7Ew{zrcdARrABKK-=(r9}o!_9ipGDAvGwHF_GxSHP5{8s*4 z(-naZ(o!s3T*@V4XE_1db5jfZ%g>T~v=Pjk_)f$Oyw8T6AG5ip1SuTKgj~y#wU}IU z!NLYg6}%ePdjX^ziit@bf&6RM)8cAgf%R8Y4=}ndGifE;^n_L&adf!b4kXc!^Il0< zry8H9WM6;#!YYHINghWKJE?YNZr|ynSjeX%o7F*7e`GnY4({-tt+Rm7zWsIRuux@k zmzrN@SA$@<=1u5mQ4glqK>INSAJw;rD_R)P?aTt(V?;Dmvx3HIfxd@c;f;W`H&3ue z=_Wz%@lzChq3&j$OEj+>{n&{hw??!DXcIU%mz9G8t)ommZet2qeCQed*15>fB+RUc z10EC>9ehF8LFQOsZxB8VXE610mZ1kJwR09f1%84g?*98brPkCYz^04F_5B0Y@%M z9`%z!0NtBfqVIii1T@#UssVqcX{=%*ZVL12t^g*PGIxZ4Rp?TcFB+v%D6{W zm~TsN3AmF7JH5Y=t+do#2zucdQ&Fv?IcxaQW!HZXJ%>*7K+^>>TSpLXTr4ES+#R4@8)u7GZM#>YK# zp+?+t!wZyjiWo)j7Aj$c1470}?-uBBUc-|L*m0Ly;riJRzXdAi^R|T|>};P(QbdS3 zxkzMb{@icU{22ZEFD2PtC)+@a7nCl%X!pvQI>C!WTH?ndo@}IOZenK~9AalCCSqsU zV;13yHTw_87f_H5`=w4{XTff$Y-bXZZ08xsqYhu}X?MtbB_yhBXMxLaB2m7ZSvT1{ z0+-&KCdL!fZRJ~R?D;T%I+$v75Y7)GP*onbl*WmI=f*JMtRS@HFMPSu#SEqce8ZGLrGm zw{wDmMTB|59jXsn{fR8h70Kdr7T{%K3HdRLx+FW0X2kLbpB)(H=3rrnhat;e94yw2 zKKlrY?}~^6Z{rI2YC3qD>es=$4}_Oha~@!QsoMGgP?k~BnY?5La$ZOl$wb_7YJGqP zkd-5EoouXz#-8VHdx#Ku1F{rq%X zI;xO;@^uBta;lUFC?FN5P<2KyLv@Dtd48-%QD8&Kx$)OGe*w|;)HSl{W0nNkcLwJ1 z)FusH%iQ82V;daKka}(MAM@RXJQX!aiC3QUbjJ6B3hvIaV#ds)5zjX&!$_JbgMF=q z`7UB9`UC*bE%k@5#zBTuCC{4)m8iNwAY!2#S#fB`2OAw`|Za0UBD$WJ`+Y&oM88v_?m3JH0 z?=W2_mXEVt)kbn#J(YQ|l+We|JI+HgU~Ufc?i`L!zRj(&v5iup_wl{c{Z#<2|D< z?2?a=6P}P|AzZvuaBl^T;BV2@X>(Gg_cNiBAb%EYCQqsWaQf+aB3euV&Ab<}Cm7I~ zRIa%X#}dvbP9a*$T4o~YWXmvgXti);{L~TtjrN&$J9-@2(nD_aW;hS#AqZMzsFN<}oE`S}=S8>|Ok2>ZIs0od zK3N!J)XB&60Py7`Z#Lv}W2VMd%5&(ocsx?!jlyIyr~&3x+&>96umW{^27->Z%ij5b zvRGqLU62zRV;p^VI3as^Qs*})BDHoC{2wJrqGQGPKzH(QNRdU>$v_!HHUJJxgrp&4 zJw(CxRPhOP+N@B@&JCr3`Chd4wy#JbCaJs&V<=YBm|E#L^qX`vwerl?P+K8Kn{(!wv?R=@UZiNrrS!r^vW8rf$`>Y+UJJ2Hi`MO(sjK^Wu#t};kk??izI=`SG)pnA0 zZ`xKK;HlD1!ww9(g&bFnVEj{tiv{=XL{mXSgO`{y@PM-rr?HPc6S>MWioWjbec+jNafxx^l!hlZEPw2&XHJKO9NuNjDE8M+H0 zH)vIcINo60Wkldl5fd89-jJm!5=E3qOeeeX2^01};c^!quLqzG=RZsg_^{@d7H}tG zEXQv+7xvd{zM#90Uou8TM+TH^gjl0@whxdQaw(N>;d?9yR%npN>joMx@l;Ycd1i;H zQ70A`dxq%n6a=Zgun5S2ct_LMm?AE@qF9$v1aeer z{cIVpEiU{*L3}k(7l&km3CUv&n&T zO?@c8vfz2%${9!-tJf)H>jI)xv88sl)DYp^2=^JGy8kxvfHppH{G!;lOF?bec|6~l z$$WZ~1FuCLlVazUmfn+O;jWO^uPE@jy}zIxJUgJa0Rtu3Q^Br26?t)lpK+Q*zSqfK zLXuJ;x)GLZ2J5>=diXp%#0PJo^mT}vqxB+LC=CdPEb~#BqdtLudLEZtP<0nw$gMGX zNq|em2t@n<9z6HQdptYld~-N1_<(+r8tujU=NXij@DCV#z=4h&I^2icS*z`3EP&rt6q%V9}bp@{lu6S($0T5nDJ(bgzAF#;Kh;u2=;*>jOK_&csHM zrl@O>_GG81^E5u-;RsHZP`~fV%9F)PJ4+9_``xF796~rDRH@IC)_h+et&GG}{8;7X zUwX|o_s37+z}fDzy1c{wwQ*}SuND=v%Ew&=7p(kYarfFBG$ylE3)8tjD>+Mz&)u(p zItq(O87A<@XS_EnR&Z&))E_Bq71kEx7rvm<7*?vkuouxt(ZA6zZ2_e5-0ws;5G>eQ zX`gxKKMICL;xrsdkBZXI>L*fa6|cenBEQ3{6d_lmij@>}j5XT;Rtuc*+qc8(`vhZO*PGCzG>n(X(&P=`s|4vv|89I=~s zYKx0|o9baYPu)`azSnU{(Bu1wB6iU~=e}%Y3XA=!T1skEim~yDCbldxaGQkMlVjKi zbo!zm_D)^6*=wq*f7Vz}7d`Zo^YzV!^H(AobCj?=Qw|(2HAry&3;b5|7i^QHt>!*` z^9oL2q{TbUaBqBqm47E*qOSBJp1Vvc_p0XMIkn{E2^&S7R}jNppa^C*KH*^xeJK}) z9RUA69J>Qr3t3GXtJ|%K(F{_%xRlnwP!v+PST?wKJhvEUe9t}fWq*i4aSZ#eOp36} z1$E?RoAlnd#P&^YzLsh7;Rkj|0#`-`=V zVQ5%QD=yxmBX~l#VbJ!q@`oKV4lEgcr|2?XV*9(XZZ*{&Y)>X}E8$A!UpZmI6~VW* ziv4qafu2;5MTlz3Y^oZ{VPX_@wC>dIAJvp!%1tIZ`XY+WK+^L-ng@vR2~#~$EP=T2 z@chh#nv7#;B#c~wESz-|Y5CL#;}7h3-u{|j&+e4F{O@ljUd~pBR4QK3iLPHV0!AXYSS@PUG~Zoe4Hz45c){CKR-`h z-9(hLtlFOMbk$W_+uANU=SW{2J7jC{BgkA8>?`ST%Vb~v14ZVm+x3a~j?RQdo&*ci z15evZ+*0oAvW8H@m}`!sRQ|M~|2oFD3Rv5sS^nsLfNtckeox>jt~tL15&{1#fQo~` zx6%uW4;>}CY}ajlp7u4+QPPle1ANRbG|?58uLH#Mx>-o3i}{8{nc6tILI6zzU&ik~sQ%4}14Pufr}m;<#^3uPtK~b9 z6Thcm`87abZtpD*4dVz>2AWGmzpva4pWbG=EvV)*|8oCmRIF7 zaV`fP@TKzppvbQweTuZl6nNty5%8F~A4blSYSR%Z)WVlUGQ!nCmj2X6E_C<_VhGkB z{=++&iAg{j_IOi3`l40Vrtku)q{uC=D3YCe2DTdQoayY}@QoDo<_o^vpzGLVicwrU zaVHxHfZFqsjnkYW=*xDeUqdFdP(qEhyxrp$o}Twk0hXRg0R zxEUJYEm4$TK?|ib24#$mDeu8p(217vK1H4{WJwOHPXY)X?y)m=(s8KnFC#&`A<+WC z5Y544NIeHLod8TdPF1O7>^t%shl@UAlq~2Fuw9Q`W2-8>5?) zf-p*y#t6U#}4?PVd>kIGC9!=u>_0nTh>Rn>2*_)-=z22MVei$=Cf zvi&+}kIidXTkImvaskfn7T0A`IQw!zY5Kw?3d-L9#o)qs|Ha^7#<2#Y{z9B&x6}*P zG>bo}(rd`=Q_<&EMKHW0$?(#&BWdxpleAhI@U9W;j=6FMFq}tLe~&^6{)6 zf8A!s`WpEtbjR#~qxey)akB8lAzL9DWh_f2TAEs?&MijH-T#_qT0X`-?$qJQv1tt)*-LvrFOmRvg8;W_8Ch4+q+6Lr#k%ZL``TVl0_SPjNp)j_C^h5*={Xax+;6I`__xOKAZ)6JT9I7W^o;`R% zp5N`e80rV)mf&uTm&R^PyT+Y=wN4(pM?dy8@N8qrdUzx{m*$8u9wMmUPGjysOzeo| zh0F#!#hF(Hr}vNHQiMC!xx5J)wj5#VgywY^QZ1v@IyLS|wT2fes_9-432V=7nF@A0lnuo=>MMzGMOv|P}?15k;$7K_N| zGc&RSsWkpm{w`V1lQPZElU6!?|GsJLkJR?w!-RX6_gQj&TdMxltn%LyxX;v#Z4F`G zC_lp~O5sVh!PCo4`64Ef^C>BGvurPhv*7T0IJ7h-lN;t7*+hbx>Ypkk=)bCvDJC)% zb~jq{vcYBIMxVMFFn+8bJFnucY`}X&caDCol6oq!$cGk(VR_iFDE$L(2y8x z%c&)AKT2R{If&8&2u>m`(K1kCKq;3_xNb~UZE{bafYK&Tb95_9krSv@Hv;cMtG#57 znPVlmJc)Aq+-H?kTog>sqhy>)Fa3gV!iSFZc2u5EP*2o|VP_)}?RYG3PUXng|MVY$ znN>+PN@xGmf4m4?)z~-1-HAwz<@~o6-UfJVmU$>R?W#*ATOa9P$R85C^tVxL+JwI6 zNK^~9FC%d}r;J+^JXBER)=q}e_41TbalIOT;sNv)vmbD5`;Zhlsu}EJQWdx3aHIb? z6&TmZ4gOo{id%F#8bfNTB}lPu>g%9Rzy_(fHh>x=2DSkuGq17UkS~{?oMx$$3%vl- znm1mJ!V^)mUikwA&6D5PNKY(s9dB)%sLKjrUzT6`u`~*vH5m;#MeUR9v>{q(+3sjH zIGicZ$t&jhueEts{(1)T0>al(?o+A}WpOV&tN6bAM0C7n@alaOJtnD`6%<(>w#pyN zTgnowGU6xN^`Wl?t_0)dy0;o#iR5LSRNO*-`?sXteM#QBChR&L%W=4F{VBFTchc2eMB1mPn zTquHTs#GlBXZF)ET{jyDaq%e7%Q8yD$M{O*=3^w2JN4E~ zfj^XM=@8-O!IZ;$zEI&Pa`VwxliXELHWDPY71vNBe~-WQyitrfIziaUagM0oQd&WB z!rf7cy0bFk=B)-p3u45q-bgfTdkJbe$E|lkZEn=@BVVdRe&P*?mjhum1|0-p1kn%H ztoVBD;#36+7$~;_(UUHjN)HLD*d_vP#9bB8EIw2BYxlziR%!`NgK7VN#PsxC1P>O}FbQjSFK%PU~723h$Q z@*4I@q=~`<`B{VQcq?k#JRb3t5d*kg$sa$y^);^6P%(@vtC2a7%Y!xR-lP^kM#SpM zaO`TUni(|H(abSw;z6vdi@3i!HpL0@)*x4uhd%E8MNl;>)1-;J8&x|pYjh4zG#Auv zL)@$Ig5q769)?og0iznrmcby~k*=fk{xJ7ChVF#vkL1OMQ$zJMP#Vp&a4ymThe&&f z^n_xG)}Wvf=C9ES(!vp6cc)Oski&4GwWu*m#n;VbD{=Nokuu8}yBqM1ae>i(MLMn` zRCkHkC^ys+LTJzSnS|atH&X&jgoIHgk)3hR(6dEY=|3HVAc$8`qsvDiK)Jvj*sZ%j z9Uh=8xu@rjiVNx#(ysf`(p*#xS*Sy9;WKI_8*zHqM~G65cbF2#QMi zbv9VFKA8nXrAS{#LQ#O!(V>9Uk-9>R8y|!cK$>2cNL3y-M@Gr~`FT)8$pLB_6_~I& zl(9#}4Na)AV}5vIy&1+bIFNOWx|aQ5Y^$2354R=eDY)b~)NX;e8|7X~*1E*y8M}|O zfJ4WhWhJG0L~(hrifO)Gh9`5ge<8!{q!VZLir(dai(0RQ4OKDclWn7*) z*+x=tWv)9q-WsXs?mS@w7XOCR?YgaLD?841cnqFS&g^l9u&%REg6iS-C)dX;*V;5g zN+=IkN-a`@PME&tD2mUo67f#4LJH5&bJ%P9;z9Cd;!VbnUq@($Njta`%YpgQRV}2JDXAsJ=Cfs+ zi{Ho;r-;E+eh+->P%;9COF60EIqfYdP&o=7Ey8ub_AFM;pl9PzvcT^D=Tc{kqNA6>r$K$m#|?1`k!D=_@oKjY1Wry9!NfF-_(PjVA34EPo7=|{K1FQ9AXqz*YpOb?#4s!t?x`v7YE1!QJ`e&b z0!kRR5y2uRYnU9h(QM&6T#;lG_T=bD;e(XD9!FvVN$7hZ9l9?3eco+(i$h3hNF3 z%|RdLmD@;qhCr71+YoRrDjaF3m<2nK=aR{stZo=#w5NpXXokX0;lrvJ9`*$0idLr| z%cNCHI|c6jUzTqt5ojP;V95ZEHdarzALZ@xcwOUqV+)foA|FDV1R5$vxfdrwB)8x% zy@Uk=Nv6&`^-tUKut3{`br)*MKEIoLA6#}h7(7)|6)n-#MZh5(kpQXTdFkamBxq=P z&@u~GWJq_d`@>;Rf9h&YRi7Exuk4}U0k8~2!;6wOWVIq>SjoReNu8Mg;wsWv*(>FE zAHVR-RUc~R7bm)jt0v%XC(^HXn-USAE;R<~VgnSeiz&-|ovpEHTsRM3(O+xUp0VFA z!bg#Ro6f{{Pn?LN(cYPy<7A4Uy&BCP&n{p|9n@AK{;Bd3Z}-n5`A!1&1&cbQfe~Yu z7WNgci5n=?SrfgwjFH(n|M)yheGIN_Myd}58qY$hdps-j?>s9E3XgF#(g^5=Dmpm9 z5JkBqy=%PJmK9HeGjRdVSY3QxwaJ?Pqw$qjm8#LeE)3`tP2*$@QQC+9MkNmeO`^5h zoEV$uB?DCq``P>iR(VQky`U8H4s}-26EJMrelOjJ9*hAj8vFZ7lUup}`;JF!&L%0{} z%Y{Zeb@6CgG*r4VObs+!L_Z%1v^j^JutUM_{_h}(g1fJ@#g;Zcc^0mx>Ghh!P%T(m z?dxQGvZ_>;Vbn9h0m^I*V*~efL~l}4b5SA7z{^CB8o_+1=;2uOS!$b>VR<-%ZjSn7 z-UEfxb#Q9D!cNmCPPn-A{6}En-3VpPRet|geDE(Wz%sjkt5*stzY|Y}Hp-I2#bMec z3fEnTqg1=pkX?;f^v$DM4$!=eUIoQthV=b2vuSGWu)?0nali7B{o)+x^uPL-=mEg? zz6&2wY{Z=D0Z^fwk4fI&_;<>k`5Ci8mm#UQiIrlO?0``$uhVaOfMWmu{s!{-6Doc#jOLpQDs4nc}Qz4LevtEQ&p#Gx9+bH*wL72fc|e zUsD&ihpO{)E#DJUrWPjD^So2fxU~$eN7B$W>F;Fb6~20wBz5AE;_tX2+k!%Zjzb4A zz_+*TuQ|BVq2jV%L&eid{Lz3sRF;!YtT}&mSIL-rO9@m(r zFLw=|dSg9=T1-jIdTHL9dI1W zvWA7q$|ca8r{q;Q1d(Fv3EIbxcR03Xp5eubWug4?a*3b)Wcjk3Zt)FSlDs4%D%(n0 zg>{*%m6FQ~EDf!Bif|!H%`wWuqOqZzw%01YK`H?6o8hTgWIDoYjaWsBZ+Cm3nPFr! zCxl6?zG}*V4$--Eh*{BFwOKrdBUH&V6m7th`(GBWPB{tigZR}1UP)l$urN)DShxVy zXDP}+(b4jJ8CIns3NR_iYB(PaE5SV>A>tlLdoPw%uJ!p>^Z>6c^@eV}^AM_cB_cc8mVs6ZFM;N?$3H9Rc$~VXu#s zri-m}0-`+S4Hh+83*vbov%-Uxu*nX5e8^LpK+3YJNiR})pGXdjbE$aPRZs~+cA*dn zJ?9fJ-nS*eMUU0jl`q%hoU$i`jWiwMq>ZvO=|}ipEF_~}qhcP0IqkxEqn;Lxl+*r+ zbLJyd7yKUf zpWv7f6mV(UV%uR0$HM7cw#f73MAyUFFL(d6SkIN2=gpaI4{S}?((*r^USzb3bXD;} z6Ibac?!;HKOX{jJeN}kd)LO5EQ};b6xD9o5JF}<`D9|V0-_ybb?THDN>USN zsdidSRi`f}i@AQFVBkn7>M|8XHH#+(X2hDhh{qad#!f-!H=(wFASi0p{S5(phJ?*=;}m~R|qRI zmo$tW-35Rh|6u z%gpm3>Is^;Fc9-OQ@IrzJ56a{Aj1$ho=UziObEh!0Bbw{C&7o9O-;4%!|arFE_X8! zS#&mX6~`UcSRK9FTKP(pT)7-pUmd-7@j^vIu8WoS$~6TdRkDyxwAkvJjMk2a$Vh*5x=bo*(d+j6sGac_6LfB5 zk=$wd=S$C+!J;9sPwQAx=Ar5W;wWl$L568R5Q$IIgDdMl1VBf3pH&t-K6EXJ@P5^| z_L|vb*&)`#j0w8vTPGBf-C@jf1}{%et)O@PXidpbT~elywX3YIvay9cF;tuob`a!n z%5k*Km<%G)rbGDDk`~Jzyd~6;)%3cgw0yZ|xc<#hI0$k=5!@>@~CH;dbZKn`02PMX)HQbNlJawePQJXuOAh1)JJZMnIg{EMla=bg<( zI!J3X4X)$|EsL&-CTu@bNhyQ*(jnxk%*g)a<=QlLa-hVXLdVU{SE~VItx>)-9#c=3 zTDuL~k&a38S$$Ttz*obmsw_H=p8^F1_?U?qfrQD88jIk}B1jtQo@e)7tuRsPxbI-I zeGd@;LQfkB3VuA6jl#%6*zK9hoMW;Dx($Tm(oW3PAhmk+Y%d;s(}um$u%D>%5f;18 zvOn2#JSrsA7l5nzX34IG)UFoQj&A5Qx88`pZ)n)kEz#$9Vhtq7L0M&6d`%Q}`jjn4 zw+!D!{Y^zy@M87VlZ&D$43vR=Vi6>Ly`au4=;e6&z{vJEaedLokkEZ?too>mnZ`Rq1LN_ykh%4Y zzL=p-+kN$T5wU=9JluC^y2$es>@iT?XjUH5JkQR_y;_XAKONpt`&*1m_Hw#Jb z6D`LxVmdHoZHLdfbO{e_tzbQ?t(yo*DwFs7&%wmDr|#ZAf+^;SiYYJ?NnSFPeCG+J zxK0;*a44fZ-wbgh=%YkZ#EF*Jxs$bcCbdH_i5R2QQU<83POj*esmQ4U4rTY~KXg?- zN)$)%Y|`;`A{xp?IW@l9eY4Cc$4bE}jK|7FX{|&uNB=&#FV@?jUT#QQ zF+ib$c`_l^30Dm6W$j6FrkBLgp)61NIaQlEHF0kUwTY-@t-{=kOS zPr1e8Rg0NB0RfXHp1y}fG{t!E#y=cVC+BKVv0EcYR)Zx-g$FeX#`@>o9s7eRVrXwC zHQlzCC^_w<&_zF^hPO81i{6rJn(H_G%R&a}v9MHOvJa z7ise<>dAEFLjCA<`iPaJLEBMN_AxUyip=1r%KJB}a54t0#jC6_0};tGdR-;dLafU# za32M;#+{Lky$;6~$E>JfP@}&=i+6%r!GYrcZE$8)kidG(w#G9*o#v_#TL^OKPS3jH)crkPSBH{~q?)~Zm zv&v>YwUVinFJMQ7r<|)K>@S%o=+yJlITv{m$$OC-tH5b1dBEgFkHO`)!lvqsHH68o zKmDbpCN*y-Bs8!DNoxjO}O_CA7yp-okc#Av^dy6jkd}lx&m`LFOnJBQ0%44^-OJ z&h3s$9QAiDKpv2`49l$27#($DT>~MH?gcUpo);k{hSu)u+^x;-vfzdGv3xsl>%Mrs zF7kfiDDV=WvckA9{Fj`Jy}*&N<6ctF`{9j#U-3iMxYgUNOQv9pP@;no+%1a70;a)V zp^=Qx&7BX*KiLH6@5Y6;%r4|z?oq*r$d$J^X)kQwXl2eyF+<0`z|A+Afrp!B?jp~N zs)->0gzef6Z(D=fr+qr)*5y;9nON9E^R(R(0q`8rIOjRPA!vD3!{gZoA=uH(%RA*M zJIN?sxdz8|@Ni9mhF$E<^AjB3yI^JVs$xuhtD~}_)(m4)54J3*MJx7d7X7dPWbCe z6ced`6UMgRf4`0Iwq!vq{jtF644PI~c@7e$Qm_p9%t6 z;CeZ7C2D81E7k$w#O7xxpbO2oQ5Wa6PF&*t1tveHaMA6vPhpwRqJ7*}bG=F)k>*7y z7n<{Q!v!lAre^QKYcY)-9)_eIZ>ys1I+U%h+4zkpyFt#1fpM=ng=x_rY5zqj6T##LHE)aPwjzS=JlX_rB{yBMCM`HqX=Y9&~ zaCKF{vAezS9MveWJ<-mUVEr`tD>m!;oNVzBTawXKjh+75+x-m=4TbkPR0&_x9n!1vyx2L9~)AKFq;~6Q>NU%g{prD&+)&F6?c1rp@YZf7k zt>>aQw>A+APg#VH^C>bv3fue?swVo!<6dLFwneTgrXtEbUyu*f+U6LQ@)8nk4IEZr zt7%`|^!&3nb+;e|X3pi@M&h&5nV_ySuWwER1~Z%w=x#LtxRyzY_!&lFt+GTU~<#qH|Y{Wa8h6n>$0>A5yO_*_x)6n z;w1qIt4Rax%vOme=+g zDn3oC>aUIXTz=Jm+#N$uY$gJawi{3cg$7oI7nGpLP!$87QuE6S=c>z*AjSDkV>#CA zY88h1t!Vg~RXsXoxioWKw|*smwGSitOGt~%JGN0Sju=gQa=P$;^$O=1`C%{X&t5_) zVs`Qj?+uP;eU?_@Ds>LSTITQS9Oq|ympU$mWKv)HU{nN0fmwL)3AaUrD~y4yPb zBM?aXoZl|CrM0Wt_Qd$|{i6stXPmC`_#)=;2ULRc-E&!>7P>D#)WbPQxpejm-v&f^ z14pCU%awqN>*nlz>BOM)^KU2r9l9gt#lASwtm_AEW`$~n{8TAvlxk+6CIEw5Pl`(5 zgQ`go{140fx1u5>3C`Wj>M zCj|^71B-2IuCGdvGf8}tNf8RzEd&P$r1ANuyiIKu$}mv0sj(CML+EMOhl20KM18o(}O3rlOY)qG!nkqvL$lX{iX86<}=Rh&VrJd^(Fx`w+eCJXirjA5t_CuDK zZ{t{zY=h-@l`xbeH=N+U-N!jp#}A9>kf&$vkwU4q`i|z{XduUdF=3UoMJuIBJ<*1OAtt92XVB$VU? zR~zZN2H3A=`XeMn8|A3<>(A+a0W_{8C6}h1H<|hv9*uR<^_*;km0X?bBwATk`H)qV z6|%+1Sau@lD67*RsEsMP&nMeUx=P}UK2f4aQNLQ7aGDR}>99r}oojGqtQ}k62)XkO ze(ow-;uE~w_wZ%3c45lfXhvi?{$k)?nllG6`l7eB-k{5fq@sAqXjp%Z@oFBWg}|eZ z(MxTez>QcQMn}rzSq57=mR(6|TDPVltaA{~V3@!nHDxJmhdN|NtF;uT9|;#olMXJd z*(Ql6rpej`8;3zmC#(tV3~EBOQ%PzD7!#Cm`XE*T9~(IBu`KfhnI^VqPHAcCiMSUZ z--(iXtI6hfFrc2)xb@){R(pv|=iJ=JCyJcm!MLtNdlEF0k}m_*V-vN2u}epeS1LT1 zJrL6y)%hKF*voQ5_0$c&380iWNjdWPXCSC+60nsGDLC>vaen~$H|5A1pP(DSJ_aEO zHIlNSe0?tUR1qdbbjvh=IVX<&>vi{GG#E4zvf&B@G?NqpbWH|6vSA%v&{aRV97(4j zF~6|acPYScuy0t2XRyELl%ve*sl{jiwrr8MP)_d|N-U`z3P`sW@~Wplc=f$i*05*4 zgEQQWxbtr!_pT8JwRrkflg{qOg4rN_b!BgEkmbCFOG zw}5VFJ|B?oH3^8MsRRNJ&e6udD$57?0v-Vu8tg00AP}vOl{=d-y#hhLz*AZWYXzPX z!@6(3wLHLgq)Esa$&?l31Y{npl**yr0qkW@*=eU*_>7Lw8HN#S zA_<Ic(-u#V?flkGn;(46i= zoV%`&V!ixDgqB6{oIfsR@FSdX*BOH~u#_-?OJ4K^S~WCIy%~LCWL;^$)LxeP&k1)$ zPIAuAD~Rn=EDe$+ks~~xq=Os`uwlb}3rd#2GTxUp?FA;D1SK!UCJ$5yQ5}MiM-kN* z61TyU<$_2;M(cZldnJI7QL*DWQ{0N>FBO2h%n%}8xi{5Eo0`ZTFy8>Z$?r|~7Y3&% zu>P;--z^NDnusf-aL95t_PT(f^{Rcu9^d=iEi@PjQXXSfPNVL=ct&GsD^SQPan8f0 z1?}Ny3OX^cnIUKE&WJR)C)u&pgUsL6ht1E7ggD9DFKJ^wr_k_gR`c6t41c+>@5kR; z%qqMh5%Trp;^egg3@zp( zK4Ai7&%`3!!?)!1dVF~BPx{%zbo(q&30fYPmYPyz6FwsQA`9D$>4d;97O0KV7gNue3EpP|2xCW`=JcTyt||@GF|Suq<~dPO*-RbClXi zy4KL!CD1Hp_gaQMc_heqD&h0(G#c$4e*PxouUEya%4HXlyw!nXly1#SNj#}*Gd@M* zc?buw^)}4;2D0eW%P#N9zR|=F_1d$U=n5A!)E}d{Tcs0woqRwx_bO0xbMnUhVQ=iJZ#-CY)S1@nu3J9ERs=LwTe8Hd_mS$TGc<2d|}oA zL{-C+OQWeiXRa+8;G#jIrJ)FUpEU~eNTcPBb;2_d@9NiCdPrpz3G|WuMcc@m--sAi z5lLEPv|l}~?!-zJ3OO?-1Zjt;J>U@y#!G;6M)uPd0Rdjiyk{rtj~y1-j;mBHA`EI! z2#Gvd891oI7cbH29fKhepsr3;a)s^-B1gUlwBxN>h9?RE=z59@*m{6lD?x;R@9J*Y z8TDdy$f^r7qBdTwhd?EQdp;)2BaEGxLFt)T9^*V>f6t|SL^^@IMsU0G3fzgPK#v5R zdw$VG&Mofbs#5MTYiR$80V4v&iaL>Zo3MMS?2T2usYfmEd}1{=^%e<9Af+87toVqb zjy{hvS$W7?dSwPdCCWWHA3{<@Ie8!@9potRT^NgkJbCn_2wFi`sd0~5b-9szd3BrA zJo$MKDg^B)Mzp5DXA<9j4nAc)+Fe}gcQ&WS-2Gw?VMY3+IFLw;w0iUmt0MG0XMN9oo+)EyG0HtDd1!s`dv?FI% zz6{4#yE@+in>`O(KFfxAR0+Z(G-Yt6%3r^#&pYd7@pO#XBWrPn|PYlEl zcT>R>$2rP>p!o)&fly%*f^egJF_o`};=WD?KLJseul9$CjDfWSsM22&5D(lP2cd(| zT+pfNw|3|S=%*(^ZJi66-3S`H^zH$tUP>r=!o)LZ%Appts|`@{!=-M~o28ciKC}=< zG{ju9+jS69ESqGCpfhjLptAxvVqXXr;Ro3hgKuZR(})9}1`@diEgL`vkN_b6o&+Gj zHP?eD|J6ok4J}>&bmlLbknU0lpuGJCpp?K08i6$hBqq;yKvM$E=qaBq7qNf~lWVaD z%!M&GAL09;6OaWs8jyvpjAX3{A+3ub}^iqL?KJ0OF=Dtc`@@ zMw%|6RkzO}kZCBvk!jii;+n`{;gv<363A^|Jb!j|qoyNmZf$^FLe7+ZfA(r3hpQDF zzLcc_?12qK#W59t$K_J_n#R?j*%H14f0um3`4Wu?*BW?+BoGqg$L94CH>J(N79| zt1z-K%*tnSwG)80^df({jr6TmPh1JKxCB!OYAetz$Yzk zGbK1t99B#yT;@p@@#~U>h`&~Gy$z#8lZtpRv|Z1{5%%+Ns+MBde}#uqUO3^4q~J?0 zTn7TE`=!MW&#DT#yYm*T6!)E_SqFo0?CUeiO9kf1BwWb4F=tfWD4%r1vGp$9v4gxY zK99}=yAw~J`VpYPi`+>3_;!(1gyYYist+5h#(dm8%`*qrZWoL5Z3>Nx*1xgSnJ#qD zDwmM2K8N*ianCnA^q139-7MWX?F4;yN{*cXzFNutin*xX5*)9)(y?M*L=p$+Q;V9V zF+6Ri->ERoSa)3DrzHs=>6Mvs+FZX=Cz=+9*DnVTJ`US0ZFx#^h7#IVyC`l4?{iIm zskaZ#F4mepcn@@b#~}04Siwv@va08n4ZW%v2Rf@C3`-@yv6T z*^)2^A@f&W`P__8S{RZ6H8+u_LRnMko64e4@+m6$8*v6V3IBr`mEKq7#d7w;TVZC< z$Qq@H?|~#aME zxunrMl>uk5oH%;M_sjZ6V~Nr%^e4H_J0{}Zp9 zH4{ygQ36T@e+LXY#cq1W`oAmQ#1j6kc;lN-glEqzLqZBq6BKFC;d+}xbV}*u5iB`q z(ur@;HA_|Ts-Y!Jyz&G_1h8-cBLY+lfo3N$sM~SXIBm#hR=ZO1m1;w=x_ZNANT9-v z{?_c~x;TmjWjAJ!{U5JWwiKH3zr9XgYg0}ex99#T8}ihLdA~VXW2ZCzaWZKmOa8Ao zL4K&#t8K3r)h|q%naROdwSEd1n->vIaITNQSJnT@0VqJ_`B$9a?~9gQZPh6FhEvxj z!l(`p(u<7Np^fgR<}2_hwVjO|-M14}$>+J9=h6Qb1E2+FSAFu%rje@*Wx$()#~czM zOFHt~7tT}_#!CeHd-V0n$nrG2#!@@(Mb6#p`;5#b_~RhhJ$|l5=6XrHq4}3|0)GC+ zUH_c{Geg4Wdzx$A6f?C0ulXD|(~US_6aYHkt$+1j29%_ovE_clY>d8JOhetkfQ=u`AG_o1VF@iP0)uykWC$KwUG><0@Vc&heM)Zp1*;Ky1#oq5Nk7St9p z-#3kZ;rwMuqm-$uRFr`vmZhE}H-5jRoWGW)U?u*II-EKkJ-dZp_s)%_3xZ055iY9N6z3cm;xN4-J22V8*m(`F1wM>_t zp|;EIQIY1pIKhF3Z^X7O`e>(nOhUKv(T{o@iv5{Lx0^Bfu3tcyFP5v_(hpZm@$WPM z)c->o0Q#|$8pkWp`S0i%?AY`b&lV5sXtr+1?FO*}3e$vJ{hbMMdflow`Az&l;>p{j zKzC&9u&z^|A||(wXvb&X(yYjDr=|M}!!6CW={HsLUo+GKUKhWp?6Z&Bv{ip+@8XQi zk58E@q+ee0LL|k-;s(9V=gf|1*G!1Ne>ndmcWW~LNA3n$hjSvOw13hb_^ah-n$dqG z04VTdCqdsZW1pR@ZWU{{F?R&o@p!&2SO2vG)NWlfgSWyVeQX*?2XYm!%-hIq7! zUCIe_%SN3CAF){i2K&<1w~s{HfMK@(aF+4kU%W18{iL&lN2FZfL?5{ir^jAyEr(b_ zwES$9X!L1@!SiH>dz9H;ZvdNrjCV>Zu27z@FN^(e1uPO zr%g*@wyGqT*M-8W&EOBO=|%?s#y5yS zqp;S|?R<<%(>0?8sK=^^zMm8pbo`RC@c70{?azuuVUqc8I`Xk6RLPHZpjQ_S(cJa%XFa}-eMH`#6O4!j36DKTAvg_HikH@D z_+Sk@QiCPwRq>^cRuslQk7c$EEqu^M2g%dzA>OSnrC#KJ@g}L~_vMQv%N3%I#Z^k< z%c*gK;Vhu=%R&IrDxs2ng4k!Qep=5Na&<0?=jY#0@m}7r^e(L;MQ-brC4;{yxHe@Z zcjv-}6wLYNBJFur)GrzS96H|@jh{awz?&uYD@CTpPc#4K3YjZ?VwR|Ij;K$BLALP= zat6=fZ^jxqpMfy=kZoPjF9T6df*g8Gb&b-d+2^JGtF{Q$IsLMeFES;%ZQ)bnBN}~Z zSbdpTHFB%{QQU$OR1lC5BJni$Sl3@vhne0oaBH$tm!n#u_2p%&lPn$rNON2u3=a3H zlGmXoewgc^$IeIrOQJurUUkIO0j?q3o~I34m8T6eSRMB}srN$saN<$9L^O{b_s3cl zlitzR9`=J<)h2_9`u*?DZlPQf^2tsy09n@ zY!LRZL6Di74S$$zGk5Zu=Psqm^pH}@f50#YATS3A8t5S$nBD$)jtzN!=Mb8d(8g!j zeg1H9--fkVGgm#2fKQNVl%c+w`8U6_XWep^$RBAn1);~TrpL5kie|CwMm2!DOl`MD z-B54V5{->8o)mA5;`gwrM@!e^wM;{HeCZo9L?Ml0^0K+LG4Q28X9^vo0u?=-WAu-1Sw)KnFf*hJIfxbY^_ar9iHlv^a^x7rpf{Ws$#lio>U7$PBu%A+4GBwVp7*6Z%tJ6%-9ZptYGTpxYAi3Eol=`Wfqxb#UM@+DHYtjrl>G@XUA| zFIqc}jfj;~WYDhkj$LBeR9^E*a3v@rhfQTMs1BNSR(KF)m>Q%H3YzDWmUV^P`tu2% zPCZMtTL`E~Rqn#@1si<$89l`~(&oc+tTfMZoVeNM;mo2Emiqo{Kv^R!ar-nZa?!v~ z_bht5W=|XJ)2OkTlH!x@&nI3MLa0?2{O8uQjuYT5aE+Vx`cuLPd-yZx&FORaQ@ShGOrAnvux_I)62_Kl(c!O5VqCcgp;g&y9=S9LplOqJdH>Tz*%P$Z&d8lSw$gSCe3^3(()6DZh-TSj?f@ z5O2)IL<>lA2p>30;FDu?#VU{QC$acZz;MDEbwJ%RMbi1DU&FJ)Y$k z!51<;g3qMzH`}FTw4~&o`hLf(5&F8D(6yCBwpF9swYMs)8J~x7_!ThNey$Dy%Zw#V zf(B~gQd$Wj*P4e>^mp=fw3}iRIQbP2J>wMEKFqY@S%q~_wE$<~A7#MC*AECXh&(Cq zBb4bvb<}AyNyOI9$|2qyG|we8$=U5xbE{ zgNWlNevXvxmYxEP2qBIf!W7!}wG(fNkLGqOu-Hn#$!kD*4Nl^)f1EzP@9eL*Cv(P2 zRX~q_`%%t{vx~p%#3Nnl7o1)lpN3UnS~#4nk*uUGf?=Kx8*A@n*-``=nx|wa1Jd2& z$3p%4rwX2O5C>gvim7*3!D`W!haBWr-ToU7tu*EU;i1?n2j08;aB0nqGIW~T=bz7S zbEZ4XxUZi-2MSPk<<9(7aU4~4nK9ODGo#dn2OvGXq1oZ(S*KcdANeKje!(bC)>il{ zE24=^et~yz`iW|eu=UDPDCB2PIVd%D;D2&b_y#F-S2$R$Zu}PSWdb)K?zfc;di)L- z6?I;813^N{$VW_SNDGg0t!fhJoM2I9X~DH- z5ypI@sW2&jx!;`feV_mBsNU82sVIBBf}VH7Nd`%p9KFu4f~C2G~5@9tJh8G9;PEhVRmMN$yY zB-u%aS=STmW103pLwPfEH1{V_-Ip+mT5?>EGKY4-3F`7gc?D5NJD233`~HG^SP5hYaj*7+84v%N3T&CaT3@tA~h5>MyN}Wstq;NvJ8*#W+`AFFu?#ty+71OjT+u+h`@Nmvs*J+=8 zpz+LjY6^C@y2ZaI%t3YY+p3F?T)^RBcl06Nq2abgbGJe(qlAngIWZ(hPLZ#KrYuit6YIYUS(SU^4(9DKYF`xJR~wASu-+9$Z}|MW4wbAT{nC3d8E)Ne}di zY^F>=EPKfgAoZK#4i7e{MS-+`1HA^T!y(?IX*XYV2}=Aa{w7>m5}d`)hhr0z@>Bgy zm>$yFJCczG0VMAam(!M}?kUvViq+h6w|7ZH;x9?;33nY-@p+K63Bk+>BxBCmsQt8w zHQ%$Kg}o@E@P#+9N=Y9UVny&>abm^j7x3Nw5I8>&tk)F`qFXf~C{)_j7Y70=GY_AnXueqrlq?%$77=OaPLoHeVGu z5c!bxy!LaUdE?E}euoy#;j6=SO}2cz!fg(}K=zv#GT7Hm^Vc~aQ#jQeD>~8=#Ba(Z zj=Ih_WA$Tgy>^DetDu(1aon#zo~_Z>@2Tanp1eSxOF62WC)8MI58m_4I%UAzDw_v7 zEF8yoT2YfKL1Z=83RYC5AayA-*oDVz^}_TPl5P0KAx9Sh78MQ@_7iNH-N#lTlCNJ$7qZE|s@RuOArRH@(<6|T~CwOfe12*Rkq zkP~jgRjaVh|{j~uaMIXU`II9t_0H!Z@WRqsxjqe?I6ayy>97@fH{y7G*aK7 zV;9*Qka>{OHR4je7)hKj1|J(B)loiB`cSQoM$Q|vE0$G}p#ef$OMRQ**hOp_K2Z2#)OE?v{1@BNS84I zItUgGg{?#jgU!UCX%?;Z`yP%MNqEy^Ngm{I1OoVj*y~O4JLp!nI5Ye(WGin!tkv@#97+Q$XBRvKZGCGjgDT>Nv?HZCe*#j3YNe_44u6p5#^&{vsW8Q{5R*(nCIc$7r z-iE9&mxSgzAWs|IGxLEy$UD&i5%0D;v#nVuyX%2QtQ-Fc^kM=U^dgy{_e7r*zO+TX zJ)+d}*uG(LdrNg~KrJNT4I*V@KbxJsR~gsH3*;IioXNc1OWZGCdWN{WcH#*%mpyek z-xlgby;rxlh<&%rLM@I|Kv!TC`jqeF&}D9cK_4E z<5u8j*ZrJQRVLouk3jwOgJdlm$5Uh32)inm4dg%LZVF_Yf{LPc>4uUu6oztD=nl$X zQS_Gm2A;EV5pO7-ATaP${?7W!1MscVo7?%3u-$^FfkUJoHSnE2W6*7MT-A=)(hZkY zve3dUj)Cqg^TrYk?+n;^(?&g2MP(RYDRxx$3qx0(o)6MmH72&Ed zz?MIr)t<+tx6EZY;H$+RDZU9!`w{>dn#dGD9C|B&9lNI*zthCK9e7ahjFebriqDgOk8^ZyFtDu-?T@l!L@LHN0oKPUPljnwBF?rZZP`EPg5c8=7*bS>mt-X?HR{xTU;Hc#0|zg z(5U|ddRRAtQ+x49avM`q|EwJXUcfCMY1rQalojJ;N-HWhhkv;}+e_8!YnYWOzg%S@lpZU7I^?&43AA z(Q=o9zH+`-`x(pf`a}~nFh*=%=rlq$We)?mXc3s&Mh2s|&A_w(?Y+2*nan&k=6t5= zP5R2T-4Z8EHcHQzOV0y4wq2Kr1kZwTuU{syR1xs%ZNJb9TO^VTceD$V7Sm{((T>D_ zA;+`*Qu(-xc9!c_JOe2>p^x6$h0jId+I6O7$ds6VL^5bgA zh*h7BUbr(^@YzugK5#8kQqB~}U&?9s3Sn4(aEmEjcCcH{jBz-#HC($xD2KD3&B1-Z z#&a-*xFttmz}$GoN9iN8t~-MIYz#@{(OP6!xZ=2}Qc+#kHAzxgHb{0uJ85Zso}l6C z*h0ybBzRQ>wQJ^&q_Kgqe-_aZ4`)JZrZ^+E`mOH>W%@hYN(x=++3M5t3D5?K!MmHh zDK}l-X=hULMsWaKH_>~@Nkq`?D+|^S$d39}YV$gZOgMQ5bOSeZ?Ci^!ESv}FV3yHB zrYP|zh-fzg$qZh!sY0v~oylpLA-<|l=b^VyjNJPX6=pQGva^#ynBvvXk2HZ+N$E|damiMfPs`?IXSX^WWz*5+nokIWtnPAG9GVs30a z>3f=7pQ}bLTZNKxQ!db5Nd)+gEOU9aR6dq0&p<}~y4fjahrwSVyCN*bFTe52vdj*o zo#dh@iV(OtWX3AN_@G&O79hUxxG^0)HDxpY(DECcE9m6zvpRgg7=Y8r)c?DoM*C{d zg@S2N*fJe!C3ip@Tq7UuS_?Wwq%SwT^XuD}vF$d?ug#X7pDZ?#eIdH$-R*PV<*~0= zx9lj}*tmH9GW@Z1?lJdyc^~$^xvk^m#d7B9+xP2vTc;(EvlClT zY{Q`wuvNJ1zvzF{RFAY2N2Yu=xg;fo9MzhJiU2ITU)FB0KJY#*B+LF7xbrf0J1M#I z-VSw3=`S=>*IDZ|O(;ey={2Ld*>(_cT7?6J>sV%f-od_B2KT%6@*zw_Z*r&ZLs;+C zJ|2=jOfQLq7ir70RQ-kVjLoD5og`=*v+21OtDbIVWMd`L*(P?fagI&+U=_e7juhNV zwWPlI`IQv!oB;a_LI`F1smdN|WqyPM_CuGWLVUhniYkXgn9T6?2_^Z@X{}=-P zMTG2=emf{d73ikGX|m_(6~=pXmp{bL4E`at#rbAy1}@Vi*b_Oq%hj@u;bzN7b+>{d zLxe1s;dtDZXQ)MQQ)9h+jU83hq}~M<&!64)M$JrmsV>K}Gk-dud5KJ%y9Q4rqU(h^aNbU!sMsw z1Ivpz{+5=8XwdXL0`n5`apxertA_;T^+4r;@9gOZKmMAvt5z^jK*5Ob7mrG?UBhdg z@&TQ1D;v5X)G1ut)A-ai$d=rX#Q;62nO2XY@KaTl$HEqc7ADp;WT~~B3M36Dw7Z$% zWHt2Lq1>QVN;KDyG&2}H`ck)1jvSbYX%#fun5Q{4D&V&`u{=HIR|K3Jh+NsUG^%J# z`favrt0GgV3f_q*=VILY(ayXbqn=A7rYTwq4XMpaQ^w>gC5;;h?y4=l$w{MU&|i62 zNMRawwqjhGU65S$L9`>rG^0bJ*F(j24&nLrAZyOy&}~%`zpj=49P;3VHxJ$r=0bf! zD}Gb!OeI2JxE7+0f;=OsE4Og^avQ#zqxqJ0#y2=d5L*>f&_7$jjiTxP7Js>8bG50l z23wI)&A)|z%ve&i+G3QIcMWFRHhBNJSL-0XK4gg|XY?5vfRJ>md?RD>6L}ckd~R5kXa?RhEu0?)2oW@}kgYJ?hVy{c!L%P%EUM)jBD|bbf#{Z~ zLQLvD6PD&Nu>&p+soH!K`gRQTt*Ejwr}}eZEvJhGu#~ms)$|&>0Lx8d+*XP(Vd%3d zhm46HH?=U965%&t`I2Bg7yH2Iok37C`CuTEAP4sNXL$e`Yz+=Q;@-o@Yey{-O zGMVidp$Y1o2T_nPslD2tR)SCULf^Hng{tFWvLvT6&HD-~4?Zwb81KwJ#Ps+6q-ZR% z{6DX;G3u3*E@ewI-4fG5S_L93EC(MM*uR6*wBy z_}gjJ{}lL^1^+7W+0p(M_%;Rq6!>YB|5@N){r@lUw?V<^Kp1O7qCl{gWh24$jgR$K zX|R^j=I)nLs(Pf7un-Et|KI}5<-KDNGcfGUF7o4|q15YJXGpHUYAiu)Tker*H4aU9gdM--W|& z2<*U|1yw%y@XG&0K)QYZ3jy(1{zX9Uf-bMp>V{b8J8C!S=@b7(K-L`59ZS)z`ssVw z%$kjUhbSvWE?;RRouWpM`*y@=7&^7=sgwsoR7k8i3xW5;V=x0`YFD-~EY^6|&BoW= z1e#}kX0m9=i+(T$7}-s`$)%yt>&z$J=Nt8we`b z>Po%x9lJff!lJ#N-@OI39kj=k^G?pUnQD$6J!FEeybsyHrQo0Tp$=+v2?vrIEUwgTn)ou zuk*q)x>M#rG9|DvXWNKaUU2MO!w=M7>UGgWWs5poXC|x_7gn@HyOPhDC^v@} zBVX$@ooVfHt2kVUer*XK@quH4I4`o)eEgJ@$7R~Z$o<$*h2r+My4YD1sNy#$Qhe88 zP}!Yk{b$6>j|Crz@!I@mSEA-I1imR}RVz{9f;6tbFCnV%xs!rYeQJC8W5-l38>Pi! zHAT?@$W#)5Fh(q);4VZ^jaC_UaGkApU+M1nR4=*MSG@-*kUipVIJWo-;@5wAc5K{z zLcL$CNPPYmFodno|KGt7bI*T(Au;Tiwb{R5$mIV83O0VL`!5%1FgwvplX zrjAw@H(ug)AJpfp&&IVxPeo34{AScgQ5YX(^Bv5!bx~xMG=|f#c#GS?XZi!d#`17I z4G_j;pcd&d4E54J!@bOiHdmlvsqnqUPr4x~xP8rwxbQ)51Glp_#S8VX(ffaBd&{7> zx&{lkad&rjcXzko?(PJ4cPF@OfItWy+=E+i3-0djw|U=pX1s0mb z!|Cq5_OsU7s%8iCr*p~=WiKrC6QzGj4|RF{4#pyl8)tFc%NGh6NeIx6HKcNe_`PCS zoZT$6Is~3tLv-NI-t(YL_)qo46DQ~7$X~c^3{9aD+{C`|Uj7NH#}qv!x{(&d2T${c z)gJ!RoF*u;I6YZ=%7}nZvhgpIq00j2;CKO(H54vv<{G&#S3*o+`;%vltaI>#O*Yso+#D({^0316gN!uuic&yR3(Ql_uwCF1Ap`{K<^`NaZoZ^|31p{6%~jCo zeRjO*&8#RuQdVwLb0E!J6_ft9Dt^yJ{B*t6S}n?+T&IXhx{2LHnSG~&HDgo1wpPpP zzs-e?|6?vxE&116=*Rz_3sDiBD4uX{ojpKj+;jA&i^oXF>2W`hQqXoAx=qr*qKJ;` z5vsjkfQG$IZ#Cg67NHXTB!1B~CY52&l^y0!Ete=G-}>Fsu+5qcPHsv4`lqqX77x{5 z0w=+YwTNJQ23}8D@7dog9Erc7VXK^BwX^)oFlMApP+6lEeB|wd-{=2(^pO(}dwDnUij_u8wZUKNq?@+*0_c_w8WH?LW>;wvOvr3!=25XSt~#{9i|7p5xdOVXu)1!}$)HG{re`Srr(vi-U*h}W}R)z-s+DdR}Gh{Xq-$EiaDZnrnDM1w!dgK z_fToBCoN;PgKp!L(rkaEJQy;z$5bTOD+hsBX#4fjJ@!N$o-on_7hT;l z`6ng^k}5=}+h`}ZjTLQ7@ZY(<)*CA{-1FG+ zhA<-%<48Y;tk&T!v*bE^3)j-(ARj#bAvRqf#3uM}VpFj3H?ax(huF~nMQkGfCN`0t z(TtgB(4JsHfuxsUEV<{|mb`p8cA-;xrf!nmc2~d|xt%|tKGA@F&~9zb6zT`1ArSlQ z5gBnz$AGAlF2X8NkQT^}or9O5)=O2Ep%x}l`_etj%+D|%NmPXj>qDKyjNdCsAeMib zgO4S)1H8o9UZr&hrd$8aM~{!YO{E*(o8RH@p=6D->4k04&UWnhqYLshD%W>P>~eI= z#8u!1Q(5s7Rtv6DwU(Exz{97=BQi1Q)` zrf~s^;)(1R;wRBmpwJm`_*!CKq_Qv>6F`VV4H`)bggAjbhBYMOC=!GPGGoXgY!0`cdg%q z-w+n8ZeVqBJ-VTS_ksp%MIK&K!RX@yf7=HB-o=9I&ahF#VKx95fp=#df&zmTk|cFd zfr$*bnY<8vT9e4grYjz=DBFC6(*T@na1hoJ#K2gJUjQ7rl5A+4^xvGeB27c1y*6ir zA=g-d-yk`btpG>F4ZX+{5N;$*7lJX$t-#+U2q3wGehWmoT^~#-xUaP;+`wfQGBytr zJeHBKF?Nce`yg=YIB*I$o}G{J@ECwPgYN*x^M#C93OJr27M*#B1Fc!)0^n#|3QU{3 zoXP|rZhk^q9egZt32w4x6Y3N zM}fAT zdHiVTdfb9|Z-INe&V1Em(>?v%nQ@ZO{ctJXn!f!c295xY&<@i%8Xp3pnCDep6nS5j zN0DS3PI=Zs`WE4b1ubV!N`Je}RJ|^`Lc;*U-KcWri}4tc+`1lApEJg>fEi z3wPh@7yLJwp+VyHSfCxnJHVm@L%Hra+G|}KwWBybm!EFYhzPv}>l{elmsKRZZV<*B zF5}u18r4O-!CfY5a8d;NX75R4&*y(;3oSzzF9E*18f=jY@a2`u&9cP6Yv?Ri?F#BF z2#~v=H@|*XDq)X@6x2a0!ZN;8x?wE{wVG+c9i(@ERCu%Fbbt(s<7a`k6Ymh)gAC=o ze6-9;wd zML@`6R68CBk+1j<)FucY zUSXdwGvHZcgfDfSqqjl8te?QVCeVuAQSlO!=N1C%9mWzu?tLVLmvSy2ax4ntsY&_x z@kRgZ$G6-LYbY>oCz@)ueCDmS-d5qNtU`Cj%~s{CI!xQ0Z#-|6ud&>XvapUk2v)&$ z^lJvgOFC0Y45aOlW~H+wV`j|`LCLhqV7%2-^zF@X?0LaA<7CJK>k^4@oufmLrJ#al@7CuM)xVMXg@iYD5vLT;our(=U?|CA>0%iqh=da)Vk7+ylZ~DHOT(% z2q1v6Yfsg{L~Mh+$i^9uT-sr*E3qfHAvS5ybqoGKeuXz)`F_ekPc zm8sP`?jUgofjW(0oYmrgpQ04}8z*Crg>(-=yyBcCyPDlFf8eG<7v^M3McL=_+SP|ovQBpXz%Ufyz`VCLWu%FT1;N~C<`PsMR}>Qu`t`aak*$}IhFQ{! z-XaltATTjFk&$B=i_Fc=K6~$Q@sD&9<=qHmCK0}V%>gy`A#W_Z8zp304EfLL%q=tF zxzIsIjM2`|w{`Niunqm6WgXJvtJYW{`0T0sL#fj?ARk((!t+N!{Odl(AM?Wk0555+@{2Z&L{%e!OlH&h2^hln)BB6}bc~irk zHK7YiB2dx)Kh!;nAc=ug`=3wn5V{@ipBG#FHXBwhBpS}#1RSeCWLVQu)Jw@tJvc52*h;yQyr zEj~USK$)OmsTLJ>r+@9j@ZaomD#R&?_n14B|6m>VbDVs`4|+e8mdf;fzR!Hc!_(%J zID*$z4R)>A#nz$-5E0TDa_870wTTd#&O8HhOJw#b{RGl>GZ4jF7MRL%7~?H-0MyhHLPC*rLx$rE*;Ca)?{L~dkvwQG|@WQy%Jn^3gV zq~PjfIf+=r$ty5T$nfYxz$b8rQF+@pHyS;4sHl+6v(kR+Vp>i=9i1=HYn{alNN>?C zR27o2Ns6Q<05S^!!@{<;sY(ZH&4WLUlutKQC-|A6TSmYA+(NgaNMbC*I%6WD(*@1x z+B>jwSs^W_kdA*jxy5|?a+SL9G@?4`Bo4}OTKl2a6~0YfqZOn7i3{!|K6*-v{tb1K z-}*S84{U#uwx)SutO25nN`R-PRL}Kdm+4Fy>Q}j+;)Y)wtow?rhJ4J~oX(>WL`2K; zG!K$EwgtdXHEhT_!)=Z43%$h*7|U+ZUGcdiOig%Q=k}-Xj} z8nB-WjS~8N87VG5hMOr{+KIv7&sc(uqMg4t-}yOu=NZ9a*o!$Rl`NRg7NDRVJbPvR zap~m7V9zZwPl~Ny5QwAZ`kHV0BQN|YSg_s$mmwtFEcgdul#t>(dr?P@OP1`^XY>%{ zfclcri}hBF*K8*KI(480S@{K(v9(?Z>l+CP8J5B%ARa@ zZ`QQ;6q!$T+!z-Im#(%p$O&5;cv18zUOAS(J_lJQMT(=M9Bg)xW3Z?R?9qvP4?KIl zl!|i-V}C}KBw*aV4{eMAwj8V}+;wsL$!ZR_?Ur7$95|+bG6H-uBKy>RJe=#_8HC~WRH^JqSbvCBhzNxrVXuHhr z2JDpue=AygGl7090=~mZ1;70E{;uGC5W`xj&MkP~{sDlb(VF!PXOwjt(%1aR#}2t6 z4h>83Z3;4oc}9E*1I6>;bb~*`pY#S4PbfiRP7=Ggp#o}WTEc3cWrd4d2*H$w-@M}C zF7Vwh@G*#GTwWnJk{f&w8pD58L_AE$HHZ$+6tPmDNza0W9nlFc2tm&`C+1Pcc&9A2 zpz@2nUBxc8*vE{r>7R8NoUHk{9iMb~*FK}ptFQX>d(8yF#po@})v>7(1nl^D;&Nlm zHHxy9cH`B4zv1uV1p0S}bB_3tQW+sN_pZyPOi*6OJOGGm9`m0e&Mh0%vo^HncfR%s z)^FqyZsvt*4;Yrd`A4;tjIOqo=qP^oPPI_qytCWXRITdcw4@V^ z_?OuwQZ^6hmn&z=E(`Q+m#lu)9cUWF`E`9pzjO9`Wogh`6#COrGQwxGn$RRz&m0Fv z81p!;Ct9Dc%l_k}#Oe$8F0Vh4w6UXgx9!*1uh<-pPko!v{q=ALK7g1%Yiz>fuc8R# z;LUu%=>p(%yITEoAQNap&$|c0;@Y?3w+&MpG$Sg3=iPSOyP&%^3VF zIavKjR``A)CHfggz*=ognT%G~^^ePXUzg4+LGp!M$Lcc$w15Vp0HwZYy$KqfI1!l? ziPu#}f{2FKLur4>JaUx0IVAh>p(eB>rk^jgjHooKXp%0Hj02{w9mSp&!;)(<$Y4WP zABL#1ZordN7+OW~tvzd+RtVGV7inX57(AhZok?)zUJK%D^3WmC8vtZ!=73M{%ojtw z$rw1;&ihgRtsqPatzI(PIIq}}{a_^(lv$h+>mfp%Tt`W1zwmWhC zJHZOTynaV`!s>O%djC9ms-FY)zRnq+`tV-F1P>qB4SY-tyw^8oSEt6Gi`YCp^S)6brMWW%!JWuhH$b^_gsyk!(c zt|g#({8Vr?#Qxk35JlL~i^1_-YWZdSbDL!4G+JecgZ=fnlWBupcGs=_LX-U`2=_2>7$z zA_(|>k|dlu$%_Yfi4yv-+wRDZDKC!}ULcZ)S#HNdqo`~z)pYwlK-5FrrBs6s?5GZZH-2x-f zwu5-6L{v>8gShUHB8g_o~f51h^KtTL=#?!gut0I z3f)nn`4xM7#PW64z0W-xX#;d1A0LIt`X=7Tz+N0KVn1s$q)je{Zh5v!zV#Y+D7_UXTAs=XYnJKxMK=!R0>qdu7nR&k@v+#ycy)cb!cr zF9$u8tGBM^yjIqpE3qAqXP9fN6j|;j-T}Av;+yZtZHh;Cdw#j9PW{3KV`ud*uFE59 zu;GapjshaSu9O6_)hBV_D%9>>Ulk{tfmtVjW*6kR7O#wc#4y;!hBoBGJL$s|aX(HW3!L=_@4u@k4ET>==L15;Xp0J;V4<$?n z0uiPmpwYmp02{DFI0iG~ASYctiIm{dFWvq-7x*1Qvbk^AuMOOBe9@0!$Z*tsU7>w| z$e!@NM|26m5v9LuG6XRqbUf;PfAIHVyyuJuo~$}DL5NQ;zZStKpJ%U&iL!?KNiI(s zwsp}(oY5tR(HY)W`zu) zj)T0$yTl0G;)CA1FViF^7Dj%)vzXfbbsHKy5!m-l`Uj_(tM1yCgU(xghFNFJ!g|(W+HkyTdXn0oDNJoYM{X9IQgt$E8v{Bm!n|KeaXPcqOJU2+ ztEREg7y@@6t}dwEDSyy)x`%2J_CnEdn=j$w?5gXiSJX@s)h}y1-c~2t_p2Lxi?7VR z$j)kQJdcHEEgak=-$D<2H#ugea+>&h05v#YRdfRVEa(ZEUc)mw0;g-7--EQ-mAt>C z04@aiZkd2~jT%q`tH$+&z9|bY%aoMLHC9zr2pSfPTe7h9g*87QYeI+tGrqF+JwS*4 zyB?o*WMZbuZX--e?vh5~!Itt~wjPM@54cE81i3uwmOL%Sk)lXynJi^u0tIn7O7P7m zvy#HNh^Wz7vWDiUq>AhWT=-u9zF@1)0xcQ_pDinbhTzsHbq(?~?HF^f&@S2PIV!>_ zvm#*(W!QeRgd$Bk%+P#6*uE+$GemV}-%bscxxKGpJ4G-qa122=a|^54EiB;)j}uB&L3$N*Piess*O01T%*N zQF}{Dlinv^8U(LmTQ8^I@Y+XvmK& zM)AliO24$*eQ!@LOZoofCZ&?Yq7-JxZ@qpPNo06@bs~84h{V(Avy*mwt#_nDa zj2v6?JX$opN^|g3)zr{A+&PJ0SQ zMktqI6d`06FNi(Tj*MBoNlhc>mJgD)FwT}izTV9_PPtP9Nd7qIFF!TV3KgZT*sbT> zht#LnX`9z_ViIb6D{Mr^v-L}37lu?$ue7NsCOZzwoLt{Z*+euRTa>V z&Fhv4HoH|^`q85Ma4Os@WJs9iy*Un*Mf``#mak)Ze{ZO+k()4n=gq66c&r-9g(-zA zR)gi!9IN%ITh^byljb9*R}0P-fW1c)#Ua3j%$)~x7PL*PO$(fGnyXdbpo){Oj2{*K< znCJy>au@5E(&x&rnqw7uRmaM9HEPJ;`^})#Vt`BT4O->wYQWT;4n5GNWmCJS5tDi~ z2y|%`=)sTl18I;DrJ-Z~NPku@5gHVFUmn}Af+P0u%ZL^K(bY$Ap$7&SrTq0CC=Gfs zP#o_UhgaN0BM>_YqBO`(jZMsnY)$AMW+a@*4i45x2v6;Fi;QIDz+XU*&m8Iq2n7S0 zv3%e|!9;P45)xgf@$WLCi5YD?3&LojRQ#8OL2(eT8toXnNmOb5i7{t{5i@gj;L#tk zfZTK(5GOxHQJ)4T&q@Kj7dQ~il9q<0!OU1ofJZFCl&Y>W#K|D6IK;_OPvESXz_`?x z;hi%Jp*L(tn!?WWZ*m~8I7%-`)phyN zhAx;m*mC!A0+9H!Wv9=FN`b28vlg1XuP4IYstpx-Pd?)1AIypjkw^3af^w9<)Fhpq^@;_F5Ct&e zRL6*?C3ZKmWRT`;#dEG_=T-Wt$F1wLy@JNuT-?;Di?-2zM`uW5VcpkpFLh#m5{xim1(uj2rTOhw7LIB zfNBiw!Skx;33mxoN47GxWbTZz;_jWcbZ$VV0V-ErchqJ=G^7x;Gq69fUpVoARxGx+ zM=%3E10G#O1K+IMqT8at|4O+xcEEJxBPBLoiM}%B2-XPJSdX?`{`1XcQN7W4>tJHV z6Ula6g1RZpdAU_RLbSym_%B&@=vZkyf(CJG8n0|)Xa&>o^f4d@gf%I7>e6Fku3;YH zps&rhUNjDYg}OC;M5&e-b{K_!h(2+k)xxfllR2Qxkz(3H3y9CQZaIMVWe0c!dVXC4 zxdh!>)$D5nVyojL2?N6U>3i`QKC#>pk;)mK}AtnWuRq>``rnW0I=qfGQ#q~_xkS{ePnk9VH;v^t0EU9 zM}A;EP*A~0$i8J_7r?6`+*vL;m-sm-3veY7=nL{FO9Ap&+J6-5YfOb6*m)%h@VkCq zguO|%+d(+sVt_u-bO$^!pg`W}(yof%d|Vp2^TPFs``en|ofA{x!7P|X2+&0c$o5}k zMSqeoXzW{ooshtYMEsIOb{#r5r8+)62+yV#C!7vO_WDAA>_v?dJ|tG|ibjTX6$6I) ziQ{{GC{$o52n-M>lfz4EMV^46hBIseo8!1Zseu=yYJMFAVH5)RhbY_&%z%Xhadc!j z5J!6xbawBj{}?2a$uqp1#bLSp+;{t+dK*4WbO_c3h0xX%1xJgnnBEbFI+mjsqqNl^ z|61=rGyPw5W%4h&61G#C)LjqJgCHx&2IqbHhq91u3u~{EMJOA^kYc20jJtXKk}T1S zmMg>Lrjc46C7w*yT3D<$SC|eiJR;{Y=IWc`3e;ZJ=vw?%k_@xv$US0for0&-mlhBC|i5A>I1$k5aN!c2f>NEe>uBTBX27%ChmiNn1sI{vJ#RpH@k= zlSYn{@c~YEj%xcS$T11cVq0G0m<)93~{Uwxs)wiX00zu&_k+68rl6%yjwC6gDv;k2( zqMcG}m%3k&R*c5_zxGtoGs3Fd4ex0wQ6M&hOYZxSS%wR!7{y)zrbtoTLC7G;^lciM zBu50#+)rGRYbPZr_h2~TINN6J4DC2)Zv<+VUrM4MuxfVQ7M+je@%weizf$tfRLi%U z{n8{r{|mE_IEDS$;-YAhlhXXWbYxpp84*BSGs)Z>sc@H&ti|()@B!?-tLp*f^JnsFV*Aom8p-uG~|Ve zFn0*vXcFXWFkUG-sIg>O@3C8BbtsJM3 z8C1Y5Nr~ImgThCags+LIJq@ojSmVH;Lt`gl8EKKd=)Z?bh+LmvW8Ui5E=(5B@4`c_ zd+N^$eCCe@#8wdmJzO6M`_JmnlB-kwn#1vflH584pMtdd`ctf^FQyMZml!^L)^q&7 zjg$^ef>{=ZTsHjSinf2T@DP5*yRm82!i z{8y@EP6)W`OYINb+QpzM1Jcb_rDt3|&mVZab5RS3FUiO@?*&pUQ#RFQYPudy8#A}zl&Dw-7tpH7)Iak z%|E=9ttoQpyjSN%)K@6(u2W~QtGDqEZgOp+3ymC8--L|!hse>PT<$8!&?l(0;oJU? zTuHohLh;{5%JE+xXeLD```@FPPakN8;oeJA>YAtM?tGE`d*}`t!xUDrF3?7Rdu6bs zw$f%omNc2RkI`y2?ujJQminh=K_=>NI71npk{28gRv|=LDyC`g?;PA&Amo>w^f2E` zlJOcvAhkk3L~`9(^-yP3YZV1YL@_h96ZcSom{lSqT2{Uraz<)uv8k<(7Hb~fe=x68z7-}-YunruLQ z;Gl);lL#=Y6~pwY#Wt(mhZE$85m3ZdmqRsmNRTSDi&lnf*-;eAnAViRQU&w)@3k3p z3sxx?%IYe|7OkZy#|#tZRlY$+8!-`m(Q%}j8tQ_^5bCU@y}trsY3%zu1SBEgS7$Xf z0?(NY;adv;Emzs9fl`m7bH`e+GWuC6LtBq~mr!sdqw`5T5Fc5RUuhN1eG-3VNoa7q z#2XkC9Ayo>I(bZXnJXAAuac5WqR~RlIH;_sB&e1kvXL;W8Hd{{DKTa7abK+|kJdgJhu&tZs zpfF{tTS54Q6|UhVTM`at3u7hRpM`hZl*UwyqR!=&UE9j*=Uxii?S;PvC}4BmmJ->Y@1D8 zZ_qDxqk7-P`HD1rtmsmU93zbBs=GCcvE-IUvejo`yY*-zxu*1J&7%Ta99={VgL-zN zi6VP|1`#9h0*e5Lze+mxQ2#0E0CsX@0^OM_T1KHX$UqGW>}%Moi$Rf8(aBjps6i<8 zRym13l{ikp2-{cFeE&-_2A!Il)SiAwMzsp&sp#BTw#qGjx;ruPhgV;Al_=wyWcp{> z9PC@opC_n{iE8!U+IQ|yza~qk+I)W%tBv+@o^%kvs2+A1u}{ZHNP@X1(d8+Xe|kRt zUBxl{waP(Zp;D%$3)3RMsw+#Z!7t`4a`1aIwF&UcH*Htt?_}**YBbxA1~Wy?7PfSG zt>8=*>s`lVnoFg-`s=!Uf$2+GI{&umOKBz(Kg7*ZYI-;AOs*#B5L7pkc4F|Ke$_ua zrh^GQZv-YU5g(8*NW9|mcEo5ajpv>Eb}&K;jth>>?Md~ke?CBO?-Cw(mRzFpLwkO~ z5Rfjp)hgqBZ~Cn2OFg&8*w`+-Tyfxy{>Fe2`p0z%q~WmSmr}L?EkeT~U%A2r9!3FY zM+@*t@P-gl5b946TQ?>w#AV}H;K6@Pyf&yHf_0PvgTbI}h)}*sP$N=g_5ECh!pxod z$OE~u`xlVM(oo*l5$6*um-LzAjbn#40%OQR^`}h0nxBV)9N<&{pC{wDfxi;()_Ht% z1K(=>>R=z)SKZf9;HSn>ldU^Qz6z^>^0V4bBIfJSc~{uBtxwOl+v@mn|u4!xNID`u?3WQ@$FIbsiV3`fDD zXO|_BAsY^PGYxnf_OpM1_V|VgCLaa?lx%F>VY;rvKXw5G4p9$5S_)u?jvlPj05S)l zt5^%3c?(k?727EY1q<5=ZBQ`mHV_3%QWWCj8LlQYqrP#%?nt=U<|WQ?qrNcSuYMJ5 zf5=8b@>cQZtIJ-%BfIWoH*FOf;KU?O8NOkHcW&g|&G3LkFqO64=$?FQUC0{18rb@y z`w3wvsE6b9n=PpShVPVXDy0(mzATu8W5~!oo~}LO?z6vH7)^?BRS^iQG8gCpeYypZ8ZQfG7a(}Fz+wgC?G2_owbp1zGJXM0k~yNNB|Yd*R+%&+TH`JSnLQm9V%YBQeE= zKYWkPch#k!%a}ep_<2nc)PP=J13yO05un#5wKd)C(h~*U?eh?O5`@G*n3$$Q=SD_J zA_}IQqCN^HtOM$&q{AGNP?{obE~7a@iGbSnqb-EA-S%z}SnxF6mvMW`l&7cZrr|x= z6Q%xuct^`j@|4-SNaJe(%`p_u5P@sJGgmih8y-V4sYJfoCmiy!1@KahqZM0&3v35? z0ZVjtAcp9~??A3ZEtrkgb|nM7JfgfbO=!U;f#Me64HUOGpa-j(lOGDM;ut7yHD>R* z+b=+W!66#}#qB0=XBnWl4aEYATa8(gR=`x4g*euKh+AA+d<$q-Dl_TUXlQPdhcB3+ z8zCow8)8w+;6%Vu4s72qhWNxz-<8d!ePO;}6FZf{1@-}=!iEHeO|+b#z9>O|40bWONL7pnXo>EV&kO15a&;TuW0Dut%5}*eZwt)7J%X0Cz zu>I3H0TKZewxEd6poqwhKw;bcA#4M|>2;j>=@hy>7z2N$Q?} zK4%d?UmY00DT*1e1vDdrZk5!>8N@MNJW^C#NhR%T^1DgOzu-9uys32iZAf=$q)2jQF5(2ZeP0pC$){soo?l<>bWnBEBI>6Jx;!ZH+Pq zu@7NOwqp|(eVQ;osoCN`ySzx7<&7YeVs|h^tA%bys^HEj*-=f71aFd9z7_qI0&F?1 zYx>%3hB}sdCGcSygn0=`T9DulrXsYGQMxj4=@jIQUGG~M@nEeul%^QhQXtZZQw)Pe z2UNk>)KlI3QN>cM30uyqLcwUWNARhyjnf~u&F&O&1Cjfq+DmBqRe_$LGcp`A=FB%f zauemJ)=fx&PS0=-9bDjjr825qIr9MZNtJ9X%&qRq%U(+M>UpGMVZ5J|@SkqOhQC@6 ze4l_V2+cReD~QgeUAykYr4wH6;+H7_cL5PdUu=~M$qWuVz$-`EFdIh?jvXkj{fzXp7bN2{y>FGa=)YH%`QQe6d85 zkKTqXSs2fa{;>nezJ+}kf-|J)1PyaJ-U@UDy2@c$k=ipZ$9-aC-5l0Kt0-qPrK}|o z(wIT2M_l));OlA~ScZ~g@H+h^w`bi$=ruf>$=zBbU8J3RXaqU<$&+2H_T*WE3GcgF zMm4;@s&kvG>unlCtT`_x5w8wd;^0XNoAq?(N+B(Z6a@n3W>!~N4=Iyg+|A00gO zKXq{5-oNVLG<2X2hM~1F3-$|H6PLcs{UTLflEpoJGePYi!h7(ClOP}UCj3H~p8MYA z;K!tsbpHW`U(!~64h%$Bb4G02AM^91GX3BQ%6-dTk!m6G4i~q@LCaxE59Na>rgQrE zz3a5V@`(+t-v8-33?Va+qL!nIr~2uAx>n=z{L6OrIL%_+hk?btL2p7`w^Qm5P!vxu z9JsfwUT=PR?OHyycv~03gMXUv?u{|-TE;{C4K>mF8nd~yfpg@r4e{S4vB1AbV(i?> z_2|xv_2#iymBF?n*HYfx$L&U&>|C>0uK90vJ52&}hf{HJ^x33t;gvKZt#i)sKa#o% zh#1V;aGg*ZdWjC@?dN<^JTwPkicxo=3h zI4omnyuFOKyt5R=rY5ULk!bL)lv%al@xS3KnxqUplx%xeC$5E02;_Q^3!PkVQDGvY z|12!_n66Hie`bO) zfTKO#0M;oXvXYXE>~3IAurYK;p8z7k^gUkTt850>6sn&{YY9qB5y^RK4+7A5SRnfC zUyX-?{}>Pd(YEW3^70W?e_nRL{X_zuI4LeGBiJB;jR`rl-={H&Fc>fk=kY1(bN)-$ zR)&G?19k1Wq0Uy7(0J;UbQ6BQ_sHCH!nYBtog^W@Rf?_iJ}%9wJrl2!*TMdBQTE&` z@+n|BSL*P(M-d2YpHMEXYRn`N$8NN-W1h+-qddl89&1^Ez3yxSSZvQG@o41k6{$H( z^B}Yj?MOLL1os?z75&ouiE!-I>%3OxegDdz8^mu2oZgsBv1YQZ9Op@9_DoGUtxyoS zqiA=EWN-X_{-%t3mh){qny|lJIpTf3@?hOJDywoK9Ov6sIx|nmYYR65ns|#9 zDZgMY-MO5~uhVRpOaY%NIR80TijM|&_h5eQutAkZkZ0!ZOnC-}40{D_o zp*W5_xF--%nA``SvRFtXqq4@@%K(BnZ~ zw23-5{)Anh3zvFa89cx8<5J_xWyDXTxi6w|E-Dz9IAt9um<^=dkevw~(0cW*m$Pl5f+R{uCv^!2SJ}=lhJkS15^RXr(v!IUyg}Dj|=jY&)GsMeTy%CPS$pk?zzb zupjWlEK#t~)0@veBK5;=4k2gwLUBH0cnqhxih)Ex;ohNM>9g8Ko5@DwtR{$-ZR=p$ zREO@(HWM?#_oj3>K~+=V6`qIYQ~3#cQB}g<*W;u}3$BK@Y=tsHVex2g5#w1IwF=s{ zYz)h|4sC$J8iBzL3F1BE7g5x@C{}hynA@!0moXj!T(&qbkTn}JrSWTp0eY0DB(!1L zC2_9E`weB2cwZ#NVRgipCnJtkFBX7%9KnyHn?hy6W8s1YU7NNb1%|fDZuV)8k&aDk ztxS5@WC_a0=UvzxVHwoh0U!eBzhiPyrY?$1f!SsM0TK0^{*%O^7}xdDgQmjV{s0D4 zP*Q^Ao-e9|iYup_qd^ z^$W9CU_k&}60as$-zF@_`Y*cBNzg2ACd_$W&b0>w6({!}ALw86cB`({-16imu4c4L z+oxyHH$k>5_x5aT1(&5`D?TL;KPqLHVKsB{;OQYk_IBfo zjh-_KG7t;-&TS7ru9pXCCB5ipy$f*j5#?E4bxklq>JPoI4Jebm^WBD#JmMvFkvz7s zKk^|c5?|x7cDY2zOFyqkgqCw^uXpf5Wo$WmKD1{~JPAq|r%-bfy!oUOdwEj`;8?&K zP?2kBMq-}MOl#UZXWUQE_G;DHwC#Yy;2uuifK^zQjtm#=?&V>%@JIx$z3=WqNi4vt zY+#e`OjMXbKzMYbe=$h-2CUO8Sm={jXq4;Lo@H%ai4=|fW2V3GCjw!CQ$G>i#LlV? z)qJ9A?wLMa16X_V&TdkN`m!C-Et(-Bmz<{h+#Oj(w2g?WTP2Mz9@zNaTWLVO5Y9i4 z#V_G)wiv$7e7V1kgeh8hT+=a?dv<6_=1jZ|CEK35FTf*6(X2+-n$Zl}+1rX@^{cg7 zy}2%bmA0sBXW`)#7~sG)^neRD+12H&qckwAIyN71kDJ4pdPv?a4cr$L>mjeXdaXD? zL{+??eFk5CBs}&^i;|PxlBrry?bwehdE6>2(akRm^GhF$2bbF(nekaBgsc+Lc`zO) zsz^H(@WUOB+3{bRuiOW$D>fGKc@&}i^c;OF4B@<|u35P*t??r?fn+M?F z4x(V0>Z|VMX$oCG1vh{mNiW1_$`V0sO;XgQzxuWRNna5Vtx^F97noDGb3{U4A&8Jn z&_f^61wY6RGk$kYAz#yFR2vT{_0jxRoQH(5f=i)+63FnVL8xpm=W`N*A&Bz?nc6Y? zP2bP7nuD{LqtIHg$jzU}h}>QfLz2j0iHk^*$(XUDq~*rOe7zx0R!qjjg=!{f3QD2p zsLA3s)wXp3c{+x%0VX@b`Sg#}l|SWA8r1{?q;{n_mD6vng*K^65Oi&HI|Hjr;h!Cd z`Qm#BBG=|t!FCIXsN2Hpj$708g&wb6;nvp@ekJa82nxJ)yu5}v=zCsImh;hD>ilj( zX>+5tthVKOo@kUzvt1fJP;gfreQSRD{6V01=hjB0j9)#vg!uW+&5ydEdShJ5fOmPK z=KFY;W9gse{(E}^(^H>5b^9397>m(Ld#ZzB_j-sqf)|rmCML)89~A~UaWWSzApzDM zXAec|>Jz`E&m{t;dkN5VpF%y^Re)=Tfm~CK&4e?8)D%LKDpwCJTcoFscTc12zq3r7oadHvDmoD7mh%}NDW~61f{J({;f4>g>Vh(zV|7groQ{Z7 zg%1$Oo}FEj1nT?uW;S))0G7xsk?iJI`r^OpO7K~w-+qBVMPH%r!iB3KhDZ=tzOY{N zKTd3yic)p9r7qD0o}KMC&jUAF!|;`kfyd45v?2HXW<uEKlx|u~x2U~}o%E#r&52M1^-eNR*|BcI*QHXS=Wvn*ay1k2 zF%nM*pDDI-=CzFa8U7JXPfSX=jse-$ZR|I-rlJi36-Ki)dRWQ*_R`?{QPTzKOW$iP zczqKXgne+%U-Wr9N-ga9&mWE6k_((V^jbmOg;iDFWWcT+;JTjF%lNT5tY8dudy(q{ zUzZ3ciL`}hq9bpl!ONk<9kNPi<rQ<^e0HrqC#(b&pPM&|u;wCNsTU`z{rU`!_=lkt9^Y;ZoBPG*7KjuJEO zok=wy+TSb0*}=Z}Pp5>3Z@?|i`9J-ur0skkD=H>X*T9VcAzcDbKwJRU`*9gy?OGG~ z!p<-mVBPoy#k&ssfbaIoKXm&lkL8J5MKX|P-2@#84w1)TS)s;Yi*vb5G$uz`2Gzeo zSTZw2N*L=Xjaq|D3=w87{!-n#YF$JTYFUozp@EeIzq$$uRzET|?R&#tN}lRC#l?*s zZcn$8e@pA2xWRjNJ>bn??J3CNF;Yr%UyBua^c4uHE5^f4F;-%ATc0Je#9Rzq6*8>eF~tY??jkF94j(Ir4e1F3wzihE|rF^ITl zlm;gc^#^fFY5EpSViiv47)gtMpB}07m#85#?o6@YgW48EI2S=R64H7u|LVxTMh3M3B!bn^NnNBg2>^2lr6Em z%I8gyLt9@-^uh0>Ukp#-(TAn6m@BM7qoFRLKN(BTf8j%NHXgS27rd-W|QzZN$!a^&H?i^4X+xFQ5KW z59-|p{v3}2_8gA{^&D^6W_A+*0d^iuB9x(&oErre`IFE^9GNE>Mjff+mWeiGFl!`2c^fPrC-28NQkw zF1pKq?M9yobB={Q0h@u!k1Gj95F(T!`ctpL8Yh)jFADM{CX+9UDj#fQB^L-wBPR&U z{{YNrM#1cPFzsRW>rCN$sx&42v)mH81!)VWS_pt5=OMu{<|$Um zxHY<53KXKOHro_wKX_BqvY#_NjwqEB`U!_3KU+YI9*-BC@D(!;v}($re-Qu%$#Abe zP-al=Cx9PUfQeO{X)wyLQULq(N)KQH7y~4OP$?mr;7DMB04Q9*-r)58O+fXfSM0AT zF9AkO!1M&1zI5?MY4|Bs0mivWSdc+LBnTQ9RjH$^dPquU;X$2_ZCX9-l7;jm79dvz zWEufUTZA9%IRf{}<7g2ze_FTa8-TA9iZ53YlkQAJd6w<+4 zP$J`P^%1B*<-zmA|GA*Zi|4!JJEHrky#9q*_z0qeb)FzQq_g(ZMGtcR1pl02ckuoJ zC7n$J5t2aNff-5>A9;kp5&U|pOcWX{dTZ=n$?5>9-Lv6C*tSE6oz!Hkoz!5ComAyN zkh`fS9dtp`;&hD8cC3NdYXV7-`SHU3*tK5|o8Q9Ek*ll#O&q~gpQwF?{MRUV4a4@GnHPT?=iVFg*J2lyrtG3+09Lk{W>0MswnKU4OF zb$b&62wcAr5a=lSy6WO{-4zCkRpEf9q9VNfBGav z#JxcdL+9UnDDWL}#iURxV5hr34^7GLok3_S+?`r!pE9HEZ0I>ezQ z*f07Y=u3L-90PKS2)Jnc)Ovm?L;*TAy?h~Yd$Dc%N(Y@{N=t@DvNn6MX|k_14nLz> z?AmTc;aUWwKyFA^?Wfad@&J5~w?2UT72f-JB))vfk$ppm14ehQTmQv%7yobS-7-G` zuj*j0_S#zHx$KS-`L!7M8Nd!+*z9uvie1`j_CpwC7{mTogaz2dg@GxdCYTfa8%LH; z|AnvJHq6A$Iq#q8H=9H39%B$N(`SZ1af1e-d6p~r*nC<-&}AmO(aqtu$5<+(^D z=q8}dGco{^3==Ws;WvvErAGf^{B3FyFrj*xKs#!-$_kze*W++x%$g3bE$keNp4PeB zu=d{p0UqKqaq@vUmQ; zBae=m8D;-Z3H-bLodmwdsc;Rbu)^Lr6lycD0IqfS?*(v>e`4?uXRcR*5#vX^u$|3`JwlhX zr)Q@Xqqd#{Q2R#P8MQ8aT^yo&Ah&yKFoV-Y?({0&jK^clc#&n=I-mG_PBjoAxo#zD z*#Ee?uqgem|J~K~3qB;vq}#)n+r$vUX=tQh=)M6FmFB$ih|TqTP;*q97zC{yerg&p z`WG8>v^{O=o|^eQ)Be6f{n4i~4Z`L>$Ke~~<^JWgk{%p@))=9U41^H7kHH*Bsf|c8Dug9ui_XplA&hbvJzK}L|-Ur#oNgJjN zRD8GGyx#8S6g0IPc~G4X3_mYm3}xa_CaC`yZ^$xWz^+N$wU~`hl`b}TUYeUCvsgH} zZ{L^CcZB6?y(7%COeX2dy)Uy&zIO!wl%9Za(NOATY7ol8DlFW@sL)XZYB-C`Jwja*N_8;m=<^CAchMYt&r| z28vVdWKgg%%XA5mO4rqe2)!ZJQURBQc!7>$fk@CE>7r*YnaU=00q zprjL^h(IqZmcnw0TY7Xr)ac}(f#zacW|T`~`EdyN-PeD0C3oU62tnVAWD5e~4WfhpEg(TfQS(euOi> zED+HhCfoWKi$TW$$PmrtTZ~1N?%SSyMKWS);;-l4GJxTwF#wO&a2>_zFnEyt+FH`} z8K$gK;_}&scfBdCTLvhUv@B18%_MfNB($COXdQg+KU|=i_AvI0IT17K1gN}GzT#=YZFTYLv z>5-|3*f>rj`RkvE@nUYw^jTt_|HLDW4MuG2?A`2w*uT=;Yoq59rJ;C(q;rJZ&qO() zpFi#hNvlOTb}5c1l0`X%&n-%lYQ~X@5p}A`Q4RPTVCN$v+(O=Z>$1PhB0q!rexnv| zEHt}wx|ch7lQ`Xbsr|*A6bn9Y!TfS#KyWsKIw*rro^>5eJ}6zbupRg_rQj>V2=&K89vWgRoNds}=9 ziEOPz;;kHE>@>G1))ILQ@`29@|k30=HIQtp3C6P>y z3QKW+WfT|&`z~#IRmhkCxh1eS5KAdKM|eX>GI${JJuw<7l@i=Pu`>hV5++g~A3-W} zBzEW(*L*t~8$lvN6=pP9m=^u}mM`n)0w;f~S3^Z)UdZoLXSuk5MNWQ5-aD-H>c%^v zqC3j6%)|w_*r=#%N`F0dZZx zpzDV>1gCc%*Q5lLD$6gv=Xf;p^@gZC85lt}aYaSC#7`_cZ_JaS=ulcLRMKeeV?uOA zBDRYfohrnPbT}}&Z7HZhc&>1pJOP)(K>*R+eBHza^%T1Qgh`R)_bgF8lc;x;9ApWh zzz|m7eaz=?{)QhA?FFh0$AIseF%$!We~&a&urc0uBM$^l)5jef=T*AER%M(fkP<~v zH6<|&s!^D^DZR4R6=+lF%30M**q%mQPIYq5fH`y~4Hho18C?99CFu(-wB(Xj%6t16 zqou4|Dpj+Rk$Sr|Kc>~$=-+^FrQOt<{`n3@&ZEQfG+A4c2%mCD>Z>v$HlcZ)-vj>EW1fg%38)#SOW%>d~ay zIt9CYR#dm92Z@?I%|F#+L;!Z)rIqgBds$g*7D(GxY6DD4i;)1Y^UH|BE4EZZw-p}xxd;wATjGK==nFYz`w(w$$8~n=ME&S*j*8=+OY9a%3+Tx+##wqkdI zs+;pdp|&)Wk#i0%x`bT~1k0ZvspwX#sKPE4)>=#o@hp<{bQ{T8gie&{@9){dEXrS6 zb5e+~7n)YODY%uAI11Ve^>Z84H2~!d9tbSi*I$O!8SAF%F!fyVMXP&v%fM~!%`aCv zcW1Egmv)%BZpg_s;Z9m$C2q0?ijE)4*((z?>urWBE&;;VV{JGIC#X)9s#AOic#MT} z&ni!fC2|^-rIpJtuGxw;)JErfxUOt0sK$}0oJ46he5RvKYc>j(%1c#Q%DgC^fBK_@ z{81FTkOygoJv1b%z1Ekz6@T4&JJTOxD<(m0{dvyiuJ%Q#A6DvBjDKZ0JGN8!a152R z($84gx6r2+{`<~iKMM$?5%bZT{^JOI+aJZ)vzPLM0VGQqT2wC{8Cvr1d2tjy^kyP? zWu@qZcTidKEM-2h)y#JX3u4FJbQc*&F0b9b*9T2@TE|5S+qTD7zhqp!7_r%X&fdS7 z?B``KFXbgkjqAEi?o7AFvzlpjHGb&)4U)@Qu7w1{P4PlHVCf2k@WVNzGxA8?l6UU=ATlvmc*^XvmZ-q@-w%P zbROh%_?%)hse)HSl8xsyenCg0kw!}4jSgOt5|X(?T*a#i4RAfMv!J8dC?%C(ESgqP z?cKVU^c)~e7tfHcvqm^k!pN4;O&==+*WoMK5s&G+R>%*BT-%k^>uk4&o=HElW0L9_Hx zNBa05jy=?_#IJ|Az0T5nD%rz{q$^qo-Y()R|l*(n)x^cEm z4p;_k(5jG+WJ!GY-^#E@f6hr$u5X0Z=b+Srjb!E|x6kG?@li%dy(>)d(Ll^~GK3}K zN~!^!KP|F1Ir4q}W;R$I)zBF1GUJ+R&byFfQWV_7eCmjn%U7xMqay)~M#bDmwgIpC zSY+O35PfP>&EQ2^9%O#nSgPKKqBQpJeA)fb9_&OOOsdhBv~&pANHBVvzjR9?>Im7Q z8j1>@;47^Y#HrF!)8U5TE0y|02F$J0Z>LlN1d-_hl|E;RYS0R^{_o2nVCeBVeCOM% zRawkW$?6XX3j8>Qn2{H3BDH9mxJAIwA@k^5Nvr7G5zFWr#*U#AlvDAc6K;DZ7Mfbf z)6u&PLC?wcQ5IR?dQuMFah-kVn0dEe&hx$dlGZmD-{zGzyO;a3+)l%vH|_;Ap$kun zZ-!kQh2n#lMXiX_^NT!jM;FkDV2S!$4X?S zoO!fvlP~5|W=~m-bjPG$Euj}q5xEe?YPXZ>aE+Kq@&ZPWCKfTPs2VY=tftH9IwT(p zJ=i!+Rx_W$d^k9*QV^`RX}mbCYS302y2Kv~lNy+<^xd2EdNHlh3N0RLF~Sm7SQ01I zR3Rouz1Svo`mVR9pk9GI&v1r<8*ED&pW?oR%L4%y6nT_k`zBUK2cW?o&&(8|_fdEr zJ#d^9`{ZzTZ5%s_G^#svOn*AajS=D)sUG4_F`R42me6rV*rge4na6zOnF zVF=wY%#3-A4p*=(95RP<0$8ci-7W|byMR8|rHEhRiOKIY#`?_}!R=0!Rzy`f`cNtr zQMPhJijeJG)yP}Ky=EoSdqUr$Kg-Kv3;hl;5En(XmVG0yZQ-rJ7xRjMxhd(gwJ}CJ z#8Y+n@ZKYowY4#>x7RHQIa+w~t~Szff{IBb1F1@%h1^5mQ}{KUFtH&KhwI1QnXA2b zsU)lIs@Y-`Rm^hWXWGDEgZlwK(NX!{ZYB@VY?|BXz@Uw6)lD57N3J>xsv(coUlAe7 zT{Rl%^my!6_?sT3A++w{I#npXDOSg8yf z^6bHy4V_o4z-It{83T7LwEry)U$a$QbJ-L^9}#K%j51a;NL`Ss=?kDf#gm`wj6v^| z4bn2HI`nR@mXK&y%hK#HwcmJ(g@Y?F3nDLwZi{u3ybrGPLlsK+&c7HeUm}&2gJ_Ah znA_oSX0yeC?yI9!LS7)FQPR%N4C|9gsZ~F-29fE4cI|=_yv3SUiYmNPdct1BAVs*& z-hXaDykA@kWwChhr|(2c!l506M*R>gbwYYHCe2_}&rPPi=|1);_P+XXBChz~RDJ#g z*}{LS`h-~JB2t8Phwnb?te93hzo`7=cOeaVcxaIe|4r9p6TUJgKsnJtH(n<9nytg- zBlyOGuu|-w5K_xS#yl;++}f`~13iciY&;=me{7EjI728&t&AVJx^q1o=Xg0AyVQc*4lcahJ+!^`VE)}bh<&tk%0fA^6vyl7G9CfZcpDE zl9kRRVG`|jk^Kii5XMrEik+Wq{(gir;pz}?mpnDToI$gI9&D1hO7Idgfk*fD#KMb= zTW5wAi;dsQu~C$5Q6NV=fAk<`-`=ZQeOUDqH@*=DCny!!b8m?G=O|Y z0U9Sd)Vc+g>8*fD%l*C7mxD6z@k13XcI=>#M^Y^AyE&omD zziUB)h5CMD)N<|~A3snw@<~fJ2-o>Ur3NXulm6KV;~TB@jU`J|o}oo*@h{pqd7VrI z#{yaS!>w#j5i!tLXsnY`ambcP|z(TCc+DEP_lo>czO2l{l? zos1OWQasIu*K=_h^VYFks4~tHFlGxW?oY!|-WZNa*-0anvcIbt4xf(EM32Vx@|DQ$ zoY_wn+62n>KC?>*pT?`NnM{>v3fF8Nm`YuQ9x;^V#4lp}s&F7vUm4=Hua($!?IKuf zQEOTa*H%`|yG@V@+0m=m3!)pBz9V-E1?YTEY9{!VAkLcv_(ehLe$M0#Jk}DtsIh6# z5}rlrj)b-$zT#3UBfiquT-|TzC~OkYpV6Cm@8tK{{?d+8B7LvjtM9P7{gDsO!nA6X z$0fdOJYjzFO`Bye28{u)%rCrAN}O4lam4qg+zDAQ`6Ldcn>1X9fow*idygQ9WqkRo z8~E>jxE)?Aemm<|8)MKa?&wUF4FWiKfpKz`^}ldh)CzD) zbY_Ui?Y>4iXfX(vX0m!P>v~m$K4oS%PAaq>(Jwn~#rN3=;jRVt4%V+K;?B5|E#{78 zYMz5WX@=t)4|8^!PDx=vE~IDqf)k`?`?(g&(!4sohc0c~tUCQyRosL1S*Y%4J4i?p zF5!4BTjt9b+;0RItJl9PtcP+X^TMkB((c?L?n-4yb}h=`<6I$a8;Zm=XJ~ttQX%eI zz*k$9!wG>;!+xOe4ReJ&I{>AgR*ulS&3wp6_J=ZT0!(dvu69UGV!$~vIgiT;2YYX7 ziEdVn<|?On!ya96rSV9|yX&eB5di?ck1$+;v$!_&BVf{e)CJWr^b0pIcBJGdg>EjU zj+p3XqOPN%ti+4|>8o>pcjsD%O5N1=bC|4-WO5(n`u+wb6t6*CF8q;M`2Nq=^V(jv z?QSbwoq*nt%YI-Q+Z;qmkoCo2*lb0Oh-zB{8%kC?CXO0>C!PI?4)s7BijSe~XuSXu zvqd(km@LuFESNG zFx>{bqJ~7Zfyka4J)o(Lyc&p_P_4B=CFWNzCp%BA;lCN@#O#mJ>d;2{%^()U?9H6( zl1DuFxYW^hWGnNeFa9pnCii>cJlwVZ@=cUS&;&Cvj4KN$dB41y?^7Z1wq)Jt!AWI$ zo|Z#Ui`FP&b$H8E?})kcX8HSt_8660)}bQ)swK6a{b+G)&B4XnY%|Wq>A+P~vv`Rp zp7W`3eQqmwAbxZ+B=50H-F-X&E0dadNy1tcY0ZWS9)_uk32}WQ`axYkF0%j zP1}qC_nD*hv{z*_-fUFAG>OxpG4^Dq$G&^1o+VSA>9H!!D&+>6tIFEE>b={c(II@( zq}1WLfLonCt;8wy^v%_{yS(|X#M10^tKT+bN)zO~IZ@I}pFgF^`Pg7s`q$E2T181o zCkienL03vq4ztGxTrAqbwf7Gsj}98W>bsI{Sh9$92v@oF&s{bCQnWW;OB!KC=ro%V;QK_y^NHo)ry(Ybq7Bc zDU@83WGR#jr8A_9mGNIrpwM%ulP{X)v{ls9UygD#A}es+nIuxO+?foTaXpR(XzY$g zi)rjmG9*+t7m_3@Hdm4&)V4O5laxDtn^Dv_CrxL&AI9)yH1$vNWmvx;@SiG`E~2x4 zPK>7D^^K3C;}1yXHT;;q4f4oHKbsWL2plW2W|j!a3N-CkVTYWQs$gmASLn~SK9v#5 z$exQ40*V{nMR8D|zS>lG*q4`;-90Lh?y&sE?K3vXGLywD2g%#;SnI{8kXkndA*xXr zqph5LSlYChBaud$be}Lh1g_#smUSaqF2w7Ucz8ldnVFw|e-L&%+0sb+-IjL@9Nm^b zb2N1;Fgc=pJ2aUOnDx4N`7=ZR<%dK@`k0MGM%`SsM27R>r9_7D!w<=fU;Q?c8P=!O zlBbQ{orZohQdlgZ^fUCk+K96#hKsvQw#rQ6Q93Z)y)Gz;&kEgbT* zfG5dz$`H4Q_*3PMP?Usw(oY$-TFO7$0#pQ*C=)OX{*$B%CLU!rP0ZgMQ()5nz;Paa zqys*D?*lUvoEl}GESy5nbWSd2jTPp$m~3Qj{R8%`)0sDMLCd_-!0(AK)VVBFI3YOM zLrk#PEY14kvdmw}_3L@9N!Gc3Z$+Q0D}7AhB)9w4*IzH-ow+7d{63#E(@JzjRnY{vZFtc0670sBTZBy2R8mWHv<*{JlzJ(XxG`-kdzl|q$Z2BQoq=ulL%pHeaEjAljwQG^I8GK5DHPsYM8ZZj z?~Ms{V{hJI!mUH0<<7*7xM~c_gjh3XNRiB-Dktoyirqm4twYG*m0In6;iJ*2*iyW2nUv=M^>~quBCnN$I6g0kdnU&u=1W!%QfyhaNotVo)HOLD|Yw>-jtgkaR}@y(f; zy?^>#{_2tF_1nz$a(xLfC5Q~}8_IwxS^u_FNHB(YXL)9ztEu!YqN5)XHrgbDGy+f zlv{omQbOLFRR*6?4lX4jn||cAV8JYChn^I*9w`!=M^+S+5g#Ft5m$z`;wlt+EeC3K z{Q7&>ULLD0Cb88cd@DrqYGj#_SstvCx`}u8B6y4mSw0-W%w8cV}s~ITE&tn72p`wTulgOVMi}nw~FJox~{mTi?NO) z!{tjDl)rdBc_c>D5PU3AKsA7SB(&OofPA3)>Zw{x^8F~|g#QuZ?O1kXm z514?hf1mSO#3$M8cp$xzVP`>5k6UHU_v&0-c!i2cYsEon*k0ahv_t#>t1__@{&%r3 zpoabxvX2ADA*He4pYY&-Z1*7+#xFhy_JbiuEPYZq+khMEIbKdz5BKt5+Xk1bl_URR zEuA2!m_mGUOwTz$ujH1}>Wbt~-!6x0hF@t4J)UU!L-k^*Ld98C5(ti)K!7|}ew+hx zGJThh75Ja{`3kr=VOc0Pgh7sL4A%Pi$YoZz#)EA2Y7KJ5PE&hA`vYnj8jqLDOj4*A zhTo%^X1-+AP8#F)?qVHih-r}$H{};9syE^d>RNWkGU+9_7k-9UVP1x6Iu-c1b%m>X zxOLqu`iQqWp4Ta19(k*^+0RKCwc_MHWpGv3J*zy7Bm~17m!`cuAu$V4T^h2}ePWM` zORgU_^rx;10~3$8z-uE7P@XFn^WSh6*Pcuvy6mtmfHf}E_!4ahDI_`YA^0Ow8o$x}oHLo2-X(b5zXur&{9~gc4u6)>0P*f*BIO)C@glWe-G?(%MMz z%f-ZTkS>*9Tde4oI^-2T^ZTJ@+}+jJSRH4eL+N_Q*kbr7ZuO_+tXhuWpRB)OH2t1t zJPjY^col~&72N+)xi4L$E0=js3DR?tVpllg~K9g@0#0 zA8*7(kb>|yi859C*yjj7DLx(g+X=Os*5S9BhtlQx4X}tzmHLr)e`I-HuB!SZy(;u1 zHxIVVU4*gm;P&4snLo(9HSvom9E;J#@wE`)=nFEnBUjj=0Yp!56eZFI$BtyxNEBOfbS~CLqWJYx3_&^EFROahC9% z5WQd!fc(Up$+sTOEn9z!x)x7fJ&}|@XV&rq#{s#uMeYQe+Oa{0^z_6b3r*N$gcXZS zUV5`zferUbr*`K<<2yELn;wdUvmTkJ4a`dBCnLd}(s|*N{bD4{oh7cW-vjM{wB+cU zXZ-tbyaNW{K6Hq`t>L7fRg_cPQbxtMBa)($`ThjQ`Tup9dFNsfAiX1uuaL7s>!<-R z?<^`bO(emqfq6&7L)HJ5PQ6?5HwUbhg!g5AROa9#o41Iu~Y)>6tSclO^mrvELOmUE^h%6Sg47=m@ym9$eqXL zfw=e%+%s=6-ehDh9peCh!8K(`RM}!7fs7}rblK#=pD+)06pzBT@*?BAoOp{e=hDv~ z;b?DC%yd`Q?JQXOdtqGzxONQsCr?O?qT;*24m_(Q-U(W2?#OR#--anH)B$qc`1<3= z;R#~iFd_(4hkt45Z0N{u5OfC=oLVY7b1OgWEhf|&-b3pPkvd*zhG2!!&!ald2;+YI z_#Js{q-M!i;87y}{&>xDhKP@)g5qOKDJCV%Q0r#sd`e}0UwDu9i*H>cQQ@*N$3*Sq za(s45Y(Ep=FVRO_KP%cKiM`^>0nO>^K(vI2iHvvY6(L-$K3tsJv@CXn$bB7;Ve)$E zI7N0}xIrISXX`4yQ@Pv+F1}oh0E*iYKWL`Fp7^m(*k8pV10^s@JW(a{J93yoDkSfG zc5M79udgga@!TlcDex7mya&J;(2ifNz*gZMV84D3nernUOp_ALT=E6X zG$FHqS3?Q%3lI#oO8F0ciB*7k5A+?r$%&dN`im8tl+1^-jO&ojW!H%ndo4D&AfE!s zvg7Lb_M?mvp|GU2?U-B-i{@rRD9uWhC}pwA-xPv{Cf-=3AJ%=O=)%__$!mV0Ppk*! z4lI)(op(Bgz@2hI)J_o-)>2a7(OgnM33R&NyJYI?yt}y zCJ6-bW6?+8ZJGC6GbHV4KA%>Cjz9-F*7EKoxw9v z!2xSHBI$d-qA&dh4XD??!zG|4sR^|tsV+CEMPo%#$ff^ zgrr^EfYyW!X6ibo!pfg+!R1PihN9C6p+)vOeH%~rH-5KC=G5?dPI1%6z}9@yUkEA> zsaOoL)1`Px*^tV7iUh>4C*YKT5i>t$f@NVY;H6)97$aUx`VQT08X!uyQq5c;(?@u7 zE*?Zt=)pG397q)$G?MA{)ilxz%+sSJm`fb(bFW*dn^|L+oY4- z=2s=pvl~*fo>1walifV5PYpl@dyh-B5a1@VK%}L-`-QiXan#2Gr5Yb{)CWuM76CTT zBfVM+K-*Mjw4eDQ%h(+hlm${61g^NA%<;S8FE;E|&C1wQ?(h!`q7PzQ5}$C19Lcu+ z-w4mAgs-kqLV_~bBLqI3gUkhp&f`afmpy+C^UvEuYj8R+j0*~^3W`H(?ntKLI<^YT zM@Wxn880aKs(x2NHiAw}Jr1&_=AbU9t3I03R@PrQS+91eyPjV9+JsRq>Dh%@$r1xU zZnltx1xq=Ar958(BI5v#47~~^ypFv}2+4B?@eT%o(|o}F0UEmEcTk>LZ*-u!D=#Dg z0t_a;FF1}6JU|d4x-KdZ*eb9?+Tl;cC6BE0%U)LlLe`01BhNg0NHI7-=7oLt$G}x8 z9}X&t`0~L?rg7CzB+r;L4A0_HPA;xBvmOytQu4x=lD}aiVy7{mz%LvNj7BTmIX(Hf6+Y?jC)64&4^YyYcMOFn%M$pZsG9$DQ#; zOu9__p;2ITtSYyH$zWsT18_i&UGe-^&nU!XykG^3K72eyCO3jHLcs~71Bjac)-|F^ zc5Yz&K8gWE*^1N&Vu*2^{eQKMJb-N@%{)=5Uxw!f3@l;^lMIm!KOdOY3djW|Ix+6~ zod4B3V&i#m7Li~+I~CU1qgBcOH=?z~`X7n5c;^3vXlbA^Lp+9&N-9Pu7O#tM)|G4( zwgkJTB9XPWD7k~QMWGy=D)j1TYg9jsHFQ{#S?iuLM0Qu_; zl6oM8J}jnZ;vPGd+GjJRw}oeQz30CA-vEqB+t|6-?uXfy*oRmB57Nu&Z3kP*BQOF7B+t_@Qn zg50NTkJeMxnm;O+F{D&67&@^bf4O|)ZN0}WwaBFG&V8=6$h`Y%=2ERSBT(<2*Bx&1 z+)TO(SUr8`Y|O0xvN1CB|6g;}|Ba3D@&C0kmeqfF zqDT0kWnsePg8#!+(*K34{Jw=z{kxG7g&t^R{O~`}at9}J5D01@KhLT0!rYFdYMey} zDh&R{3-NqpqHBzLsk4`OV*U3A7h^0<(|Hy$R?XC}agXyZmo1(&DK*(2c1d$bRw(7x zTv@a#W>>?3Try>0Gs6?m-bwAavB&yMMV)2tC99K!3JsCs;h9m+8Q0MpOP9fXaUWXHGTEiU0}!L15T2QgdwL_+3HhCp9fC(2%%+ zKkyG+kr{Mt{$*Sr91Rfu4XXVwJ0k9NJ>@vij%XI-Eow&O3etrYz4hY{;8M86=a(@y z^d!w))C`@nkYcC21!jflDCUXK1KyO$q#lCh+b&Tyr@1m-pyYf{51Vd@-}T!^zU39Q z#%o?fRu`+?M>ipp`zhOk{Gqc3aU++Q#XO_&rNJ{GxMXtx!NtLFHBxME`PCm z9Ye7CC-cU^6J-?rS+c|6IKzOTw|8RC1>Aqu3|(?&h!0+xR64}02Mk@}w2qUx!L(w+ zfHkv*GYl$VuYU=#^|$`2Za|Vq@FaKE@`_=h*%pVC*fG$p^|w6zY$y5g7Vt`h1#*o>{g zdSJaP%t+f4XQt;o#ovKpxw~DM7zZNfEw5xL$fb8Nb{!DmnDosuLFVDr1u-%~Xp%R` zA*CA`OnO1FqX{`qFv`rge*c~X%(t&J(jx&UYu}G|clqZnHdyw6>mdf0XeUlf@`zFX z?)trk^*9#F%NMz>7S(3EQ=I6uT7S-dqS8b1=33qVnl%!{z~RM^w7ln_zqOv($Bz`_ zc4UcOU~n|SDn~07EE*AFj3H^U%M_g8Wh{L1PO`b1=FFxAep{6`$aPeZ(E=S)h%1(Eu)3N5_91mXT4$`C3CfauQ{H0`4ypaxf zEMY!=&zdo|Y%!k^^9dTP8d9+h!R-YZy zP8s<>;}r96|65K0(cZ%3{G%&sCis8HDd%fP&pyckYR16W3+H08-$saW&Lo8J!WUd% z(qbXV#3fyW(MK~zmZ(OSSCiI%xhHEY>*Uiy&=;vnY2byfGnYsj*f?Z-nPD66>wQSo z7ek_xu5gQs6Yvaid{!O%>!eT5Yu=j=N4FkJ?0&qQ*H8nuAtH#0zb-i*9e)6=9VRSA z2g7@wQScG7LF{s_Cb3t&& z#?V***ZcJbqf6LZ2!950Htt2a72B6_aQt-lk1(^w2_DZ~n4qB?NtfuqJ2swSAufww z)*G7*=Pfn5U5)#LO2Jm~DXP6r+7JI1yGHFI=jHt*Sf$(AdAe$?*Bmbcceie<6JTQ4 z+@E159zWg6&#KsBOl>pzP`4Np5+-v3NxWc|-m8Gx*XxQ3!CA5=sIr$=GjT!h^h z>ZaFHpEbh5U(-dsc;k4vuF{>`;!7H4#=l#jUi$#mjzK*h}uNLVtP0 z3m7sZa0ZK4x~5sR9ZFCtCnT`!QvO>Q#6IXw=x7Ew4t;1YeG#3sEdwS*rpO4TrO_W;vv}~gNO1oPE5{(C4T20d@ zkZ0I1DpAXAEFaR&Sb&wH!QzH`CL^X#pm{w|)$k?p8=mMToKJT0*Z*#GnqmdA8$r}j$J)(X=`YDyZP~h}NM?_%H zf1Tcl{a>axurvR^(;H=#@PGe(dZYChX_D-}rZ?;tNYT3q* zEQU(5eP+L^pzeGmqD=jwalnlDs0BJP+C~m&xWInF^$qCX+j>OQ@_xCF-_;5oDBypMWqUMj z=f!K!Uds&@2zGu<@~&Olxov*t{-NgEJy@<2Km{*Ib1QY4K{gj*U_DB=;g@#Phcui) zXPi(N`^NxLw_iw~z9l%(3yvq)x{_o2`UUJX&z3Ry-foMF|5F~iNr%oYHh9}b#$v{K z1;dPSS2oWI4W5>?elvLgyKr8s8?g_Xmi+ewM*CF~QYZD-NXhi0@a1&b)wJ8oC+ahr zNNT;bXB3?``6DFYvcF8{*>!AIT0)VQlI9)-#j?yOZ%csT0QFysSENQ<3zVlY2=FUx<7gDDIGxWwvce52E= zKHjElKHjXF{8YlzCR=+S-nsfT%b@58E1|#zRg$8*TD?-116DVPNo z!|rob=_5cksHanNO#!T6RMSGt{vnC<@gTIFqrSaZ0{`IJaDM0eURk$^@Rr;aSU~f3 zM8_2cK9(K@ePL=an3D+(CnB)Q>AG$DrnUSN4YDs zsAeSZNQ@-z(det5?^hCXKuWD#$rdNoXJ<}y>m~f8uUP-it03BJTSB?i6xg|qpQZs} z~?&%LUIy&@Z;l1G!{z zjCKDy+-IGWN035~bSpRLzNwDQy*Y2B?=lr9I2{e^;aano$&hB?sj!{V?$sQn*|Qz> zW!9ovyY3OwK`ntX9jn0y8zlbRGp)U=rsJ0?Gm^F1N>yZzI@B=H?x8h&G)Hp_0=RMA zSivf=PiBH0f2kR!!7Hg+a2%@K-%pfY4)*-3^0h`00d6qW`qvxv4CB5M~AAejko3uy^*$)4{r~hBc|2qkM|s|S*4?2wNG~eThq&L zFDRorYt!^7ke1^L2H{0aErW-idVTnRvPtJuH8M6R?+cqv&M4J>N((l5SEzJNVJz{{ z-%BrCaD78A<5tXYf9YB)2oPs>q|bdu72_uutp)MeC~1dv_=O=P1wXubKcFrjIACbbyAn%4 zHrbh#iv(8@KXj>gUYw{JYH0Gav^U06sw(uu%~y*95wMvm>Yod9{$m%Ev70>muqWil zF8inMmEPG;&$qdT!AJF}qN|UiI($g&2{LkP_19k)*1Xd;yNtr&7Yh_OypB4at!FL2 zR=L(o#u&)My4#x?GqWp~MI;;>xw{Y>j9Rl?XgdT`%4za95JI8ac@qX`zsr!V*crKh zF)M4zn4Ov;`$fC4kl}Iqe~+aSg<$a(^BLe!wJy0YFK==H8!&A|M{T32ust@W%t3Y4 z_8QpQidxFLt5L;mcIIqG4l~qkTFXwIe|ZtFl&N6b-WIxuL|DVD<~CzZjnZ^;q&UX% z?(WSj`UyKxGEvBX91()whvQ`Q`**-~|C`$#GD^zr=o!20hD;{8t!jhWy2|~|vC6q} z4}Okci!Mx6gB^1k16XjJU3p7ij_r`tsdM+{!;gy~uN&rnW+yNW1a?sqA!D!%Fd;~8 zH;DTDst!!N;Ke|CpJx9l&OM*?+>`#zviNB1Hat^X>x_i`onr1rw0W@>=yW#ZN|WM= zASOk=X_F`tccw(!a?KYm5NYW<=+KBBYBa?U`yF-N4w?QaW-j4mHBiEVlXAq~#Up_t z*7~{Z+NuPFde?noM!nvrn6r&xhhOQ_=_J)!VJtWCD%F+py#suYZ}hH$5mw;f@BBUNT=I<#eRaGs zPX;O}YfA=swovdd1+Uyn6iCKkG?32i0g;NiduYph1`5R2uIdSFl9_Q#9dh)M{TPTt z*;aZrb#wY-mi0_^mkbR*N?&mnw)|*2LLFP>G?e3|YR1RHjmmp!cNPj~QA4AW#_2Bi zY1{2-KE2%?u{>{IWXe2zv=#U_>XlFIzuaTC0AZl;907?TJ`&`n0u$i&@O-Tj5y;Dh z1U{C^%F@W)PVes}t6%n}KNDB{zoCHje2z}n0=w6ADS3-v+kuTfytQTfM`_^{=Y7ts zH?_vB`P1i8y3UOad#sxc2K!orGvW~dAaN$qnp_x>T- z*88T!?Cc7k7m*Ng@|$1>_@n=Z!l_n3sLniRfHh zB33yS=`;^#QO5T34H*4`Ns_%S_{9wPh1QDOFm|?+w_q;)ax!LR(sxw@tWJHLkmOV> z)x4Z)bPu7;((Tuc#w;qIXtOQ2Np!0RWj-zX`C$B>_C3-Hg`V0UEzyjefueSZGIt;|J^VjS1$Q`6uoZ=8wn{F1F2&%wGV~7Hfg=s0}?PAC%O+ogyzL8 zg};ZX{D-@k2;+4E?u7*RC>$DoD*CGXc|TrGLww;j;sr*{HU9{)$szv~ViA4|vB%9{ z@0|V+7j_+p-<(DE-#E)rjw|o~ikP=+Zt;TbFWUG!IwC{i*Mqy9IWTD2D`i^-Hvzlk zDDf~VB+%k6@Q;$WKp9quGV{4lqT~Cvflbsea{N$G6HW?zl+F6GrR*x@jHAA^dj)Rs z@Ba}pkNH1B<{#PAdg$1{LBN~RH(p#ufQj?Iw|}eUQJp zUz_CZD~QCmP)&F9yfh=G&n;JWMGOeSW%C7kZSQhON6}guh!TCMi{z0MIl0(DrVyw_d6HU3x6TZE}Mj@IzE=^cOz z)Me$NAfO(>xZ%SQ729`}PW@n{$X;-;70BLj$%C&Sw{Hh``oXtvwGP|liayi=Y)5n( zaPpJ>F3%F9>VH`gP?`96cA2y z4f&Et75R@)x>?^1D~|@XaUyzhKG6F3~1cOIz1udc_aW zm58ckuhU2kl-q0P!Uj!)d!#B^j)?RZeBcF(P8*Lo3$sSa%}=a(_pvh?miWlr1xWRW z;+UO=@Wee z5sX#_)qQ@hkWROR@sFKDU&KAI;Hp;)fMyzgEhg_F!q%n1Eu`<@0;VstYs*d<^8`dD zBiaPoOU7x@#q?E>%OR7!XyIJJyE?cGSfc3K`hyRj0HzMS{g`g+3f8V_=B^IBrUS|X z$xk8mxfi?pv)P{GLFtf50}A$Wz$X9RC^^^t}f-W>(_g&elgZzCg>QP(r)si);}y~)r<~x>&|CoNPm8hq*H1l5k_BhEYxgtn z?EF)N?OY5!cN?%_dxOG?Sk{EXKMGIpl0RiJd)}ycqUfrS`Yc2!!{f{ zrlXCfBNrlkaN4_;KtMX(gUcfrOC5AhWY*aRe|~yQxe{UYwq=IlXrcCf@&aJEvkmn! z{}#3h4z$C4a_bM`AB49~E%pc`jQy}izqZ)c2_)Z?`t;yghk;_T$-v?z3)1TGOt}!b zP9T%=HJy0H>Zfd0wIf+JPr8?}%8%+eGM-gZ9vpM1B&?q%7X#0JOqmfx+ADuLhNJcX z?1lSl9B9f4ARRu^`D{plBxuIpVcq?V_uAa6nYWS+;8S#>S-Q zF=a@zp#$Bk6R- z9Yf&JYk@b(LFyo-B;&}I#R3>NON^G zZ)djGZO9fG0%lZYJTk~<3ZBp-&pMDuoio!vI<4D=u-MjjT{s-fe1G4|Q^ z6}D!gnYFfeN~1}+KF>WgsfH&y@|#;$E(^}$>tkQ$(7f}w$~9AeQj+nI6H0`jM~_+7 zX=`awx3QYQ3OmzZ9=8rUg=68-0t{AbE=4m*)X;*<~$|1QJYw6b?#R!$0Tlq zRuisdZk~6G_Y-=yYrZfELv1jrVu=bKv0m%PQ{C_o%i|MEs8qBu z3??)n!ooAz(FI`aM19hiOJ7&k2fX=-`XY7aRV|A>fUF8Z`xyc}bf!O*`|zfj z)#8gV(?{1Nb>dhq%tXG8ZhhmtuFN@G8t7O?zV6CKY^Nng&lUq0)E%G*D==H6(Cne9 z&}X>_AZL3=M-8Z;e%`nse%`t{*@nsMMRRj>P5Y5T&vGWYp`{ zeBmeW?>e(T;2DypSAx$i^!6ky1PIn}QI$@ejAjpaH@FH-ot&LU7q#=aOGAsQg1=r2t4J zYKJl~W|cNF-LCe5=G3ZzBEstsS1J<4cVmcq2(L@5fuPm|O{RCUp4iB}-nMIOpt16O z=*6TMXsmD_fW*I3OJ!e(#*AJY@B$McZ7HF?%X_tYuz9^eXC?ig&Pr}N0rN!ok}+qL z48k59lnAGGG~Ecvqp~Dop-Y5{4C&sBEnIO#P*LQS&zInqf*q|U#YU*eWSK$tek02$ z!ALYtNU*Zc`+rr3gEqsepXFxt2JfvD|HPRoOUe&eP^vw*IxvuZib;hiuyYRqhKJ7j z@jB&Vt)w+~9Mb4M$Td192*-1&AP;)V zHR^mh@NjtY^!kM6jxmm1tv}c_=_Z!*o|v@`DCiTTLmisxL-3%n36V{jf4ADCGz||c zVu{#^5;-)O)7AII%cShV3DJ34?vBTQQR-mouesJJok>X}?Po2j>9}4>p5R+Ol-tt1 zUNTT)WB`!;Hb8V+pZB&R?QjCGUSRZV2Cnn7Mj|PhoJy$t830*Be~)w+oAUDqX!4)_ zqaf{^wTXd1DF&B@<}1I~hwl=nC!zu!*b*KMYKQ#FSA#ob*Sr}%Rj!0YM3@eO0t|6O z$36kasbm^;^C4Wum~mRZE}#I%slXevD);}?RzIu11dx4mD8uunh5_f6{LL%s9a4*} zvTwb2dy1_XUt0WEFYZ26Zd%ot>UN#WJoKJsX6o+$XSUw%!z&Tr|8)*TJ0SXXF^kM>ghk$rmqan4ecFTmNHpq?Dr=L9fI$nGbdR zoijufxj2#(7l=z3(wNaC4DyH{Sr?($ji8XlZB$T2#r9(-gW}F8I}lGDxP2ue2xQuR zh;wGA^X7ez_(zd7M8Y85mFiECwScf}8t{*}Y`I0>?UNpI*?o$+{^p8_33KlN(l0Qu zksAU4s8~-}Dg$bpS6ZfhVjL-4Bn_~8x<$b?1mOfLhrxD2)PM-UCq@mp|KoV9#6>+k zU^#!a{k9KuJnZ35JOMKuKZ|MvVe0VSXu?fpR?eRb_>vRXUlPtV;RNXTh&syQBfjk` zoCf9-bCZ>#2b=O5N+>H|!3B9;pd}gfW;qMeYzR$vQ-#;w3(%lYU?^IbUC4wK^S7OZ z!-(Jq6SnsIii&MY>&%-9mjYsh0c!8X#smP#MpkDOq5-;0kzNucD9+ow=wE)$A*pAf zpi-xIO;@zPZdSRWSfVpNek!uHryFg&!y||QU(1CoR&&b1-bGmJ>kae0ep93+T|$;ahds^rxFGTljZ_xX1DdqwNBFI*hj$B^cX zt8K%~#nbMwmtJIDkx%8jLsjh2r`=l1O!*#34tSMk=*4?cm0jOt&6>1QTL&{L(&8-6 zMjG!Mi||~}SKegE@mA4pQGC9H!fEP=t9gT;Wv~>Gs4(X^Ma%;vWLmwAaBca6qr1Hj zD(5Cy4)0qn>SA{1#+;%{Mm z7AZJquRvIoJ9bv;{?U3y@Po z9Co2rT{$mKujcc+<%I#wov%)0+OrU0HCzHpw8N^^(z%S3csU=^>X2*@VfBbmwntZJ zZARCAO`r~gUsg{pD=;>zkI(`BFz}dbUW%-6L+=j2LpX7ZcPx*b|3f`{L zb2q)<+O6LTD@7BgMPb&*#+z|M?jC$u-A@HvY*Lh|C<7|FHJ=ujSjQm@2bv~kuFVRa zRWe#Ncyu-zhZqS&xjIrEcsYm&L7c%Misuw>OIn*Nx5w`sd=&*d!r^o1>Q^uQrf6g4 z@C+ywf!iPbQmqv2mSzG4dif82o(|`WcR6YUb>S+~<*YtIPLD9e_UU3y-oD3IZKGGe z1j#^6kiCyY^&4xHZ7PRM_dg-dn>2sY6>d4)>0}eZNj~3L62Wm*4!u27g#7gAGb9qg znTq3xtasTK86Z|yV4JCT(0}uRU(CXM3YYiws~R`2)CYWBg6B$$)nDWtXft`64Z1JT z$Lxc3;ostrT$HR{peyG11OyDeyiW_9Ia3Ad8pG`5YVl?h8R(f+nroEy|IWjFGYD<6j47ZRp@V|l+x_Y8=K7ImdJqA|r5YznfkpdixoleuuHGA7{ zFPv|+DFAQ4v3whGds88SH0eKIg@caN$3}m23M_XKV2M%V4#6R!Y523K2eP2{sX^yf z+uhf*HO)g#@_z|&$L za~PVoIZ5S;f#u23&3#T9xuPYZkE|aveD>N^g}RGet=!aOWQrG#EM#9xlT3=B73#&U zqSZT-=5VuU7cD6UrzW|vo}U>l=rgj$kw!|1qG*8YYre>mJ0SAmRhs;l`G@HGUcC}_ zK?t(@UB_tjHd*>A+;#PjpISOP_X0B8SmXKsZR$!)FmK(S|QES&>yk(V5U(#(eC&jwJITJ-9EY7 zgbK%_f)0uy!G#gXy0$$z0?J=ixhHZL-4?R1! zj{_=!p=K=5-{a7Q(n;b!E3=Wi;8=xd zN4uZNbvQB;$d|SR$Vv*STqVm9I!I3vu9@i;b~I@F%jZoy6PRi1IYitZ6H8npRDSpl zfN;mhbN+1MVcgu+ZgvbWC{L7`h-pZcQK$uMdJFc{xO=?5BNzNsWf5n3)<+l{C%=AB zO&-FBZIaWn^#9Ic;Cl)x$r#K2T69IsbS`&_kQe=txH5%XVwri}$o-SWgNa=k)CWR+77;O8on2K1 zP4J?qO_KJPgN>?`CMztY7iYZ9mpI(WrFBx zh_T7=B;nQ0b=|%***ts7X5B>gyOB3qWKT+c$dCgHjfqMy_Q}y?(LH%ZSlSXL31Zd- zcly9-wlr1hEHt%%dEkGRL7ru{uusY$5hcT`I!8I2!af2B*}zq_LEtKye74{eN3qXw zd203#hznB!`Ull&JPEl^>2wlLX{j+M-E|>@n}co-a(S-F+d5$+OjB}P7W8d=DjOU| zOz10iy@oW6j3!KH#E2^gMI2^IW*lZqaRy@U;kRUB_qHDTrtZ-zDL0ZxvgD)E~S7{ ztd^k5*{oV9>dpxQYu$eAX9ZS7;kRf$*4$B&Aq%Im=KjuJ{Nck;tr2c2YQn(cH5jmX z4F)V;qbvXAPr^yR*+p+3pbEf4)acahQ3-Ooc9c@$oeL6^hi`%!O^S+x5icItxCQyx0>m}*ZEy^*va3y( z$>I+w`mNX zuxUFIurNmFURP^DevmyV82OT{LnYKM`bSnuU_I;b`Ph*^X)cmPRwG`aJXL0=r6ei8 z6X1C^5Azzfb$EH#gwo`wZyE?j6N$@QFUYuCwRNaGfCBovbe;SER=N&B2{#AUt_5`d zvvyreUuvL|Az4=5pa^BT4hku193VI(q1+AG;0=LSbrI1GErvo{af>-0Hy5ZN#xmI< zp+H}_>-PN{Jr9+pjb`|KHO@LTzL)Hrtl{U5{Sgr|dB^&FV3+u0U6w@v3WrLSTc4J) zkH<&Dd1@o{yN2eu6f3ZVy|1ACw-Ra_0SlpwTLN@T+JojVoH zYT51vEy|FrGnMb8E21-~vzlFdaKYUPlb{q`Yua@u?5!HjkZd3AZB$jy7m72gaqu6y zetZ)>TrrZNUJHqA@w@x~wu9F2+Pe-D)pDM4A5~xaa$DWTo>eGvf0JPu`CBMLAvA!; zq4E=uuAn4>lKj}-PsL>fS%Kf5kVeyX#Y+EIRCcKSP&VsK0;qJO&`i?D|Wix$Wln3fh}`L)T7^g-_A%6*Lh} zO4Dhmiv#6u6gVccALzOk`=e#ilRgH^Y$V6GKChdjR$I-JQU%>vO3oQ)`$HeJ=K90J z%drzmk{AAQ}k(;S@>PK7Fff#W!+PWN%uf-nFmgs`_#Vt zOF^1^Z!t|bukrc&#Q6a^Ds(Y(%YiR&-=Y_^}N@!o!BqLP&_~dNvMwt zQs715AxpwpZ^a532tQq z{hGD_4m{5K!9Gx6uqP13926Qs1p2pp z)M0tpmb$KwU#t;b(WcFn;cY0*+~c4WcfxT7YsPQs5~{ogNNxOz1<&CCr?q+yvC0*; z)hyzI8#|&Ezbv(boWX4r5V=ImLXqnR>&HF7xQ`84HDXBqmm|HJSCpyhVe*$FJ^4Ph z+0)YEr1zfR`u=*Pk*juKL8I3hkwD*c&J**Kd^1{Y$$51H?VxrWL7FrjbO=N;W#p;* z{sw{B!^EE4Ef0cU$podL)=^Kp*TNVSWG#{5u-bFH4O^&aLXv94<4B`?$-S3TYLmyr zq4Yp3G~lj9D=*^kp3hHCtw7?bhtMsyT`G%Az^%%}PmkJebu3Lt$F%ID#2-&(iOa2JG4W!Axlhv}{BxPYCAH&6INJ!WUUo@U35V8IC~iWK0mT6ikh z3Et+fP(35QmF`7ZSr6RSYW)Jh0iVuN)cSH`?~YuYYyu}zXBXTETh zu=@gk9y$18FW6dM%(7%Q(<2O%{T6JUZ?eKf&y8|xkGp!{Tv4BAR5;NWxUkx*)I26obBT`&^a_&Pw7#vl zmE~~&30F#5qz_d%BBDRSN_@kFC>UpO^nwPK9jH;F+S3d0kJ+^XGxn9`!__uK!qp6# zV-`GUle$h*yVT2WAr)LWRGtCR8nGn|UbRkTi(i8tMi{|Zp4AE)na#_rkqQv#1VX;&!>=_^B7Sn>}y@a&q<*M_wQ zWD5fXKH7kUP_U@%L@E@3oi(C7wW<`qF4&E7cYPUu}TQ>;XN2 zjrdZh&b;QdN>FQH_>Bn^Zr0gZygtOBxzOJm@`*2Jbn>A7Y)N0Q`a<96I}H;E?Ifh+ zUBna=;qaAH*DezATb$KPh&ZTA3cs1Q%!@YChquxJB;N=Mq)xK!Opm_ZusO%$>RqqM zFqo5bXXY@NrRKPmRYnk=5p8^~tfsnX~SOwDb0)UPMq4{IOd0(c`!L`;PZ)E3u37V0N*Mytgap)~Qd z4Mfu0VyV#a04XlPP)sL5edA37P)rWe8O?*^@O2q(uZkuFU0W@Q6W#vVKyJn;YuErb zP3EglIp zj|@)=6M1gsBze6zrHtDA0)6+dN5zw(9{aXWuTeU~scy8=Z01?Y$%SF!tWjb*(ZR^r z1yf+atkhu;%hNn6Ucppo;}II$Q)YQcSluIbq>`+231$whqYb@aueL8?#2}U5Y}uII zx8;y1gHrOwT6hg@xc#^$k7s5|P*c=W;%H2O!@NASVo?;%VGbGl^hYd&+ZqbaX@9ZR zMuyv^13O|1#u>5TOpOx{aGt=xOe9EWTxPExqxvztqZy;oI%p=uY<5rQSI=vKPIhfo zDa+bIMr?W=IosO3(8jX91SJSLLR@;Q+QRIRzJ#JgD7k$ZLZsw@0V5dzg#)OZiAil? z%(Rh=0kd{}^0Lt=qLmteA_pef7x;+21aC|zxnxyb`ZOvpcI{aae6~Goeyhv9r>nhi zR7L=eTQHQ`aZq0(@TpO=|NYb?!q?%sPc+GoXQ)*_weO{3TlTOv3b9Ua0%0^f_?a9{ z4soMDx>o&?@N*rapQWkOr`)c<`;)MH=Q`+3aaQwOznE!Z9|Pv;?&M|RXGE*H=4r#6 zmcGO`aU>KqP0ldTq}~8oa9WN;DD{0AP9pWEFWjjUyLyRwLhK4UfajJ1c?2_#IJeih zI{+%9@@E2z^dX}~R91iv+BsAM?nJwc-701SOALB+-~&{(R@{(MQFgX3bi!If5(Lgf zHW_=W0oFNo1DfnHE*w9krxo$@tC+jrCKUNTs(^Hy_FaA`Xn1dcI5;ZsgC+-l(8%PW zkC$L@J~7%+NT!{ZjwbbpwrbzNnG}pc%pk2^9;?@Rq2JB5iNVPchIT}RSsNn6)%Y;GXrz#q7m@HeZLnoDFfJk~rFEUys5;xK9C`?Isw?{5U8u4ftv_|NUwVN&Ec|#q^`VzB`t0 zuxQ1x7N1RLkris>n#*3FKgp1WL<3_RCYDyY_6IYIvv!`V&jeMvYQ-&mSRbGr<_8ET9jvjYWzppm|hYhE85f&Xe zQ!@=>_h_)3gHU@O39`v#ky@uu`y<#u^|jVv;?KTmc72AVlMj)3z;kY)8bhR*lky>$ zFdS#Pv(M9R?-+NQZL3xkWNz?Od!%~o2E2pL5ACSIL7CG6M6r1td*w9d*OzoN*8EKz zg@fRiPq^DOhqACc8#n+pX`>WVpzY&EFmiwfm)755xDAJIoOwP`gm&nT5{QPyv%O#J z?}r#Sj?&tO6Y8J9Tu*KxZ_DL4ZDR$T%v*>0+Drkj5gmYbhn)0!?+T#5&}NWtnF@l% zeV)fzON0TQQZSV-zQy%%q>Q(JSW{1N?RmA`I1MTd{*=m9peyTW-DO<{MY|JKM81>b zR3z)B&fn5xUBRp&a3cL$7Ro>Z$Nf|ok=rtk^J;l(ntWFORA*bb{uMPcTE?SrkQK9A zCcr@Ih`Ea3f|6qo2lBobV_6NK2b4}tpoQBqVpHqJk!E4>LpMCwemYB23`Lbhl0su$Z4S%1{aRqx5;`N;7!#*z(K8(U0 zaOqn2yt*7H=v`A&vY3`rmB;D6y;pL(6=4@Hz&;r>vLIW@v?_{4tLqmYM&{R85GgP} zbbc4v#E6k7RJuObP+fzioCYXnb{v!e8xxecC2o;R^zvbw5zmFL$7(Pyn?kG5{3y9; zYw*#76yjkbco+}3H;i8wBL#~I+#5cT(3BeRiJ!Prg%`G%CW!LJnf8VhXkm=~+_NQw z9lh7{`dR4x!U(u>%k9mHO%N0M+Gh#;rH(&0?m2-V_-lt{Kmhzm_MO&bg?1Yv><;sV zU&kFd=@*eed}KxEZHd)>CiMw@m>i-9Zv;bE3+)MjOfO0U4a%1<)e2ov%C4nEtOb=G znDWY0uhRe~s>?&<5BJaaJ8uvn0)%fr5uU#G8*goHNdZ#&k3jDEr7zK);zx|A;``9p z2O-AGL}a%XVIhOzjHXD^NCISrJ@TU8AU08i(*W4W>~GZjsQ?;xlvI-;$lt{^S%VZq zBUW%mN{eAVCk+jbKlg+nAtEa_@!JfCv&ASqm6(IQfqpHP1iJNRydpstPvcZ`_gUa% zv})~?SiRCL8`-KB$Z(Bt;jYL7;_VJZhqlo`hd&Y}HR)JDO% z%7L_!0pCuS0z!-)5E8eWItniH_7CttIZ&tVuJ{}}cuDsnm*YEtB8!3?m5_bUYD#lO zdtxR^(l5Tnp)`T3bnraDvkyFj83mDB`>|oQ-Hz%PaKP7=1%AmfATR(0?Q^YkngumS z(+#0Htg0g_S{m13{o;A@gbVXTbl9da(U_LFP1idhA|4RGOREAWl-%G;jyZ)_n>dOd z#%xf*UcUih0#RSF4@-el3x&D-A?$(k+INQY*W`ysxNeA@1S%Tu&dPY;q>)sSnv0TWB^vIdW!Jlu=Tmn-l+24!W zoLaXYnA}dBTxM?4U#pUkTp>kFu?`WhE$q*$%7=Uzih8;%_=lAPSsHBU1a+*^|#Ry4lu?JSZ>>+DkYLTW9%9q7ijt}*9C8xR3 zopmf$#G5?Mki6Y4#?OhP2_wA5zpsceNt3;c$xnVZwhsy9lgdU>YC?azIgun?omS%e zpzzd4e2=Ia48!9#*;(Oq9l6zuhLak2y$n6(@gSfqy{#bI73EBm5a?_H*fVH)5N3?W zp^SA-o#&3*OAx3vb}qRYeJ@61la(DMdZzx>rE?+l6BLywlo?PM^-_nFaI=qknFt+$ zLsGPdV|?t^bGFe!?$pPeR;8DaTo1PSE#5B!N<3f`WBDyhQPrW}z%ko)@p!|}zXa_l z;gK&(y^W?>pq#0t5SLs<=Zj?MJ#`%7Z5E-ap8H`MFUtPV(_{91dKPrmxKg!{-ZibbBS?541qHXL} zA#l{!p6lUYlk+G3qD0M+7ibFJ{{U^k?7=;cI=*q59w`IDjK#ga5BXt%@7Ff$=JvFa zy>{ymiBw%O*G_baGR6*rH=KqSJ+>0b4;nN6ko;OhrMz-JS+4zK5kk+k!4qf=PSej4 zr*(9C<-pW(EFsz>tKZ>TWrsmkAShPlZ;EG@Pp$VB>*r&0Y9tBo7MFJvAliE` zx0aQ=Alk2buB(&5B7t#!LPql#`moO_Aceq#@|pJhxyokkf^d4eq!bkKwvFTPMjc4L zPgjVS8|vWQ0KVm=TqwWj2kaL=Y{)e7P8&3vH}7*|nJ$V<#0REQBq9-+q*t{`d_xRO z)#$f3ZQhq{{%IoKvb%_X$k1tT@HNoY%RXr^)b4(8BB5o_)(cbHpE!X{jdng{HM~q69nDG;QKxcjRTy1A=CJzv9ka{S1xB`sEETF+sOR!S zE)b^A=tN>$NFDtu)TPlW!~MguxULYYIY=6K;2HJ{Gh_*jj^zX2w~O)f`V`uY9Mu@= z`V^YxGrnYCnsA-e_DhjtOCC&fgc01z6Vizau){P3&l0Nxl-c_5OJj8gVKk5wKV@)2 z6vV=Z1=WHI-+j9pCAnn~rY(?VUaDNJP zgLhD8PnWII_%)fm&3Y|ii$DvYqRfLp3s@>xb5zC6Q**CUK}ttPgT} z->rNLCg%M8S#%de)pLeaz9T>)`v&Jfm1zUCMhpm zS|Oij-{1cvkQb=fZe(v*P086>mh6q@G$(p$es(MKto`iQ&7fe%U_TWP{T_m?<%S(q zPc)b`HWPSC|22bF@f1 z$drYj8p&>z$-+9V9I&+~Q7mbyZCZCc6-kqZ*Her3+Wg_FAN`v7WovU0zOGoDm}9*t zQhnl6KS6#!rf@FRDB{2IA4;Mpkg9vLTW~in_gHD#@x=d4=+}V*_|5pXrHKx`~t|HRJ-{hwp%Z|Ss*Bq z1R6jq7;i_Q{Y4Ulfaul2R-Qz?y%k#EGS+2alBIn_70OwTn(wgk27;XCdH4I`of|aq zZa_wc1*NSoaIw>6fcGf97Ry)zh++%xGQ|gY0p~{ZH#l9L$A%(Pc8O9wTNgx8lGj_G zdve}Am~j!F!=|7L`e$l|u+Ioi1x}D2LIM32x&9|j{`aRky^v;8Pjm+wmFuCwj_eQJ z8&9Ap1G&N+UeGu!H|v^$c#al)bj{RBVuSwf^B4FWH225pf33r$lcQ`yU6SL zNdhX?>)Z(`9F!a@@-@6AV>*mNZS<3DjxlL*?^CEp_l{<9tuD@yBKT6$B{=_9{7~6A z9lS4MA6#8uU-k={Pf4F^1XJKbow|J-GHoGGGLeJ9M{tpMZ~>3JdHmn=#2K_V_LLNl zaLi7~C`N^?vX0+kfqc5@?gonf6YC&}{1R=nEo&ffy0hF%R0gUt5VBd!pf;NSW1~WM z19*236DS5XgLmiHKXI<94w7US^gnT~YsWuuu9f=S*#$rPdJTa_r)vl&q3JGken4aW z(7|8Y@1R`+Q2RxvrJqN{tQzSf#dnKjg#814KPW$^<|Lf0M6n33LT@UQV!Lrn5{-qF z^(y>$OsKMii-Y$ZHMuoGF`@ThrKWVEwM-VmH52oR`8Ftsh-;g)HeYngXPNUI8&O&! z{F=rDT!U=)i9zjTs=t8D$5pxI(}X!%tH3(nIwW9ch7tSDL!{^Vx^yyR^QnOiw@*O3 zJJ0ZEer;x?K|Pg>Xq2W151`L>=1Z;ce&2ZO4D@UItJZiwIb}83cYB(}ILbFdA)2Il z)&-xDN#q}HKHoI(F4;n}fB;-dp$wJwg#{Ifs0`FqGAOvd?zte+Vvpk;y{rpL5dmRp ze^-4)FvY$cL*Rw~@HurtkVZ~-u-YTpnV}n{E<{6Du%DuxU4d&EKh|yzAUnITfw&9C z8iK!y<*04YOYwYi4FUH^;vQT%j}rNo!AT*0hCW~TRCo_-ZH|%S%rdCqzDK|Ekod(j z@^jC-!Y8=U^vfi5hUKwpU!>PN?uUW>tz2g&ioDns!_Vhk_Tq|Y`z^#$@!-};5qpAW z7S-2uRWp8Sr|q*XhA}CQ*PD2!L4Dgl8L>P0;ixY9##cxA1_Tff(reXMUew+D-x#s+ z&lb$*M>>jU?-=K=97^##3l+te4jl3NY9P)puG9u5QRI4`723$j2S+=(S_vMPn(-V1 zmn)_gZw{xXlGour->r<1Vj`9U5%=e0<`Q^&`)}%ms0eGmr(!JrcPqydJwke13W|cV zg+HBLdzvf_)n_ZaE|$RCWF!$3C~zALw?flL+dgO^C3^brV)=Kyb5=0cZItKecD)MmO@COlcL70ZR;%xPqUT#~P_->$)_fw(=dhTEG)wSwNuGps+2 z1K8y5_W0DXCBVCChbX#Gzmz*ouCmg_G`^l{6;~FC&io1sCw@xj=t{o7ofAKTTbNG8 zF0Fxe{-ilUg;crtMNPr9S)vTS4(UWPg?0Eq{3jJOb+457%$_wCflMK@&V3_;_7Xf& z%gL}^6jTwphc$bt?oWsEnch>i3_ojAPS4(y4nfa`K=Yh*VkeQ6m1T?B>}_LKHoXP6 z@CnU3XQ;rOq}o8Fvf4BO8^pXih{K?jhH8?})6F1rndVbZqrFq*3%kYNlo~AT?kf== z*|*uS7HcgKlO>8CIz1S+WWuf710y*>%hJzI6P*bT7i?|uxI1th7jtVkf^_Cm?eb1V zD_52$8x^ycfxk5#6dKq_k1r@X9<;K|iMKY??5NZnQvQPj&s0xZ9tOdQrFlb{U>r{s z12OA-co_=uAbFQ zGo_y2=8$Bn+%2+QFjI8&^fJt!H*(lb%_y2QiP3c^i_b(>(}xD01W9>wzYCp2Z%$8T zV~{lDYmlHWT;HK-=si#UA$CUs+J>mgcq(6o#i;Q_bbmydF`t_otw#@-4&jtVJ$2(X zFw`4-;h2<8TO38hvmiIMoMw&-4K0wev}7L6oM53cHE+dY>(9Lg7ADpf=5dTi%S!*B~BparyD6bkJ5GZl?) zKk?`F-L*Row;EcvDg+lGEz4~-mcb1vy^Z#cmCs^pTN!;qN2rYYf>>Ee?8jcKdag3O zJbMo?lk+}#VsF?2$$`CeJkY&2kmKMyp% z-ALGs;Yw_)p53lI$Vq-tmGLAMs9?Dpl!t^bH2} ziwD1NczDrw72+w6yk4{B+ztjTm%Ig~r>n1hb{O|yfs8+EpL$Ef2Jj)1*vjU6A+I?@ z;ccpg^+yZSqI6Cm;9wLh;{5shwVOD>BA@aqHRbUs;IU8S0+u!Q+_W}$cVy}H>60pu zehb-2|6R!5A03shh_SUP4fhe$UsMv&!i@I#gF!ct!`9;5vOJ@1sdq57TMaLV#Mcg|hg1a_$UQMPgCx8JiV=o;jwS$;o3{Ov&bB%Wf@{pWSV? zpl-*Y(b>O%JK!(i{&Dg}^Y;qBlRI^dritIohg*rc6i@D1DZcjQzvK1-+ftf{ak4*n z9Rl=92}NKe4fcj>i>?;bTi804UWeGDv=Y!of=hVHS1QEM>&a6u^*ZWkr1Sxak`rGc zR&s^>02bKRm=YE*VwbZeKTEL;k5kDq`8mg?#YJbtINP4MUVoyd<^ zlNg><6RbcQbA6D|T=3#~wxMpVqdRZ}{Y#J5A(b2+e5Cor$t%(=S2K%a25sxImqm6T>MdvnqwVj5K^tP< zGuOh-pS1r+oEuxP9@F>B=mY`n z0J}5KEW+ZS*xdz)-C@Z0LMs}D<}s^)@V)5Y;d`$x&mZ`X4aL8uG9<2p^c%jT{X2Y* z`3>JglKuz2w}h{d|2?rW225;##jd{mJ+YDTzY`mZ|B=|xDGB~dp1v{DVUiU1OP2H zNLD1np=iOwB`j*&)MPxv_7{Xu}h_F`{4*{=Or$mzQts=jy>0lJWJJ{k12rJcLc;g zZ|S1LLD2pM3UWC%R|M@P=`F%9Gq&Y`!m*>`I_xLl2d335`4@^U2~&v@>#Peuxa7(5 z&K<%c2~W#$6YRpxpH&#s&TK(xW|pA64^cbe=q$oEMFr zN`|&9(0p<)7r5Gq)jN0VWbZ;AVQX^h?n#B}d}!FtX<4_3x`*M!r-rxqB$|9bxX8|o z*H)eui}TWQK>ndI;N}E7FSj@Dq`??Bv2$!Xd1%DFW#93LGlA5snHTIbV0G=APk)ab z;tE}WIPuyP?A}#bCdG2(p>ek*s=tGuPlzSjO_xf|^QJfNL-l$jI@s4a$V_=w82x+M zZMa^u6Mty7wd4Z!+Vf}h!Z)T0&K59r>P!_DF{P17FqB~^$+#OYFw(OCWr6!mZWt+% zS-!s_m}m~ue`3U4c##vHp#HcOB(in3irR4$TkoWvTF~!)j>MfsDbG3FF(CwH7v*0O z|2zt-1x zen4s5{o+m`xOwa6ypan{yWLtu#Okl4!MNVU+Jm%eW1ckY_?sQkc+;Z_8`(JX&!Hwj z&cb4Md{~#d;#qM{ceedbmIAzZ2d6gXIM_SGtL#n{X4IL#XG7{<5eBwvr|2i$a)-hO z&XQLSVL~LqrOXifsO|7*j{`?OS+anhL&o~ZdO*XA}VuL zRSYajrr5;HEPwp7VzDS$3*AuE@VB#~tNJmtrVB3sl&%KV>)W z1IN)tV)#1tcO@jLtfD}HrPko2_28RUttqgTUUSjeaL(m5m{5-7!HbY0+`o=~mt=?) zLeRuC(`&0bfFFpHP8v?AlgaYgst11UV-A%yq5u&LqelI`-)cSoLNJKXJI#?Ef}d zYZ3d~J^7+O{4zFJAhDHxp${XmN=}tuG=`;zlfHq(^kI_1=+xj{QXLI8BL7Mv z1LHw=Zf&`eu!3I)yR$fY$r?xX|3MOJs1^T$B+!5Vm*t%DKP~4dnkQN8_GQ|oHbvR5 zoVf+8vzR@V?*6`>HQ9l}ORj#FZwzNWiZxO{$p^@e=!@9*sYpW^lXvv-SeraK%0s>SC* z&HJz;EO9*sG3sHfS%a6r598;^x80x6+6V40fnk}jyZ8Ir(Y@hUaN(|W&}_g!El{M{ z#?obpTKnKPLcuY&UBA+Juan+n6chlk&p}Q6$S!o6zTY7*z2D|M08{|H<~U{ov8P}k7^&qAdi}3REhsGSpAHkfKii=s z>GwhZX>Km8lAt|+ok_{SwHZ`WTBQ0{3@{>9m7BwX8?HNi>JE1=;+ZRAy!KDI5?4{+ znB&n5&-z;vBu8wPCtjRqoQ8F5BQ#Do=$Y@v!$K7P;cDXOn1ag!@VZ zg{BHL=@`&Ok!s8x(vG((4U9&xIG5t}W7v8ddxGlnACQ|eKHu_OuS6+QUsl#y7OJWs zSuB6lt$ft9GSZ0TR&vUdzs_B=#7MxgPw3CN*qdC_qci1!dt$;j?@uU>MmWNF(~ zIU8u+eap?d0PsgBD=fNoEV;L_JlIw~$D5tZVy%b7FO?B+@^4g4)%$8#PU7})n1yxf zj6-Ngu3IFwmrFmorDRZuN^Ba=uDmanJ)T=0ple(sXq-c?4*(jS`^*lm%Pgkc081;? zO6wcU@RgG|Y8JSvlpaKJCYIzV29S2{>5yA2Nx>vm;$Xm`M|yrD&4y^qIwT&^Gmfm2 z9mC13*zke9s@Pz#eir<9*$LpTpWEzp){jhw33AGHwKXZkvIkEouTh7@TkU~chafg5 zf7`>G^(m8o3a2f}8rpfBoO$9g^L=@pXB91nT?|>j1l#DY3T|98Jb;DMI7<{C29mT; z`onNy9r=InsHv~5{rnD+Uqqk$sf0&pKA!8H@~x@hCA#lZ6|+#xPWRhS)-_JGb@t|A zJFM3%_eMkT(nRC)3h)2HfHEU6M9!izWCVC_G|FfOIC4Ta*Zv*b-9|0W+ub5Kss_M9G8E z6E`i8rPhCETiA+qlx@dy`;Cszth+uz8be&&_~X|?^s9WKaZ><}Yo!!nZdbO6^*PfR z?&gZ3Rs5fkHJyuvmop*e9nP#MMQ9EFb08sN zaZZGpD>hc4?;;(lkxv`JP=4UL>aCH#9FMc7Z%RaFFwg+LCA@V-ej94CuWm{VT$yb1 zB+MeZx_t_a>>d*|kRx~Ub}N0oak;wtQvmhx5(Qh<>GSw)`%um+X$)uBoCz^#64I{- z#46fA=@qh>Jz7clF$cj=X}0oGNV!~4F|wMn+SQN7#CAT?zYsz0G{Ai}$5=riMr-fV zMq1w2c&TOurpW4kn(~Z_W zqXH_m(sNHHJCYJT#bjs+)nI>@Vt-GIlFg#2?|#FjsZYi?`_yTO5ln(-S)YtsdAdJS z1g%GZXr9pMWT*IA*>?AbXduRn`s2Gd^jGx%{>^E>zSP^Wmh(34%jo}Y0(QpdB(U({ z7QRa~6eV~`A5kVig*B9?M}>uT=3a05YYTO#Ku;1Y zJTyW~aCl!@5-U{$KSD`R!n6XbO}7Gzd0?naPSA6Ea&@V!7O`E;d>jX{eJtqx1yfUrEb>-xyjf^h!W3J#}LD7v?2cH-Iu z6?T}qW5I$7XQ+?Rzt$GSV3!UJUaG~j8{c4$jx=geXlNjJ-%jU?cW5k^gh?4*i%vNT z_-%(J|Ngc^)sRgp-kjzLxh5~x9BL_x02F&RgdlQ=Ps=Pqf4{ZSqWvcv*BhM>yG0iN zvUR&DE<;m`z{6w(*w__clddp?FnAT2p7F?(0gs&zB z9E8zVk>_1(lolBn>*&Tv%a!d{HKNkbol-RQm7T9D{JYWy2y-jnqiL>)XgC{bVi|5Z z7K~vA*HUBJ%A{ut1aoG0j`^Xs?CZgk8^hjl4qnxR1ams2$>D2_j>Kjc``z05ZFw1_ z6~{L{UGIeqnpna!dnJziXGt5%)UiYx{`o;(uK3S)T} z6lNiG=hWn!)VGlHu};%@S+9~xRN|!0j%G(vtcQ2MEdf4EHMPk+?;nWTw4?Ni>faFc z%*)vB+cz^OfQp(I9b&`P*eTykt<-mR%jdB)RryjQalN_m*v7P930zusz8xav#V;Sn@EfEfr>eS@VxTlSgmt9V#vy2yjzKwZG z{pHQRst2ppwf)fZb(mT2nYX6cxK7?lxAdXI^o5WPG;r78^RAYSzXb%>?dTC%l*pLe z%ktCg!0EI^^f}g97tA}+(syyzC2JT?i}WJQ!;uw_$`&`kM(`AqD%aryl;e>XBUG37 z`pCO)S=xekUTEJelFynHRXDLqD=YIoYoes9<&_LP?ESDM5LtFMl010i_tF3X2D#a2R9)TqFTS9#v_2pN%rP z1pd0ooOf#v+u6CDCi5p<%~RtSmK(To-B>LI2Xmo{_7Wd{=kl_G#kLf-xdEiK*RRii zyZedarYisK?w8<}F3G;zp8%Ip)jAbUr8yt`g@Vmao}D!+*E!^Hns#xPi{4=;{8OAP zj}Nczd#|8wJ0^{x{-baYJ>BTkg~|j7TdKkR$?>PVxmE}ekdRZhZLBNn>5{?kw4#u- z2nd#(=?7ePGjco)tQyP9nImL7YFht>b?``3WrId5(Ajqhw!+UgBzkzS|FQRt)S*7T zy1nVJM!nxZy>7R^c6YbxdrxecE)YcAA>F6n`g>1wAK)I=jTi?YG8IxP>hYy1OJoEDj;#6 zw^G?=0u=qW>P92_YwebI;~F7+-@C=zbO@joAdBa=4Qz3P(w8NGl&lSJ+X)cQZTSCN zh#67P4MxEBAUgIWw9#}z6|UBl*nEgR{S^cFQL#UNg{a8M*MHEF|#PC5#D& zAjaqyyf#M_%ye~`;lz>%{#a9hg}y`ec_-%^O3 zqf{ZY1TXraooidhnzgd5k?D|O^id~q9QaM(+?0fXbp*7St+UR0zCp5(P8fCM4nR*d z&fYyLqlY?VI&WC-$Ul4l)b0TJho=q-c5~wq3VK2>=A6f$jY-Gb%$BrdmBL?QD9Q|W zvA;j@#9{NRu*Flb3ura%+(+PX+IvBhxPZh>n*z+(9Z1tT>IApl=dZDQ5HA|xQk0|uAoJzyzIacrFH>Iu zG=9dP*-vkvU?U#l0ItW_&B}!6xxr8JI!xrZL{F+9BZ&(joB!+9WWp<0n`~_rUTM}5 z%`S2LS)@XNOlFi*yQtYL@F5_ijMzJ~>b;Q<pcE{5|jE(pJ_?G`C4rpLRP z-)KG!Zn&+I<3@;-*h&Xa)I}&$)w=d{0s|2Lno%wA7mH@0Dh1@bdP=={+tlO$5S}By zy2EWH-T`w1 zQ06!9>h5LQ6ToL@?kwOhG7+k%8bg7&dt25X_ro~Tr%X!n$t%`lY}xB^F8r8jI9w^% zA!WHcRZKD#@CMDYgej>mkp0f?mz^8z93H##qGT`y;>g!TKTp5bNG_Nl!Fl$|cjmA- z;VN<`*fi^qjyK8FZTWa^p>?=-8H~F2={iQt$=;ZqlhwSQ?3yJlLc32S^r>B9T9oV+ z7n&cD7g~~}o-M;b^1a8?aJVr^Mg1@dT=*-DXu6@LGao+41gzf6l$XgYo)^E^V-ad$ zjie-Y{+P}0p?@)c#`Z}m`^ z<#N{NFy>PTHmq)Yqj0Mw>3-aAD(ym??=@AYb`Ms22kaiE2AVcUYw?DQCLI@|1F?Ty zvd8Yx6L61bM6`H?KV=Q$0|M|@RnA9U{9CQ?Cp> zz4?L45?I3bo$oi~oyx|0rMuLv`bjx3{*JVDOo%tRQfi{Ij057bDHX8T2@=s(@`&l9 z{?SYayz<^50Gnc_!GOH*->YD2TIqx(Ny)2bT3&v>z|wUx3lzwq(-^D&&P)EVty!;9O8_847t1pnS?tU4$%}J|eyihcWBLGm9=tCCw#nt( z3g$taW<{A2d1zw2%1yH?T@&_Za$M^MX)I7%H>tz4lD9sI}NK@w_P!?nIr z1ogHDi?U>(iHO=0>(I!2Mna$9v@ZRQk7p*^#*{~m440G1dnY3+Y9+VrDL=%5ir#qZ znl^_%dEejNg+F=OFj5`?nE{2L-TJ;EVjQDSYYzBfuf>%3iEuk?MgmXFv%5>2(|_gk z>)?|cT*9m<$GPj+e->xk^qCbe2Gqy5gYv}>LXHVZ;npRXF*89E^;O}^8`tSGA)b5} zRPewwKGd(0(pEh(Q9#AYVl{QjB0s-J)}9%@ekq0GWIYJ!T41rt_#PEkN{U9s*E$qA zBK@7$684Lbn2MTVH}GI^qHa9oZIt{dttiacrSZe1n8#!Z|B%tUugrc)Mw7B^vx{lM z84Z`;{@%NeX6#1-GnF%5kX&Qeu2!akC$;No7{yavs7$cou<#OD9#Qy-4s`$g*6*Ps z8X}7cqh4;}*Vv7}iaI^)hknIWd4Z}J8u^k8(BDpcZcO$jOXf`AbteLE=J(O%5pa(k z9{MkcsGZ|G$r^}96_&h`s$o(?sWN@U6Qv04GWdSrbO)5#UJ${fm}V31;h5Kd&)Pe+I!-5pPp;(JD9kF^*@4fwe*2U8>;GJFI`O4&0-9(5{ z(eDlE-wzJ3sRuZ)>Y05MJ7aC3%IaDYRrvb64sZ~d`Yx=s@N^X4U9zRDKj74sGoe2g zg5Dm6XDmvEYB@SlC0lhd5HzniC)!>m^=v5k*uN%e!{$&Ltzs-SZmdT9@`3{ zpT4$JV_;Z#DL#X)WhlU70OL-}vLB7pxADcM+Xb+^)$7X+a3BNzYIFpl0Ui-hxo))* zLTe}tbWBBmk<)0+zVeXnpMuPi!b{_Y)+n~M&>uK>5ipg=ER`$7U{)60U^Mrh{Ir!o zjboX;0g6tGenaCzlTJne+FolZ%xZinM-r1`s??H3#UV~Ruin<$hl%3P*{V`r6tQ+^ zH5eYKbhRqJEY$j~MA+Xgjv=d6DU~N4oYL1aF0T;8v;zDvVxbYQph@GbA0 z-Vmc9rNPeJYW$E0nW;4-zHQ0AvEtwF@<0aUoEqk^>fsdBfob{TIiu z{nWa=9vId(V)3Yvy688fRDJ4=K;O~($?euiyRv23Nbw4FmZ}En@ROAn*uu>+l*FJ^ zs=XD8K$qp78uBlVt0OX0^P6p3CM;Coozrm3yFGs9NF=%N%3Hj182#GE{O=0Wbhn{* z@yAn1u!G&aKU=Z#%23Jm$BSE5V0uuR4&vfbRIa-5X&`_2GB`7>e0;cHpRnj3oT{cg zDvlkiuon)@k7EUsR`(VL=z8btqQ?9~}Edc!nK3>W{ zYH>-rxC+G+!85X!hA}Haa%A}z@uPy71Z%^_kgi5 z2eu6PeU9usD(TK(^)&d?%t|4PBR?e9F=&_%03wU=K+bn)#!eLj{}YZ)e>a-;+a9PT zFG4tjv1u5FJ|}G+$2S6SwyFWP2|`OMREW>+gAxosY2RN3ZfXSkA*Lfw_d@Zy9KyLn zlTM(S()WgSf+w(iXVR=1;!|x7=j?X5A-)hFo5*a5_MH~eTWqq^`FB+5#^@cHj)}J}uL|c~@t?~=70|tK_%U5qV{wM0AMvlbn{iZBm4sCBo$K}Oe7x>o68e03=5Xk z?#_bT)gz>CwT=V?aQ5 z@d3CmL%DDpW5#Kt7}Rs;9~y2&nQorDQPxq5+PWO?_9v|@`c3pxyT}1#r`M2cu!_%8 zGBmQTp98=i>23#)`Ev6%AH9!&zwHIM0{E!()P3CvU~e~m-4P|@XZc+M`&Tn!x%R(! zbKWnHubu!-ADjSlhp)%K-cQU4d)16wpJ(x_Dwgxr&OKdIUoT_RCcEC=9*-ZhDjeya zh4uH<=%#y@T-&3e7$7He!&G?Qa%^Mc{ZdV~a9VU!h%esmCbaS82QJm`Rx&IX&d=+( zGwwVfC_y`FMe4V40MazsEBD$5D8c#1ZkbG!3ALSIj^p;8)W#Obf~l`%-rs8gk?#(G z?M>k0o!1dGwsx?29sD^{ES3MdU!N!Xb*%W1U!s#=X4wdjw0fG*d496XQZn`4j!~Mv zs~3dt+~tWHVNncd2wVzH(-cq!oPZ zV=UeeUZ=&A5vCW!hif=unu#@kG8Ae_s=lY0I8(>%4wPdC06XpOp@Az(vu#qVv`<;f zY0bQ~yTMkw*EFAtNy*vyKZ`wB6rZH&YC0Pdrd)tx3b)B;-s@<1_^@N%Aa~;Xul39T zdsPLt+*y!!rqnElW%goPwXP)7-{JmPlaInIYSeWJu{j&;JM<|~S2j9~U85Xomewd` zoj8O@CAc+GYVpN%S_5FD^^VLj9nN@Ow*yMb&w{1imH@5eqB~j6$dujNtCNMIi3b7U+h#=3O)r}CdPD}Fln5`n?nR*Gso%k z(W}%B_n2#@aWR%#{@7@X*~rTGz*UkA4^lU*HO~zgPrMdQ5=1Ul0F!zp@#8*O{@J3- zzGRXxlTb~Rj$nXa&V6mU2pdbiZBAn=xwsc6oS-pTx_ZYh$T$Js4mw;t*GiB{GJN}& z8C+3QqHMf4-F6x`!TL!kV2@y@f}T2CRE2Fk+uXDWwWIYkF~#+f zCp{gIKD}!T$dM9Ia-(!d;lIc0>}=b1l~Vl96QbRKRu!oFH?<_k)w6By z-H|rEct)u;D=7k_N~lqbcpVht{0@XXaUs!2=VYcRc&bbV2{jjf zq_oecu-TWo9ba4|JV=?5?sBEvk5Q0FGeF1l;XhA7c7pMaIa1FW(|U0f*doF+Ur*w;tabnd&(bXn$efT_Ig znKoWn>Wh`oxHXSN_#TxJR3?RVyue|T!qj>p%?~EJi-kzIXCnm zc6ME8;vRlL!JnYbVF(biI$@!5ClxL_KbN#!4Lxbx*SmY`Z&$9D$uM0dKL(Xj#H5)i z*n9(qk5X7)PQ*wVGqJC!F`NtWCbBBu3lw{tg>5x_?-G>-gS)kq8M12?O@=v$q>-L8 zniZqz{q5n|1tA!@#_J^trhiO_CGpx)BG&1Y%A( z+h%a^rD$Js+?KrpvGWS={PCdnnMLsTp*(9q8aZze7*U>Sc*PiRp3D}G#VVBCox+D` zEtsC-y8IuO!|H5iiufJrg24XnGk9hTu*N~D%H6Ah%gWqvm z)r&QHuHSO42r@maQlDsQn8r#pSsl7Tp`GyfMBh&?sSA2Fk81@s=TSHv>SPO|Wb>4XTxT$!d^7X|Z5Y+dgP{uc3nc z@{RNmM@MR(#-vCw%M=_e@3ju@V#cV3h-aaON;r5;y#fjQ?1eTGj&WklRu6SFY;zt9 zMLEgnR0I*|ZNdPe>g15d6>0j-xv#kITvEB%V*KJJH0&gNa#OVY?{K3RbWn zuO{uIGKE#Q>Q)VTccu4fP#FyjJr_+nG>yxrm zG`B90YPE@B2n)D)vI+_2!jZjpD1i7Lmi3N~aMWw`BXk{cpdO zDBhv*G^YoSIhd*Yk9bSHsd+ecR;r)S{<2+CRZ+g@5>%l~ zhfd4%lKi~U8W?bQ;-N^&K%Fli`}r;)xMJ+2thQ5vy+klQQyz18p;qT4?kI&#L*7uR zse-m?KV@>7f#2AVb7xCflXX;6QvD}xP#&jth+}|?U*d1fQmeO+y|7|lHS;fv`L&sI zy1`cIH3!G@pU)nX6pA87C>12@j}eg6CY=I0eZsG#}=wi0TQ3A zf%b>{Yn5v-AIh{j({zbRFCXLfu(MM9QBj-J>-UTCJ+phNUzNrauvAC7>258fX1yVy z!ergV8W|M*$#L=)WC)Pv8lLX$W?ouS&YW6~p{-E87G~qLv;XDPGa) zA|VPB+ovoh)nYcFt4COB%+GI~D~r1R5-FHy`y?Lp`Jg|c=M-4N)>+IC+EPzHoM_PqZ)anw2mMweK=na4*O3*;h$@ZavIzJtN5V@OY<8VK^JzoqtLY zd@H0n-_Xk)0#8~$fiAl~z=g_V?7ys9jv#waiPzs_lpOe}x_vXnx>4rY&vmbQE^XKY z`wvh?r14*Il@)~uf0vt0HDIm3M@ea#gYboua|;c)xw#CP?j&ojBw5;*5_71r{=z zUpWQ7p!2(;^Tw53z0}n{w(QMkO=`qrWx{`yyw23c6rWqnoQ$mv&KdDEP-*!dQWVDD zw;wzJ#!+b^ZPc2$`V+&^rZRX(MVrT9)N)jdTGAwyX<7=V=(MO@V)aww&|Oc$yI3-z zfssoe`#ALIZ#<;+*P7{KscMR}%)%34an{qe=Fgmm>E{ zT!3$t`bdI(iRG}g+#i+`nW?X}S=x`61D8*30d0f;$uQ+bg!xWq`h#0gDwJ7xNfY+U zUQQo=?0B^&mO)KL30fbV=CL_32Xkz9@1_t!|BX2d(X?d)naJI@J5JQQ*Z zQ|pJJXK#mY6fKZ}l5lk`Q0@D#;hh|BSr34BwapcvE8ceVHua-WmgiZaj4C-K zLMmso5z{Y~n0Zx(=XqO1h4I?3aRU3KAGmYX6Y^Kb+XY?gd90mR0G{d*&;8nijvo}> z4Zp+zD|1;T^lF<(Q(`*svTu*7nk6x_PC4c?>+eL+>u+qz`8TN=>4{Uq!^D%yPF*L~)5I_}8(T^f0G~ z)LAHARFjb1T~KUsv3BBS%^cEIEo#>ywSZ?!2De;AxlvT;hL)3PgTwW8?mO^aj{A*!*LGljQ;Kq34PN z??;yaH8&iIomZxU8$s0?UE2z=9eo3JXY&I*{l_c9j0XeF1>bH=Zq|SURW;;-vBa>5 zDm;6e6~&QjS{|_*laEAE{8_YB(sU9;9wPmxVC-TM_RZp691nh)d`BH$H#AZ|$Dq+e zLcB+Dwp$S0Y0D(Fg=a_Ipp?>^#TAq{SdthihN#g+y54U{@o#rG#jtuo?|@=j&hgLr z^v-vEJT#mTcJWLZ-yFh11`~R5;WZf?K6sj#bLZwgt!IOqJ%f=a=2kaCTD2GG_#K9+ zoq3#l)0%H1traRMn7Ps;5rl}glEyoY*WJc=Mn3%X2;p?WU>RMB9b@+}9@~lQ=`)dh zI!OOa6J7(VB6afO3_dVYRt21&7R zlx-X5!t|Ts4RL>rmA~b7M(W-Jbv?!PpFC8sh9Dp-S7FqEdqr?UTN9Z`rvH^p|3#l* z`TOXVh^%%ziPdQRfh8_d>o_s6=$zM#?F_EXUyk+}IsQ(dOdC_V8X(f||Jm91X5-z! z%XHq|4wWnW-*FjHjzr&S(cWJIRcVIgRRtJtWWX&(L@>LMJvL(VVLOh%YNiPlmttvv?QmZn)lMm)X~cSDg1>(Q|V;4 z?~LLOmb}T*g?i))5u**8sY-HlFwM5|T$k7yZ8h+(*yyHFH_{0t?wFuxujd0S17q2q zOB@P{qa^kXJx0P*37)0M<1O*Yq~lgc08(Un<5ZU`i<{q+ld=;J;&MK?0B%QW_dU|N zvHNA_I@Swb6FP&dha;IH-spBx`^j?s5KH~(jm-IHJmo`1S6kWgTHEiX?5j>r*QF|m zELwPnb4{K_jZAivf|Y#a?<1teLPoXa@qxB(AgW#V`IFonbGwc@CNCZcwc#1jBZYSy z-Kxu@i$872Px_*50wz2MUAlW~i0cZH? zz8mcXjg-lJF<>1{2R9TrU1igC#eeU!Bn(|}S8Pq>zY3=KJuWT|MZ%3xS8kB=>vzUA z;V87KqxXw`;=X*IL^iH}b!NbEc5ioW$~<2HqP65S5;p8b?q*Br!Vegn6((C;#7X}% zJXb$s*&T<0zu!cE_tR^8X=-XZVrGJAzJ#p)c@jsT*Z%Oc4UU_FgJ2xs+)X1>GVPbv zd5ZK08=$#1)-zTgy0ip_AWsg{9Ozf9SxtO*9WlmgS?-=6rF@Okw-fJbt#yvaQ~6a2 zvazAc+I{8Ted(i8jal1E*?~zYe;TzyPc2-<90uZ3L+)du_1vuIV~|kYR6s zXCD*;;D$dQyWVeKTEOHyh%RREqJsUEJwDbFM;q~j^=}~#2u@}`q9#k3grw!Sabs0o z?6x1~iQNc!`ZJILh>khvkfDgZ~H35okoXVmkfsJd>dEv&7CJ73~ z*BI%f3%1`UM^8#u8K*uxz0VHRQT}{2|7M}R^$U1ZV1UWhJT}?P{qCTB{(ws6O)rgjWIAvnu z*ZT|E>d3J3Vrx@EWU9T5=h!EcXiq6(+)XUBNId4GYvQoSt(EPsRyUy+%_u?a#vCx7*7-^E6M8G(wA z3gCYGsu|#!0~`1j*QCXnKr!?~M?i6Le8;IifldPYpF^lp5^_LXl$!K{)SUNP%3(lL zfie56IlCSlii`;}VW(fxUtLv6nLC$E{Mhfo5bpgc~!kBFug zzn!r=7T3<_z_x$20m(BR$@V#Q-L76xlR)?HUFFk~Pi1 zW9r*o7yY5b-`$=)DiB?8-MK}%sHo8i_u-Xj=FjnhTh=lBttCG#r4&Fe^J#-|;E*hj zAHOOu;Y0Pk{-?`^#J8XpA4vj8zHgepiaimfg}9o|#|wLMyvHrRxyv0bb(4-5*I`tB;S&th^Wim24y~@jVtV05R zE&S>Q&9Sl~^SlIr`#h(YwEy-+KvpB7`3`r)$cO5Q@mIQ*MdhIn|gxK14_B9r2^uj6^@dj#BA9 zwS29>sYy7mddM+|de|Cb2cQ+|QeUQ?sY)sM{!&c59&W%0m-(G}G@LBNZpX@fwt4aJvQA3z4|;hG8NwtibrY3|Bbt2Q-a5FA$-}x2K@e zh(o+aoDD7+!CX`~;#R*INm{CnXE--fp;+HE*$1zD{pvSe?{Gpr9E8p9?~zl3>2&V< z9Z5B)Xt3HgCS%ttT3UdT?N5B5j@L^gV`I&*9jXJ(TDMLR0ruX$=uXIgCwv}GhY%dF zl7oM(jhwb#9BKakr|!MPC~`DkArG@(iFFVIPhdfJF+=Xp6pU7`loz)eSy|cs z{-)h;WTYf<*AjdKV0Dg}RrsVF)RL-W9>wwp2y4QA@9x0pnm{kP)js6yu(YUNF+G7X z#F#Qa)-0op9$3fiVbnD*X17NL-5X)sJkHBVw5Kq(*;Mr}>H)j(?U1f3+$&`iO*~ zsUrI=5pFTsYBj3nhT;=vqKD|pz@!cSnEf5Hokz3+(kt%-SdWajI-)3ub+IehbF+eK zZXupt7=o0Y7UI+c7EMNAiw%Npe~>mG6$682yzs1O)o+&4n9YV~7dxKnXjmUQ(Z(wT z(*7J`#}$O5)>zPH&N^2%zfeTN?JN~WASfzI9xU1SEp7l*DIRITkS<^Yd0SK?X4U&! zFkGw0EXP_m!c=CtZ^U4Cl;5qxAB8qhY>XhD@GPwekQvzq_}}vEeqXPTs=v#oTDY6v*E9(nr_uX^%!y+rmrMS^f;GC0Aih<6?5@bW6?)5yys zopVz6_?kJ6Dy2Z-{HgSg<~w?TNJ&M z1$1Cm3=z@8)hHlsQ6|_h6;3d`UuEaq7Op!zR!c!32^sytaEE4CuMuhac^7A^t2Jz9 zL;7m{HSxZF5$v}ew#P`<=O7yZduQ$4WMN2FALkXv(rlS!d~7t|!@fqIa+5vJ!?fqd??t!I3pXV%Zkx$F7{Z&-F5fuOHt zGMp~~JnO?i?8XC6Jvr*zX6oBYU3HLt06)m+ix{cr1sawfy4nscy-CV9!pE8WHCA{B9e~ej^Cb@LiAq2q$uD&Cn9Dq)IGuAy=f08^2Iq9Ps)1`tf zlHL1bd=|<0B2H&}B{g3YG^S<~Vn<(N0auJutG|m7#V^O*OX~BB5i~i>e2o_3lg9;I zJ1I|e7t5T&D*-A0M^I-_3_iZp07jSS?gt<~4w9!7l87XbIl>ZK#QJw#&1hi4k3#BO zb^>C;-fN$>6v~wjr&(f1;yndALLMVr0#bQA#~1>T5!pXey0R$gU3iA0_7ePY?EFWW zJ$Mto{=+DEK9O7{B0mz8B$*-KK1D6pTHN3 z3;2<662otDH8CmhczgQ>{<>*XWa(x{Ziv|hd3z5;z?4bq&*j`&qxF?Imj92gcMOl@ zdBa9y+qP}n#>Td7+qRu-Y}?-0#>U*(n3LcCJ@|C4p8hm7-Boin>P9`cas#Yyc<61x zDW34)U}%;{9n5qP#Y+)5$UMEH8jLGZ$wYCO(sspqV zA2Z;$x;axAH#IgjzYTh7qhuHAeP-^q$jSH{MW560k;`~L+e+6HyiQ}VcmVBEGm0)P zp2;+d$7sPzKQ%tu7<8CPOM~5UGq`+Br+n*vm1|an`2s?ahT**6$G^|}VuNhb8tpPR zbTf-zrGsr=hP-<_^sw6i=5!D7^}#q`b_S5jK7ffM2Uq{ZHo(c<-MRP%us-sj>I;yx zj`ujcS5y}Fbqno#brEDFg2qC;T2CXXd@8)b*jVZo|JD2nN9Q~lsSW=04>M|S$Do9* zF+ZbOb~yTkO`epNS^K-U*{PbTT=t%3PGsw+n4XPixw)yiGj}PSS0}~9&LQ+Yrq|e7 zBAr{7Ig`|Q6|VZU7WBt0CnogQHZhv`4tnhegZO+d6o&$?KMk*3YKi{IFZ&R}Z$U@2 z%uoDVN^EyyW1hcyB}3D&F)MTn>8qMa*ckQc6xlkJyEnyJX_^)Atu-)I>&cW428lVS zF|ih+La${Ifo{M9l0|1G5QWuacwA6jx)2l{)0ixdAf2gS6UUh(Pw5T7d{@M{-BP1e z9c33f7k|vOI;+(EJp)yY)SxC8kllWhb!Qt8phv6n^5}F^U#l}RG z3T$y=9|hA&bDdJt^E7^f*CRM$C^at*zH*IS$QF+1D0>`kq8CpANZkdfftTb0Jv~`V z5Q;enFZ{}9YML=QkJP)O@$XHQk1mP92ut27>!gUy`60$zSK$L=N>7aop zJ6;iy4KFwPF2$uK47tE7ukia7(S!;Ni&a~YfL*g!k}EQbtPe^9!2MxFLc8L^zCP%R ziC4-D&`(~;1nE75oKUIc10n0`^yOV|cEhIp!} z;O*k1$ccFxICHdG;tYL+Fvzk8JbYfY0E7DzD2Mc!2S)JSKB}S3P8b=%CTN5+3KocBJf0>qCw~=7lCE$gpTbr)YqA_6 z)nM&Bj(&?L=C*N>(8U*`Zqvjyzg(eoZI#$>d(!;QHLS;OBd|NSR5aRPRac1vd;YWD}YgVFL0Psy^9xSo)v`{YiI;c6PX zj@S>>Rqq2=Wx|x$=S>>sD)HqU_QYntZ+5XI!XB^7`CGKV3Yv;RjaZ)mSKo7!$HyC` zs{qcs>3V=ln<(T^%X2`-CIE5_=l6$7*<2ic;fW%)j@V6ktW~Xn&6iiL;88jK`Df7d zj!K-M_>TX@$A(4gyP!{?{w~lD3^h3V1cVG${@(|T;a-IZu)DCmuodaw+t^S% zX9i>DBbtSi5Dvoo6H(@RyUBE_#4R6d_0(To@~|f2?}Fm7uglZp-vR7PfB{N-{X)v;?|G%Cyx-g6dlP{R zk;k?!^+e-P4ioIA&96UG?}-YiRh5x2uo%PZ# zuZ3|n?oiOxswX>wDaK#Us%GWvC4&@U4tke19!6?KMkcZIr-X{Gsb zwg3G6t^RE0C&Ie6?7gU&8vjuzHMh+*M=bVW@>=s+pX)&890TUuI!kB3ytg3Ix*_5= zyJG9zw;JFs0$}Dhp#4ckR%pFn5<4!>?^w=OoFsBke?Uf2tLBd&R;WJN7(-a~FVI`@ zx%lu~=DSGSyh^J=FUZsI#lXzeexSxa&U<*P;9jj#d;q`DrPccIF|XexU&&v2_i0&d zMzm3-<)2aUZmBO9#+jotNKp5HAzx#NS!5?5)TQ~>aK!R*M5c%Ph|9lt-5Z=dKi8u8 ze);UFKMdf!pN<7QMXvnZ4a5F!>o*uf{tl5(Ifnh9vCnbo`3UzrA4GdlGRJVL)qYIL^7)xI-(N!0N9{Gf8`mWU?PBb?{J0Lswotx6w zD?gE@P~z(Vup$H0j3z7a=XoxAE!!7ip0wpsJ1CZxJc0n@TMBlq6Nz_#F|h$(L$JXn>V0=Mx->mX zYJm0jte%bti?aqR#aLLzPzazHo?-kE7nK+@`8AnrT+?sdK4RrNQ zKLO>ZlsteJiF~bZ0K4Z`DX*^2BjRdFe=~x|wfUY0rw{z8Hx{L8b_qR9rPSrn{m^?! zR>qcOwj*1@*#c=Q=_c&r~h4*Kv>Ie z(L#%TOnbr`o}1Jn(;ujvhp13_vRwQJ<|jt3$QgI$hKn;nwLwZ4X0+Bxsy8p>2d3!FJM01JgCVgH9C!_O+U4^#DHYXbucXmTmMy= zkj)PGARDa8dT<lnM+N$_zKk|4pDpt=j*4LoJcRD>Ev}|O@i*EGrc-*;*U*`Kj%Kt#4f|bn~gTL7e5AgI5 z;vNwwhnPU(XN`!A@fQzQSJXLH(?>>()oJR(!Bg}XJl6_t?t^o2gE9_S30~sW>aAGq z$y}=2@H1+)cEqR+mgSY^%KsherE#jOr6jjDwaBE&bzq41Mq$(k?7ScET#wHIKJQHM zo-^|FwMs%y@(x61Jlqr>Ve%{J4RHqQIQw_}KYbt04^MN}2=_ie->wt-KVR?0wR%2; zoSuE7&Glz`V43UpD-aAR?!XWu^0@xIByX%Y_|ja)&B|qscy!($ijiv@_!G)~TV~ki z|3NpNc!mit?nw$IX;R6pPmAcfG^B;gQ})mp3(+$3)Z4CeGn@J6w$TfEC2soqAN;PT zZiesXAg%Up-Cm9DAtIei-kG{R_`LVITWx`J;JKva5Qi+xiBC|lll#tSngOB&W6I(C z?LSQxX&IvZKCzTym_^e5K55md^EHJTvO%V8#=abzjZx{{<$|i-;kzoPm6{ba%SPXeMBwEk0=B=!R^V5=#9u)c;F*;$>_ah z^+50N>JlNg#y6ppk>%u5dKcl_8HY5GOF~3iUiBU*V)v&MD$?5re*%kihsrl95)@|V zlc>X5|1?DK(TeVf2vDP&2K&tBg*Q4t->>bP(Fd}#*7MY;+Qn!&b1FXJC zmKiCu`!fytC*X1B-$#?*#Rn4!)+^A?Fgu00fxe6vUB8TrpJGH8{@1L?AlLHC*#C<} z&{F6UbLt#ZT-4jJBzcYTKBNioJ}SNR6oXo8fEVrW^z$37_ff081(>mI7>U6O;94WP zawD|(^B{Ox{ZT5cw68U07BNnxNh#&R*4Po=k8-ozXZ#j{0{#{f#GmIx<-Kwm&RSGuO8K! zIn~ux&4TU?J@iX$9fNb|+7Oh>jlp%!#@*>9Ml)l-XJst<)l++S$pl` zlfol!d$wo!2F}L`;Vjd7r*EN{=gyoRE~YOhB<_fRcF6{ z<{oNwHjp1OAp?jqH5QO2dn4Ygptd<)!@n^uOZgZ7w%K4_HXA$qe_tNrOpUx>5lQ|( zXCFOsWZ=^I{;r2M&yA&am0<*`?gLZ@FrO8e-vGMsaxXvnIqsIb0ko{+?;uQm!0QMG zK+i4{wY>hIn&xWfv;zo-Fb&f!QEkiU44T{|(hMmFlV}iW$7slH?@)pM?n-$t+i3yawiwyD@d~XKZs#2CF|s zvK;Vh&q@AiS2raV&v22X8yIc1cs`+Q-J|eg!Fu%<9A5{Rrwr-zAC_-^6Y4(CHNOND z{B)P|ak&{Vif2<|fcwR3`?$9mdfm~>sGh`ek2pMm>5|uS6=Dw?h+ZpoUV>PX{;3yo>J$Q=h7%h_Dr;F;I|t5V4=yz; zx%;%w-~IcFuvwaf9v zc@Tcu<38Q?V7Z8iIsw9>w^_gw_`UbLa8HpeG`LQ!xy+@!stmcvxrUApqSpAtA!s#n z9>X$^tJYFZzQE~Vf>>Th?wDd5Uu0KNbALiHbS!LSZA{AFVPdxh#6hTf8es}&ox^Ed zgNUn~8WL=WH2SRt!t8G~{X7fx$$IEB90{-?Dv6jlQhSqEHNS4zhhc+-mDaBM+>|&x zCJw_h9s>iKuI}TzZ1K;%URhy9(1MkF!=|fii^DLaB8|hyrf=CZ@7bvLFqHi3+tsRv zD@#?C4O3Un9h)9vHQDkG=$LF3JSHV-3?>d8hn7vUn>ftiB~QKQuT0z_y1aS$sfaR5{IeiEX?G zB?{T@;2JWK4-)V%ltbjXCSw&J%0s?C>M`-J>!*{u<$gWbhvX*cj3KdTn&`iyCdgREV_-Kv#JF>~JiHSRGFm}$MX42V|yjT<~RBYAMvhAcfqg9CTpqNTAFl1hIt1m zaswAF7kZP#oa>BsySyFyOEbef&(8v1p=b(kT4RX!{14m173~9Em>~-%@atN4f*|1e zoRo`&zaJFuoRk})zc+y!;qzhGp};=Y!2k37P+8|qI$n%3+9%O`4S|V206BL=jJ4W) z6|UJ_&z)vPWSVo@BUWrtbB0geJU@B^d^{$qNb9rYSiwch!ozPc<~<;bTR7c;RGZ)# znZNl5HRKzZJ_P6@%(ehLP!-bvoUG&Bhc_$GLeGmgw9z{fd>X94QmTgS_(NH(zwrk? zu7GCc+_Q$;X;_O`T!Ulx8L#52RUjzJ5&ow;vc_C0viK6txS85xj!~qR^)gN( zbP(^fcG;gxJaWw1ZSwd`@h%gs$Ww*0C)W@~kFmKav`$p=?!rIO( zUz%R95a`4v&HAVm>!F|Jm)(;~U z?m?I-$t=$)^0J)W#hleII8mP7{keXV%2~cixyQRvzno~ho~kd>;y>0yIue>D;z#&Gj$X+EUL9D_ zNLBHx=IN2NqLd%a^qpz*n zW_xV4IJ7LP#@?LWPOyOuDQXzRn4RT;#iV8^kAS*>HqP{oTZL=`^$bnMf`YX+2yc}K zaLfnBJSOr%P~YX2gF6ES27ifyfjvYDhFy;$LFt5x9=J?m()>Y5{97<_1eY25%hcJ2 zriNkxc}t(i#xmRbE$x%yRb)Oac{_80NIPI227;=BAw%Jjip5;~DZFMEs)|#Ld~hRi zK9D}Ngwa47KRVOl;WGV7#I&>Sx^2UK2#i-DeuXAdE8ynvUw`tXT8N-alv)h=%D=rW zrChnz)a-AJ#Qd0Jus|hM%vos6-^nu#|VS5D(@H;W1xPuwtJ4j}`52(ZBhO~W8 z`$!BBXRWDSqBp+fD~eRgqZI)|uWUT{6G+69c2ue#=i%sd5{Rw6*xYKNkzY>3Kjau- zsiZ(zFN?4v9oPhk$Y}3wJ?gk_x#?^nU60W5K>;;Lrh`HfL-&2v{mvmLSgyXAM#=_! z-UE6XZpKeH)`1z2HkUw`CmKI6<12|Jy} zPEs*;YsiQrsqHgh~1In^sPbha=Qj}9m7%!_9ICq#1yp`t^hzY z-Fp%x>fJBC8;L~h4EH6HbSf=|<*&5JKx7gitV9MNgF5G)gpdL_|HIdmTzr3#ry`}& z2I&M?L15s;H&fnkcoYpGr}-FBViqs^k7KlSqvDMMHP#9reZ|b3*x5PA6(vYWj)8w^ z_;_f*c48=@-|Mc*HB zvP1J!)p2^ql}ZRpPtbPjXZXfq7AA=`EQ&iR;31wc)j|i&?1U*&8Ulx1{4&9EAfa9o zOa?x&wpO|W#oumYBN!2*Mp9HM#2rErmm3k2OPFY3g4!hR#HyoTa#E)3V#b6WoSQ3Y zDlH|JK)}M?6nM!iDh7-Vj5GZjz76lBI2>*hbdWoeTFxy)w}vO-J;64FqmCE@)Kg`x z3+%ey1N@(50mi^ha?=`=cLBY%b}ruBZSsX~BB`|);Y;9*H=sr3(8VwBwVCk6lcUXI zRQ(~TH;^Rm5;FF3q%CUBq0?b^Chk0{31^No_;fH4R;pPEph#KoNnqQaw&d5_Mk5I4IPV!~(iWal0I=M8JzP(SNsFgJv z1U?SLq#5h|wN6W+la_LjL`htsvM~h4FcF68!2Il1MpWG_2U7}YGY2f|LP@SPxyRy~ zk+r`cza%Gzt+6`%#Oy{U`eJqEzP^02#4D)8t2n1VZS_1=zf-Eg=&B9z+1ifJ49R6U zrva*8ee;Jli>s`CKgRyH`m;gG*ZL=fZK>rtDAbBh1CH1d-ktPC!%fs<9?>Od!5nX=5PYHHp}ji$t76_Ge1FtUjqvc9W6SPU*Zm{AEYunpJZ~yaJAO?2Mr| z7=9Qp=6*;RWnyJ*Vji}zGkP%uCx-w~iSC)R0F|z_O+lqa<2T# zq%f?qiA|w|P2+=0xL@<=kj-)NiUCP|5P!n*=W$TG+7id|ejY*HMmgq{$LQ-Kp7vD- z+7hwqnuzj7eyAs0ZyJfcrRjCq^rb7yaTS#Wjpj${0}N8sHO;PCLb{26&Qy}G%9*@G zdHyOgEP~5LDr!_NNc4J1Iq6Zjh{`Ff_xslW%DxVFW)0n;;X@UqU75F5YPhHKbC(}q z=deO)K~F4FzMfa2Uv$_-^~bGBcIh%o7H99Xqp;G06M88=$11ar90kIz9aKiu*TUAq zqm@^{WIschW!Y9e3yorkQ+TOlIWmrc*sIB}*?3Y3vj~IbhG_L?g>Qpn#@YNSg6uzP zt86KYu`=JbHZS3)Aq9TRB+R0RpFMVV;B)V9J`m+;$KR3hXt6fbYZ}CMWw+^|538Fj zTY0xT@N)!R1bE%=9JoEy?{KdCI8QyzNwmelCS)PxVq4_4%SYHU#yfL~mT(PgIQoXr_>JgVO#RMHW|RPS)78KtyH;jJ9Qbfg}HUMw6rs8IkhLqwY--I>df9CJ}2PqRcrQe*E34l3fTLIp7d z`PRpMen!NK3>Y$%zt^L?Tf*E)a0 z!kd5oLQp~uPM#{*g*(LgOHj|^Wfw`gHgE^KS}!eyIITBi#n~4O;VBndK2;%lx1ZA0 za_5C1BZ~96?@kZRDO5fU9}2_7cuh@2+5)GJu%vSb(8_2|v__iUCY+_-MjbNax z-}ZCbdz?3w>kM@9&g-B1!`of^^p1mXO?}!O{6`22=)0}|V|km@d70Wgd-?c^X=Z&3 z_=z<74B+4rybNIH>gMA;{~0KwLjOg9?xW&|LC`k^#DE3fV*R^pdm4Eiae^ktIAcDz zRY4-SL@L+5*8-lPMmu2@Cq=H$)x~S(P{c0QLzLhnb}@w$w{vIDKLMvO-aE15P?=Tf z?Q4Iye%zsq3`fUZPR6+iDaXK9Q~3Acsgyo1|9FVKyjY0a^&kcU;xVn@k6om!_K&e} z`=(#S0c5-aKC;UYO)Ut(?%nzD@{jDErywHqMX8CmlAU)jBa!M5c#3Pi_OnJS_2_>nt@`!U_cm)8lD^F2P#2UxD4 z(iA}-B_l@w(Nv>@;Oa}M!GFC$1bD`{F0q-aBOw{yfRTldqRiQq$1VP1G##p<<3FT% zaNIAEeZ=vQ)F8DQYsRfF?>qk0g}m3>AA9{*y+2zYoqsnHn_M)Jq!~>?T=M=Mv8iYx zUU7}RiyM4{&1PSiP_^Ul?t2}thRVwd=U=$?(zOTJkB;-lv`8@{`O;J*a+WQq<4!ubZTf!Eb7kO7Wwn1}&DtqjM zqm`eLc{%}XLkWc|cGQuvY`hyGN|_n#*uD60LfS#W`5N8ng~8?J*sSv^@GWEqkbwal zQwL3?{N~}*=slVBo$*>3ZPYGpR;}~)K_;6vd zOV@Vgd{8AjgrC1@fRD2bU&J?4`te@*oq$#@vMX(@=K|25eei|8|6 zib-v_O+9|DcJW>fjGsJmdg*gCFBFgai5}()mgkBUmgS%RL22BoD@B9g*zH%9n!)gp z^cP1cTJl0~w56_%VVA;mIXarDj2}gQby^vw-fQ?(*43A(uP)&(zaocTU!P}Z@D|Oi z^3+SN2(Ef?wTd33clp2~vtGBb_;@!*ZK8!{T_vC{6rnzeph52A@^BvDH}n@|w^5F} zHvRgiv`|I@9gLTzm0a}sTL85GMrzK~puMNspt2?lUmDgoJ+CS$Tc{$x>rS?ibt{iD z=a0aFpOp{bzKA=2yI#$B<^iT5W{Ou?UJcb+dT5VpoSZObI?K~|6=1n12iZ0UUqwSH zMt*Tb)}S_o4YD^{T_pRh3h_yjtj>XlRt9iQZR=j z1fJIF3xk3x;UrJ2MQJt*Q@UUKi82Dy9tR~dKqrfjB1%GK5)(qJQYh`hg7|>bswBl8 z2TIVxM)eHU8!N&+0{sPUoT!EKs`f7r&GN~Iaqz`Z2|_oEeRDYl+lvLoGHeHVLCH-? zi9FNrJ!)l!B$c5gNCtG}B1ooiX4yjgEfmI4rVoDP(kEVNZH;w+T01>%tc(mDCmR7f z+uxTBx#>ittRWJD0$#2fk|On_4n{Xy2x6$+OoRfLw_|9ulMw338bTl}O~{ z4FR=iciZ=!t0>e3xBIZ*u0!IvR~I=>30I0_<}7i-dOeWU#x7VGw&RW_S&))zIT% z9Lz@3B4Gn?#XP7g2pV$|---M=6-opUH2y$%hu#vA{yJK5amm0C1d9-nk*gicLsI81 zVEuat7IY;Cv=qKBto{z=Y&j^&xHpw?+)X_kAb{xo7_Yk0Ogy}^Zuaj4G}hyL-<)LZwnyuzYI-U{jkoA`=#ysef$Jo?!Z>W3^}k8KH~SIs z-2^28w8G;V@*m`D@KK+D^?o4~*#I0Q#2m#a><7Mm#sJIbiu^CP}pPyYUh zKcuqNqxkc8i|)8zc+Isdj~m&hhusKRIUzYi0j+Mu^`nIRUtS0{`H{@ONU;1tBZNOz z1}N?R&gVysGX))kWZbtXg_;@07LH-h2( z-3XiOk^Q%Swg1_ID20Z`Gc-BOYrZXPx1pFX3>}vl|EdUMNYNvP|F z(+qPZ@4vR^_4Lid{T`=HlcoX!aSV>m|IDY}Eiuz~AEQSJ2wXqwXnpHn zz146^p9=UJ!-`b(aL!V=ewV1Q^<)~SGL%{@o4M1fWrOqQT%oqWOH{+j^vh){HCPwR zHtSL#sRbD{-IYG#iXFRSbvM^3nQK0V1w+#?`)a>p$Xamo0Xs#$60fS30M+VHZSrl% zG^G!jwEEO{1|ors@#WOrR-Sq*$47X|$4>T}HXA(^{+W69Q>Vgz(RJMQkZv(}MZM)S zD>=B=qP5F8t(5F=E1OU!0{^gR&zD1+ zI!Ufpx)hTc%&s>S4EPio8;^YoEkO{FzhoL6oWz%46f0+{t1Z5e0e(V>pHN3QngsTU z)=-*SSy`*CSJ(+hVe5;|6=b6p>T-LBew*jk*p4+Hz0FY{PMnJ%vWndx(+fap`+Ywg z9encnd_UYjUWpU{wntPBKP}ATpM;hCkX&ORg~>Vp-78N`J1skg)#nX zXLf$W72_r_LUn#TEQzSRGM$O}^M{rT%7q67k+!)^E$#t(sXmDM6gcxvYyzzT#HMhL za^mU z4qRzCEV5>e^X6pZFFjQSu&W8syCAe&EConUpO*&SZ>;N5g0(24TpvWmpP(DIXL_@c=qPB`r_C~C@jPD%kbv|{Nr&dtDGl7-K=x-As3SseH zJCrLCZ^py1C`E-%Z5IqE9=GZl3^&{LMSS8}}+C@S|{`9Uhn`s?$MzG%_l_AkA!hA}xw{ zpmyQf?prh1TWL8WI^J|q@aGYuQx{P;+{FhAY|Ov*CtVVYboQc$mD!ua>9t1r1dHin z;0XIMfL691CFAu{r#sAcX^4wO-+o8&al;hB9}#o=hC6lXIUc{X!EuW%^=*yEnvri< z;BO~wPaxm)FAr~)iF|+nFYm(h4b2Gg*IsN=Ea(j;zLmg}I>&T?Ga5vV6!#4DbaDLimfbB?3bHGd!U{(>b zeB;vqGb7_i%-tGc&G@n)t>gS^3F?5|=r6vTH0SH>$~PLJy3UO<-XRL`UWD58b6J$o z-AaZuliiGDw3q6m#PC*pg#@d$;@4r(GVX7Gb*B+}_WH)cZ;V!$e`82X;U+=j3MEOK zG<8J4yL!iK+&fp$OiUSjF^7dLn7Ry?6(I~ zj>xh3i12>UJa3Lr3^1g^AQEtN=8yKp_-eRvuszo(5hSMAlghoFi*!N20*~O7sZ%AB zyH|AXCYve)=Y*JYD8^fhfD@NES+8?5X(!}&DeGy2M9}>UU9d}ZDV~pe0V2lZ_-0k| z9@LvwmDco&3Euv<=k*UXZ%h`*;kc{a%}i{J`@NZfuc+5Kj}GsSdWtPD44MAX#2Zv+ zM(RDyH`JZkUCVg==OdUUrfsq!jY1T3jTj^&?p0F6KkVOQxF&nTS?QGJ`sP`u74DtZ z?UODL=#yS!IPg(So@qBR22&BF{Io`wof+vYv{NDPlkv&q(Jc~Nne@annFr!;xx9{W zF@n1^w-J%+Ww#UIcnEx?I0|=3(-SKHsCuYa!BG#}I?8HUaMmOFvp^Gz2hjiQ zwPI`ZjU?Iz18b90SS!?18Ku(ti$fii7gF)jp&TTy!qp@n_0dpikD$RA?#L#B8}kpL zxAk_?r_t7G6N_BWfn9Cw$WnN}>7SYDdd~ku@8!)OkY!{v+`@ zCUMx_CsY5$N}y3d`-1*OuVe1E6UpH9j+T37>&d!!2$4ri+ zih!;f={7}tT+m}cBw5Fuho}`TW8=Bep`sqCPC+sB;*4ZmorMhIE{PnfDZCd_HyfUm zxC&+m!Y($*rjEAD_;7DVsUV8+7qT(-M{Gu#8I-)(%~*VZ;yB1!g2G1vL;Qc|o1P}I zEomf|6!$71xUFfj8~~G`EjBWtp<-D4mLdOOjs4>OX{%@!>>}b*M`DX}+_kF9`P*jZ zSL%**TQN(r%F7~YN1v?6hO`@u@b`-@A39fe0 zqsiR`EuFKA9ER4jm12z;eR}T{`qGckumFn@Gyhtc>v9i4mY+LuBxhgqEW zK;1{6Yp;Lc1zK?!1GsyRHPlv=i9yIe^C(rOh!}z+-~^D%uZr2Uy^D)0RXTID=negL zonV^$>aH&xIC;P9rC^?}C)w8!%3b{t1p0%FjaXj z9fZiriY1aY4iq}o5-+3(H?5I0(w`+h&`=OmBj|xjeGYht+^kTmfeqRLYH2Qe(tK%7 z0ZyTnxxaWlHG}4dhRrjsQyd363T=EE^ZSe3nV<}7`6XY;Mev_nar5~ow+CoW zDzE!0uNAO+x*qBkBfwsfx1p`3s@3P~jIS^;MjMhF^?I;twRaHc$1~#SBpaAR>pogCRl@@r zMzLSN)xvbs5hT;o3q7cR%Wac6p_P)CMBXuN7W}YBMq4;PD4EuR>aoGPa)mi6?1vc1 zd-1LuF}9Hz!Yil#NP**RY+O{BNIX2EnvtF3u!@ZG1b^@Kr+%a$S5$H06b*sc`2yZq zd2hhH1^mgGE#}8LuBvNkErhe7}Iz5o`K@y{&4T5Uwt3v zFY>c@kftcs*M@wGgD?yK_W0d?+PEzO*VwKl=NyI#2>T+$%gx;YQF`=eOkde}`==&w z6>*kt+PR7Sy~Jpv9Oecb6-Bz4IW^%^k%VpTilRH90#yup+i^ z#}GbMo?8-Se}nv*B`f_WQ@A`E0G|Qi1fzFi*ar+T%~r~-G7)^SntK@$5j4WBL@jAI{uMCP7PoUp>F|u5S+y8s-w<(X^apIf*}@pc zChta9jt@Pso`U*EJqXrExd4wWVq#;(VLC6=+KI>JwV8Ek3GjoBlX#t?kb zbhh_Id7U>PMlwmt^Cm@ckmu`qj}wEF*qKQE7rFB8m^}Y7h$BjTal2TJeJ`Lxs#l^c zq(9%l@lI(>UIo#0Zq(d*fkONVeGwr}l?{q)$lvkdqW%vL!NcO;pZU1}OliT$@c#3$ zyY)-m<=+ak%@L7Aso`X-U0aKy`1O`O1WKz`%pefRYW3$W2m9#iKr5G}u^V#iRpOVN zRhSShm`^EEAhKzXwDzC?HMm)1k9n!kC0KVM>XpKYbS;{#KDDPEZ|L2g@AsAz%VdwW za9fW$T#Io-sBwceJdn&YnE)key5XELzL1h3=@?Lf5N2+G%6vV=uy58+Ez#JS=y)hI z#g8~?Kjst?J71gJ6|QP0^+Q|i+5CUFv-#G!J2bP_Ai*j6Q+W)v!s;Pc+e9&= zR1NUgK2wUaJ{=(p^{$6H{%jIxMaRrq{<>)Ka~O~IjB*42Zam9Nbq{;DxSd-TYTbZq zVJ-Cx+ued%ZN2K%h}U8L_Dy-;1Jxnjh4JV6f<#v#QT$2Yj!_gm-%3}ZTND~%6&fTM zKdA-_*gd*oc~<2@_x{G&znL8(w*?Z}Iom|8AXO8qi`uS8yzB~HA>23-f{a+RKHoAu z_x@>k5I**JPg?C730r z{ZtF}I*qdH{r)5BK@t~FLv>&H0pV`LE!E&!qvsVTS}rWqvP2Pdac7+9aBDj%G(^u(P7#GK0ss4@#Qkz}9Mv=&N%!jsR5JV8tz zoc@hDfhz`H&K~97Dz4?)Cj_?Cp4%O7PhhzsbxJ9lt{UOGO(z6z2U*V zFGG}0FH6fUzvJHzWd8ethPYj%Es38M1*^a3MenU&y^GtcAm}|g89YZ*0~-(q7DzR+ zY-b$3VLbfr$3~!xb;NO4xC^Q`vXQ@#ccQ#YXe6P{rDgWBT2h~WNgGlt4pYECg%fYl zq!0W6e5MS?+5+>mVEy-e}S@v=JT zaFuUvX)1Aw_Hq%bshUI7rYb5ImHB~x z^C4*`C854*rqLckufM+H?BnqtEfW_ai3xUb)aleqT}tStr$o^92xl}(dM-eEXg1vx z^tQXi3i=^;T3z2iv~`ZTv5~rcvfW^MB9Y>vArqo+p!@T>PC`*IS;_M+Nle-*B*yh5 zB?qzSxXpf^RuovS&=%lwDIr6?8dLl8gLT6Hkkjr3qri$g2Dy&>(cylRlqIhxS4;*a zEUIS#*R2^kD*r@UqAyv}Q4>iVP7)tZMy%cJ_KBOo)OJCrH-$kbDI#Bz>we1V>zFG0CS+E{zm*_! zG#197G7TjFv7V3u`6=L0f%06RTQfzA*lJnD%L%36HLN)gFvUdD-MVt8_bg>$qkSEl zMAr;)?XK){z<2$K6<&KNT*-&K2pj@KyL^*BqC3VQi(M3YiZI7~4 zW|43Njt$tz)jvw|)f1q>j>NISxWj!L1Y0od3?TG<0{4_#O~j973@JvTNSwqA5>sX% zZppf{C^@&k$YZzj`5;uxC=HMu#a9|ATTDMy3$f_EKqT${qQwb;|6B^i6?L z)cB0dc9nP?x!=%RXLdxuP+#D@nQMX$S=1HBPwEKT5_?F0R3{YuKt~^`#TnX{oO|C> zk1l3K$MD-xKU0u~0U;GJSGw86k6b|lu+KXW%d8^XZ?h8dbfB9Y%*2dsibEzsEB(QknLWVp3l}Cx|huLr6UZ5&A(S%TC(|m3`ah9zPu^|1Y-Q zG02i=ZPPB>HoI)wwry9JZQFL2ZFJeTtuEU(_S8A^%|yIkOyrLp`Ey6CT$$^}_4Isl z8bC%=_Vm~pO^WyL^|<$xs}x}T#~l~nX477zzwt>GibVuS}{w4M{FjQLWNY4Z(@ zr7e^tRunD=>4c1sAfT~Gl4c%^M7TBG5qq(2PK^ekfeRxYXfhJ#sXBz?g;=kUb z!Cqd-b{~_mK;(mN^yh@FQC;m5s8w4P;+hm3cpIFxVzJqV zSO7~nnTDq7;Kqs}oo}*|+^$urh@w&SBlNK*!x{Fkm`Pcq9-2dKj}VPi`Ge8?9mp-l za;HGV#7280lX5INEu(WG*va|Gb$U64j5QWjm8EFg_y8hk+9s>jIHhuTYZ30 zOA18xG-zA~=fM7b|zpv&N0RsmkcV(D*yP z+K|hTn>7&IJP5hk_TXb1>-!R0Wh>xg4*b2>qf-++#F7du%4S3#}o4> z2sdjVK64)x!(Z3va!tj4>h7^(2 zawCWoTuBIpU=UL>kyS0nW|-%|E9}+y1K3~RJ3j0E!WMC<-yT#g3wr&94SYYXdI@0k zoP#V^iRq7-Cr6ni;w(hs)*2)tgI~ekE;FLmPZ7L=S*WBt`o*xisa!6#Y#%?qzK7Q{ zKYMp>EC2!m0{A}vGyajCqnqsy{{9CGD9=%{_ncu%J*_Q_H>e@rfcJyKDG^Q@lvEn; zmX~XEwqKK$=V>y;IbIK+P~~+F*M$EroIm;Pnajy@_LqFn?~WnVR1`0t6GSDBe!1ID zVjoXv$P?d(d!LA*sc40je1R_Bvp**%mbY8umO%0Nr~Amp+2ho9`;`Y1A-V}Rua|U* z+L>H|*I2-pZhW{|r`ESK;33)(Ffs6cDZohgiVILGBfw&FAf(2dhe8p^xc`lRXrt$7 z?aS_S4xnTaC?s6{WMYc5@tvdQ8bBMo@q7}AyC*qWZ{q%M0A&3h@oqyxax?-!SO|xT zsjDYZX?z3!xQn!zVKU?eI6zdoRldpN^b$bJYtQoQlt?j#C39gr)!vODJ|URr7)s1c z)UK-?KEHnI{8ATk-)Mj7^a26HwMg@d$&rqcIw--9(?b5w)J{hzQ%kQeDri`Y8GX|oVb zJD&C~%kGl9oP#5^fbq!+VeBobrK=8%gs9N5kV-S&#Nyx5!d4w+OwY6x?);PGTC(!p zGw7*&dA7NvdaV5Yby~IUI3oWrH-i6$J-QRv)t$r@nza;7YfN=42$+Y*q%+A!+i-tq zBuRNxdYj@66>K|fHaEiP?-aOR`d4t}W+g9cU@;n>N$LiQ^VzLbhjy6O(;K~3R7iDJ$0TU1~V-chpYk^eZnhht4^snU;2mg2o8r1jk@nj5JlUHlRisKfh*e+(+#Pp-w z#5^XLRe>fKR!iRW#Y#Ks0CroX{Aa$nJE`h8rur@XB!hRwN?|78Ts*&pMy#7F#7%O8 z23t@un*boLHE~ZIli#@6**SS;m%WRVi-9cw_B+)z{~y@Q=z9_UendAhec4b?LqT*C zdRv(~QRD|zRrgUskpXhqnkhgo%viOOF1z+q-D|Zojyk<5!EL%l)x~b{1Y^?4MVYnHVo0x=iSQIsqDH7M8we0cjG| z>X1#}OqRp`cu)FGI(K@)SGS~M6{#be$rfvb1V{ z`aaY*JjM{U88@fk9h8}+eK92od-18}0=E1>56&!{maxKl~cP@yopXKmV$bQTb z*syhgbNjP|K1X?zc$I?=j;lUiOI~W6B{WMfj*9`i5cW=18M(dye8j8r3G3wim$Qc= z$~s^NI=~-^<*xx0gH#sge$}W^8I!)7?=q2pPcl{>sjDZ>$-ryK=LTYe0&206eo@!y z($x~%MQL#*bq$kW5f*?vev=6CUp`bj%}nm_8r@p0!Q$(e~|%j08w%S zdKM|ZE(?`a+DY>#SX17L*%16u>MG013=nt7PKb61)njp5Fyi^#4!>=S%#Rby!Th1& zdzM=YeQl;sRtn^OI4{ayZ zIBOD>|Hn^ZY3vD>`u-wKH6)fOv^e?1wLdMaS? zx%3T-w=irUA`lR~fOnK=8v$1X4@KxFtd|z(+kXa3v9yh@bMP9M)RrL@Ru+42lFub^ zaArE;_@WQ|CE?+_bA+N}8K;DH=H(1XBVt-k&rd#sUlh$61%Lox?juX_CUZ_vypx3y zBc**eL4dy{M<8ll%yvjufZ+;6V~H;XczGWwvEO67BeEpr#SxQDuyk2Af6QCBFD7HX z0_!1^i{j06C2Nd*q)q5viHZlEd7HHaM-e3(#vN8<6H$YM74$sM%51bCBa%ML5DoXO5mc01HPZVB~I zjLuFqqP-z(H3WAV=q?s1#ud`LoxZpW31K{!F#n9H`V7;MyR5eSrl!Tb|}4Zd{QqN7DRTA6sjx91TeN45ODD#8E0+iASwe{btefL2i{K{}z0eAJu) z%XBdh7SKBc2gIz6zhOu|Cp1Ng_14nuH6{6M;3i;chsL7ln2ktFx>GbZ;1sR0TXZuK z2YLyD%LMTX*0I}kv&2-Jw3)|arAwxA4;g~X?uJ|}wJRHk!g9!n&F(bn^EV}P85q;| z79{)mz?)2;>|Le9S9b~je@Cvew=kReNl_}LZP<7|=HVt-X9zK7CMj=p{=?#t=`$o8 zr*lmBtS;m@=8;sc5{~_}7W+PVvn<^vBAj}J7aUi?mX*uOQZ8j@(OGQ3azBM$gG++$YlP%Ne!`Wvol2xls~D)9NEeqbe~%inTER#RZJSh?psS zTN+lNIxacj_WVm02R}N#hc+~Jv>Df8?1;ve`}+|D?)^*C7KlauXbFX9TKu}>mlc!% zEtJ2)EAD;=Gs-Qg7FHWg<-Efd+$w`P^%6z4cqH#ND>zpb45$2`Q>EnWrSOIJj?TJ2 z*}nzl;B$?P%%gkHGM`1r4 zdEzTcJKWEaAODDmIvHEj+YIP65Q%IyW;~^3$=tZ5Z_Q|XQ|rZ^j4G0G;^~7TnAlKu z((SE81Bcdgl#S_>CgP~wzl$SvaxrZOl92x`fn|zDlW8;z3SM~4IxQBF= z;_eMO&r2*)oGe2YsVOGk7}SfvspjZTRPX)C*N0ZH8kAZa zPk@wn$UI{%n+UW$+jt1$`Zvnc5bBZXn0w4SJM$rMh0Eyv&C%%clUeR-R`u=guDpf- zBKYxe@Ns+J*%9)?Q&KmigSoropu?bjoFrV(aSXCt3dTC3_hTD8v7I5V01zl1b1P^~ z4_0AS7rmabzPXb=7_9*iUel=ML7ydbuak``Wk=tPhC974-3ZGd#&IXDJRYuP-Y+;W%=H&U8%LD5B1 z@e!CSq)11e60az4AU2+>zPug;UVIv;=+%@P^=g@~Tu|IegBI;3`w+E3c^o6`R9^F% z`Xx!>s(dBYB<&nnC5B6y#hl4fEM0oCUsAEG5=2O^xvPxJ;5Q?Y5dL0^&MAi;cm#4* z@T_c+k#*JjLo)wH#NmPR$z?x0IMB`v8xV6W!rvBl2pV!f@Dm9bp_26zu?fiqtpEh& zgLlkfysrTAwdUj17L?HD(q%X~8bxA)p()@gt4eJ+IP_>VeMauqxMr`>fb*bYtBk`@ zV{jTCsLHY2Y$_>gzLFCeW+>kN188I3J@wJr{sbvq>pR=sN9-B;J`;^n4?gKz8OYZq zyI{jr?05>F!0jqkBx3mU$KRr3Ruz^MuHnhpGg;XQ{~)J{<>FRw5HzekqJMfn9XWzl zbi0+~s|BtR5a&o~SwekixEXWozB**;UIDce&VpZu@VuFw&Ot#Sld9yUM^DiR9vOzt zB>8;uLwyvIL@-1}*$WRhFq^8+GY-24sQ-y+2u9j#?gwRY-5u^I#SgysSqO3F1kDNs zea#Z_iCU|r2fdk$qic#-SMprly@t0(C6bj1q4OH%(Ck(l5@{6Yb!m5z1`Ts0iB-vd z-o(14lt|y0;b+7hZbsW0kxl&8E646o7z?1@2gREvCofbgjkD3bkE_j7WCe! zZM_RYr(46H%ndypE070vosXo)<|jzkTp!=_3Pc`nUUVRmQYoan1Qm%lU@}?5kKXZ* zS>55~RMrg^PdO%v3dW}C@@qj83*tFJLGiw--=}0pttrC$lL{iMNDax@cb^#JSwAkq zB*sQ8`If{a!8(>(4B^besM__I1ut2``n&VEK0!)P=M~nC5rF;B&a7E1ZS~6v0c9NP z?p?8XzZe^f1oa(G3dqUVBBC5_1=de2o_a>|JNp(ELTROCH<|6QEundGf+9m>Up48T zRw{;*(m?>i7xQMPh*iy~0*wy+j4|o%C`=;4He?xlTxlIHswCaIah^sZTD)dZm~OpO zOpr!R(PX9HFb(BaXvq~@b-XlH_8QDTA`arEsGKllOi^GQL#4NdVB1s%SGaW-Wfwg_ z#~6>}kV=;B!rVkB4GrEl9Nf0mN$g9o`i$-yQ5)p$0#Ap67LB-PHuT~HRig+62=E0iaq)`RNMgwx4nT7z>!XIL>8(WH{Y znMC-Re3n!_oJgyue~>ZK;DiU8#956#q>9^2OpE%DvY*D@_sCKcV>*enL5fnQZp9bq zFP<3A2myJWLeI;|+NyMQ9h6g61gF^2 zsbxRt;Gh3Uu)X|+iw02H7hwLC`6HR;SkmX&YZrgqmA^M4m|R=VivjQfFG!m~e1aZS zJ$*D&T8c33CYL0hepJkv5GVifd{z^G`2~%luK&en(fqAS^utRvbf^8e=##j7m6wGS zQ6mq({ludIr+rmpHtdVXt8N%29_DBtA-F9CF zO4nYRZYld)Y$Z&b5}hFVMygFbc9w>E<=i zw`gP5q-!(sfQR_u`py@O?SgfvWLWeYXNcDRgj-3oc$fX&E`^Z^mp=e<9>yCm3_eDA5k97X)L8L?H5hhu!(E( z{iB3UH2(jTFd+hOldH=t4F#rFngDsi-e>#|wCqB7Y{?oa3^?-iw zraih#S&ld0Bfb0Q+Dad`$Amc`0+xH#>4WPPU)&I>Wp$WHk>H{K6%j~)u+a&^WdA5D zjBMo8aSb_pU2ko+D8UO`mx?M$w}c%zn>aAtu~U!1IZ*ri0|TUHfa)^VXA{FrCw~>9 zhS#LE&!Vs5CTn+J)o*B~MgV6|V(z>J%}F#X#YP|{-V9#czO@+Tp0>tLaJEWMMzMbp zHAAS#DxWNnv9G^TOFa!==x4+zwFi@dGrkJFt;F%=ywzsSSi)4x%+Du6V(NlaoFj)} z>ZhuJ3PeI|Q2Xg|EadN$fIT!kK0aV-vpE5CO4_-DPFL@bm?M;L>+vpAJD54-{ z-^ZQyaFt8()UYLfxaMg`DBTBOcDvWTHI_8G@bzXVPXIhF$r<&f!Boo6tjYeF+m2!@ znJ=fQE+<#Z!tCWQF-p>0DHJ=<`h2moz=xhk;;xKOL|S!-F3ay|Fc|QBMKwxQbj6d4 zvcOdLojINC6|gz*Q37+63goUl1no-i4poc0bqR_-tTMT*mzCm)`=DO3`5bbkO}p>% zu7$ZNylH5??GTG=jsr4|lZoH$*=Bv)Xr&=r|4hQb<8aQD%d=ANNqS^ip}b4FEH%cH z^~e;QxdH$`6|!ag*HkE$Wcr<=HwCl;Sl}78TW#ET7|{Vbqv^e}LrY zW5coQV1z7*uGd%@o{cVqZlQBTZpia96sKT4?2s9wunJ$$S0%KE=j%&gmTMDow1{~M zDG%=s;)DKa3vU7oe^`rHxopST+a7hW!lvAkywd5lxU>h+TQiS!m{J`%%OrIQCoxRQ zQ=Ffm4Qa{vE2Mox|hr87@w2ksH;d^aD7YLyt zm@4Pm3FYREJhgWsp$k^vzQ}in;oOw>`z~_6QnpIN&Uyz097BkX10- zJ^*w-4FCvl&DLT~PSHvvwGjeRU=DqQ^ zpejb^u74Lq&Wy$_D3RZRbk*_d5a^W-0Nih5HM7PHxlv%f5zY}19<;Gu%TwJ z@Gt1dg=90j*E)m7&Ncv+m1l^5SL5BbXpVwqm-+L)FPufansZ)CR;Sn2xPvA{+0jhQ zrrPVHx{<2Z%s+U&SG?IxwKkcbu$R_EDCLq<7k7P)UbuoM{zl_E{h*#+81V2*LeZa} z%xS$qw^6xXa%U%bqj{rVerJoI@6a$<)UhjnYiTZTcB}t!!N9*OoKs1uaRwJCpC2Wf5dj*zs=b`x!eO&F zGXeTlPNbaZ_bmDHL=#9k`o9i%3srP@h~1#`gA28PDLNB zWJWsaX|oOQa-?A>G{+vzs?jkN{8H6WFpebOb$%VHYx3GMZ)f*9NEsn3;OF=vypx@< zx6b*lvZ9xrDV-HLo$YLPyzp~P&^g`E`Aob6m=6_)qIrEA^OnDRzxwaS9R~pJ4!pcp zU;Z3-nz^N6xkXwETwdCFg#vY;hmWst?;)x5YorGP34?W91rAa45d!^-Z95m-30_g! z{W5vCPe28;8IG3L3hP@~l4Dlkdf6DzUw>y78rx zopt%6@X|aMpOP2=-@*|7?YT@x|7_L_({sc6 z7}FvUF|3g_`YuvPIk1SNdEpCWcY3yw%TwGP#SKkPyb`AUb*X%9B-dVbwuaC9eaa~` z9>Y*BFdmJDlm@+SJx|?lJ;9&30u@kT}q_X*4 zCA!}{&%@Ccs6pqhH-+aEf9mwH`HChU-|!WO=rJzmVP1l>*~^*D#(ww~03&jjYNJ__ zuQv)Vn%V8yWo#9m+vz@O>Cla@Ys|;i9v>YnZ+8%-c59AI4Jrsa(!cl88ue*;bVhd_2d8nFt3wMUDxVo0ELW&1obbC&ykM`yZtWfHI~ zK=2$s{%K$_diC~WxR^ioCZM$BL*gi{=i|Jdp6hVxyDqhv9=R^JBk8MH>5=yxFLfey zDLqgldR?)^{JmYt2pQ$yDPOcAEyJfdt{0l-+82#46$+PIS}8U`vZ4-}hpH{mq9`@u zD`5m%Nh!q#(Um!n7y519cVWF7&vIHLzRk~;8}%;Wg7^29<{cRG4Un6gP5uZd>L1st z`gyV{Y5>+joU333&LbG7UmY%dPXeuXh{y)(9aL>|CM_=Rt!H669#g+deSKG}X(ije z!P|CC>7z!^&Pdxu{QCP%*Jfalu=Vo1OyRl-atEn8bUkvN9@BE? zz<_pB**rxnmrQ&iM2NNFHdDkXz%TKxE63QSdL>M2GlA=H<}zK=aOYhZ6DW!oEHmrw zq?_icilQ|>Ex9S~K88!rol`GAPL6%`cL;bs(vBUkyN~0($Dg6+gIA&*c&O)CDdS~= z)@)~Urf$PLQZGb4mHAKlGymM`+}lCwSO{lx$^rc^>I)FUDs4>QYo?ayBH!dRca~$BZfiV&Yt-G<9fSOgYK-yke1Ee98=#PYkR zi-Mkxf+CEBPsh+-RfFFtv4YJ7lEC$MCxMXqX}OiGl-3i^u|Q)P&2aGZf!W@Pkb&s6 zo8Z*Y0p`xTi_`F`xv3l+z2Qnd6P3RvV`4T3;N1pq(8jHmkCn$;OWX6`&dXux7r?HA zrk4l#$4Ad;v9d}Ltrm4l$mANSevS5Eo4)C^;(2m#{yp|`%x&hkpV;-nHPuk?off6g zP?D%S-Q7LKOAK23hH?+RsFE z#OGzB7_%TpXPgRaMsSl7E04-j1HSt^UA>qZF^Yaa4U=IZ{58oK`1hY`t>l$?ple()5m!1Eg$dGW$*HUVALl-J}y4#+rO;r zYvsE$(xUl`gJ-O$Jb#akYbM}NE2Oo8Q0b}b2m2lvf4m9iOm>p3PIzO&MocUxsa1JY z$!o=*;AjyvX>O90X5kbNWiN{?=$a_8La>0R1g#pgbk|hd70YaBFTO=b^aL|ms^93{ z@_EXNBoF3+QYCn85z{T^yJhAOwUX4}A7wChSL-wE4!e_C_QyzHkq=4F{!_AKLc69O zLVYX#z*yBXDZ1qiN}STsmGkt~-;6x;9LiCC3jqh6-i8hN$N0oWm_?>FMNe!~{!gA* z<|^{67sGgO6D~$}=bOkR(iLp7OQ|vYGq=dNr0E-OB7z+ZBkn9}y=~;4yOCDXh7BmN zgv`C>ap8FrYe@4>xJi~2*ipc|CHWP}np&&E#k!q*TNZx7$}yB$W63)Pj~F|D!;vf%Y7i_Ie;x|qye?fC-dGt<346pN3^Kx<>&tfl6x?HDn)i_F~9dqh9E zHhs;8AKD4yWKLw;?JaIwT;Y`c&=|(u`AwGsbX$pN@We6X7;h#|tAjF4P*m&wq zEPH?FVCjS)go-rb^Sc#YX*!!u*4Z@6C12@{28pLV89Q0YOY?PjdMA8Vc!&PjnRj61 zwOs{}=cYdQ8pQsaX$1`v6Yh9TCy&k*gXPK%V1&{9e*E{-&Y|QmIYL2I4^rb(Fygv>a!ybUH^>;C)BNlFSuxR!l{IMQPT4V9 zc(}uK5JG>Hd6u=i2t>N`jpx7Ml#h++HMFGbUEnXqukYzsO5>c$b^w&s{yxDIkd-E$ z3Gwta^Qy7;LW#Gp20BSkWLLV=x3Rmwa(DbHD){dK=J_KmYy1j6LhEhy3U@ffxTouu z$i$J<@qF<|RBkO;man8)proQ*98Zx(^>>pm5w?%Moq2!@_F3_grs{z{cy(5}hmm7C zaQU65vGIl23qap$&~j$~)kVMt)Tu=QdeLvrg2iz-xw--L=lpK9F5G%9?MzNEV=WrW zp{Tjvk+T`{-57%ECI?c;1$`4>&ceydrPoWvrk-}8phGaKoE;A?bcoGrY`Ub$ ztu#arnL~jR2+@()&B#gfL($o>ggWD_xP^(>51i@%n+Jh z{zCBX5Ow*=5m+elmd+4{8)3Ykju(Yzb3J!IpU^Hc35ES-f=*9hj`yZ_^$767&n3x9 zcNBUjAN&+=AMJ3ybiYB*(+VdLEBa@0(~vjagaY{h6RRkqnHbydmS1{hQImAP%_=$a|1mcgb;KC^~chKlhA;8Wg z2C>*1kWJkIhR;Qc|Fk3TwV@7CP_ulyBjDH7bzKf9#Tns0#`3aHfK0EtF{@@X!QM^A z0{i&`{KA_|edFpfdSY5t>yOrkV4f6S{`#b zPLG0wNfr31!hRa~+H-*zUo={9q#M0Y*obvU6qQ^7-$X`>?|NXZva5w6vOTzlkY+Tc z*ZT7By+@Y&0u5*|QBxzFYXw8099n;LvCFTHx2SIvfkTY{L9fi|KHi#2V9Z*PJT$nk zuhsSfRPI`Y+#yB?AbDZNW1ZgA7%#dXrX+!fhd_dpsJ$_FK8rXW0{8CG!!|hFE|&Ly;(jOT zH!f-`A3Ub{s4D_pvaKuYn9+({NM=4ib1Vu+?}3=ZClzKg3lo1(fa^!wd5u?O&&7FR zZ@ub(edhcCKwdT;#g_o&XmIj>kAIZ_3zk*kR=+&>ci$}Vf5iX`kYkg#z$|luO+wbH z9FX3*Ft0SIy|{314w$YtEvABn8~UeeO@^#&^v;~o(xwf}$hys`1GgjOT6}wyOmaKY zy0F>z^pw@2y(IVaKXFO#aUoQVE*xj{w`Rj%E3V>==gq6j8=9Wr?OdpEzP~OH!8!&F zlBjO}G`+d2Toc5zcf@nG5%RS-`NYS^Gl!?wxV_j*4^m&P6Plx0nqHz(LABKEV)Di} z0-_C2^-Tdh^4kC}R$d3 z#fq#(W3JPu!D9GEM99C-BP&-n{onij-oio>$l2U2o}zmyRY`ak<=&|rs@UJpA5xiR z?>mEsw|?a%JE1;L{RFje^Nn$YvHA<}W0@Ira@+z$_S+_nB1|BpNQz6t{0DOn2tzy3 zhSCMm%4D7N_ec_3g7#UXmco?h6ACCAvY()DGTZ~im=!cgxqP7ZzAgDsx7g-DEP|Ll zweYWSfdXTkTWg>Li`ayL@-M}WVGKgz&!cUlf+WRU4U-iNg?T5k<|c*jp4;mM73RB_9nX$b1F?Uc!c{a01h*`=*`{^aB`|D9xs&qRl z)3t21j*_0Mo0i|g4KV#{jN+#~)q%Hr zohicFZMAe~pe{1Qq=93NWh>lQrO5?TjxOkCztwbAa-Em!#JRrQ8C`s3!(3?iqwP3j zpMc6=DB3&Z{d^Ylb<}D-hyhM@l@kK5wNShdm8~|?U~o)sOG;V}gWDkGmxwsJzD34? zzpxmAuv~v%L>ML^#KtJ224%2u=s>?xhORDy(?iHH6M78U4Era@FzR<}ngvbWSeT^F z={b^x%pNx;pv_1>;)k)$guC+`HMg5mf*vn^P;H%K8z{`Iy%LW>Q=pQ_r=Jv9j}R2y zeyuy>1EM_3qAUpf-HM=(`!C&X=82S{{LY7yMr>rH6loDyzc^P#m8Hk9B5~%MCL%u{ zjr$Y)Y^`+CgWdnYgxV^0HvD?V`k;R(*9y-({n09aj&xwlsc*w6_ zS7C1-zFAv<=-!2Gk)Kh{m#@E?0Rpx+{%cbYO-FzOsJDjVWyO2|CiYESr<{~nKC$CY zM4qgP%WnA_u7i(3P4%lGC@z>E-eQMo0ns;vI%g7JQzs+}NoBV1eRpKZ(iN?3C5qI&~V zICYDRCLP!kRqpoOyyS-a#07Keed@6csl>Gd2a&$scYzAHJnzz~X60}6DH+-m-Af#Q z#E}KDdhYvqp z1yc@Nh>agYqPsD;GH8bq=^VMOV9ST2HHg&;7#iW`3doXnK<=C+uNJJL(=z2lq&bt^ zhJxF`SkdkT)7d$VbQ8VGn=z*zpEbb>lKs1ewX~I-l=Y(We}8d32{<0&ygVI z^9~x>zdxwn0PI)*x?ey~$P$+yvB0aO!C${8Iz1#BfoxakpjDxND8`~e?i;#-%hS5R zj+?B43O_I&*q-uVb08P-lYJrG-Kk{KRRr;W-o*Gq?*&}Hoqbv!Ui_Qxds6`la^L1`c_e(d%ua0goo-DX0?FbaZM})(qWCyH8{r}$A z_cQmsFAJ!gBn(0L!kUsJB?tmYSxH^*I=dc2o}N)K``!CtQ`-*t;Pf*%ED3IpaAgNE zfyw-d!bv%NisU~A^5{Kp35i^18Fs`{!Q@!te7+Ud_d_A?bbVsCa%~kc7eZV;NX%>lq5Vz%eBChHH624?i;}oP~iyeaICvga~ze zQ|s?HwZQ_zkH=2t8W&ei3vV!ECiks)e|vnS3Cm40xaiII^hvq7Qtv*x6wqgw7_`Gz zSRe?*_nnQ8&yN!Tp_T?6a;%e!&3!Guq-p$&hpi7^OLAo8VEI%SV-7@&&Kl70qO<;gmiZrG!QMc5a(nF)+}Movv&N`6M(4; z1K1b;vccldA9tqZF7IdUzZsm1Z>Z1tn*{NH{M!5Ij9DqBTdg`Akk{Exfz$*B2=K^k z0eoHlz58!9m~*$iuLA)T>etkyUW@crKG0p(SvhJ^&60>^ov<1yxmL+s85$T;K5Icc zQKgTQgdos9d_cGX$@t$27<((g46vH9bU4qjGCC&ph~FUo^iYVtnqY-`!Dq=s_ewrx z1Zq6pa9B?y?DNi0Vj)j>%sY)f`RX6q7{i=y+j73b^L5`mWN-dY z-DVkQH0r9Mt@^}>LSbGv8MQ=1n1%es#MKC=pI{BEz~i{tmGmW?r8EiJJVTA_QI0jN zxFT%S-dfv6kx9mD3q1kn|CW1de*F$#k*I7>DO3;IYl^hWA>`h>Nx|1ak3=Ph?WJ;3 z2<~e0^8UWFy?57z zx^EZ52qW})OM;t&C(4TrAUi-c@hx8y{CCwLNNE8@_Ifbj-p&1|2rHE1Y^K=Dl(DKO zVUYhQwDrGgwQE|+`41rxx7?_m#wC7Uq^-2CZ00N98Hjs>qN>Qw+h|S9o&xoE2Pk1g zyv>TOXTF)?A&LK6KN7pnS&??7YDc&cA5R=t%lke-cbd$j4D%>DH1U?UM8{&!7oOs; zXscO3c?F#R8Sl<&8EVBwVgTrel<sRa1%ugNAPmHa zd7>y|b7~X=#1484WehKC1e1O&b)jyFR1&N>d=D)W6@#|{L>8zL^^ee$Eh^i$(>J31c1Ml|r9l!G3nD%v&M|1YK8K@Z}|Gzu3 zHmssGFigQ^8@QykaSeQVGjKC|{t84B6YgA3kV{q7`?8v^!6w&IF$hwA9oZrB>%D8;=G8SH zr~s+KR4`UWw6lW|{6Y}1Moh6FIXk)Wt7$+hL7S6_X`r2H{!o{jm^FM{yol1(uVr@r zT6US0wRO#62BIjX3v7p#8pxc3B8{)_YJrig*eYs3- zX2O?}rt~us%GuUVER(CN))2SYhe1oK=U|-VFHo{4zqIJg8sv{3Z1O!Tu4zdvSuPk090>j@JG8|pm&JE1#nMt6* z*tc@vLaJ&3CO_wshNcFwkAR&L6w%}6< z?5INmncPYoN-?kW@+ePIkB!Uc3EmgNF8#ChGGM5!^OlAP{;*O09cs?c7fX5Y??y-Y;3Yr6nH zl7Mo{^m1hz5_-MvtP^5~`y_tnJGoW9C>bcei=lX3{sHIpLM$6QS!Z|%JiEaV+)zy-*SbF!h0bd$-hFM~7qc_iklVu8`}g@aJA zYI(?-eN%&wG=`H2mg`R$8FCvhL@WbiAaN)y55`Ois6zR3F^-U+eI#b%6~&U{sE*GJ zh+6E=itT0zR2dR4RvbSb3mu0sK$7$NPKq}n$hyQU<8z#y57w3umDqVpg08@*>FHuq zpvf!Idwgx(8M@&bNb4D186PKji>4z}2>4h(lM`Tc7SbgCJ3FWgg;6b5<@9o;AP zh1OSgQwD3Tr(|#H2mi^db{mH^t8W2b{9O1SBd}lU55SvLVKCPXC7XZ!bJUo2>HET4 zqCSE1^g8EV^Pea!O8HXioTsCOP#?+O1NpCmn={D*kK3+W&is_Ubr$xI>`fJEJZ%bl z#pux4)XwbADL(pR*3iPfK?GA$#)RJM2BP?0lLiUx@1T>mx$A5-iGrD|ZY;VB2eFSC zY0B9_BD(fI`bFUDr(|#clNF z7O!DS({;EJ3-KO;z__1-m*d;}lf%;j6DblBd4%8+R%%86-qQ8^Fr4x>4W?xC_THL6pO}eljLZqErBxJDk@Vh?n_F{%|#@ z=R{Zwvd&XZvnz%=v81Y|-k%LLLWy#aIJ~LtW?yU44szxr9z?M$RWCJ&&xcTVcKp zaU8?&613*53a|}L!juOcC8<1`8;ieKpw0ZDM!+R9q!-J;>qY-<_3F)&cW{&oez#Xq z#$tQ|F;z+9Mq?A*^kj!H`l+-&9Zzc-N{VPfmoCjt>;_&R|1ATHy4xFsgf%+O(P)*l zg;l&}pxYni17+6Ilq>q>r^AFJ%3XBIUlPmobtvcoR?-aB{66bA2ktD@X6@#M06hv# z7piyXzzkJ4{T^MiM&|}N#N`)n9IVY7u72a4fer%qSDr{B%2sCI;^*fukM4R%NiS5@p7 znhlrfVtcsa zX6{<^3qQ@i+|9rSS3mB#H0toI6I#W%zzvxLCryLw(o5K$vdRhF#Ua)2NtI{&aLRzL z>o2niVz>)QZG*H%>kHK6&xu9tg0<6|Ov5U)#p*?Zixm)(cv5>FPQ8*7;l%V9Evtcj zTZLVtUR2kOx0Ap%M769>9a<^4no0%(;X~H+{wo>X0?p_(VC3C7-5VPwn~K!ri(*Zr z&de@v5T^pEQDqP26K|r|hly7PO@>0e$?-?}mJm5Ypd(d%jQ5e{o#MRjoO7izzH~+2 zlnE<55-vsu+qTuQZFHPY(m}_zZQFLbW81cE+pMqd|MMPvCv}uD zc8y)v-nrIXzd0Au{f$%v+xn?zSSs{xo4hqu&EKl*pz8zSnj(ErDqaY*()Y^VvOol< zxu(udIl_mqqawBQJ{F`zQgc$JwTPKHa|1Mc9_W?3^S7%_bURk5o3Z}5nbS;pnV80( z5RpFAG{2bEJJdZ?UCanS3SioM-nUh$(`j+O*pOIG*~@P8Gz}L>*u0GrgHrwD}m z7(D&bvV*nE?yeV-+RJPar$QbjQ&7OxeT07F_}sUCR` zcuacfL5NeD(E$Kr`0-}|=p{^wCm?jBe+#i;Ur`AdTaxqqvIItDjTaydyN4HAta(l3 zehLYxMKzg3!_p1;3o-@KSWfA#F@7E7*!e;ZGRo&P@{W8u2I-NhEEJV!|Dn8cdsI&a zfWo(9I)Pvg`Q;H1Qy%m{11q|l82%$wAsREU@|z#LofrffSBjYCXFoSu6AKQRYpQ-Y zV}yhAz9=&riKvfH54)`WC~i2CS5TFJ&!aw}ZlSC1px`ro-72lje}w zMvoM&q`o8yYyo~oConC3;D#Mdt(KB}dB)KUG1XJ$6ub!(Vw(~PM+%<^_2qN(oB?g} zida+;9HhGo@DfJMzB>(=0phqow#g8!gx1_SV04~Nq;1%ltiYIJJgb;Yb<)SP&PckO zgL43*e#?cUpAaJth+nYr3P-*I?qt{iBWDg?fC01IvhhazWVyCA$!=}dTMX5tQD4b3 z8B^7{O9)EDG8#)3nXy`yJI-9b&z|#|S&8SKq&5cDf=AW<&g@eNCiw3G@CZNmOOoiRK~nM$l9! zP8=R-?D4RH;JORvt|WmBUfV(#=)wLw`MwX4=edV}#?Dg!CpJ&FM1ziJ_2^~7OQPir zSj}L>c`n=KhnYUhRu(4BtB~{Xw*S)XAGfnYg&@i#&l8*JPWg%L(_^HEr8U4r=p_13 zNeX@nHzc=D(ZPpl?^!V97W;)7?u~VP`v8-jn@x~&3CaQt3|lkJ8a_CTMEOR4M$u)9 zJvIyNft4tz|6Iw`!X(TG>Osh{i6?$iBLkY>c89crk)twYm}c|y@1HNu&6>m5(lv>Z zXN_n}%hFZ9NjT#T=;74^*p!KcY-By%D*nOyVT~P3zBft2_-H~W0$U?6+RSvZ@racH zZ4WTce!)bKITrr=K-;1!-uya@zFfyy3Pzjd;^p6>uJpgPLHato#k#cuP9C)oG&`np zrdvXOze~{lLt-yu2rK6=ATzGXEmaW3xt8pqT4wWdg%SvR~jdR zpN=H^9*`?f8Z;LbBv3^xa;nyc-M&|s|L8vT1zn!}${GHaG=oZ<2o3lG`|7XNPk@a{ zwW)8dNti|nL27z$#|0xK(B+LE{5swDBU=*H2da5ESCK@|^!2u-4>s#ayBcUVj*%w` zSp1*gv6`;?6%zoLH!9$1n3I|RcA6Km{jX2YF$LJMYBw6qI_*gkC3z@p94NM!P3ZyB zeZQu!Uszs0b4jE`jfUwN6mk_HfS-s)mM`f$b2c!}Nt||GJ@2=D^6&_mSB| z!9TQ}2Te!KQ@&Cp9@6U-!D5gt>rV6NTmwtj9lL-H!rO#pzn3IQ7 zT+AKUS6Tm)j?AjpS@wa-SSkw6j`70>m~2y2$AfruV^MmsFcqWdVDI$cC5<{xLJDMx zWS*fq_T2tcjPDs(xPCxRX-VlYJ^>br z8lJy_=vV4DpuTr)T;9z&;cV|KzoCasA*mo$Za-IQvG6HL9T*J{@2Pt`!yvu0aNWDS z8s!SRHlUawcI{=t;gj)4#fmjODF|oLw8lwtILb@_j8>f<40?$F?HOA!^l8{^;{SP)C&oqHbtH!(5 z2i6hacJ{rFP(Jh~l}+4|wLc@iXX|jlRT!|xMH`Vd4pUf@PO6{BuC zKDBe^CEsaxb;hTEz2P2v78*q803$~B%jbSs?rfJmlmgG#@`Ey6tTgjOUITJxx=AxcMt~CtS#)P z4C?E$%o|Zr7$L>)falnR`{#%nCT{({gNc$)YGlY6S?-%Mz8R2euzi;^xC1 zES0{oI6gDeYJ{GaP~+S!=uN|18+tLz);jeY4LjVMeVz(b+L$5ervryFt~>~&bG)+; z?SjSxZG^X~27asHg!*0R-XJwRI9|jKp$GKFbhB zGfrcc`d;1-Z0pje1>h3iKOrru4065 z@X@8W;OK-CK8X>M?g6i_7OUd`1%AYF04APeEK<#aN;fOV(6-qLo<(q#RB~300tVTN z+$MgH!f-b%D-Pn%Qlz<9t!ww+e~^CsFrZ8c(X?9~Dr^2U51Vy~m_d570n_)3{F<2~ z`==gM__}k@5d0~Gmz~@A2K>^dV}RgKCdMWO5d~CY2oa1@&W36ii z6y;0bnY1U1u zfR~x|*BzibcMo9v=Pp)oy#5dlrI7fJ#;Y{Jw!&j)vvx-3nrVczgDkdHR|fi>r{g74 zWyA_l+634y3JD1Qc>~Hvc4>f_@aQ+248U`GPkLbLgU}i9PlfJK_TSe1GCu7FC;&od zEhj?w9^I(ldBuIHcXlav6K58@03?3ITR?}FHRcQRG+_s zTXxM#6;HJ~z&Vr|_D)jK(51L{!t#mFFM~l|=QDT=)(biFm?h~^cDBa+UKUUXl)T>xkyy6DNQn{qP-%_9ImoAOI&TIahbL+UzN zKJ;hXFSxgW+@0vj>-vA2>#q3Q9_fMYS1&4okl=Kp&z2Ect8~H#L(Y+3mh`^ObiUQR zmLbZ|3lIS6>p$T3h(PM`fD5o830xR}ZMt^9b9HaH-@Fc3_-|e0;B60x_BOWscmQ(J zb6%hQ>OJuFe1{3ZZMp^*a0M4&PoyJr3z1j(xsz?oc6Fe3%-1Ulh%5f`b9Z&!dIs(DT=^cTLZeNhWHkKQnbv1kp|FWEy&4z$_E z>c5QWgSPt6X5xiIf7QrKhmK0dBdoXT@s;}{#5z%5-5uJXg@rextyt7Lb>5>?i5)1U zxU6u*JJ9lRLgOY#+Fcb)Qu8QUE1v9BMa*qLqAvZ+rAEdc^1fB902`wwJAhO4Znm&E zmnHHpu^Toy3*Bi5V`OKhLp*2%2W90u&N6riBC}D$h(0S1As1-V_md5-$w>_HI;7fj zky1>xi@T@VYT%?r|Y9P>~ctI{LZjiDuYh@XxSf+)l? zl9-N5wL}BBCn77`mi`=7F`Ox*0zGL{Cgl2`^%#xi45O_-cu5)9Nms9FE#)50>Bdl% zwg+k>$OX2Z7H&Bw0qj@)z^b7^{qtt`wRH)&)s3G${(zO1YYs|+%KNR*AsL^9Ul76+ z(UXR8&h$Q+tb)r+kH5`G7t&J`ohVHs?>Ywsr5=eqb+FR&LR1;Gg;vls@z6cO=Il6v zO?}fOhtx-hO@Ur#qaUyED0xq0c>rV7&{xU?c@okB|yaWW9=q%1DMx~==$eEKCy z$shi)=b-2p-M8QBvESz81J5^rbMqx{kT!Gp$3w66lZv(p8NDj{q0+Tw7;cd-&p#w@ z=x2g)PMAyfLk61m?twlb)S;v!7?>|zKP(MQ-A3qQfe$a8`!UU`DF)13qq5{p{w;2^ zJtoj&X7W8Wc=}M5h5{`kcA5N`9b_G!I4oJonz55@6#9lA;SpR7kuvY+kcf=ySU*>o z!RKZR2mce6$grl>b2GC5LfXv?Ur`;VZ& zdBaNTKas60SL?`}8e%%rdr56tjw!fS(*Wd&oRblkIDXc8!aV{SM0)Z%0T1OdfUQaL zCxAWg7==;oq_^|JjemNZymoUY(re6`=|c_x8=>nf8h7NIV~{fAT*l0^zJ0z8*O_hz zb5$m-J>qH%$I%??3>_dbyyhu>^B*s1SyF=^{zo5b!EEz!_Bhk_;a90Q4IlgAl$W(D z5BXs&I~eG%&M2SZf)m1W!~bDO^F#l`kk+#b*}?89b$O6PnH-UOaTH)A^xt&0-$EmZ zufq^la5B0?dAL|<6Lzrg+*#r6Gf+W#NkAT#DHD~5$2sY^#nPFX!+~-OQIjI}H}8k= zC%veg~!bY=xYzA*w$xPbx?GjBjwX0<|mraXJ(7PFzL$HWTc z%pM2r=+g6>MJ%hMHe-=`o)OWP|&Oic1H6A@C{+Y?) z$V!o)lcBruZv>G;-xWphu1LaCuDYM7JHMj1+wJyeY)QVq?so|P^Ju^gYEWC!yxvU1 zjDpwv&9QiaIa9x6_@Qqo%3^|_D$T>l-Gw}ENT$kFmsM&HLm+BW2nXE$8xZJGt~UF0eD7qnur7Q& zzEU}nvojgLQmlWb_-!BQiqZBF5qfCh*qdY~zOcJ2;V!w^CtKhAqr~F<)|W++AQQ;| z?L`OOIdtYo^W6f%`Tt1+nO9e!EacHv*mJKFZfV$e*l9Q0;>=`sRf@V;M0f$%z;?f( zHrq!?BkvcPxJh+3gVQzh(TfC%`@ArCSGZbVd&+=zId-Hvk*7763Og8*D@8QeLz_6? zE-HhgfmCOCo5 zW(g`oAjrWko?nc9zeT^C5~WwT|4zRdv0*-vTlq}f<;9O+2ha2ihAtW)ee8#yi?IHY z$MrzrUXZ)ZX=_ts0v{`nc99Vo#!ch;&-H^e9h>;P(~z_bFd_4GT`-S~tU%OSjHqS(h}^z0klWh^_M*HLG*er+$7AsxbtShk-;eBIftw z?dyg|LUoI#fzt0rKuNy@K-B^Ae2`9hpge3)L3j|rul(&k3U!1-Ib-Xg4>fN~n762= z1xOm+ON29;djp(V0_6aA7K;}kwo!TJM~TFQ=Tv8v(r;a2UqZxldYIeQCtMmPlil4$ zb35u7yEvwc-HI|5F%$ni#&c*KYQE(j8$r8+?-}UIVlpWVh-=pSkgm zDfH*&PUZEs?N7bG_rnQ9FlomFy2rvwT(8pKe5^v*H%m&?9Q9qJ_nR`j|2Vn;hL(Um z_EYp_%pg(ZIdjE={up~Q z{4T8Q!4ve^x4dfls?{_D^9I!p%mLLGHB#)B#_y2wKv{%AB)Ua8p0w|aH-7@RS|+v z7;$#pE;FMlP(bE@?+B~CBkh^;r}O_+)2Ki7Pd!mbi*wSJH#IE+DJ!Y7>W6&ma9EX# zbUcky+KmWu*q=ghAO&iXPzu{rzrhI?Qc~orLDqfwIEPIv!6@4+bS4zN+6zA;F|;m@ z2|3&x*W|=5U_-j*09`<<0cs7!&hUM4GBkQt$ttqVEK^9~>hA`aP|cjd4YF9{C5#`6 zcTT2g>$D&<YS^}w`5fD;LD>CX_n2X~` zj}!Csy>daBlr%)5XcukbEudKyO^!$_tjdzo;V z4kp710C)#9x18!gqb1gJ{hIXo=m!#b@j4g&2<-Ps)TaV$LI_Y4zp~Rn=r!+d5Cf(8 zz%J|ibo2Yi+Tt#rkSdNFLLxl1=y?7-IgpDRYI5m`yc})9$-yzviqzV67@l~Q6Y{XT~2>&X(M{*RaxLjYC9{6DH5OEG4jZ3s=NAysQlqWH|L|p zF;}9)<^9N6FKEDt!!DNW^J&Tw`|t!V+BK1)-CI^%aXrFFUY@RyeoW!6)D%Sa_T z<8l=6T=W7w&nva}ho#S7 z#i+O5i-ldd^d`%h&@s3l%sYaZWboviF-RDpjauZuSqJFqDfYMEy`Cxts;0IsbO(u} zuBOes5g5`-lJu|*5v(TJv$KQce(J+=0~TP!WI^OCI%N<)bN=3?2H8E=hn5|1coVITf&aETq%nswUfnVT8ZmJbCqI z>k~YlaLkOQt$xJ&vQpP-Ys}YJbWxZptP}QI#*3OpP;D{UEJiadBAm*DmrwXG6B%-J zHXGG@I;kV@6)6gxyx4$72u)=vR3Cn*5bI3I=q>x2&S;G* zApCXCKV7>S%)?gfLZ}R}1;5X(_^YvN;!96M{isE)8Jo{aBwbR~l5djFrD5BwAPqV{ z;#V}9Bc1mpM9vb_c;F(0vGJWDMSN6GRiqJ^F*zUVkodLWegQ_sim}ads1`CO;rw3| zxPy zHXR_y8P@I#c#o;S0O(Uv_p8&f3oGx4}jF3Xe$bcy@CZ2VQq z=&>ht(vKoTEjagoof~}}RkK=?CbpIZzO;XaZd!zNG`vvP?l*!cT9P(lST=?|G?xjq z=>?Ui(obYC0hmS8z>cLs`(H6Fn;4Q74h`1d;v|qm6eNU>ihD8?f@ysI>-#vF4}vUhx5lomSNS@>YC089F{;MQ~j}>{Y`w)AAY!Q)nJ2E+; z4ucJ9VX)mCDFpN#a)uQ<9Bbc%lY4R<@i(=OwiaJ%44&$5e2#lGHScv%HgLhGR#l$fLllOU1eYKFlArmHMP( zdv!vHHOUpoMXG21@O`>P^~85W%x8M3qI5@FA12lNnjuWetn8V;gOs?r|3^6-AHSfK zvrUe%k(-3{^x3}V6!r$iPCgQn4+S2pIctTt8(AytGP&CcTqadFEw>j~_7~75(kd`= zWGS-MMMvMKILC>@-4gTW;I2OMvOf7*2EJsv3{d+40p zqxny9SH{o?SP94+`iGUFC449n38PqCbQ4IXk~;(oT98)+&@ko5pTdWj3x#m@X7P?b zh847Q-FLWV=O#*yk^R@&Ld{czJzr(npU6Lroc)iF`~Z!4j{w5#oTl#{paVke+njjH z%#hLs?jtxTt&P9=#cfoW+IQ*k*LmXEv@eVUSRFJUvAMLaQ#2uTD+KG>D#AAT*qEwO z@7qxaOPT!Wr{1LL{_Dhr9MnI%I?jOyFj<0N2%$@t{cP)@osC>T^wW-5?W@`lEytYX z+&eMFK@Su)^?)W|XH*P!Z?{RE|M^)^0m$MPz^_@$lenzyA$JMk-1+UOZA{{a0%Cft zSP=XkN~jG+)hcY9YigR{xc*?VD? z8<2q7C-ed2bFm|_@ClM4>cy1+IC01#dy5Acmsl=zu(#I8#%dmsbQbo=i?j}A&C9IY zOD-s}LJ1qF0o1)TV%vD#p5Nt5@89kgh`}^YNLdnL4EI&3uCDE@i!3WensqNIrcrN2 z@7tLut6k&qXHT_Buzli(q^*xI)O=Rw;9+u)oo!@+XOzpY zS^R;HE@Dq_qRX#a0FW&P1Y;yYg5PX+{`TtfPXxyCym+ok+$yIO9mWBHWriHF3e109 zhJdHNtEH!H1U|PgFC@WDx!iNW?7@QH%@w2fB(HHc;dRODP0U15e_+sK6YDRjx?e6th1fn%^LgdnW_U?{*veH4FIX#XqEW6 z53ENf0Y_=25f)8zNQK)|B|L%O)D1M)yI25$n}vQ9OS49>C)_ zd2SINziPY*w+oO;9fdq7X{uPg2&Ctreu3H4Y0V;#3X2WP?}2GYa?zl;t8w0&V8Ty< zMIAraU3FNJxeS8sq{fJceKqOh`N{HoT4a9nz?iNp>#%h#wIHgByBd2sFBQ{Bl%}=B z?KJT9zdrI~7u(1-(dY{wv*znZV&cri2KW-aXV|vmNEc)pJQup*p}QIrlp#oLI(Dnq zknkzYRIzoRVC(ycZUeY!M!AT+m)l@1YL^s85pJ(}e3ltb>ks35L0IvtW`k`wsJu0i zyT0p!%M|yD_BOj)41cYCTKb%hhfU4tEe~s&r;b(obGHk`3Mf^7rY^C8PFiBB6Yuow z$30IPI2bNpyCbrgnnZ)gGFd$e&S3RaS6BG(zfO^3B@`;A;ouByeKj1_+mV#jX`V_I zvh|UAM9zEmsUrv+tRdCHmEx%(q^eU8{u{|rdyMHuhi#XF#x+UU*C^Mb?y(W&XpZ90*}YWm4TQ0>e`&d} zUKWVP&CIH177w8IfBQ-I{2mJoh>NFx``365gUL2`CDyTWR{GS+>KsS&Sh$SqK87_d z9}X9wp)O)fiISKm_H)Z)ga_+WITDL=CX;tO_lCH<`yZ~hdo71Jtx0;O4VfJS1Oj}Y zKd*ayNn?z`Vs=!sbFqR7H*;+(m!&i+*fw zIfE4=o$TDWZgXnK2G)|zG<)TX+=(8Q#teQbjI8(N?n`RK6bSy00-j}L0T&wCIDw`( zW2;I*FIVZTvBJAR{8p|NIgR0+wz_T`+5=_|@Exk9EcjL8bUHty{BS_r3wa+3TW!hr5K#r?;{Yh?h|&otY}GIJM%^me?VVtB11~OH9ooA>)Pk z^9qSWPDctJplm!g_zbxU)%QDADs#Tk*Tq75nu?=96n9G0|NHNSp-`}_kw48qv^g#8 zVBFnW{5FCH&JvnCTfOX6E?aOOiD*;D-Rkibh*~7|j0>{NAD%d+sGdeEaiVc(` z=?l1nTH7XoP0)oi=yw9#xf7EN$wTI5eOv$$K(S`*{7B%e5M&bLf{O6yEcU38C z)qJTj-k1tUF@vB&Vi_hoVs~a%|Q?tJ5lHF=StByts}~#;igm-eMc2fuag$WAL`B`Ex=azgB0kb4IZel*yd-?4~!YG>T7NMJI)9)gaJBJ_@l zG%jVJypyUy!DyVszJGn&__Wup8X(R`SWwnqaVT-2 z#_Hy0_vs`*_zzc=Q~gTS3A1NVF0nkZ`|5dmc%@4I4&UA1;%s%zpGnU74OJ24eHbbz zcvPKjCdw|s{}HL< zSK$UB=ZkU-aKyaftkP-2&uM~To_+n2z8&w7ot-h)gj{fqTfj+MOMIVKl(J(P^^wo3 zT}~a$EVyE%31g-;Ucbxl+ zC(oLo_@;1X--cIMG3fF34*o!PxLYeE`Qj?1xP7YHrLjw1D9ge*wgAa+IRrPLYF>0l zYhSqu>x!{iiR42?YTfg%gOkpUtzqczCvRRlDt=E;p-RLoO_IdzjDgzA1Oi^_GF2P& zrU($mQq{2u#^h9I_R)jFY~A+P!TB8XDnw^`-)fbV*w=ivL)I+({F=euV`!qfggjs6D<@rXM zAW=cJ4^zXO68Osq=4Iw=#iFl2?sTYT%wvff9s?xY_6*HI6Bz4zsjKBqd)q#-w3yyv zPO_qR7$39j@kf)m+yNMPCLBR8KY0>c458c$f(Q6>k|b#<%{{ddv% z!iZ2FVEFk(rsq}K+rStnPCw`<4hYDJ73@s|14V>k)zAO(jX!kG9!XT`Ib7=PlmCvp zz4fsenaiwO6$+x=_oo3pr1K#R+{oxvMt`XhT#1SYAACtIB!iy`s{Z4NV_Tx)D}z<1 z)&UdBhAbO~aySkSK`gw(afMF#J@7h^GcwhF3a?*py+>hF(X)!tVz3MGit`Q~SuCf; zkk5H9uK{_mhb-7lGJoI<%mvBc6M!$K-bnx)_cpCna(@nivTRw_m;lA_8dLB;b8-Ra z1`|g`;)zOX$BJS1_OP|7_J-n+s7(>p*ydNck3P5t6^2jY8?a1T z2JexH%0c-(>*X&uhl8qXs`&V;V4uNN-=)+V*J@~0uH@!j^#`2CL@OZl)@(3_$bIur$7_5g*O$%@~u8wUNto&R+hPsO3_3pDa*Dc z{FRi5KKslkrhW2F(x#HoBMetL#`fs=iUkanSN#43{PFsjr$)djmzPlqiK3F_C`d2< z%Vdw;+aRztySpuJnUIn&QUa3Q`3B%>Yhol4#@=Secqt)L6HOgj@}XxmaW&v92KHlQ zkZVMlXG_OkZN{Q|!I;d5{4^Pf>OEJAGI*Z;>w)M@*3a&6!&Nb{*ny&BU~nKt0(F!e;sa|jOOuNL$T5;i0Cz9_Qz8! zSzRhm!36kjq1R!8up@0Oj+WHF@qO|?U#;oTsRZ3CM16tNOP0a&!=`I3g1X8OOn*T} zHpS+vE8TX+4Z+?S_-dw1{Gb9Zi*$*Xx)vqdV?9FR}Vmqq-iz(n6KAY zU?jH#de3&0&PMP=NKUohl}>u1=pcm)!ZIR=WdYYUh^}hj9LS3SJ+4=oBP7qt3yc9a zxx7@bqjIT7$3v>=mnBz^b+bkOB%ZWqEFETW*6Mb=V(#J!Z`{#C$MPL`%M9*QPh6W}qw0NtsMm;36{(eva!dUG|l3m|>tUK*AKI_&VW zvrG4dV%Ia!(ZLbNzCm z`Ah$is&S|m+9Q3&JMP}H1DsH{%*HAM+H_(dr$tMzH%R)7<8M;Gn+owmTA}71>Fnxi z%bSECrhfryybnBC^O?7BTQzg1e!!@4?Ls)$M70=Bc7aP{8C@l(FoN2Ly-A6`8N1J> zkp3P=w;gY6$=*m2X$gvd!U2>0_Z_KVq~}IQLgg~0gkxME3--9yN@<*k-E9k?X+;Bn zu?Rpp0f=sW_rd(5znddO>gCna(t>n{dpDb#TrJ9(fDdn_PQaGOd`S6hSsy9*{J3dM zc2L#M_m4HOGTT8s%FBy8P+f2)Y=6F|qM*(ZWqUz0>n0&@dtsxri`xE*6da`F*c_`67rz9>))} zsczZv)z5U{Na^`C3{eUj;&>x9agCMWE1j#dJr%@K7*i!`O%Z;k_ANd5IS`rZqP{`~ zgKi(@gC39u)tMI+{KV$Ex`9IHu}djB++*=)B11f>baGtNx!|j-D~>`Qb6-adMh}zF z1UjgkaHQ<5|A6NGFEd<8qybB7?V$9aI)pf*_ypE=Nn`jeo zyz;*~QA*k2avhH@xIOhrkDSb>bv3WWdCaBSw)@o)W|?wjl_NkhclF^?;)G1KtOpbs zVZI|)v7orV8qBv`wMg6TO&pSNM2V4}V^mro)7nM{iJ)4n5|Dxt8iU#ybpAu80K;Ie zEk|HG9DbMOFB)b7T_=X=?_<(gVW-E@*NqWlE!4-kXoUHIpi!G{wT@095V3?fD{LrG zkiFpZ2y%RjEE+RpWp|_36OP~CNIfOzvF;{xis%JY{quHxS1(@PKR`ZeIl>!Q%Hw2P z@K3=n;g{js__^`aOgFks^NU|A%pio@KurWf07}V|ePfn44ZTl?)i2sk4navoaAs%C zTgoj*5SuSv<^u+Pb~k_ata`i!4CpdN6n$~XZRgGUNN!Wj;ES?Of3wMIqd7OROKU+z zxl$KE$?XMc{=}BAS7K_sfcwZ1+X@i7g3Q$@y-FwZ#>+J7hW?gA=u$F$iFWM}^QVkOiN>WP*h~Iza<~=eeE1tW zzULo@Bz_TP^(!9wtV4MiHst4hJ z?R<`SV^?f9|N8d5gfT9^6Ra%C&Hk?89{s0~{S^A6rZL?1s(yYR<+~%~O*19c<#s!9 zzMg1YS5pU1`z+lj3!jLnK0 zu5s>w66F;P;k!Od4o#vmy8m@i7|X5BF^u{d@AcCb5ga=q;X(tt?Wuf+;NCB`n7hC% zWzh^%u@lAQ>C~3zlJ4alHd{`rN%iPAMv}(zOwrd&bhxr|?@w6dS!1#^+o|#@3y&20)UzHJFf2m}tTW+p-ut)hZKdE)Y|`>Y`lXR+ z3PS=Znc)QZVzt(j1n46tYS<|-?saeyQ2pPhqZY=+H(zl+Y|NmiU^u(@HTki$KAp$I>??fcB{UWB0r-|b^1*>WahUNdM=g@=h?TwGx97U+N@&R6GR>Ukb* zfg`oLS?j7UqIMvUm#tz?!Wmv+5Z>h(SBbIb#kQ%Dg?U9C=Qow<=Of~0aNn1LdW>JS z6ofNqX)=SC7L@;xIhK)Gu{pQ;xFF46?vGjYKkCJl{X*C0?9u03SNG$r`U=rS zz0`pI4r$N2O@pu4>kZcN#un*+52)*?RYB(!p(Fa}&1JcrRhS;}i_J~^e^2O-!}nI~ ztN1uTsYal$Eahw#eJUK$)z#G(JZM@uewZM%@eab*Z#F@a_B9=`#;N;}zv>-@UwD{^E=T(sDSh^IVMe8D>ZK+PR#VxN71r-pbmvb1$IIQm@Hw#TR^;}O- zOe243cTLu)fr-2Mc6oaEs?oIxnqCKK90u0g-Tf1O863)Pyna z-gQ3Ej2&JLwg+>BI7&6sNivzl$N; zrQZ8XHVkNQ@B80#yOn=#=Xy_^fo%n*ExR~=2#_kNT#lb1oZh3Y5QQ!1-unH<3OAs| zzr(zTaaO4=9e?`-ndRS|Fda6hRU44o>SII*^rJ)dgQvprs6F*3fbnTCzO>hq`XXM5 zdM7R}`VR4(y~!zl3fkLo-7Xebbx3k$|KAh5guD#AC*S*kSFDuKf%kQ%5%@Ln_otw|^f{ta<+l~TR#sRe{=b%NlE@zOlxk$?~k=>+&<{v%< zDkr`GA_t|!^WqDjEzrqpCndXoIcB?6t(c;mu5J+?&)pw-26+bWr82+A>dd@8P=n&E z&p4qtM$~&ru!0-@cFCbjTggup<2}4*;KcASDKR(ATjEGgGGcmJH^#H-Pa>7Zkc{Ah zrYuR3&NFG4FLWHiAtw>-bkAGrgUrFe-oIXd?y8 z9NT#KsIyGakuq3O?)5h2SNN|uZowx`RK0-Jtaz-`S1 zM=vq=Uu{fwD9qKx2l)N)a*sX-?!O|s({@ATph89=eVEwfLLyUjp`W1A{K!PG{lHGC zYn^e;|6n*I2>M~v8E+|iC~O=KWu{-I{T`IXW~uo_ zI}uYR;WOVRmqGVnAf6yEC~fOq)Aa64i+$|H`!KOMwsYfAOr4}#2A6(40J0)=?PiI! z3Qhr$Hy!Z-VyGBAVSow2R>6I5i9rVYhoWr__!$=b?@kj=UQhY&)lKQo0SD)AX29S)If-VyH13E& zA_uO&l<(ah6wTe)KW$R13mFp?^zvEZGNk#ULC(Mjw4f}egu{ag2EK(oCwa`Tan#Vj zPTSeq#6SM|>O$I5m{OWamO|Uyp1+4XgfID!9;Pc&D6FD{uz@rDUM+CRw+|mDascZ|hjTy8gVUw|iKQkxd|Y zPgmw`5r>yuUHad1kp)RD_vodzYg)l%hn8Jkc#s7~9rfb#uxonPtprI;$JDTuAqPg7 zDw{uTB19x~->CTiHi&ZeK<=UuxIwf5L=I-f|7{R#WA~@14W`$c;HWGKDF+oyfno*9 zpOqhRn|j+GhpHBwUw(Q5JJlFEg*#b^qH-=CNval@ggeO?I-NIJX`;YoD=}12eXm>t zFl7Hf0FXd$zefb5E$OoAFqb>+dZXT`167|mL}QF@#Jf8CuCP;L3TV_*#&xNe9Kma( zOuzU~VIvjK4hd!NN#wKhkOXibZlsilD(s`(npbchOc0#SI(W;ZKU>^yLq#nDI8}Y* zI!&3WlTc7bWGLt(m%#^*Tn4YE|BtKZo0Y*!^m|h>c&RRLQU;ol+K-ybMNH!J24?^T&L;(>(DEpwYz9t#{ccx0V zQA8Q&iITeUH%Px!B2wh_)$dnv76RhA*`AEx-HtXq_7E6BKZ2fj2OJMUAGsh30PJ(L z;iAAJv55514Id%L8zwMlDM$Hsk)u2pytLFET}P{No{m330uDW-Z8QB+U48u>>>=-^ z%>HmQ#Lf^v*Od)a)FTz?o5ep<@Nwz4Rpn4f)JJwg81P@z6zRu$cO}kuHzd3UBx3r7 z9=Qc@0I^>M*mqHYd>8rgT@k}s+>q+6D4`h}j-eMK=*rX^MPnLcSV$d-=>&1;m4teD zFl4bCqg>U14I$H12bhWG(zH*_u5xH=#a5j5e<;T@ECb__1Gy=E_2Sdju%meLhXgBM=F~6d$n|A!j#YXR3rm+g*6}i{PE-Y zC2v#^rrBt9jwXIGjC?=)(YF3*Lb4gZb&{GyKKQn>Jq z%t&g|Lk{E%h7Hl3h?j9u5F%NmBu)Hgy(I%-bx-BGD|oDv&;!Pi*hT%|PdLyAa${5{ zBs3N-COtHS{I`(zR@ELaWRD=z@LjymR?QqR=>u0i>_Mj<(BMt-tloXIw`ACnidE0u z)Tfk`M@CCIsI>BBP*n6)=}AzY>DCmE#XmKB6c&H_jq2C*AT0Cg;ux$mejMZ2pS5c0`Ke!RXQm8)|IUEmXd`hDcE)7YCv!jv~*NJ2h~{Av=P^?|r3?z*}w7wTD$ zBA@YV43EU@eazP0X8?jQ>Jf7NYpAy8f^o-BT!9-%L5Pw9KB*<4oOD|G1o(2jPSFs= z$Ne;ROC>0hqRQl#()FL*sE^7H-svT+OuVhq)VpQs0uJ}IMnEGl8U?Xj(VgYQ_7yFw zfMVZ4>IhF+h~S7nJc9Ba^`!PMb2oUtZ$I*owAJ+garJz&=ldo4y{Ys4QeEEU`F<50 z-_-ej@xE{Ve7|_#r=9Ox%F^3XmR_y&23P6h@D-3J3Iw`&)Yt|81-fOzv8yAl#R)5g zMX(S0a4Z6?`e`*neE!e%l*KSI!_^5mB5y;9{6n=wad~F@u;tUB_hKgA7q713^h3fHFUKU8UYW{J)@?RFH zD{D0eQU|)|B?*He^rfI9q25~~iU5olp+gadGP&oP5EN~$>&cue9ek@9%V?>mh!fz& z8&R@B^r6E@h}=3j^E^|?B@e}W7QcIq?#v~c&gh$Lg&rR9HN*ep!VFPHh0=*{vPuW9 zwJAb*7!RVU6yPnP09R>#L93D2$BRz8B4WN#?YY~owk{L7xCBuVJj7>p1Q~Y3 z8jTr2B@h(xL#oUj@c~pip;-Jz_MhSo<5i?S+0{0Doa+<)d!O$OZ0XFvSf+gPL(CfQ>J;Ck|l?UEGbR2 zEJA0}`G9nN5EVF5^RSNq3&VCX8NpmLKzZra1HdBEi#f}w{|m^#ua zC!J;|MY_&xV)!r?!$;3j46i?H(02BC+xz;^Vt6gNfVQfW4=aY(n=@y7dOKCHDbJPW zL27qCm~D6N#Jlt1v)rB6pHXK!d%W#^edyhJEqQXbs*?}9JFhpN%=Ywl>h7$Zn40a> z?%bYjcRq-B=g!0K&fBp$uRqbu_VoC{Idr92a59TtIcGInsYTzKZPB;mMSt+z7QKqy ztnsn0Y#pguUCNPHn0!Zg5Q<Ml#>{@g#yEaETacd-bc-5YFeAKO=~Nj z);6EkxERO{WkzK=ZZ#xA_reaP?_fsgMl;{}!pM2i>50Ae4xavh8t^|mZB zAFn#)63|Tf%Qa{7mun|BpjP3cYvdQ*xxjOFoh9q6wE|?E77%J$PRuDi(Js5gxH?^G z0is(>W%yw-TbClNS#1q(ZW#)F=v}4bDaszdL{5mRX_#S!_qjO&;JA32Cp7~b*5N$AQFgvJIls)HYK zNLkfvTlnh=H~+Xpq|XN^yrvRvO4U;Obn)iho3qO|-4AE)-+Y;)f)c3J^jD7Roq8jd zzp3#aum9_x#5cak@4v@B9zicv8q1)qen&22$q$16q)Qm4;Hp*UaL(jnrP)6+mI&00 zWTZgR>$D0Iy71-TlN%ifMCI1{-!FmqN^1XC8mhyRuOz2PE!z8|D8e=CJOpN-f6$Uj%7Lm7x9K*zpFana>B_P=wI_G)a1WV~NZ zf`b~4+uN&yk3?7+Ihcqy_nwDTwiYU(kBb<@o-$w}pV8C&UPOoFCXNW4c>|L3`%jTN z@J`tyBTY`D$t#)oNN+XYQ}9XW!f`O7Uv|GU|J)NhlOev;Xw*h=0exqPLhwGuA=yd! zm`V^KR-)())gwf=0pSa&cUREb_cZ=CO@&3C?(BR8=P-mLzWBgb@ETDk#4;%n|4Zgj z_`35Ih?p**{;j3|l7Ug2{}SKYuY7+VW0y;G0X(Gbc1R-6jhDTV=PB-!eENnjVjqq~ z4g`;H{FFqYgLIse9ICO`f4!=Na^STD&Q-4Bo@HErzk*-PDm2D^7C#KWf~$%H8S{|Q zWR2cYhDH?~o@Qh~;n3%Tc!<{Jj?2IDO76Wn8wGy^ypmaP#G^tU@Ojn1iDPMoLg+BL z)D0>3>Vj6)!gvn@S``RG;y}tfE0aV`=oTCdLNtI3eI#mX#dY4#Gkt49SFvy8DY`zU z>AUF;7<>g?81SQb98CHZ{60jYSdwva_>_NRT8YGjl{t5+7~*{vA|z`gstfss_XKGb z+EY1+ruW|fzES82qJdv8-hqH%UrI;GS;mYR%SwGw)T30N6eT!gg35?WOhxcj4^mOb zO2S@BqmkrE!9@Jv6?o8>QbXicGV_Rp`Ktb#Op%jgke6laUZuJ* zUyR?$^IseRDb94+yT;~QFZ*7dj-8pNghOclIZjfg5SlO>f{wAzf=_x~1Q8tuA-OH&diBu0i^dr~=Zv%;K<63`5Z%ud zc+F4!DiPaz6-{kyr%K4cxD$F;mE(|ui=jnXVmIwIhJA4Hk2guKHgmpZXD)Suwk11} zBzl|u+k(bRem4;FH2XK-`U8G$_%Oi!?c$%tvXz?8o0;S5ZrcaD#>)<4!( z>G!YORFwf-e16-smZ@C03mtJBqmvRgW>XJVlMIfj)w0J2aowinV~Bd_%pE-}zp9T; z>(pBLcE&!eg@0%K_G&q4;z#Ad?tSDj9C-2JoJxU7`3Xk?v5XRcpEXDNr2-&FwT@CY zp`a`7$0=#V0UA0(jKmJo-1SB9nNO)Mn+zhlvDOx{Q)3|kI#3&jntl!%uQ9GxU# z>;coj0{7%B_+F3{K@HVABIQO(n^sLQk{bkHU&_j0WQ8)-n<`B!xIw)kA=k+olc#4WCM82& zS&~-dn@(t&Hlw9_ot)A6m&7cmW-!x#-Iaj2b*W0?OfK?36+z4)Z*X$RmF56DttB|uF{@Sq5k$=aiEt=O{l@K&ya99kZVfhr`N>L!UoS)SSl*(AG zAyl@ppCw7oK+-e8@1uyxKN1cx4SKODLD(qcu@j|)mWpL@6qc$9d6RS9$9I<^dnx8@ zN-0+R0pFLSK&PK`P{?o}JJN(lfJ>B664}yxFe01@xn!)0oyqIhC?5F8HQ(f)Kt72E zL#Z*Lh(&?a8DeU1d^{FW?pp8~zv-b{KtgB8w;MTVp$S2%iC&^x%xHb5C25y%IUpuq;naq&77i3{6%xb6HPx$X!3l=*h{&182b1)rzSFLQ z-_ncw{lvwcp3StnuNw0*4nZGck5Au-xH9(BYonPx3Z@Tx*z={ofb@U;IF3G7li*ydoZ)kKBq_K}ZxN@I1vUQ11t?&XV=0 zzRY=5;zRwSU@SYrp)CF=0B|=ZjUJ&I+|yg-`Br_zunOiXlLW7wN=@8rrK(lf}zMEE$F&$rci8 zF8O&yI*nkT1JHm-V`zw^_~E-VhMr;nkA;F>4U7Q!1%c=cRj_)X4w5c#qGNi^+a?ja zL3l7sIK78_)W@RW#So5_)>Ci^KbDx%K!`1=1x*r?OLX%Zjgd!0DG39UNTkC#O~*dI zj$AwLm_aNgK)E91$f?CJbm zh5Z4o>V&lvp`D!?_*8(Ems%0&Hj}0ddkQ9o|9~6G?4)Fj!ZAiS`=SN4@(X*SXgb*% zs2=)GPeAhUo}g~JJJQWeYHf-@PAs0a6PIv?5frXT8=f}0-0>+;!6CG&fSA}-6Z_2- zAYnceuJGjaMUKzF&X|gfuax*OZ?X z3lmqsRpxcRxKG7W%Y1g3WoEGDYx8US9Pic!5q1&XA4s8AqYQ>zc#hofRA_uH6TTx? zZ9<_6W!7pD6QmLf9KR$CeEcR_N<+qA7bHFO_^KO%BNE;luCKTS9pA^sjAvd zA(D=9+QmAC*4jUc!GYq4LhWT-#OspZ!z0docgnp-ON-brs!-}gmf#peITXigTjh=F zLIz_~d+62z7_vLz-7FmF1T&ub06}^OZlKr^Tq5@YekRsOMh;#=GQ#l|5nJ}l`<3i7 zRiFezl=wH$XK_(9`G({yY1fw=$IplQoG5k!&_n#D1arDed&W70nEqAL z11xQsDDc_S{)$CYFpm?lZ`y5YwGc%V)szXpR-?)nK6YFw_?+|ueDUG~Vd%w+Q>ExK z-H(+SMVuBS+!Um^s@2@^uEuZGYA)(U1GU&Lkn+_LkHxg!jcKRcN)dSRqM$=BUYOn$ zMeB(-`Qk+(MP9tn3o1Xgavr7jOIW=0f;-*W`EPNqmh&Bih(;r%bnvRmGBtfez?JFw zlPIWk(rF`)xEO?1jR}}5mLvAnnZej};yMZv{0Po5zoVdi+>iKGsbM?k$A0OdfkKKY zL%xH4BVII$bvXMGO^$b^+%I;I3&bGyuYlu0JW?8l)14gv05H}v|5L}d2(<$U#{2(w z{C%cA&wKjcpca7d?<+JwPUZBz5@a5_K;l!fh+E~fB0Es|_U(K9QTcVbQcqvJ03QiJ zx5$Z<$2aVeaau-M$9t`)sfyPxZOt~_+;p`FMc?CRhIonKzQ*A zC-uIxj?`ephQ9d!pZj4CI{SM&JFPnS)i*L|MY%zU{cKgUIJk*-%tP!-A3Tc3oK2c9pDm1*QVsF}NKXLaq?aCn768tW z5Na0VQw+aw*>U%o-zm!#^zO&P=gA3WL6NSRJ>`DX}2zI$Z z{kW_L({)?qCh*($RFnmXl+{Hq2majI7-Emh4`@GZpy3>_!U%oF}91p|tLN zq2$7k_(-Tt%O9V0&jQMm2i1tp!T2o(m_w~oj9WhioyJc%iM7yMoDN!y$yq!{VywJ{m>%MU zkm}KuNz(exPJUliDt%{&oa@w5I{O!&G_g~8XaJqN)EfMrg7e>`Y)QLy$b}pqOB3G& zaa=w7mp-L(U`~7#+uMW_4HT+701aQ+vwv}@rM#GbtC~SR(zeovSQLHc3o0zEYG%zUC4x-0D5VU?$Y-PvWCfevW3f8$?;H3F9&`0i^oeOygoHkaAp3 zxg^xb5?bU|q&VCt{;gMujP|QR#-KBlYeQG%6%ttqk@Mf=;^YTs9kyI_)RbYyla89a zAF(G`uJoEiMm-UBF05IxT|@y@=_DhDMT^vyRpVkc<wnT-WxDz*H2fAmY9ozL0L4OSB0!udzh)e6xjc-0nc`R=sz0xF{KB*RmLZ?wF;eB| zS$rN7k|7BsLXhHAh^?Al2L`gFt*Bq$kC{@LZ$pVxpauh47VuS}PEk@Nry0bkkDKz} z3yS>rL@>RcE+v&~65C+UGgVSa`Da0iRE(4;t4)OF`3BQr5(#1)z~#n9B6+xH*ttfG zrb1*nDEh=FXerGh7G81tos*WdUzE2fVgYqjPULy}hewBfL&<X^dO?l#5v0X1dT?@6ovM1I;>5&7?z(eUJ zpxh{=h15k34Amw;V?RCNQy~_9WN;(4KmmdI3#Z@%@#PtmljOr3U)S&adhzxY?9-?hM$xW}zp>#)_T9XOpv z?bvN~YVH0}54O<>bPjs^ntnfpBns$06_aOtP2aB#_yhaOCK=NG_t*&uC4E+ZMMgoy z&_3@6-JiYRefse(K&PNmk;iW_FVeNdRmTnJ{^EOgr$A`+cYqM>4rX_$v&<=I{)Be` zn5bJj1+RtrW2fi{lyq$eD9(`c@+0<;(t!v7I@gjCr<&Bdl2BWAzBs`_rZ2$w)4Wwk z)@0!(&rO>Nw^y6v>4y4**!WIOHO9U;PD6J7+bapx7#9-c z#z!F?;y`b{=fBCr{ZRVktL+BBc=OWknz`LsDV5_CHCVYHYYCQUJH=kCoB&3#ep^M$ zCp$1O_1eY`tfDbI61gDeR}N0F2SdK@LIQ;gpb?x>H)N~`;ZY|PCB4KE6uNH6Pux0# ziFEN4HUeqCWCUC_TSp=zee<;0XdRqfNnuh5m0!G&xkX;QNL=jeLcOR&m~`7tI-z`l z#7R_L`~my!ne5k<@|TQ#(XJYzCk@?`w2l!t#N%gD;XqX1*?BF%o(lO$d6Mdb`Ix^L zh-%joN7ea1S8^760w=`L`zf@rpO8zsfE3@W$yTpYa3rC==bzU*K;Gm|#tQOQYFoHrsyzja7Vn+D}9o4FA$t`dit- zJ(8cooHJZ*iJX;OVDA9K-xSkyMv5q)EJSbwJ{4-(SCqJ;upN zz8bCeld%-}5JXx9LYZTP&gM(IR8?X^5&0lAlJdCe6XmO}(j^sY9nubsh6{EzF>H}2 z6#59mOHOTqg1XY4rLs(nTlL$0(U9qtd6$aC&i3ANwlflL%lRTv;(`>zuNgjbdS!?mbWgXBzj#Uek)#YEC4R z4i~XXwZJ96n(@h(F8KoPDo-F7j~V%HjgcDi-B>AXOlBs_~#d z9$oe&`x>gklq`t*tJqtSPo6xI(Ti^T#6_|9=<0X5RP%daU#*N(GQd^Jrc}e?dvhhT zG23?0xHfLpkMgN(42YI(UA3?$cLf?7P4tdDh9M>qE#@2mosdv!7##CkH(+-Gl2iAf zQcCV}>d*YbROdeBtsAGv)T)Ez3THs1^6JJ5RIOXhdNXZ)agGF~3O7h~cK#NqtR=_(uBr9Uqw-gHY6fY`9(cX+Eqz-4R){64uMzI%n+1+ zTAfoWrd6mn;sIiSvdFoXwh$~`&l0Crhxk+|#)c?9ka4y&5N%8h(QazQGK`m_;$H*e zCr%g>)-oefRbDl0$zsIXMK8C!VlyC-;Q&1}gkC@4ej}Hm%q48@t-4F;yv&j{)Gi@d zQwCy^P%M?~J{D(wSY{)O<5c(eGD%2dAGp%Vh?B#~w6m$%i9LXlX^-b8)&+@Ul2}hE zXYJxqNC`Sd~Q zNRoV7%qy`ct3@jS?o}QL7QCih9Ap+D=`X=){fpy38vCR%Bg1!JtTVA%M2=}H-3XG* zAUb&6uz`k=heWmyR4*)OaFy^V1lLJdw4GKfb*iz$K3}p>WI+;)O#ln=U?9CY6KU$J zTpJ9JP#t_~w+@P|7rqmJsC3S?`K7V^&n;4 z5P1Z;T^b4FXg~70BEFTuQP8ExA-+rR^}+S8FF#izU+nsyD^;^W4^sN_Kc6dq%U=9k zsm78_JP`l&x$^B(yFtI~iOdbV*%wpvR#PTX=cG9kb9Hhk|4YxAhewz%vu2}FFq6`9 z2QCCPQgcX$mBg;Xn_tf^|v_PODD)pxrq+ z>>M1oTjH*V{p+Ofl>qw4qcQX9)m(9)GejTp2$6^xUzGRwjQs$e>w!Q8*?E=sMYL`4 zEo$r)K+pR@rwTd;^1nv?BsKH$iS|(yG+QTCaM0k792Y#(X;eY;s8t2LwMPBqXiw8O zo~aqQw9=1fkp#)SR|YT|pm^64mqzeb4>Y|jN7&al8kx0Nzz1v?B040VD=VE$dg46{ zuaO`jUl-Zks4@t#droM&m&3{_Xd8|3E*u(v-)4Gw$N$bXjrmNs8p+>2L*W>D+2xU5 zVTw#afc?Qox$v7s*VBt4>Ab<>N%|mk%Z#}P>K1Q0zWA*=xv@|NgH39zl{A)HMI(6o z8ZvlJurEZXG5Zxx4RPZ=5xXAWN_?S{s-&Lz$iJK+UO*`8V?H63O7`N9#AlcIPtjzj z@xP4wBDOp8j<{C(iWvTT7)&5LKQr<7c>mG_VzT3v5Bq1C8zg>V0_|u3!)p)w8D8pR z&r1t~cC!ka&EqO)9-nZ2ZXTytzR$mt6QsOhvl-b+i|-64?Y*pEDB~ls`Mx4vH1gAQ zRR*bYvS~ps_QqW<&T$ACDi`M3x$WM#77#ZhjaMVDTca;~dz_SwMoPG*Ha(6Xx)g;N z(exvl<9~$wgq;R;d`+Rg5KooNs!+1_V8W7!w>O%hHW#W&oT_1h3zdmdx=$L&kZZgWT$tdFPWJZxuoiAtP5xu)iWfSq>2enK5kh_Z6&pp)YBz3m(g%M=#nuC#o@VJR?8bIMQK;97EJur32*ab zWpcbKG|ewRSD?f2I1%g%2z1I|UR;9h5%#YV5?e`cCH*O=1SoWn&mteQr3L=+x+K1i zL;|Z3`OVghr}L(j`BvsznLnG%-wi91J{{CLU+OPni4OL`HuZFIobfPN z;-pgrCnx;BdS^1OxRmA+Z~++tyFT%22VXn4w;)6`AU;Kg`NG1D=La`jYn^f;bT-C&^yu+YOokLl0@jtA3B0?L^Gph|^rpPB-3ya0!dC!|Oyate6b5}gqS zJuB0#Ob2B$z15sirnjw3w=&(z^yxBPr1V+p@&_WjbhDAVj*1?iM|dW79n5$N!5!_ZXqV6#Hn|?qy z*&%sofCMNqItD#F@XzSSVuxNnmAQ*5DPMumA7U5183pXFEC;BCS4BkUJk z4f*&l3w@`J&C$GyUGzJ4#Ug5_5Dz7xprjs7=GJROoqTf=m*)E=v-6hi&3uQax!J%z zx=}rAO~N9xNM~hGf`dG;6?Je!=;DaC+eu%~^HGR=7llYjguX|N%_l?wf;sswuf7&p zaHYR)WthApw`(0elo^=V_KR3tq({m1jBm;Ta^FcgoY^$D-=l3NqqypbxUPOr^ys}i zu&XXE?*CTo#XjTlHzn%83<6eeIf+O4{XI}40N=i)TaqznLfXh8@0EE(b+a+6tTq-P zC(@4+G^W-xzkWHy{oG=Yh5K7Ih*Cek*q_}ze9;S}%b^zRA#?~s<5{i~{015P04d5XRw;;bk4s6EyPOB(Vh zh(qr3b1slU!7>%s-=T5b>m9SvDVQ%M^+63_$}o{B=Qwq~J3eXj8?8z;``idpCS}rh zy?(RTXyhMB!bi==aw}>}ux7f!iD0v_7OsUb)pHH%;>?&?zfC+gB@^uR7M z>kG441NIaSA0EdR%ijLHE153A{}2Tysp`58D8PKgceE zxBI_{Lg6$D*Q?iUECzU=iC%X@`ckXaod*J^pp(_=&V;=e(&|pb-dR8Q0_-imb@b3m zU5l`jixy!ogRtRvFl%Ugi1M1HR&&EhpQhzUc4u0i5 z-$u*)6GCZ%vmOCXd-lr8)giSlfMD%ygka5VYNLtz&y4Ex=;UeRGPz%7_oW1QW$swu zwOzmq>`qaOWl{?TkWV_hw?-JcokrGgBBiJ>YE33E8`Il{ekKOYY5 zIU@cmBP?dhOqGNbH5+D3Q7H~O&fuWdK?Vrr?7)Y49E4pF$*ZMYj%_3;>P(W?0URLJ z^&>;9E)c5VaEX$|4SYg>M$i=^I(JS>;3>QFCNZ2q?uKJddL5e^ zcy1LI#4<|)Op9kWo|+iboIF<^*9^=h57a_5lFR?|pa1-4?fv`O>(}7tpHD}l)9et7 zl&xyY8AG9%KU_qcAVd^}diLH z)!Prxd5WgVtsfr>aQrZrM6**?Fl8UpDv=pEYKUDI`Ky8Ozx?{q&WyfMk~}FpYl_i+ zTJ2t#W~ZIZIA>RP4>4tA5W-RS-x2f~_Rwyl&Z#+bTvbh|u{D}R0Yxsujp zZ$W+m4^?%X(*y@oVQWg=oFT2iyvcdyh`@@~<*%mm?8FH^u@k)A6a0GF2|lqCyvYf6 z5oOrdL=`ov%P0HI?2|ovMwi58f177iJkv}*RK|t0 z8Jgp4`0?Yr^0Q&ZLD^)bPwsSBstqgTxk;LqMs^g3C(TykjF|8AEGNcirhb`0pR!+n zvA^0=c~+mY__=(d|39T(7u^OpL|x_HC2PlDM?Dn!h#|_cfRN*PKrpX#mx6srDBA-y zAZqTdysAurB!}w@{}MN|iI5;8`d^`t;XZaC zL*Nbn-d&l1=XgMnq6-Iu5Dj3qG+u#N-%;P@UFJ)X-|xtGD%2UG5&R8>R27Url$W{G zko&a?IgM-Hi#C;WUfeQ6qrig<`-50mRXj|Yu_jFs7W>c^!in=FZmk$m7!MR}KN`)O z;yre=dJP!_muYZeWC|~U9>PCQl9`VtzG@c#|96B!;RPNP_1z$RGj(-5k((mLxo83iLNTMmMA}!&a=3(C)%~HvHNmw?9}oTJxRG{+qJj# zYj3etx1semt9mEBE(7A)RXM4H+r3uvp0ZOHYTki#MXNMUuDJ9Pou<(vtjcqtpoE9BZ%$D zHT$dkk|N)qM1=ZOdLg-?YAZJ~<^1&a!~X;JL3{vA@A;KT=ogI;sLD7wqV0V9%W8f? z=sQCc)<(p~jD*-9?Dw&U=)U5Kn)v>{V#nGL`mTrlK`lU`IG+0ss{cWWKj|BdMx$|f za3KC|G#Z(I4;x3FqrWr{jt*Ll=0W?g{g+0obI|Vm1vFMP#)&@>Wib3pW9e^Y2lq&R z6lZt4*bgFR9B(ySb;VYULsW1N`aWS$b6jjR$ej8!ny6lcJ@(em>1xuVtyTj+g|H8O z2x_&ophSLQtZpLs^7bBw_~}YhjVfq0n^n;2bgCd*20{zBvt)SSzrdM?!fcR1a#E+m z9)Y1-praH2Dj6>H^dSy({EXgz3)4?l20mg!zjSigJkZxd7z{tgA$y91(iV~EP%D@r zrLZ~$&CYSFd2rY{JZ!g>JtSwVs07eQsr@`1TveE{vm|Q0JZ`--a(z~0lOf+n!k{Nq zhk(Ir(tKD#Hqi&>?xC*{HHxp4MY_D<2% zm4>uW32L7Z)?OCaJ~_0##Hx_q%_sW{ef9xp)H_xFFO5dZzh+Nd2SE5I$X@|i*q2B* z0Ad^c%l`n=(4Zd>#ckyud;Fih&wkDvRO-)BU`sH{%g0lf_p(YYmicLSG6V4*A?Z%a zY4^kGXm|2N7v9N9DfP}$)Qf*loiod9k!5Zw>dplk6rJnKau1kC9Vw@^z}066kPXEz z%0y6}ZJ4PeHV|h8^IEVN^xd^!rTHAvI~^w7bwdxFhMZpvkauWoX<3n;|(K* zKew7nBKXT=_)DYsOXK)UBKart^Q6238}D}^C*|`HlxHHCkc%K`zF6(Jt>AlzB=n{M z@);mL2u1E`!P!F;gK5wY$w*wu;PK6&?98PlVfKNA+3LE5*^d}zD|g@rf;**2I;Vi4 z$qqk{6sA8c>7N`<-(K=PHlY5ime8^Awiw-F^uj({jJ}N+ebd_Nge6}tiS&d*(X9f^C>S9ZLqAbAmL6p`u5(-h zEFD|0WWiEG%54TqC3>HnzOo7y0aM$e`*}Wi>iwks)xs;QGP2Mr)BK}?R_C$Y0;CTP zZlx<_=k_*cbPC{z_yYhvFS%CVCM{H)D}R1u)lAhpw9sm`&p$hBz5s;Vpo$~3>-7--Xar8qh4B6wwnXk3d|EM8H!x4^&Y zLEW0F$rG`L$xtv=OYL-^cPDUFRlWCceo)~4UibCUL86Ry$ofXb&Plq$3YY^1#W-j0v%d!n?a@9+^<&;{2spNQ zN8*0uQC3;~oQDnfoIy!7ci1QYLlAwQiaN78peaIY`>Be6EG2JtmnkHHhaMduJsYDC z4vee;oQM{XJ`5mrk<+_l$YN59NmF+o3ntBL>e{WvA?e|CNNV$W$OkN{TrH}cj^^|w zZQSpAvRu4rBwPUA?JWQ-r*>sIthucS6H25qV`7YL!B@YUxtFkmfSN1|t*rGk*XL)8 zsdkJ{n5D9k6!y7lS*@17k*k-Lt(axj%rf%U=j&!o&7oafabkM?tZ5arGApE@lEm~n zTGJ|NWol^wVBATU%9vbHtE@yrNs)$$b+smyE-b0~k)DM#*=B*r!lkc|eXKqD=BtZQ zT(qKW$|jWmFemg<$%cF(kn&O>6N`bAmIGP5AV}%J=Ufc*8VA3I-sPR|6yxt)O^<0+ zJ<3bIlvMVZP}`%lx<^@J`BasUY@H9Y>P4#5NBaFtxsNFYKhh;XGDSZ!Wk1q|Khi}x zGG#e3ku9B6P?jSp%Avt)wj_s;yi%3a656Xwv5jo0jd-)oXz!B#-rvjPt%JUO5l&GMPEhhy0xue1KRHHKj_OS#Rco0FutWh^uKt^-{3})c6)XNG zsr_cJz}*Bj0LOz}EVOL*T~SQ7+kN?;&lN6HyG{_r8ZSZSFr1$&y{bV{?_gCHefr&OGq06@np06=XEfGhx-4uEJh+7(F0Bhc-D zUoJQt>T=E%=w?^L9ePI@8g*60J`#5S9ia$yJ<(gvd8>jqWt==6>6D0`!jU!?9cgpL zBW=z)(jOKb>B5uRT=lFrXP(wqTLNHtx4~J)#%$5+9O!lHfS(1Z7NF|;n^$!;4_5&7 z4lLBOP;WL}R}2kCXhg!h?t3&k8$;~Do`<@^?Uh z;oTyY5DBojchML-=&%3zTnPzbefsrh7`iT|*T0F(2=%{zu6#4mghvJfN$^L^c4Q(2 z-p+2eg|DB^YXACjTL1gr=gPMUNXF=(3?y8>gGM_GcJMiv4qsz>Z9x&RpvZzE3yQYF z2)76*I<%n3f}+`sa59|%#~cDBgb2D4B}fOO0)HZoFvMGhFk^-WR%F|`K+s)_wJt-F zn-DXG{H3VE+dvOPmf!iO=C|!_9CW?i|HXo>HG!=kGO5O_1#Ch;EzEijFe_&PTLfnv zS)65Y*1R~&N9=}#94R>nk5Lzep#lYlg2y9Y$A}HRg?RT|L+C z>Z@JVd#yR1Ifjpl+5zS*Q?zGGtyo#li+?QW7_UBhC-!7FGsw_AjytE>JX2b%*()^^ z(w&9W;#J;+;^7p&p?ZfJvyL@u@3J>@8~h>t0&h$TT0`_+y}fIc@@U~n#8uMqo{ z{Fu-e*|;a3q>@M{jbi*vw#s4fO!i@0#$RA61jdbOM3c|t7gh9`tVy70x{@ZGqf2vU zX_UEU3E1<++Vid=&b~Kw`N+cFL6t=7s?6eEUI^{*P=9>d4YsH7UpCgA36AAqc50HV zi|n-0)18vdh; zIVQe}lA#qXo6;XvB^uT$?%2gh3&J6F6&Wt$f(*G?B~AQNMx}W5QV8`*w!bs94VOWF zHws4^&>dtE(Phc3f@_AxD&3h6VceMDrf5uXd7jKYcq;W(6{IuB8GQR^i>4nSj zg14oBtH6MbOz?~imuAgWt@TG0=RQO6V^$$6I+EFN(dF{*-Bo1&a?}m_Y)fbLYEe37 zqt4dhL0?|1ydh@XhO)e0mV0{YNu*roZr{FB(kuh)9LxK-72z7neZ418N_}1I#)TjM z$f4o2pjzulzjUJSx(`CGDS6`Bh@>icGL88?CqI7Wo!GRg4bq^|*)a-|0=6g{z%e^` zUX7hMv;Exs%naw{`q@5!J1Q(kZh9Lv#bkLqp1Pl+e3+sqoCw`s8JF+1y67Aor%xTd zac6{+MKUd0pb>$G*F?zYCI}-KvJ6V=IW?+@aH_;w7A*O zjr+ZvL}y0;($UgvS-J-(5NUGCxgNM=4a?|p$=gnzG{fR%3X8cOIA`&SZBU0V10c;& zesIYHSBV{F6hn+%TXj%|lje<9k?k}?jJay7kprr5>{h95nf|opzv_#pJra@RE@UuD+%ieDX=S_O{9`b{w8L06I<;4IA@<^%~}^ z*MN>_w^aO{dz5FP+7~90S5a%J%xTJwv>s)4m<`EJN@V}I2t*NiYi-^ThO(OoJ>wk{ zkZgFUlEO2Fiyj`(XssX~rtp2aE805x!F>Cu+1@95^5IdaAL3*{p$zA(;uOeyaYzVp zT27vK2?<6lg#)yPeT#;zvQqsxB(w_T^|R*1P~0fqeF2_oi6AFd{0#3gVUft3 zGA0koT`W(Jcn4@q&Vwj3y*-GJJJ&HIdl;lQ`3|dt`XDBO%T;R;=RV=A*$~lGzfw}Kc}`HjeD@Ck)_y7m>-s~( ztE8uwo!-vc=R9~TxZNS>F;$DAKNTE)8BfX#ws8{9AHI; zLZk9f=Jvq~+fnl0jhwanoF_FT>)Wb4%7br$I_Ov>tBTYLBh zDD-3@c-A4_-Dl;94P57Iu!y#g&frqrPyImo+^@BrD7_q%h$OkF|4hx70p8_t5BOF> zfmRqttNd|PG`1H_cs~Tjg$3(HNscDgWhhM;2`P-b$X;vvM86XFFPXt7E>g=gs&0|( zwS7z;g8%bS@t`_rXcLwwc?obs)optVm{M7I3lJWh`2vn#xwX?%ML$QVug>RPOKMM+ z2)jz*GHgr#8+B~ppE{n6nFn~y`ZZh#h`6M%ky97J&8Wv!W(zY5oXM6r(b zqmEUgb(_LM;j*@~WbZZMS{Z*@u8XeJ{b#UekMueVA2( zr@GHjWmgO(SJwzp6IH1?WYFCS08@gSu50v$KWZ6N4pTyupAAzIPm-=1A5;!|b)lps z;ZQ-=#Uf-m$XIJb=jQ4EbV+d8V`I#m3Sm_L(XwgHC$#ngUN+uX4bdP$HPhKk!Ob`mAtqel zUJ6p*rNQU22z!|6X=QB<98q`7=G2nshkB#Ay`)e2lT7$C838PjU^E+JMevRzIs0Ej z*MJC$vyPz*KMhZweAeEQXmY18Ymr|m!RvH24*@RExORgil!Tb}Cymm`+$I)~>k< zv9c<*9uvBCFupmXKiGUy(Jdh1TKP_>EsMVQM!$boI~xW3_GC)yeX19z4jwmfRN0$- zd6TFP&{1?>uvKpto7YCuMrFGL@qkH+O1W`gm4}>|?u2`0FH?Y%cwiBDdF)A&FQPP{ z_nn=xgh;SM6Pg|98tyd%8~lNlXHWE{TZGYw=u7TUnu(OIz4A+ciT&@Fk?L=?K|ANB zLVBexIO@JuWL4HefK8uf$qeg|zRRr!cSg~Yu93IAg9t0=ezMR=_lef8U>e^JT)}`v zg&8u@+mwFC(;6kgBj(N4M=!5IIAS8ZZrA3@N5?T18ur@%#wKwW?OkbjhD!&bbY&*_tb3pDl(8%-B`nN;9na< zABVWQ--`{qH%#P(v5|(E!MyDL)`aU2*N4_tT*%CdeR9Nfj`4`RJGK+8W288m^2$Ey z+F{;LWk&Ic8{!f^>a}veS@H`+7=`zXf@7I;Dc+j?U9Q05)sVM(iq@Q!5*pgc3gf~M ztbEDH{a$ukb=xy&NpUNTz1%-~ES*Dz=&1m7+^=1l90WI`*#WNyfY>A8;RY1e13R&I z`W=>Ut|HGT+h%BI6(VIwBw-_EK2rU**z@mVU*n;>#J*m;%0v}Y-aOmGR(S;E`Ce}D zIR(V~jptwIJ8mC^`jZu^q5Teg=)0~@gLvQ3ipmX|Prsk!fwU^dJQjAc3v2WqJ>DCWeBQnhX(Wh^1+=$M$9ZXHM1O_qQ%x*D%zx))6o}Ym!Z3H*p z-3cD#;kP)s3wILxg#1&J?AkV@{4gnJrB=xT(LaPFlPNsVvONfAWP9y}W=NIP3xq?y z4oK#TtT}vM5Hq>jxtoN!Ic0ymU@^RggnjBaKQG8!kvicH8sUw)Iv;`M(7!@8m`xh} z;SCH<)bA{&?K|NrL++M;kyy!@nsv2teLQi6e6K{W37(>WqE8$olv>)*O($83MA&g8UDH5VVoCb} zx|c`B%^<^uUrtv00i=7S?4V=1N!D-Txqh=VxHKhxSAAmg*sbeP^9;F2l{?cZ(VBc& zf9pGx?whHHEjA16r+QNEHV_F+-wKZ9|0L*c#x)t8m09b>Z@M*0?VCh@tSOH;2&!sq zztc|>%yf?%V!)LjC^J=fCpfO8wtHTfY^QGP;mxcSHz_8~T2$m@nThH+25-|Pl!taZ&=QLH;MXr*tmXn6KSZ`IH>mPWg4LpyuC*IYgz;&5yjSj`+BUSo z>_cX#PCvJ0UrzYKFj}e+eNxA8-ZM)9A!2+RQJwS;V+`k`%upmfR@Sy36whZ`FNG;~ z*s6?ryvU9Ba6cwulvb%l?Q*~ zT%*axcL|#BWr&&8NsRnfH`%{(eeeuG!mwuMq=nK2s#CLL)tvzKtmiYc*$rH2^?$qMSm(=|v zj7~ok0%5vUyon`k1{@X2PXx?)sD*g%ebjig`hN^qfz>Aiw!iU^TFeUi>gyf}##VMG z1W&Y*6#|9!bWfQ~UapPVgDyA%V};q2$!UcQUTXtV+a&hzXlLVK**l+V^L9T?OFFfD z;jkA5o-6-93hV}@(kpv?;VpsR#OyVy#jcqu%lek(p*Y7z-1)B+`3RATn2$`S)q&*~ z=^%eW8GgTuZTn!t)!+Mlthg9)&p(}cPu*Yp2I*MO1SPOXNTLI)Yxq_@#!zrPq@~kZ zfoyV?gG~FBa)gb%4s6wXS)n}Ll>`)l*M_wcRJ zYug&y96QgsxzaMgWqXvp#pU=_9StW|CNdJ)4^SjSVweZ?7lyM)qm3vQhC@e_x|!mN zXg}2Ad*{!ULGZ@RMud87K3lo*hOO@BeX8jAg&e-*jYP=T^zQkkc~{|IMa(@XRBZIH z@x~g@4>n@6d<%^Zy6{VesAk~fwQ)vFAsMCyXw=3NWEk2xqQ5gM=pg^OxVmy-r??5J zwcGqOAc5Wc|LG9j#uvS9!_dASLZJtEpv7R|J$1x(iQIUQXD+`CP(cTYyi{7eo6aFS z2Jmg%t*BZy1^5Y6gZ*j_h=`GJ&;s|wRs8;v2#tz>f}s{VwvL-* zFG9QTX)1$T+GC-t60hFz!@{hPHhdC>R1jBK`^KnmfU$3&&t<2;yf4IIG8hhtG!+7E zMGfs2jQ)-!d*RO$ufd=~=OC;>^=(D-!A88RUsV$mGdOe+qFNOR@dV(I&{y`<%VDH` zY758^7-T(U{7})`4Dzt5_;KHe-9aeW%~e{60qL^YOftqsW_3@`vRVbW#9wb6!kB=9 z9OWv5{(fX6uHMvM)s_*hoPHEP3YqR6xyVyEW*$@=X{i znN96Ff{}5^ZxkCA2S~e7!P032qIF!}lVN7N+FE0W)mp+lmGWdytL^3A`cmdvE2m8F zUm+6Q(3=#nqr;d9q4a{obQRyi);;%HIsUt`j!w9)6gpuVOfvHo30egClqnYTDrspw z#8_omUH|K6+M1ZrK%+{wp`1*-D%Quu=q0u_CP5v5eAB#+l%U#D{ zW=7%vFu03pzj%D25VvA5P1BW9&|yBMY$3AXN?XlM4vYAe5X0{Lvjr}x?WEabi$hw= z+7i)oS{FK}(FiI9Y59LZp{NfpO(La9{hR6ARI-EL zhD!8tu3z$T)e50-xqm05p_s7SFi||W;r^ehDZ!>yOe>2_Ibu5>q~{sURak^4u#)IE zWqc!|eQ8HY*W~<@gzpoDyPmow)hEBZ+PN&I`p!!5n_e@?fD}lqI|hU1V|D{wOiyOV zEUnzWT~{^slNXjdg(0bXo+7Ua<(aeP?D}p2RYaIRv2O28h+Ed6oqd)MX^R(Ot6rrd zI=Cej#u2xZ<;xZ!$_}j2rGd^;P#%ozX^J@vQHo?cB6E8Sa&?H;mMKXUiK>Kkk=O$$by~ zEp9)8U$$;e$0_upKk&{D2aLfi%=9aCxzUn!a+85=S!=}$#6Q<_S6( z%8|&3C|;TX>~4zL8dF(FNcv+mVO4NRP=?(ttgR9`%q43Ks!77Zc0mZg_&)|f@jjOs zs7;HACz^W7mUFsqjg^B0WKOsC3swLKAk3SS9^yyL9K>jJzoTY9ejV)n&HRe6HS$|R z>^<)Ts1e8zxbrzCAIKF-0nKVDi%Fgqt^FZPBsrXiJc@#NSgksCiVwuhN9V-!UnAkf zon83&fV3DW)AAU#5-Y#dqdIFVwDtLQqv=}&$KV2a3rR$Y4wL*FKCZX)ye!;i*k|vC z{W1iVAoi2j?f9U$ljY%%Y@d+~eUd|ee?OBn-teK-cv(O;F^tQ3p2YVDrU;haqIrn3 zG*@_jOX9g1C4u=L9+58xgdAdpLjN`haXQlSFI^%K>o99R_8a>_sy@NGJh)d1K*>IQA`3WNnz1DF_rAYM z%R8e=!%8%qz#G=UqA27}K&sU@%5y8wf1T>jEhSrnra**0tlmmlKtp6bpf>x$Zz2^X zsV+VPbP_!c*Wbe@`QC*tvgfhq5c_{~R@c{&Tk43l%kW_Rz7H4M=n-=*6 z6yE!B#q8SO#rQRTcecWMwL^K`=?3Q57Vv#%Q|vNER5}qI0m;xt)VLwn8oeEAt{;(2 zsonXEymVQE<{Z*JVty$7AAZB=JGb9Ywa8;>zOFSHhBg&8Rnv$%DTZA7O0{y~*vyuh zhuL2eQ61g&Ea@VZVnee#kryfGaY}gYT}qsYQNeRq?K}w}(R=XA3Mg3)=n;(zsG&G& z<;iP}m^J|V*?c@d?au8=lz@EBZ*FFn0|FxQ-pe1i@8@g(Z;FGfpTh6c=D>tsw=!c7 z>J_HXF%o`Ke+cz5_bc^P?n^K1T3lAh;(TPRl(o7+RipxhA`}x zZJr4R%;vY>eX0$BPs-VI>vD8*4s>Y4&D+6kfH3JyMxt`0`wYf-&hS%f4&>K$AQwI( z7R#80{*H@Ul5Go*XtHr#N;jwsCHiyZ1997%^)tT098-YVGz|*PGMr&H0ErJ`4&<|1fUOlplOugByyXNLo>3m=d3q>y* zMpi7;#@^85!q6)dfHainscL5RUP?S#nZC^fm$T3;ghtEZR7aVwWoo%0*2xB%7H(gr z0fQqeChv^1Is|`K-P?n5zB#iitBm9SRs1+NOwVE+PhtNj_r-8-*^Zq7@QH=F5hr1# zn1i88Un+F4t<>^i#i8C~hT|4~`CuM=RmHtf+-aa>zkh>siUdpFo3<)b;xAc8>!!L& zs2<-yW?{bJW@|eYUOO7BCEY6;N>7t%3pgGE?OY5_x%xQ}JCgeV6k4a;dqUd9-baDr zqIeHyJ+5w`w(mGIP8kn(u7ARR{!0Nd!%(@wh9YgnRW2g7^+Sf%`xscFP- z+ot$B7n0y}Rl}m-a7$r?`QdC`KTRAq^0fvJ2`)|DIbaUH0e`e==H}O5Q3Qiiw!do+X+(^l?P9SV@tK3? zBRQD~lm@iA=fEPMny*8%M0chmU}zr+aBUJ$d*+-yVPQ!$dd|{Ui&r}nNE$p;i#y;T zBgPWuhl39aj%^}0EU%ZulG)XsX40@Zwd z7T$u-22~Wt98=G3BN#(yGCoQkT6EPzb;5N~u=#noDn^_qq)B)=f%V=L?YhtTOZ1y^ zhhch(?e~Iupc;$;E8iLM@V^ERao>PLbX!u!hkVyqUoC5SKIy0xu}P^&dq`bMf<>C^ zCtDdFq_)&h{c6tMdav5={jM_z>T8`CGeVAo?ne>ylidLMj(pel3uxBY-Yd90%66^v z2nvLSyxchBs~2z^CHQbA`30=fu{Jgc<`aUG;{|~!{Oy2X!F{j{_iXSjn77JsyVm&Q zX|?-uFK{CaZ%PG?4n`GkO=ifA$-;V=PdDc1{&&*sZ1?)_)RE9xqBq$2{AI9yzuLi z87v{3@tp8)1acyd0_gR|(@f#gypvv867eXm&BoKJ6q9YjodS5PzDDPhYlWGbY;D{)->K`0lX+ zj?>)po$HWHkldB*sAU^r`d!A5GLG}a4$t+fZUq!G-wVfWAx2>(>8oS?npc=eY0bUM z0IJ%KG2HjYJAobmi(z5gk*^Be=^<>5$%CfIo#NVs@PL0;kg?PV#oU?K&sdOBvnU$I zmM}F2;5teqU}+$rXg$YrM@;mf_h*Fcpds`?4=&2g*tM9*Nxvna!2h7y_$IC5R3IFD z5BuaD*SMc$yLSJIw>7~wDHYSAS9tA-gN1RT21KK4Cyv(e5nhJC&SIVJbZ8=t@8veZ zn$d>omtsWck2rxHymTqUo@4zLHX18@uq8C;!GU-2Nq7vxrOIm*Do_Sc?|k<&nouS` z9$fl41cJ~j}!p^ z_15IV!`I&8FI76M|Ld)d41=q3e&N16Y8rMlM?GTVXRaaJsoIB3wY!4TX z3O%g?L*u7-afgNrwITCJ0XR3By+n9+5s@j^@tWN=w1Z}2`HA*XlscRaW?4`&smLf^ z$#vutvH?KwMVGC|2LC)AsPm^kB65zu$#EV1*ow3zr;o6Tx1i9eLrqTZ+L{~KJ|dJl zjv-hX;LcrleDVH~`YsQG`;F9>b=4i_ouEw#kBun4RD<}t|A+8Z-5zB8ku=;}7=?#x z;r?yW)seK^XQ7jOl737*B5cX8`8p{KhLn?kKtKmoA0~Wx@B23*W|0kEy(V_|h&mQ~ zYM=UGd%m-t4*0kG8c*WnShWjEtAoFDn9t-6mnO_xzq@h*{>G%@~Y)SRQ__@(zDLCxXts#Azc)sC1lF!k7%hB zr|K`!L+!rn5V0uTY$DokT}$*9K32@cqCttVHi`i*I?g*yVt%{0 zJYUcz9#41r^5GkGvviYPV+_Hbg*aD>pg5}i3bOWPHF1JIk+t{pb!NT3UYYRWjVYgF zO4+>LGJNh}^61gq_>gHiaeKaYe(9);4`}g+A{N6Wo;a13)1p4&EQPF73|y592NPz$ z+3zmW^6Cm(L+o(D1pZuMebMzP+e+y8Mi zW&0nC5ped4Ki|&bX)xa9V4xM=ZajN_xb|hh%X5vpk~OChpV-doy{|h1rw>0kW#e~N zFc*~Imzch79wPKpSx@M!p$bP)vL?{8qsVbja|sV6@6(qYVswpP5wGC(4O9Y@sun3G zJ53vjk5-Bmp}e2Gfk-@om5}-=o#0o6BL_!QbF}zEAi^mSvbFeFQGG#%V(tdY_XtzLc*Mh2>)6Q% z@-C#t-`t|4;}^d2q!e6(cDLDbv_uCG;y!;cUD=ZZ`~uEnPxQtHDScO|CPU59VoRfQ z_QFRM@90jOH?GUjsIy;!4K24V)>*a-KAP!CY?2%*nFJPZ!n}ntDJO^ zjw9PUl9fFd$wDxXGdEr@o)eQ+)yn*=AY0GRAN*BcN-WYVNOZ`P{pR!C=%z*F7aLx7 zi_SwzdE3QUQqj>88h-L?pTb61dcr$`8=Lbv@ro2a#l5AQ;a>x<*en%cqvOKqJ^nY_ zhSIeQYf+EEb+dF|L;~8DiR?j%8@{oWewV*HL1gU`X9-35h$5OZRzBIVk%nsew~E&2 z9Z>||j@!^Fa1ioF(;|iuB4^&q$vlt!`SS%2e+)GkUT!G)*=sa-SmOFHnYjdq_ewp9 z;cU8ttf!y_z7@6%ha@Mcnm8W>@&mUI;Ht;a-mo6w*L~q4d4^VT?}G_5_$K5fJknGU z+oeG>uUG-&GgZL6&1ruM@UeWn< zg3)5N4YGMR_;y_HIW3Qv`6W*vI1Ac!JJiwST@MOjI9fS)hPKZV+T!z@|CLEQ4Ul#5 zgkT>2$Bq8-`2AkP|Lz2OoJ`HL?-aQBYNF2;zuL|FiVB0QvaL|~t$QOuTL;(DD7@Bp z9;}&HcLE0&zROPK@^cwYd z=`Zz4rxkF=GA><93EMFkU10s6mB?pZ782UvbQM<+3wo(0S20@3qe~>DU;aVbInkUX z>ZAbWwWeOlb2g=O3Xk{oDceeqiuv+jrrJ;P+rz4qx0moo3QVJ|KF%D`LZssEY@R|@ z$-FH~#xf!NNo$)%dH+{?FQ3eVwAhwoo&SoI*p^axgV(@_LzP|mq@ueZIcqhoNz-1y zl!I}D-pjvR)1;_+d6~81Lc8y3QGBF~IDb5~Q-WM#%jvRas9q)~!2}8T4~{38_t&X% zZ>LV96z7v}`9JOh+&ebAHdQ{KFKBw&I0<|&D(aC8=8C5skKzueEzTy0@-0{r-{UAo zf$@;6a}5%E!n#p+LuBW2mXq5puBwX$R0pg6?kx7wtV-^7+u_241C#TMDmrBYze?RS zL=^u?$xuRCQ-lUb0h6EYMS^2WhA)yEPKU5pOMq#cIKZ?TB+!Ju+xogoyr}-&+$vdI zV#D~7hJBuBt1TB7ic4+w;27Ou<*1$u0YJWW$4h3eMrA&6G9d zHFN}VUkChAHhRL&%jW)Ar$-2;JvI1!*nD^L>A0eb=V9fEJPSa#ueAGd@`1Nk;+vAM z3QA`mEG2r2&6d8-j0`GepqNX#}h8 z(RN*3W%cXyx9O|9=YEW$&1Iz9Jn83F%J^FFn0eq)_+W4`kX1*~=l&Btmy(Q1kI-$IOQ)tfv!_b)CPb_a)lYCG zem&-R6`^5HfF~yj&|m}YUe@p`zWZ)IH&Fi(0Mw}y1+rQ+){tTBYqxuGk) z=r)dUVI|G+o&9#8eq(}49?Uy~ZHoWqbwP)4OXx;Ps#Wniu<}dSU%z2YD_hwEMjUmU z`(=MjfVCY?aVKU|C4`GG`!>dTxbvn!p$6JBos39b<5oB@Nb}-;AXxUcNbs@U=SCX` zd?iuzl9#);twVdz{_aFjLNvri#uK&e>8^zuW=m)oWy>OrNmGQweJ6E+>^m5v=2rDA zKD%x08wsMjM)MwdU$}stG@iM-zTQeP&>O7U+TONZJeR6mxw&pt>f5n$@r z*9op%o(g{3#)Ew^<(v#q81ciT5)>czU_-g56*WTr-Pxe+*-O$FoMX`f-c{?%l9A}! z0%7Lo*R>HnepFXpGei-opO9^VY_8n|d)7DBTCNm4@P%EbK180N_b6QNyVUmXD`&cL zbRFMB7;n3BhFw8B3ckiOuOQCcdxCP%itQSoVE5c@yxxc^T1AKN)O1n0e@1pg=x}i( zxZ5dtYY}n96?2E9FSMuI;w01qFX#4(6(e;;98SHo5e?Oy^i&qu*Oz0j&F!=|y))Ry zNd%*u4^A3Enwsl-9Dw25LHd>BSx_;2jEo;w(1T*4?KtV*;y$s{7xL6GCEC9*fE+$? zKy<>Z_O43G&8-q9@J)&VCUBMAsZNM;s}11GM1Q3yhMlQ2L4W>I$ksUn6+Jeu>0xR>R}n=j%0N((DflpA(m$f44=r}zvj&dB2@lh zu{z6?7VQb+Vbo8dVq6)o+5IuKMP>4!fl8JmT*6UyJeyGCK^jux-?dD=Q~#6Df+AAI(@56N(N{&eYYVyv8*P2^_5p;0O8A%A5ncd2W_APZz z(3&|H(prus^@s9)nc`G)6N*LZm>bv*CN)@N2MP8o=?i~6X=U*}SNgrX#hSN{R*Y+R zKP4r!=C;q(yIwq9U5JJ?*ly-*S}eStT3=nP<^FB#BNeSdA68Xx4LQ#&fQ-G?G!`RrZQ;bzEfuwBfAURP<=z=R^-q3M zvAoV#TW?a(Q*Jto{vfMf>uONqUQ;o3=gWK_vvHVFwW=B(KLppvHJHu__0amsZYm~b zPf(Wq=4DbGF;lAfI)ub6n?>?~IJG^jAo`F20Tn)QXVMXGyXNAwH>wO|3XZeJ$83q@ z!pmC|?07$##%yGPB5@b=Xtv)L4-XGU?U?w|dwIn-TM&f0&F+SdSE(%V#i3+Rw1;2p z_;4mg*6`4xskHD^H3-E(Whrkjl>4oF?m4zQ?ZOw{ucu}4H@otKe$h3NpP}VM!1*|6 z^qx?or?}#pR1{WY`*KT%@(o9$i^i}iLx|Ee8=-eo>eiJLVPbg?9 z=VkX>i#436VocDR?w>{%B~xm02dIX|e69T7Y`x`4jMVu@N(JIe?0vs1VRA0R9RS`3 z^kv_zz3~tz9j8Nyxp`~|sr`q1C`c7ktxv6mryjJL?=b2ERON-s;eDBe*cX#7-c(u_`G|FB^cbDRYAP5cjaK3z%V`XIfI@jDYKLyD7*H*Yf@g zrFda0$vT6H{C%pqdIS6-D_Un{WUD0gH}b=2mGlMD*#0%2L`^loT4vFGS-O&v&suHO;$z>fSOp4Fd6=WkRS#PZw1Qoruk0$UO@RS%jcjSVaOlzT0Dnk1#g|%g&ROeJ^tzCHpoM1AV_TT$5{fT zN=PUUPHR^EmxqCs4TU8|*C&;n@-3Y{bus4jvl>h7vkB>WF@1l0gb#&rU_tDy4D;E* zv4h%q?6n&+X3V*U-<9lXn~>4*7=4L6Ci-LT^?q>d@%hKEj}7(c z2>z)2kPLWV+KMZK>ycUfEM(g}0-79|><(D?8%mH^?YYPT0>a$zW-U}4?58k(Xqj?N z84+jIhQ_Rn)Y0IF`HRYAw%dw5obU+E{K()Grp_Ey#0;n28NL?z(S9+T3z>9GydYL1 zT;k;*5=r3)PnkxdoSju2pGh6GnmL|x)7#RFV>t*GJYD>i_Z7v6bGay0No$q{_dQFIUCkDtPFvloR7Gl< zE`J$k4ponYhjNL1N{-SY%i&h-JWdwlbLX(L+<`|#Fx+jk4dxhA#vxXO(p+SPw@CiE z0BYie?2$nl5RO1RJ3apCRR)$a^WMa*ZMML(>}Y-Okve)9?rk5-33Uw-v7ZB|Pwx1N zL1&$ZkwMSO7uU^mro*!{{ch>a6L&5Lqw&ndn-0E%DDA0gH<%ea-U^YaWV91AS}1IV z!i1FQ$3we81k?5P*BjB7@oupwP14$-<)`QPL8_Z1Bknis{k~ogB9C5@Ht51KXISp( zz+r;?Zmlh`_E5H+UCard?m){UB#8LS0U{0$>ERzw(84#P|9D8yj-Wk}d+(**k73=m zFiwE3(1Jv>Yk?{sU5v#R1hcPB0vJTIuY=;2kn6~+ufHx_vFpC1w$5m8_P8=$AKDED z=8SytiO;1f{W5d@&xaZRJZFY5a0#O77_HU~6wcfi__rV#OR|`|yEGf!Bjd?f(;~c$ zi(mU~rDN=noknlKJtK)99z@&rU3+h$q?>*`g@AyqvAwRc6*46ksjc2C<7v!Gy319J zc($fx<7-cc##CjFPG<#rI|3<`)GAn=d-b)H2aS3K=J0)msIupWFc?rCQ{I@>l^bp9nqO>+(od^v%jnH3_ZYU0qqHmjcc1N-4qN}V%XC#_ z_F7ZmX37}aok&iC{hwHsvT)w`U$lT~{fd+`JBn%tj{9-j^JVW7Al~+ykUd9HeM_CwI)uTnKXW#WiVV$+&}a>bo1(z z|8#p=zpJWm6drG6W;!(XQt{FvwJl81;H>1%Q132hiKI;$FWOF7bwDm1!9Ja%R<%Yh zrS%%uO)UKf1F&-`7S^kpKlo~$uK=xa@J!315TZd!SozOXHyLgsq&^i|Rh4aSY`~mX zXNIGMF01{Z&o!lrL0}9^QMNw!f=ZQTV>EL?eX$<`;@smVean*37%Z zm<}$%`NQb``e8TgoT9?tFaf1zU$*&bR<>@s2zuio(RfD`vP}P=;9{c@GOre&S=H|R z7zuvPtuX!bk64upZ-jA-k5qu&Cefa|>?BKEM!7zgc1&@y_{#d z<23&hXX)8DoHqN>oNs zhq%7W=7}avuxEZ>&fJSDzLuZX@S^cg^7S@ZGsG*jM6%2C zuIpS@^~S<)_92(UF}`0c%#yH+IH;}{VzQ~2hX0jW%_ZYKIvtlAXUW)2hKKSfU&S<0 zWzl-bLfa7jZq60f>T-8~I##Nh`cAYE z)huE393fM9G{mgTzbxQiN#Z6p`JAjI0S*M5b&DZ%M$Nl%$6ha;+5z_Fw3T?H9rfvn z?OGJ>eAQxg7tJw3EHWT$Sk#YyN$XhQ!Y#l>+YE4+*6XzpPPqdv&2}U1I zd7;{g(0Dex-q-DP((u$jEcG0pTBM0r-Ohs1@`A#RY_d_S(_ z((w8UeqRDzE-lx%zdM^%dBtFbqwUVS9$g~a-3uS76V&o>*~Vr30i5Sd{bd+^T~ije zMip+4gah-*x7s8+ef(sFZ1j>8p_T1P2QZ}(*d8wGKi}z4fLDaedv>^(z$;hN2A`{| zU64u=_*_&}8B{|H)@pDi@8)~$$;^R%ya`I&t>J-NHFfc1%Lx`ud$fJD54Bb@VBTKg z%iXa(g~TD~))Z6Wwx|AdaXuHIyVnJsRWa~cuy~r@3f_78Ekt&tk`Dg&s^s^j2HV{I z5($YNq%~YA+E)DMFxf3<^u+VpPS|kWJ=Lw@;w}Fdo2;C7Qr3AwtQ&>E3L(kW3>(YJxDxJI z;gM5Bjk3DGloD(4LB;MJRAeiz)>mcDzdmFar!JLaeUJ767MaBQv3e!W6Zy1oTKg$p z#ReI)Xe+J{qudOwq$zwy9O9d_3{pwS1wn0l640($lX@tVek$zGuV&l2gel^0)g3}C z_ik8@)ezgxIaK#``5m;|n$VZs2v#cx8#Z_LJg?1}zIw{t)gz*%|0}|CJwKPF*^Y|X zWr~fD)Gl@+N2>Oq8g9bjRVw?VVgKaXW;v)t^eM09_NOGZXjlUzGh)}(vcIvXuRVgB zS0IPahb3`r)G`MCMV1#U*7)wfdS%L6&n%0%NV(U;mf92~d6t0tiaEzmQ8)A}HtWk* zC71(a&Jqf2RP?yQmWv_Ui?uV^l0&9EnBEgH@ zvM4A_7&lH@<9a|$GKLYi0dHW_$HqoEF_fUKn1?uH!b$R(1CB`?}H=}WK~HFyfJ5MKK33xvR(bl z3?(TcSxf5uTG~=}enHHiqY3wc@hfMO6?|`gSeg9zZ72K`Z@)Y{5m=|VH%Rs}cuw(Z z`%Az+$vrCZHW*u5vK#fk4Zzeq}KQ8P0)A{4cJlN@kB4#UyRfOuBLdp&`oahQ0D*ST~ z0kbBtzqS1Ql1)=9!TkQb=*>@jZACQYku~RbF9G41H$0WcLzv!&$L-DQtCVH%)|Gz) z=z6~)Xe)9RidvQc#Y$fnF0Ls^G}brFH;nylWGt>%E`x>tq5t&kY;xPDoX7c<|zKkvK&5yeUpzc8yxYu|LY z{t@`J5o%PUii(Dc`qU2RxaH=t)wy=sdIez`<016&*zM!QPhQh;Sydeyzqvf}ImD12 zvHw-(NNa)+wlN->!89rGp_e{Rdod}j{%g#6h6)r#8+EwK*sP_z@UCpbZ*(fMwtv`; zcw0@pvlxiJu=58oF*RQ&1tt>l-;IN{`&R^{vPcPDC~m(6d><+x<52u6{HdBX5vbS0 z6d6T87egx%+YYRYZ!h3uT*S@NWRQzVhb_(rL2c0Q{y->3Zhs1aXrtOQxLfOom*@s0 zx&**{XqbLGfMuXNp^^HYU%;XA55ssFwtrW8OiS~EZJ+C8{5c|=eLtB)T8dt9I3fT2 z7|=3Rih`TsFyK0)Pgu*D?@9|zZYEdPM^$j5mY2_cXJq5TMZkzO*5vtv`_(wG7YQFm z`tj4mUP|&NGa=p6YC#d<7^z0IS8Wg>c{AFXpW4n=?@6k&W*v4H%@%cTm8#C_mM#;0 zxqEJqp}W3321vt2x-2@qV*R8)yq7Z2^tr8cB^#5sR-{*^J1L(bvDDt7*e?N2)d<+r7>4F>fKIB z>mRqW+AXEC5Jjp(0_{sI|v$7@ULm+4*PK3UMi z5;}iK?vHPa%fv{?y1)QjV&Lk#W2v?Ye2Cf`I{U;_leyP$5iisDA*=~^@#8aGQ8$*Q zpQixy?M&PR3EC--bxA?WL_h1&o4y2JoLVh;67tDi=K;0+&_EWuW)Y<{k<1 zmU_=4WEt?VX%`GRr%b}Ez4Yk8ayr#k>|M0?-x{*)W{jUSRGDiZ2Q=c}*lo$BH;!>#WrGoVN^! zx)#t$7ys@ch3f0-CaO;vcD?lPYsDf)(dtRStEC^Bc8r294QG;1@2@7sPjq=JCj z_uu^+pBa_8K$kZRYz^4##o2!S>!uCF#inlmrTM+5W|-eOHMPI7IW7`6^um9(#Vxk0 zi2o#h==R4sz`G|BK{u1QG(PV|dngCs z(S=n1760SrbYU#x&m*=LC(#K>imuKR0Be6l8hZ~rvR$bkW20BAq_k_)n6XPCR9C5bN`YR3bM&T8HPCoDfiM#cgCmKu_C_=cFF3ue-$V8g*5-@kM8WomUHxiJ1eti5BG zXkpi-SGH~2_9@%8eag1&Q?_l}wr$%sQcwErq%)FqMl$kc|ABpvz1O|wHRlXrml5qO zWthwpF-jd8#k2rM)qRcol?NyT4S=(dE*aM_Q-`5u9hHI#_`2%K^#GY|Vnb|*1Tw|) zQ(FO4zT~TT%PwOTT_D7GYrUe#y?%iQ>6$@o4A}V~YD~y+6DjWiBP8GbLhl=D%M#OP zl|;V2vC`1XXVjPtd#q+md!zVTmu*toP?Z=9Yz%{$!5laB%APK^=Uq$Nf_`*NPro$OcE!TXGFl>=Y((u=%MtA z&@DxeH%9X#NOVAgkzgiBNOXUW>|8ngnOLcP5Yce0lFdz&3N2az^iZ!${Slz^qobX- zI%WD!TwYfHvD$Ql0u=ip3Ntap_TB&&V^*LDzqbJ*ETGhVWyW~B(*dW1T3zLg6yx}h z>-_`6IM^fTSB;=ol~PPTXq8S!WF?yCu=q^LVH9Qmr>j;vwWEe1Z@W>%SyV7{-r9)%+#}q{)pMx(|cASF>-ukCB6syg<3B^Dls@wnjhoz zW02TV;4tO@xdndB@8ncIqFfZv?^W<%oC&>023rCnT!_|b0qjeY0b8J;a$W3fSGo-M zGG^oh;>210?VUkmZB9Ix&)=VHxx}qbL|(LnFlGxsxd%6)(B`loshj$`&}ept6VGto zicro~{{$ODf6-jf1mLrQ$AnRimiC|}s1M)pUEz}4S|L0)bUesv_L=6ozQS=3KAD$s z>k=o$TOe!wOB$o)E0-F$^b$?O(|0q&UY4L$v8f8?0>>vH81XDH- z!9R3Sg^a0%_fQOZPz(;h=^hp`c4xQ=ma^(73Dl*ZK^|3djZeTU`19m6{+fI;^5V!U z`#~Q0#C)@s4kG*3CU=FKg|K8eJ?IeBZVtZ%ia!j)gm_R05wKcHtBv|61~*Wlbm*lF zbw~5S2|R3OjdjV46DnZS9CVFmy_;4JiAzCwbQjUQ`!?kMgTFb-$uM@ui7Bt;>3Fy^Dzc7XNKQK3<~>jKnT?72VJAh zYgD-V4&MBCc<1V9tG3LO&&I!P9USq;o&uQS7u*?SnubD4>n^@q?pt*T=~8ZEhN%0{ zXz=FSiNumN1YqORY_1q5iTcoCG)oy+;YE0AkThXyJ zBy&Joeaz3kFK{l;&>ubAAEDq=-#6Y~Y_TwHO_eozTH^qu$dzcH*=y@8(I|R z92P^<#Zc#^n+=OKB6av=z{b8K>(7d)v9r`)*4DW_k>U`%Q9I|(=U_@ny#n}wm5Ij3nPem|9+NPmobesEdgZ+OGkZ)soTv3Zg{?2{Jl;wAg8BPt zj=@KvF#O|{pjH08>kZPGbW3qW1dQoXKf^b~bj@xom%gC6cW0c6)z+k=?#?W+RIMje zo+MTGGrDZ+Ej~;cW6iovZ{Q`kL1+cp1#FSZ6geOZ^*B)lG-c*2YMGvuV=5NnET#?ALBa9~F1F|DiuymzuGGA# z&R$iub&cawd+OQFZRhSNrSmJ8Xg$iarJdx7TN|$i_pL`I{WevQIM+H%G7Edz=}`pK zdt=&=9Bm=B1Nu`YSb@h!^8QuNpPKii@vPgNTvknDqUfR+#|YS|rmq;5i|)M3-t2sj z;oM>T{j+x6@L*eyH>i>7_7B#_HK92Yi_?pRBNdP=$Z-Xy&DsyC(7Xh4f0 z(nx_SSc>%_9%$$=kR#wItFjQ)JFK5)8Px{=aiZTuBBAZC+^pC5?Mdg4!g`AP_fu-H z?OF-CPedMf6X*?kF3qd@P}?n6aqB?0?}QST#}mZ==L>?XP*yac+ny=AQ#@_8kCjob z$x^MSU)<2@eYr<{6K~pr;^uO3MNwgyqVgbfJ2_c>FzjD_YwKcF zVP3- zNjZW?0eC$KQcrD474+i4ig8pHh-^VvHgA*0IK{a{r_9UDnXE$j$!t2k;@zzJz0jyN8SrFq{8MUwjE&i1hlSZ&$fy7C?v@#qWPGyCn%8nIgqFLzq4_8R8=BHZAOXY}u z2mt%H^eNvhN2%Yur2E#v_l)6(Z?Hbfa>U1KmOF#f@Y~JTz}Now&bMyK!sXA_2>5Zq zvyAi_w|%jjXWZ>SS{VqtkCYrAYl%s+PfLuV#FY zpLVag4e?-yi7;*x7S%Fm!NKzGGPKgnQhWL-ncn&RpH_ZZ?ZKS>v2k5_qcQU#GE28- z0#vaL^8mDxi{(P&+$s$?$68*jID5?jlaLDTZbUt44Gc5IZk+f*(GaLi>A7cHE@+t; zpp^Atx(wjr6v59bEpkYeU>89iW{Sj_y$>aO8$6d}J?++JM~R7qu-m$DzUus*C5BLdw8 zl5@<>=Tqk6M;VrY_yZ{Gf%Kc^k-%jdoiHaM4fc(W(PhPcPcF#geH=f|YkU%?j1|_< z@Nb_Wo`ahhp7Q&5l$Q`O7AD>LJ?ey0Oq|j9)VDtt&pTe6{-#GiXHMLpAGWr(zBP3} z*9Q*_&-gJ!!*$M6S{mXV8vgR}!+*UN4oLCK6JU=O)eDfSR-g3v`~<}5nUe$>Lai{V z3Y#tkE1Bi^luL0ANWDXd(KW~yBe|6IA^o#k3;s>GJn<`+y;4@NfJOKwn;u$L*^o9U zLl}S#ENaj_NnSgtRdto^l$$s9venw%n=wUwD}(JOzbguLXk0mcaH!pbp`c3d(LmQl#NXet4~iG3}y59G?c`b$x@9O}=N2gR9W|7VxS7a5-k2vmPMvEt6(P~X84)^Q@PuyC4BY9uy^5T|_u>6; z`rb39et`i}(fo_k>0RgU@c!`gKJrFig5Tosq&Det_BLD6wGGeEw+j#Vh&{$@_Rqf2o+M3RmZ=`W8Uzn1Nw2UdoxZ(5i^8H?z`*&kI0lGK46ItWy{qwJED5x_C>z|}E zR#AhQ8Q+hC$LII&TlUBIaIgTLvR(|JNCyl6SskTQ3b#QxP%Ivr3PJ3{vj{HjFlj6} z(Pm)IPpaK8r?9ivg+Tj9RmSx$$?#1>c?G@Zgq_a5lchc2M2xvj z*Qaj3DlIeTBw8{oNzXN@w&0X!rrrq>Y*rnE+>}=sl?4=2(d=WY=LA({H!l+)J*U-5 zu}VB+@QxUX zMt(D^$!{eUJNoe!mKE<#ui4m_9wEv}#CI0vPe(^fi(1srl}pA)&h=5l=66lmfG`?U z?QuyKSJ%JWoq57%4bQMQ$9p-x*Oq;hA0g1LYHylrp2H^LJ1!z!>eX8aAYJNGP1nZV zs68*~z6AK0=r$qpDg2c1tVIFOdA2lMp|<4O{z;YE-)H){DkL=x;Jz&gc1^&~aTlT; z+BY{3&Gt`A%ffEY#~TmCx6+<(QCRm=_$90G`Bdy$DVe-nq5C|M_g-n+yXgu7i<+K0 zuPK}cv$nR4<3+^|Wcq-hMFQ~NTxfZML~66)7PZ$Zs+623*DhTXt`eW*CjY!F!4NWo zf7(Nb9`M1l0tdi;hYYi(2mCh{rz0bRA{OT)JBAF;7%3w`!u}SWL}seIL!5n6QLbTb z1a4MLV#^~r7gzH4cV`!G(fAQ4oOii(aPhSX8%aRK6Yp`PI0*wYH1njrim~(tCc7oL zQAu=T*n?#sE8(^n-BWH!Pun=_0>N%3UUyyw%|}=w_y-yEd7(ZN{~JEw~Bm#Cq5vGVA*(}Bdt5KHi0$+4RME9ceB+(K36=) z4Z+YmrNUTiZ5u2xvb^F~{ZaAX0Ap6c^>b4}*`#(ithK^f0W{fQpL1|KYRjN4DISu^y2Gh1Y3=$E1nf8h zelKt6o)~<%76oab%vUv#fWqh<`%?{U3KAjvqH9wSiNr8+!-AjMqcG2+zYw+#@$$mE zhP?ag(;W>DzfkGmI{%2BHPrjY9p?RKs>bxJ-1=<|!H&cGPsGM;&AS1M)r0S%nhR=k zL&W>)5fVI5eU%zqM@*%fEWVy*O{VeWPjMYzqwd%5(%yHVPEIL=x6^ZmhCJeoFFZp% z<_2*SZOG_dZlcYQ?n(Tk5B&uv$9?(d?4!VyuiJHaJKkDa?}r)>`93|EIb+`2X4nbmt=PbS87DsXc`3O;5TkQM&HX(FP# zU6;IXid3=o@1P^&AT(LeZJ~pE`WuUQX9O|pIVMk019Fmu@-c5m|CfpCu1`Sh)G=5? zZ4K7oPYbb5XAvFX7S&}=?t#)Fa}cm-(hhS+rnV=+)F1&43FQfKrUc0=HJ68Qwg4>+&e02GP6)d-4jJTQPvJN?a7VUqW~X{b2XqqqE@+*WnUXb#5j@E7 z60)^rCl7rK!7YAR_H|CmLn&bcOcP-zSGcU01^aeVD!K(;#E-f3WXvk6@5l*R#bUS{ zCW2r$Z51STK+s6K#kAr`PIWDBM7~EEE-LwHX?`_cQFW0Lg0y;J7e*pRH-w8q8g+UHGQrT&y#xw50xRv&TNQe8BNZN^aV`=Sfgrd#xaP0+M zvin*z58!UAqs2EX8Wwj@FQ&h8NfAmrtAa>U9LY^}{yt+igC%4?HSN>W-@Y2l8UY7E ze9r^(2iA{oK_231ET_VF^rK-kgD876jHeu2Z@cQth(f6u3V&m*3e;)};}T^r)+^NPASr9T5s=yDuW5S-Za~Q& zBQ`}TiR*4-<9?+MWJXIOBzSWkG^SEeS1Jjph55comEj1`z+c}N681te)5nI5&;^CEvYMZfbnJp5_6Zoqq}VPh;#EH^iuX7k648n zhG2^s7j@DYLr>PiLcCx6EYcYhkf^#R)_tgL00I2dSrKQXOu|rcr{rGq|Lhn>ReSpN z`uQILA5Xu8MB-=f*54{cg1RqYByZgnQ4Gbh7qKeD87ry&+EPZ%l55+!t&b^jBW2by z=1k*B)Z!n8h~BK58dZaf*>Yb*9yAp661Ishb)g{q?(ZbJdo==t;79O~E%w!$VH{P> zn_zeWD-HK0g)sYwhW!B!fSzH$Wv*1!*-wR8 z{cL8r`!`oZflp4sx_$yi6!>SOodY#L0}p?yKG`Yk@(@Qr+XZr)pLA-V8lE9ZI1o;g z-A+aO$@v{R;fxZrmWZmy;FvKe7&^lKm~akGn9Q@HW!xbJK~Ojn))amhOp-pAY~x2K ze8w5n8$Zq5FXxq2`jCfO@v5qgSm^No z@w$6xY6tTy+=4gn`7^Ao-MFK`9vmo=xz85kWp-(OXarqxnNoHf;Xkl+E7@wXPK~kb zK(^RwIBtdRG&n^p_o5X~P#)*XMv8I=7*RS>{~LT78nx@kB7B?N`(tQxR8koB8~2UQ zFEr&gion22Bxb0yAb=)L)(4cbzj=Pg(}yz38Eck1-0Ww(QxQc#Lv&4ZSPpJq=C9J& zAMyIBB0M6ly7`@a(;P#iSRo{qQrdSCfX#pxtqAeV*!~`$_ma+tB5Thj=w;jK!T*@%FE)e|gg)dH zuPJ`32bb&ugfo#wg+tg`)qKX;CV;MIOFAPtm}bd#B0ly;0YK}hwLOGb3d%KVHEU&v z2`!N=l=z2vMb76s9TJL@HEp&FFD?~>S|Lg#@fkQLI-)xfEhpi-*L8`;o}&?IDqvPF zaPVZD#1UG8j_J3W{G2k3I0w5L5MWZ<52PEqfL42*{d9lox-Y|!&zMel8yy!pV}PSP z2#nh)mo*5kfu*6|{Ev$#wcJAAUx0^d$2=GtTU?i;@(W#D@mx*b90H-Ct4L`y9b;uF z-rr=vjo|GxPAzx%83^=PbA(+&d`(7?L35L|{x`4a@LMYJxrxFAfyxQ!GIT&2U2{Ef zTYgLhPAUfpDEG3LJU@ma5>ai{3?DekmKra*2W2SA;z2qH$u;QNC#4PgZRKgmw`?G= zZZ)prqPQa#Zwv~d1p1Dv`H1-Gv3h*4VzEA0Vpg+bZ^nCJ*m7Kc#Y`;PV(H2Y!@$@I zwu8v~cZwXuJLQLY_`yVyNl%pG2Fth)!xIC&@o`*68*dcI(6CTDJZ)sTOAlHhogvZ& z^!e;~AGYRV`v=UNe&TZpLOf(OjxI6_DC(#ZC+<~1Bc8q4{#$x|7V!j>C@+KSN?uVF zZ2%C0Ny$ni>OKI9B@_i)od+zCL!`0PNG9jBNZ%Dk#+;GL(oVnEz10Q>_;?9nru&L9 zFk^@_sk7W&g_qz3k?y=B5)RTry8Q)J=}J)3{lyB0ll+%>=~&4?Xh+V6TsbtQzn#od zPabne>>pG+pTf+4Ou(QC`bW{H6Q1k^LdhXJ8JxFOF0Ci)1eI|X*4D(l+p+&cVN9nU zGr?vt@HOC)gr73E9_+_LgM@h`Iem1h0JV<)Oas53+jyVque@tx) zV#JX%e&!K39J{(j=t`lyQEt-5eI_S!b8ZZT0zqGd8fwi_3luC$QVULh;1q2#3wKok zO7&t>g4;|2-hsbp^&}S_H!g5yIWH`R7dK3{ZcI*iRk&r!;ZH!;5-+2?UI}9eOimOi zpJ}32Bc8NUXN2QIHZ2na^BHCIRBeS?8U*n)tieI^U*Cw;AYz?mn|wya=$lx91nRCg zv6Lkpfl9%(GyU=w)*l!`kQ#?(YJtxcl~&mTgqfH8Q9djz?A|3$aKL;bw9$*sK_MiV zDGXb{qtJk@wh<3zGY)MpZ(bAl*NK)| zSku}p{V~5_pVPZc{%Xc40yafF*&J76hBrt71!~l-dhc;$nQEkv)Mf#dD(mgK_+(hl z{oc+vDo)Ym>K_}h4iwMKT8L*A2!*P-6PhA@OC~#z)S-AUe4ZtACO5AYPH{WZ4bw3y zz-1I+vR_kf!JHCVn)O55F@fJdk9K&R>Dw=*Q9d);3bf?knwI*TZo#V(+2-aT5fgDC z0+IxA@^4Qd;iJ;@P;J{a{q4b_LxswRf_db)Qs+`>_JT$k0wqz?Rr8f1XYpeCLN+Gx zXjlG&%ApJh||F*F9P>pt0Aj@gpIw7~9? zx>T5Xys~^NZv00i5TK7hRU%;tTlQ!p-mG!NWO~eYBo(MS2@GNAQ5|E|%fBHAt^2Jc z$X!_s!-y-6oho5SC`;BSndh@|sS2~S4I4!KqAuISdE0N2#PpE~3LBTVWL9{mw7@dn z(g=_O8b}<$H$?ytZcBhMy?fLhqz;HS)>7|7A08vL5kPtz{N|T&t8oc~^}62o1$Z>Z zaH|}Zs16M<+CQUtOH-t2n-CnIe^SYqy1$cgdsa2qvsL91Q#($q}x=t&zwyIZugJ6#gpVyR^Zw7wCgR<^-{6%I#9A8jp3xrj#pldk zpTXXzT+71R-~C?}54cR!xS1Q85S^SOR|u8$c;Ojh&=P(vR=ThrB4|UqY!HwB=Ml|0 zD+{<};XIcwzq(=L$hW)FE2$nYpwnYmiIeorhz{Ohr@jOzv3vsaj8AxL*vlr|%5I5X z65Uzq@R|ZQn1Y0+Pa)C!ibKb&hE#8@u9N1mnA}BvuQD)&`yzH@H^+V5KY`{c(B#xX zzi$6lN-Ex_T9rCYxR6{q{?=R5!;Nb?W??Gmq4Q#AwhMZu?(CwcV`q!{*!SxClI`L@ z*B>rP=_rQM*L|*8@*jQhYkV<2A1hdMo?%FEO`OGBpqnskI|k=qBR#9lls^Utgk}E{ zxPNIncu2IQu3iDP9I|11B-HRySlr3jTON{P#DpQf?*CZuIqjE4QJb@|{?QwzLGCcc)#r0@$0&7XlP;McU1k`@}8g)F^jO%{w3u2 zO>Lvq3+u%snVF>2qGMyVC5GNP|C?z)fzx2P`lmp@i`|wBy!@+S!~>3`S?_>HEWW4hnWl&MsWCCB>qs zed`wPcZH}Wd7F{>j6pDI&L(rK*I`UZ$BS9n9KSLQyE!%XI$k|z|LWQLa!C6)bUOtz zaRtEy-nom*v+E&nnX(Ee33<=Arl#Cl;C;4M+awElrEXpkg}(YSf!PV9Lvl&x|44t? z;b9ssMOlZeFF{sJk{cg!v^chY3@4fuxha;4#1D_lS~HKNuK#Vh@2esr&NptGB<-sE zO)}SxqF(d{gR9(z8LG{;Q!~GvK_-=HZcrEC@C-5HF`ecNv}4|47s@}amS5#WOyEfo zQY3L?8Td;LE^TLvalpG^#!bi#rgSg|ths_@Q(>vea@b6#v)|~aWqx@bX!8JhZ$tZg zfXd%22Ro_cybNrmzX&qPH#sbxO9QA19}bYQx)5woTvjC(Be!D9`^=l#AcihBoryMsQVnAs<4fUzrj1+&$EjDsSRz!Mvb2g z+zwB%pNImv%p8r%t`2!N%%S`o(aI`)UW&=`u1>S{YRx^O}az^Vnhm}W)F(YN7p(tC~7r-;5qhlt*ecqZRI zv`lv8aFEdy5HNcPw%w?n)vHP-Qz z-yNUCNv0$AV;%LC&*BB_H1aIGq8TNU30to@7$n%7OXeg5uZ{{@d+%EYKgWXc4r`WB z1}fo3BJ^z$mHhTte0f=fpVx0Y9oMv5iy^MHnCL;?KQn@zIASamHwB2VD;8EpPp^Vt z?6HB9cMbl+6M9zDB@6t2f_q9iaOmQ?Y(*C{3`Ds!+OU;|nZBtTYI)R5NISPuSO2J&tnkrbF$?q@Olh^v2u zCCTUlq7oU>j8cu-Q-3g|6I&y{_9)8CV_t|hOe>tiCY}I?JWGu<^n0clHdG`@4Z2h2 zy&xw<{$yL z!XcyY@z&3j1AopxafG;3P6?J1)!V4Aovy$v!|#=?|F~~1y1dwwv$3JC&tk#&zhFI3 zQ}~aNVO6d*&Q}&u*>~!iyOcqx;Je;hn*6`7L-D#yPo)cskFx&u)GyYo4~&YiJ$Fw2 zh5E?Th7>@E=%v^|qn{@S>`La#04!vRmg8?@y{1|rPka_=%*{*c`vG(*%F@ez{kX?!u+ z8LdN=Rmne3RkjYACA6u$P&Vlk=Mtbueq_|AQgBo#+ZdjvjJGjH&8sjMiob<-MdNwinhT8j<)YE!UWK2HD7qT&3_Ul`lvio}}s%ifAb_{~Q zh=^bzV4-jRuiQu32hSaMm!l!&HYw2-C=p3De*yo(a#sqzG^lv7OXTHghKG>){PTs^SMKHc${DbB zp1yhzgyAHo_q*lI1zm)s9##yTS-vTWlJ=(tQrv6Wy{|60b6}$--ELD_91-YEWHq(_ zVr}GY&i{kj99`AE^i_R2*FG>*II-W%cT#DuXjJi1(Vx%+4x)ucW(s%1KQC<_uLV%U zRJd^*^v{vHbE4Z1FXd{wL->2VBRYv^MS*x981uNtG1UcH8fIIp*id&6GBL7wU+B@; zz?-oqy1JZOmQh$D>D_2mSJ|IyiPBSxnBxA%Fld5fN~6cjD*-ven##Vx$%RK%q z4xGa@>@KDYM{`Okj&b&h(p2STyAgqxij3ISs-0SE+6q;T4Yp0X%KXo zg8v)H=IG%~WnP2AvSv|U#BN^GGeHiRz2d*)HC0SsnorV`s99{-k)T(`NVFU@N0}D? zC%mR#P>|KQVz<;;rTqL?`MJN8|I!;lP>%dLf6jiwm~;i@{ZZO3(hHe~w;Q|O{{?IO zP?iHar54~qra6APd|UN4f4EQauGv3)U+(HEeYI6p-=($dx@WNNVUCNvJ2XPV!{_b+6(_Qo$lw! z*gx_@U?WlUwiira}+kow`_WAeW6X^MhX#lr7QF;WUs=4MD1daGE z^I~|%nTPc}PW63#U3Ub{dxK&$l$;s+wr30&IEzqX>ce7dtTsj9%rrEhyHLn8m)Wu% zXf#hr<1`BN%JG#oEx&;?A4#_1@5!AfvB(RMTibC&={KT3!x3g$|FKBf(oPeXu^^b4 z;Wre{O5-Yi?k$ugM2v<>wxhlr3Z|PlVDYK${w!Vfto;wvjIUWw+s}tCFTByu9)%%} z)=H;}tmq^)g5*3$w5>HZRZwk!0asJK z7pZG|#q#!|tqO~#KDIi?8#AV;FBPzzly_O)R;5e3cP6!aV1)BY*i;r;Ssp8T0}p+h zDaMns#Z&-uSQSw6=RvTvzPa_2&&>$UTSrcm4?@jofIIMU$xzkG{A4@>qU>sBp#Xsu zLlZNxE+r3(T|%;aq_Ub}ByW#adD}TdesSsS)4CD>ocQN_s{pR7OE$#T_OnGKsaxt? zMi#+5pD`o((2I$}gY_AL+?`?XM@fE1etm6KLR&> z==^^`W;(JtxQ{o;p*B@<-h;7a4+j5#$QuXq{Mm-b1IjUifH8Z{xA9%#O}+i~r4 zL8x(On^AZUkvnhc5z}~Rl6xr$HOn*CxS}9Z4v944O-yPDPyuXoP?3I*QpwnHDZY|O zSdFrHt)ui35s*j%g$BVUGkfWQATJ5&ZoiIWax8}i-jb*%q1*pCF@soLg7ja+%-YwR zsK&#j?Uzr84|AaE*7RbXE<3R!yF-$GkW>NWZEO{)z#%gBWC&95eFK9CbX^>!pZ38- zG$V2tcm60ZF^7wo4+l$qm!D`f+4(W0I2|dRbL3yM6pKCS^E`CDe>*uoTtD9fBQGa> zxL>c|&DscXUtO>7D=$B(n$n|ErFW&uB??@gF3%72B_lUG-K#jSdE_N*vzwx{VfmN@ z;(pPWOi8C;jC3ZNn^wPWu{_nVb}dov^lg!JN+$;wYEwP*bGdLaXKT!8(jd~(r<%>& za8cVi-|rHf(3?8?8BZY0^%66oIC`Ie_9PXwR~5a2_GD|-V3TF(JS4SwjdbG?mH=t^ zwIs_}=wp#OQr)_{gOY-Up3WcRX$|&rx3WjD_BP6wS={Kd&|l+sadHi*&W;5+1$`|p z#7d9Kf{s^_x)0jY)6m6Rv=k0~boK7roOE@HUZtBQ*~y7}0g~a-iVo=|1=kETJDEe7 zR=X2-N99w*V8hq(&O-#%t-LJ2l$PLizoA|QP;Qlhm_l5Av-i-oY*B%=g7y{?l{;1^9U6s| z)s%&S!eRD?N@k8-6RA0zEr`sY$>$v$7=?{=rZ4#M4Cn_NShgJ7-G=_}8iXk30$;P? zpX=+ZE7TWXc<%qk!Y6P1usWxX3W6<{-boe-)m!ZJuQ4YV5&>tGM8H?YcQf%K~ha^ zgIY#XRae@gG6*K+|BTi@P`TUTVF#M-_YDml`uKU~hW+lynF-wGL;Y5ZIMvD-)fFn- zhDnRI0D2NmC#Ri9k4dkvb%7Odd`dboI*w%>7-BW4Y4Ip=og%b5qCTt;8mAiHa(&f0 zvw8oUNQdy=%zvm!3gunyZ~3W?d7w3~qI=Rq|FTAT2(6{-u+-(%*twTw4PV{4-1vf*LY zeB6S8cSgaf+&hBNyayCu$Cc?(nRU?99ANTedrWs+VVF=7}lI+`pu+&*x%ZIrrQ zrJHY)2OO}i$sT^al$#idCRV7C}27cEdOY zpMY~nNnDi%ef z16Piq+x;`W_JO?vh|i7)b-}&2JYmeC)qWKvogbavDzFODwjq0@ z@%J1txxavY+<|gBim)TKSS)d9RPj!rjGt+lgT5a!BQj~IWEPtvd(%PpuRfv}YYgtD zdw6}!Y9&IXvxNzT+fBjCOAOq_u>D*_!PBBX7}u>fm`KkXG*t{Ta9H$JauZgs91Qn2cTQKC)_e(diQJ6*e%Q!ctq7$~L0U(Uhc4@? zb2sz;Md;tp@7feIPX7{K2rD6YBlR~W)iv^suj4ny=)bv>W$of))i!gsg-E0Pdb>Ke>E3HbogB?e_H$A>!G5M)J(8b7NtKlG-y6 zNfd~(!=k?jl8LXAiU~W%Bm@!tBj3V|IB%w>XOOUvErD?dnkY|gN6bbaj7&L&`2G&` z1b9uH{ZQKs^qhyz)lU5==&h-&PE(y5rAYBWj5h0XUUJpza(a9H?Q)1`{K**pp7nUL zwFLfx9pRP1e80W^(~wC~{6k`6$k{Dsr-kzGjF0vtqHU4*3S4i~(d$^}r{w0}_Sfq+ zrl(XRqxaVhj+3h^)J0E2$HM84=bKd`-Ai*Nu`_w;ylrau#^=&Z&XUP!k99v{ux`hn z#{`9S@CTVK!b&LX@QxZx2@J`C&#px~{HAVAjiC75S!VjOV-v{~`UQl&P1HTXF7D(f z0RqJkUUY7@Bz4ZSgw)t)yTypsUQ6n+6*|=>(ChZp-Jrz3f!pmIizNc0gZ~KmZ%Bpm zI*~G66)&X*x(dfi2f@K{^JY*@bxhcUo;G8g?vmRoRD5PG9UQLw zS&#$mVhc0>*^$NRyAFzxcGzZ~P1HZZuEv;r_0y^Z2^n~W&>_h*IAn)Ux%@kJEj!kJtn znVub+9@B{(e4=yO;>|H7#_=Q}$xb)cS~&ABgLDRF-84TbjHO2mHc3RB`!5%0a%rpd zhw!DCrKV|P`71pgw?rCO=Y${1zP*aXCI|{myS!OC-Lby)lgQ^R`&lVJCo|yhcg_Gc zE5LXhX>BNMaJMg({Ve9LsC6!$;nqd3)OZe^xqU(nMh5wyH;IFDVz&4OK}`1|hHiXJ zkFJ^KB7xEVO)$}coXkPSGi`K0^Az^AK1u1Mp>FmN^>(cAj|!Enw-K}i&7S)5FxWe3 z2Vze8XU{no_s5Tkh8biG^7ksd)W4Rw6XbJ3llfd^_f7(0svM2s0y@hM7AJ|{w1$Wm z3$aDZGJ4v-Zq|?mWpd(ShD^n3gUCuj4THG{r-y+?8e%X8HljrW_0hUr@uG@cRhBin zeWZoG&jQk`;vFrY0iEcXv(ygQMNxzOLcinrpv+)pxHwPlo%(1l+GZIECB--qLLYil z@i}NK_!|tU>lP@xhPRC{Q(p`PnHmlz7_tsSxRS%3`p?bJr{0C!=~6|Gxu8dlu7r?(p1-Pwt%&=+LWRRDln&4XP+Hf{ayfeLU}oKqeQj%DCamf#sR~S z0E2)HzuhQc;PoL|j+hTo^&MOo)2{Dsk_1~s`24>!QbZOoUT;G zDWeFK!udJ35`6uA1X^EEzL3Ck;IjoU@Td}kBbFx(Xo&CC%oKem5(;6s?uwR`L_@XZ zg0@SMfI<(kR9+PMcPXIA z-hj*?;r`%wN|%Nktuak%`!U=f(=92E8bgBl_(v4;JJR`&*1y?7NIu1-E!4zx|L8;d02>d5K2 zRLHjFusp}m8N^-O85hzB4o(pPgdvg#h>O>eD0=P%V0$TKRXe~z5(xx23r>Ilcsi`# z5}TM7-5$W485ka_BZ4l5@! zX99nVH?IH~)E0Us4W~E-+BXX0C~I^Nvtz5HKRm?R{P_ZYpz`{xEX)i;g z$gHI<021~Vjj2p^)se7*)$398;}->9qJv6PQhW$X@eDT#sngXW*O6JF-jvj?gRM^r zBsfHP{qA-$JardMFwUqcle*%blJ9G)>BCcIUQqu^$%Eabwf7#=ODQWEvgfb#!K3bi zgrEm{e#<6h6dCS=+Zx_6RO|r}{1~X~Br8FHbCs2ZyPT&i5n`oCjOfM}P3XRLMh3m` z8uIUf_Ct+k#{YR#*YJZ91p8 z3AQQfC(hHtI601Tc(Qq;W29j_p18FKf^gE70ij3T;!dTd0O!@5)ZL{yI4T5fd1{mc z7hPDWBvoBhi|G@LrEJn|6-fR2HEB*eH8?N6%7g1H1j>Zy<+;*?bY6mRjpjK`*?qCT z2`?ZSSu4V#;#NK4ei%(furl72YVBx-27@7vQ|Z*A|HQP6Tx#r5DI+6ahIcurgIL%K znT9X&Z!m1A>4VVgkwd(WCs%PM;ZSS4^rk(%Ao|0i3(-*0k8bk%VKJ&WgENRze<{T! z#J>mRNROK=hPHaM`X~nZ@gxw*dgdClwzd?G|GC$+%8n4{KqbAfY>^?1 z*W znPif=m}D;IBt_9)&-y+@5a&^UQo?&HDyFq*3tYcjuW0=srL^KQoxY*y zOv-tQ522MI4onS8{DD7|X-*EtNjWHM(21C&kbZI~@}%eUc&uk#9Ebj4o$A;$qx~0gs482ku_N)`T`nk|E1|ap zPJhn}$n0=E1aa7Q(9l(KAoXR`&#n<48>datgu7FSUPHVwjYwb6xFphb)8Y1oiJ1|% zn8JjtZK-x`GDujFDgvPXzX|Q}wKxC1CjXJj$-H&}V;~j@OO4mO5!R1+od4%)Jul%r zP=Ro=BREk(SQ)J5hu{^4=Z6d`As3>306_NNBIF+51GEY08c6boV+k@)zlg)OGzd=% z-_Jb(k5}sgG~a$d@!0*Z*=%ou<*+f=Q$%ENlQ?5DMoa3M4<`Lu;sZfOBr6oi*7l@u z9z4I-1lX@_5WhXRu~y?VKHLXC-WJ#CwJ?5Nognm1Y1YpS*SJ4jyoXtEl zgvS1LE@+hI&~`2?IYLV)sQ~U(Klso5-qUc}PT3+MJJB{Eg1dK= zNda>h+0%M|fB(+i-f-mpO(c3}Abl3N03tUo=N=j?})n@G(OUq8hE)4V}wN>+SBBeQOS_*CKy*t=2gK zpqtKRA^i!iWfA=mHw}sPv2`LmJv#Ln;7OBmes`{Yq17rYKrHb&Je6rT(p(#!@W$=a+8Z6CF3NdP0hoWHrrwt-c~`9Nq1idU13Pyj6&*fi?>x{S!~B`wm64R z{<1Y_sKs~Il8JjAcKA(RKVOUVNJ(?gvYP_wJFQ;sH?OKGMuzBp|Fj1d^=2x0QVuy5 z2G*JCmZUyDOKxXHrrzFMgc)7Y7K{`)fvGeT5Z5x4ZTZMd0=tzkGYLhCt`Q_qa^Hkx z6$>k^r7}#V{-}}y4=36OGa=a$sO*QzM496~Oe%3I@oQP}4hJgSyprj~AOlrR0Li~8 z4{ik&O>2HlGwJ= zXTmyBEu*Hx)c9!I93<@K#d|x>)voox!NdF#^7}KvR;o960xCO7j0QqxHN0=9{xU&9 z@67vciBfW-tUd%{3H8bNLPnv>aN5#%fRc)vaYf_BbIhK4ODl4mvFY>CW)zp1Hg?b& zQCiVJortVCB11g}fzh8ip*Z0z%h$+@UzS_WIU&V}|1mg768h0klZQ3te1u7n>6Q*z z#HAul33^%wykbY9{%PB5NDP_|6}j5m0Z>+QE~;5K-moRs71Tb|D67{#Nugkj_Y} zFu9?LWCWP9Oe4Ev>P2noq3xQzDxlxL!Av|t;z@BR@AZSq|Lk{-afqCyd}xhIVCsJX z6UE_34C!&eLtT?STaM}Bh@A&Me0TYfAd89kOt$MAvy3BBs8W#c>PsgW(bv@Cq>+?- zRLo1bU#k?h@jtr7`BLUXEH`AdO+PqLgL@fC3jo!=SLPEME{7avKb~l&6P2|Zqk(?h z%@b1Mh{=YtnAhoMQxX?$L1`YavR1*)VDW-}by8Cc@wPS3&=L?$J%F8pTuncu(;3z{ z%#TSONFe=&v12rggmOm%h1|=*tILhKv zs<}?0(h!l?t{-X{FDw_;b1(>^16g|{JzgGsvVH0pNKJr$fFx7wLE8RCXO7CfUu#YZ zoxrqq?a#9Mjg^J(F00hy_a5ekRKmW}tQvtexm)m4GqlsVvxmLrOo- zkQChr;zun!GLZeSUB9%1@J0e+(kNC~r}UaVJu;Z_n}|1SRn3ya$~jxsN&)RG1an>@ zkWt{TY9X7{Kgsewc17=k(<1TV2Wt&ch!;1r5VH{Qgj5W4^HT#o^cP{fU60D-P6DJ$ zPxLio{VVz!9TW-4M&r3QurQT`KHiX^?>v4hPk;;Nxm;7`af|*>#$KJT+5MeI~@>ox* zawx2=)-M|(Tx(br2BYJpq<-|whI@k6kv6w;7yfHMht1w989ge(;9KuNvE%uT9Yqt| zciFPg$QOuOW6i*V_+L%slh8>*$`XB|5VxZ3rKAsZVNhU2H?wE?*wRU&^=LBI02}vh z?8dz*5uE<#&}mQJFlHNMZA4oU*`B92Qd|91P!l}GiYfe7Ut_qo77r-`4lql}+?`#7 z=~`W-N-({0si`e&VRN+LMI3&mVV1^+sxs{XoXBsC6j4SqLcN}}8x^Z>_(I>(B^ps- zlt)TL39%5Z%_og|g!4u~0 zoNGpN=OI?`L$}S%iLXR1yC;;HO1m@w+~c|l`v;k0b#OVQh^SXHS+t`5TwYUkjBCvHGTB&?3TZe=`CvcELg6!9pR{9lEndP!RJ%6tjlB~x4A_}-0 zLUpejl#gN7#^dP}%!79{r-Ka~6>LBGbR_r&4No_fbxRq%7y4R7N}vvyM)XD(+lWfF zcInqg9onfibU0pJB(zu$%u}=S<5ONxK3pg+n3!GIWb$7o$-s(95X_5ltu~lXOK3oD>K+bvt12Mf9*l-1OYaGg3) zE9Bq{^YWg8{v!mGPEW{(ImR%!pT^XM(U6*xv#Sv?1>XH4z`1Q1ILyb)R_@T_YplzK zGPBR(=GJyZ{+OhtwNY#+RTTl{BeuWRvR2{xP_B^Gj>=usE`m*YJepd8>0cL12_m2; z)=K4U?TQT~b^2CGYu>(;aH_$B;x$%va4}`3dPoFv9=Gd1SF&1s*f`!>ugp$uB`mgj z%$<&0T&ostS-n&-$9k;X@vB$|u(;F&rlwzJvO3zm8radh$0mbKFkn=@4_GtWjW-uo zId#4?@2yevyNV(+fSoQ=OK~K1EJso|3kXQee!?~uo9ow1{dLALuZdZTap1CBmF6_6 zDWSx1HUw`i2Aq@>YW=h*A^jqg1Z>#t)18aWx;;9P^MMo3RqPEmJ}2e99=v>1S6vw%OA=WHV9UK9;p zKdC+%+F*}3-|-Lnro*q-@5ZkV!0mef-FYD3mX?=4c7nBj$nAix{v~4JTMEza@3C-i zv97B23PY{3~0Pb(8_-Q9rB78if`=U|&%6+`gf8rq>Ck3J?%pFC;LD2+hFN231g~>U4 zAAPU9R_Z*L4g7JUxM(@pX;S*f7AX@Hskw4H4=x>yvlUjS8y)Q%s8wq*GienAg8L>O|(lZ{t=@$7(?g1Xp|V;fs+X2 zIq8(>jhpBvRU`L`D_8=1n+6A`Hb?OBM+S{t16)pn{z&SU>%<-{(Bp`q$9`kYH=_G) z%!slBoOCMMsR?1HXTQ$; z1ag>YwOXGn$%s!t_DC(dM%ZX{55(RZ`njR8yM$?U0n;$_L1m31e5V=;3x>klNR&mk z>&^0JJ-&6?94W#RMaUYVtC8NydjvvsgX^1_w9i1HNpHV}u&{^u*hG(ylTAXzElA zQ`Ep#VGDJktJmv5)HC(R`4lGPLsT_yUmf}yr1qb|9`@M_49-Mv@ zpzr&~z|zJ$_9sk(ndEd*$Z@Eu4Qy~&l?T5TwuVto@zI`T6DB~qYZ=o;O@t9BJPlkt z8*mx&o5?TEkH8lk|EAbvP& zEnFqP`Zp*D1?DB7$oeR)P^9f-QBIh#m2ga0!b4Kyb!JnAdpc$r8$_LE467Y0o>=Ub znUiT%MO7p_e3Y3XiN1t>ABk(Bw}I+iub}ivGPA*CxrYg`)j}(G%e)2DlPfIxC@fNW zuvPY~VOJEBgo}Z+?cnYDc*|x7AlQ06eBAGom#<{gA??LVg%IsMwl(xwI>t!ro10o2 zJ*dtq%kYG?`M5x8yX*TkZso1F0Ug~QTs?u{BiNmh?0i>4o{fdYYUC$s=LV?t6{jWd%@2nwr8kO17*eBE#l}Pa#5^PqE1&22WNRpiOY@zey>uAF1HoszX{_jx7;t zk4z3+x&@|xQ2Ss@zxS?fKbK7Kt)Y|=xlG_>C3m? zQS9}6Z}xq!-0REB#m~*{74Ngx?Y=!7u-Y|94})p`IH{YxBl2cevf{0aX{mqy-(Q?t&9%TKO9@ijI&`6o(4XNSyH9`F)m zyYLgGiLVp1pC90CwE+=m?X7NY`p9`%~dUOWTfU+|84JKwh=KgkCH zSV%sZM(fyNOP71=MWpfwi`;)wdR&*Z|21JlW?12OVKIqi!K17q>(y>AcJ3p#nL>GZ ziKiED>xA~QST529I!Dj7vJkCSDW2i8*jHaf_eQT`&MzP}o3d*1nbN4G_t>J>n--Tp zAHrOGsfN*>De`{HOKU-GXb?Vg3h1;8sHJsu^ZZBHXoT0HV183iANeR4o1mGFhjsA{ z$P-SXI#ZKdUrtnIIUSp{W_sJv0iak(!0Vs_bRKr^G_Jo!7yB3cZvsEv&~rDt0c9om zR=gkYWi%r=NP@eHZ8_6)+hX&uVo)L?Y0jwCGcn* z&vqN3!jgz}{(z}#Kk1Sap-jzK@39Q4p|F(7^s8VJRFjRN4cUCIhcV;wD6&Y3(S=_o zX!bD5a@%1z96m9M8~#*j`fOsWHV0EtB9!USB-Lb6xC6_Ml@#5kTBf`_LFLj^Y_3;0 zbbUdFZ8BU&hhXX2)o&868UMMZ6kk`KW3DkRzLYn29T_Pfg`q_`lk~1fVcd7Rmn>6? zkNxVZ97CcCx*%tuvM5$Og1&#Jio_%XlHm|il~7MOQr<5$i<_Jn7#%#_;bThy{1pKw zeHPAkqoXcBqtXxkWW&K(+6C}>b-cy_&VSDH4iY%2#$5%K8nTVo-?_?GvG9?!$eJ}r36MZN$NLl+(Wh+WD6k-9kokBRbkN7DEN4LQdj0yZKec^ zoQ|X}Vm{bfuV$Pf1@zp0={QvdI%21g z($qKM+R|XAls<9zrK0gjz>P3N@rbMNv)(D5M|3TU3~K)r`1fpXM1n~L{O0LfF42mCe z7MI-TLjZt_>VWBV+>b-+s8#^U%t>^o%X1oskFKG2vCjFEy+8mz6McOb7myIFA^h*v zw@vM>%Hvy37r*D@zS$3{nrZamvDb^Z{A> z{rm`|4*7Jz&Y_6<_-bs+nkLWd8OIuEP(ZNKW71?~oUL>z&&O+~td7bj5x`0psqn}Y zy+5WV6(|;F*S+mI>=IB&ibr@3rU6s^Np#9@@MU^%n7R9mzkfXEd+OIvnaeMbS=VjQ zi!WQnVMfFJbZMrFHy>3n`+Q}pTzF2;vI64a& ziUtIkEZmewCQ`u^rDFuaNG!ZT+96vY&b-x}c^*omPO75T`Ha4&TUC&VE4)h!MPw&! zfvQPpgEt_u-*QbSkaP|ppEDm{r9|VzekM#yNLIf~0u}M6T%82523AyQk7df&L&;ln zXQooel!4;nomeXYrdDr{Mw(y?>#?{_j9FnK!b+GEgjclS5#q?A;O6X$vjYYBPBC$a z;fCyA)A{ZozUzR8-@OIr@vHUE(^L-dBeSQ{y}KrXeq-HNCWMe`*BMjqGDoZNS9c@w zRmosbii^k)q&o2b!ps&FoxG~<6{${-4nDGjXwP_^1xSgY!lA#1cW=?R?4ePh+R+*0 z$M7Bx!W0sZCQ@Dxpb>GuHcG)vm`rvetUmD=@@0?U)0s5$f}?zAEYyjXkDAgRQK6f)y)llqs7HfkJQDC}zg6N3ZL{5gh*t)HIG=RM0lWybY2B`-b36<0nR*C>oowfDZR4OD`81 zm^|5sjpxcnmlr`F>7zn%N0IJdXVqTW{I#bX_p=bLB?wn=a_;sWM+sU@(A)4RDkUiS z@lJak4yXY)fq2Rwg|SSOx9}%ZWfWd3R{OV8!nm(388tFU;j?GDF)0zhF2tls+tl1; zN1!SUB#tD&mHH?clPI2agp~Y4BToAHJoN#cGK>L|XTpy{V1P>(GZ!^dzIi4OtTd4m zg)3^P+5wvuZA{>U=MS+og&9sf<3rO8D8yAjqLc6>fh?s~VONXi?D7dkYa=C2ZSJqq zr-(JRa3cF0@<_|TMMm+$yf6%16fr~7JdLj9rZ-SS)WL+XI-v%JUi5qtOd5{K-)x#X z1E`mKfAc;m*-$E@L4ZVoUueElpH$gbkJM%PYsRT228co+PLDFWzHzjVph#C|Snl6K z!vJ2(y?OoW5;#VVGJ6}U&AP=?ocVEkK*d^E@m7Sq{v@NL^jTYuzo`?mEg29^T%&kQ zP{UInizH;tM{=;8T3KdWr{@z3QFtQUGA**njACk8mUsSgle z1G!)53aTyn^JV-rDrzts+qQHGYcf70fy^|RSbP~Qwu5mTpD7eC(FpeF3rv;Lk$MC? z&=`D5lJ5wK*a_f}o&?|VEh;4wFxdQ)U<8FGeZvsxdx%5bYZ}YpNl*rUogQdkh7$=x zAK{b{$&a@XjQ6Bq3aHBt(vGXx97MCK4nUJ14BBDf63gEo8@a4gHi+lIqv{Z`<^JqA zxNq==@8=y+#AkZYBqR$|%i?W7@L8QN*c+I~!4MEw3=0@lb8qAC$VqUJGGE3>QqxQz z7++wLKie|}>9-7((jq+mWjF=v@!JBClZYc+3-X?hRSFY02DG?ZHNUBsOb!U z0(;Q5XyOYCX0N;FT|p0gJWPD8$ncP=G0wUm6xX`)HX`N!^ffp2bb3Q5obGV?E1aW~ zdz0w7Iw1{Pl?YJEnYSpyWo=w5UFKE!Xzs#zhaJt!KS7#zXt_UTamE>-X5Lou!uGAs zCHAtlU?PKBn$^#8%<7-JwBLD zIpj*>4CAi;3K1%OI=#-{p2d#vW3QNFeQwaB4a^NVC2UIN<_|`#i|h&m?CVW~Kpsm| z^am$PE9k^>OG~yu$X82g4l{S!tK>o2%~*5`7sNKA%n6GyT!@KTb(RY1F*Pn1!!-dt z99FbLWglTsq2|H?74|a3PoCSLoKo@Y^>d{`+0i!UyM`kY?kOSx;poi5g{y)IbxHR} z+wh)!B6WKO!3KJeW6ruW0^?#Xr!QW{gTKUq%2dG_>yfShc4a(JZ~WHz88rzoo^Z-_ zOYHB;AfE6+sHnpr?&5peb_Y1dE5Y^PTF1c>`dF}5Yb+TtcZA_Z$OR|}Zi!@B$Jedo z4HXwVss;R<5r#G^Xpu-eK5 z#s@Y>Mp3Poyyr2Xb2f69XjjOg4H%*+jszGIHk=9|OVxyy9yN=5kcWa@+c(e$tLb5u z{dMRsn~S>ipiwNFd$3k?Vi_%KpuR7L1}@2%PJ+2v_={g^v-U;FA7NZ*Ltjx;@(Nk- zv%ocCe9z5)((;!KyfsZR*~;n|2so2OuyVImYKdr`H9G?@he$JX!6N%HW2dckq9rtz z@EKL85P=UikyH_K6I}BbTG-z8CI&Fyx*_XsM7?G`NK94lGzGghYzs*>0o5ju=2}uU zIK4g9vkGqFuaAsu={plf|2Cg?zkpyMJ``$Mjl3axe;*>ujXZpp^{p%WSZ=Y?KcWNU z6zl5gHkRR^?>kir-Xs(hyrykrhY0vqV^%A0Ox5DxZ+)F`TuU#QA3(4P2i2z*!jYg= z%q9D0?2(osUPEAz+31MI;MSX21*@ZqS(0#QV| z^ro#L%axo1(4iIKDvABD8>z&Tk>kf@jz)3u^6Iz9M*)%`$VnNdNNoc5&^CI5Mj>r~ zPR^@S7wUv5XY=dG|ELj397L)I=_m_x1`aX*YlO2oaqORe?;Uo0kF{??+KRYGs7{K$ zkBXwA$EnK#RGxK`eLq}-vzy} z6LBpT3f>DyIF)6}Vn>)6;W40NrvE67NZ1+|H8dWHW@?;>eX_$N3KD(io-S-}E0HSyJfF1yLt7T2F)={|qzBVQYh;XF>%FEDd3r2nU!px^ zG4ufoeh%CajNpo-O>&XU{y6@s;Xg$1goJI(wVanqBCw8cODakF3V$lnKVFV$@SYYg zg@e3KX7U4Ty2h${6lnrJeMU0Yzekl4+z#o*^~?9FmfO}N zXUH6AJR%IOhX8rh9#TrPj-1yaOG(mYs1p8UhkvRPuPRi_AKp6*=Px3pj9iHJ5dbM*7oT_X5TKW8 zu*4UjpKbDv+sUhQ>z9KITY3NUmGEOnZhLQk>La4UEnJ~-^b~<4T!{)s4lB99Ubsl; z3E}FQ{B6_4(pUFyovFrWw|H+LcKki0LV)u)z+-*0)rctQ!v$t9bixszlP{$2WIU{V zLo3sQ-`*NHk5?Mp%#K1@8PC-AygRgllsfWJ`?u2`!WWO6^STQWt|yZq0%ey=7^NL9 zCH(iyDm;#{10VZ{4m?E)hVK>R-2MNhU!vwN&dZ`PfAmY;OiJf$ufyd|+(=3;VE@~` zHV5EBj{}cC1JLnqQBK0RVbZ>TcFNQJ_J_Dh9mL=DT&!w?l%zFx$PZz27a0tKQA!m+ zDm?75fC_|^cn;Ze!{Ol}QBV5#h2BA?VZluvA0H+;4}~ExDP%#(!1;Zzf|j!w0CH-* zZhTpM|AsLsY=J0q))*KV*uCE$i9WcEPUi|?zz!Ec;LZt|a~ldVTk`Z?S9`kQ$@R1Y z|B73RlpVg`7BHF;KDP5S>^F`5ZCNC%CIjh6>z|JjKkO*1-I~+fxw5+6;dMQ@?$~l8 z0pieNn;!_YP}#wV zi-W2S=c5zEZ;r7bO8>d5Ox=B$8?~pb>TevO6E$u$?hK>o;)F{nq?s_UcToG&N=(=! zY1DT9rS-U?Zk=#ErUtl1%$R*$XD^j6Vrqr880dW^ANHvPVD71%Ws{m|paNwh>{o~p zoVRN-f!X~;Ic#B0p=NAY<#+YSufzC$)#)`;44t031y;h+fc9-OIlZ*cDkM4RLdZGw z#WI@{jKmz*Qmm~AOeC-g5!&-=IPi8=Lv4dAC3^j6PTG*^pw59HKV5O#hC(B-{c8<* zQ1mIl5W$vo_4FQvnA;%>%0*8l=yRdPP7%byJKq!P~)kKMcnG zbkdX(QXCSDE8Q(ceZtSYXhpit(Mo~|OT__(7(AJ&BnyntCY)>e#6ldSjWjzIMUu7w zGEn-^f_McFGp(f}Lapwkk{TB)>MwdyvNc5cH>IgE_j`m?!b~D)Me!a7GV;8t+0`I3 zRZRfVFAD+eDk|_hw$ZN62-RT&5x2X@y!M}Zc(gjRMmBW2%but_aEoS7Wq23{aA0k+ zjX+6EN9h}Bqlm6a(?MEdtV2E$M$7V}6USQbdeHE3VL9o;r9d0qJ1hy6JsD0T9;+7a zr)z(OfUsZovRAMoU*kuGmc_LV8#m$(q+43cBf4#LGDZ%*E#ds^G z+e{lPWW5NzM6f|*)*OMMAp`%2VqPdgIQ#Mq`ZCy3+c`I!6yXqNyf0BgAJ%#5sLs5v zAO$+p@&UV;Y@|79Z_AKR%vkg}8ld&2B9L4!{^j$1_G>LLReO`x;FfdKcGNL@;d|bZR&}hElz{f7wr*XpkOc zRbLzFjHm{k7lB-cpE1icsyDt~+<^(fsoA#@^79Y$!~-m`6ua_4Kctetr`$3JnBn23)Qr~Wa^1VZ^LdD))+G?Gz$ zZ9NWp3F&8*!o<6cN>K;@lUsr>WkG}r6BdWe<3n}WmyxspAl*kbA)%oPxJj1IX| zdAo6H$fx~6A!YWMEM%)W{ccVbQPFmkrXd?^b<8XlZ^#cf4fPOjd#h|6e&MuZm>Gz* z%ws0KA??G0_|&09A_}w};~6xRJ8DS8J{~6LqyeJkaCVya8T)D{OAj1cl7DMaWo&~{ z$_)>7V;GBXsg?$bs>1}{I{_#aJcxW$Pa#kU&ZKQoOvL%HDNgB=z%>DXgCtmEkJ9(H zI`h=-0$TG@>E9gCJZ_n8A$I^@rW&j)H!n``{-UqOb|@IkG@rg3nBz2aua{C7&o z?t62c$S-n{qSNr`x6OT*31{A8;Z#2Z(w2(sTma5Pxn`Okv*yD8*pKAnf$q#% za?nwD%Wb_J=&v75H8Sx*&$kO6i7l!YjsDq2<_KP4>pey1SiAR)FV|D3*G^xxiY`56 z$h%*-)cMNcp|m;Xn@VML%sX(Mu|a5Y^i^lg0z$f!rsAM9{JnU4HP!Q`<$I=ZgBsR&2#Ul9{UHC&9(EwP|c|A&q-&C`z6H<2Rz$Q(yRNEHX`S#K~C2ba0{QrH;=T55w&Y$NA)Uq(={*0UfW~v%{FD0x__2A){ItC@f-7c zIfGf!Um$Ryg;SBV1nX3Q&wwg89d+VDEjJ;y*C0o*d*_VI{~Q!iRD$!_$K*(hx94j!E!d2iEiwXu;QHLRz(7GZb(Y zv7#C1#~QzooQL-dr~w zFTL%cka~qP!AlobM9rqgaW|Y54T0I3pgw(80V9 zEFI^P7OVFEkQi+$$FlOQ66e=c^_X!Reb92+ zc-cNW+HJ_q?j$XC_*XC;zq(#8*}HkAVM6e)X5Xi71JdeB8;pT%lgsJin!MC z0?n43qoe>9yF+b3v$_Ue41bIN-e$;oS)Mjnr!)pQI!)k~^%3i>?5gLxD~%`u zSj%&I>G#R|XO;cRMn`ML7;;wA{Ic?5pG!4_R_C?jc(p)diC&|UVevVmWfhzAq)5+6 z3$s&2HA6PiNRc?Yh9ec;wo^wNTu^h6)>3>>+ZkVm&@`fg z-k}XD$$xYnUBD?QUn!m%4jQN2vV}+t&{Le~O|S)du%hsUZ(ANFiTH%(-Tt)K6P^W8 z^s6M%O1>TAc(({psiKoQIn4tWvRQNxLNepMk}celit7a5UQGIM7M zXeg`!`Wpzbr%|e&0hb`V|Edh0t0sR5V7aKf+v?N$Czona7Ha$QdQ7gKPjFY()VCqr zVA5>I+RT7xk+E5b3GK9dB4DGuaNnXGy`8d0A3R;@mNQD7L1jOrlUCDc1Ph5jN>{@% zWeyI~3iz<=Cu8*~wX%{yGBmI)&ClDft+mM6D8glF+01qDZ@Wsp7ih)|T87qw5o@a< zY7kb%l2^=RH9NEnO?c3mlN6=Q2T|*_^pY7YwT*GU2DA$-E1noOVkM6O)PGMoLf>C| zAnA=j@h}b2MVqUmT8{U6Ie9DFIDeibe;1^chF+P8!XTBg%bkUB^X~8Bl|+s{Q$t4Y z+Ieh`L%=m}qdK`d*L05~R@K}y?zb*?=uJk-9e(DC#(-WplIt_WA`|+hxxYKiNcK$| zeWDpUA1C6)Y7d=}oAk%gb>F59 zYQUbvkG>6982$V6GC&+#K8E$B?n}6+`AD}{Odjovs5fr5BEZ2FvT$7IwSoN}voKaB zR~EEekz;L_CbY)63T)EULbrvW5atE_KY@(=-NXod>8+LoFo(%;<5J zo_f3Y7yARhe}<#~<^ciRX4);b=gZQfQxJVp3+~Z&nq5P&4<>;gzggYGHM_nu(e)t} zAfkO|8Vibs!`glV2ipx+`7_>M`t6R?;Ykvt4e+(-?`3@gA-W-rt!z3M!0>eS$? zY3e3^tfA^t)rx59R}WFwLf2r5cK_CFG=QjM?N9J4NXmz;ZrZs$3`q8ioLom(6Jf5R zq`zHR)x}OC9TIjoaN?FjUg|G01Mgh&Wj-Ps&R@Z-@ke|1ZDnY!D=<$^k?(xQKW~B;bf~)lvDC(JGAbAOhMV+;kWZc zPb;kP4g`r}tR56yWulK z9A@CYfomyx=o(;{#=-hO6$h3!-tZce>KG|br^K8_s@r}IjA#hp)*;n1C@MTTacm+6 zO7<+{cxVXIgNJ8;NahYJq@?^;!2DOh{C^iPqThPP;Z&jk*AW2k_U2~gPfQKr1^<7w zfSEq-U8efwI_@`{vwR8Q;c<@x+?+VkJ>xnjq#ujaKQZ++49cPkLw{0J~<3GvS zW6u`j?M^2ru_{+@B(`E$G%r2@9@{2k6reMz<~m?1WMOJ2mRr+CQ4?m;zHRB!)GUF`DOWEyYM()LNwCv_ThOaU_uO+G9< zAphCQ)SQy)9TkBy9}pjN#FmLf?2A6zqzMM4@z)KN!!TI>i^P<)S~)_Skk`+IVGuO$ zkJ%U{lvF|%<>wQuqlU(t``gw(Gu(HCHbZ$uxg^)u0V2&|=ruwB90Wx4YcE7#=%<|9 z!#Z;6l7>VJN=Cvp$>hu_heK*Rgw6q$FH5Q(e$D1(Jn&eURvr&;0B1(>2i6-13C}S8 zHtTq;Uh=y(^FDSUBx$Fj%zwt$Z8~Z5j5)~h)$-CEI;N$hOStIcwXI`-@_rSI0XhXE z?)1XxmsIiiPm56Qf?T7JKbga6s9`PVwG~j*)xW}fXq4Nh3KP}TiI5`LO3ZjULbA?; zaYP)eA{V|%A{PWpp|VYbE!C7kRvI_MC{na~rjHfU)-gfq?>v06w2P zz}EPw)g8d)^FyoT*UH4~7yd8Nt%}{=-McrV4-E_ltRX0e-Py7@ZxyIIib7bjOt4gS z>>dVK4sPGVsejl(gvCgwGq%YNwC}Gzw|=khuV&2cVY3TW zd2CsQm!G$M)u#X)*q2TovWXFjEG4;`>RFBP~;~r6NVnOUTyGP5mPsiZ)vz}Odt4i zJoaGd$WgI{VNCLSr4zJ0Z{aU!aft?~t$df@=IUJ

2eR=VdW7b^L@A#g)WFvUtK9 z5t;!yYWxfV!}PbcP-eTfz4NzSz4kXTjtyw5;inLli4pftTLu>5%GBVeE%SCnak5`d z!cmcn4ofL;7;7Ma;;YK^P-)4v{L&TCDmj8+;bJ{JKQ9-rb1qNVmUq;yG$poCl=CzD zqRiW}rbFWYr^=f6&}37D2dQB@eWZ=;5L~?#Q>qztWpRYTK7E0bJn_X(%&kj?l%=&m zs_esD`V;_DVUW_$0NJXePtqCh z2Om`e21uVF@O4?1Gg1lpmr$*6#qsVz4+VQiOHp-R(<9_i;T8!@VHij{#>!e{Dag+Q z_EMN**)~W=-`RT&LP^BX&2o%ogKU0_{CQ=Mo)eFvL*t@MT+e_<{&v{k zPhh0X!IKFCRG;SXzku;Mp&aSoB1{Zu2XmGjI!r|`#w#Ut+ zM8fxJ2T}F-ATEya-S>PsKJTm9rXwQrjsnooE!_jvn5En3T`+$$aNcC5(1*n-BTV>= z&MyiDh?LXW=UXKtk2oP?hL`=${)^P~GtxvZJjB;C+2^2Px1ULoY0<@sjjfkX0-q1! zFLE%QAacT07_s-G(IMvvK_*wLahj31iZef&LZ5x5&rA)<5v?|Bg8qV| z;yJACV%L(I9uL#{KU5_UH9>NW15u1xhluGtFvGU�=U1c{+Z}xc~FOoEihJ$@$YImV~MsAuULf`mx%qUc#jv(Pz8A-5I%F4_NpRPphHCJ=|e zQH>JjZx`vpz5iG!ANif!)>z?#va|6r=7>ix36pV}YJy{~&+YuR&SMpRH>gAc+T9^ghuIUx)^CVr0} zP|;`SfvxVeLc_H9`Y|U@{*a@k_On@2AZpOSK7tx%huxQ*O|)XIro(wlhgKQlF=@h2 zh){q>G;n`}x)us?sa)CAB!O30W@9KQu~)M{IL9|R&~ya4d1)0=6tp|LyMb$pL}vtP6}_`ya7sPI^H(OuLkCeC!dwii ziubDQU_aLczzE*WO7wJ)nwEP$NQVf{ziS~497~4ZLwqs~il|w#zrD6e8aXv8HD1_5 z=TLrnW=_V@#Am$Ugb45U9Z8wR5yPY)U^d8*VIKWs+=O^l7bSaAdfdeueQnZ=JtX`d zY*NBxhbyPswd9he5Vh<&l1;y)m5N0XPdf=kWb^ZN=e38}g#3s+O>+fP9RD`mJ?qSb zp(G%UU@O3?zKMB(2%_#8tETV_wh8%mwqPL+k6BCH+_o!k#&n7??<;6 zbo+mHuy<&GXy7#nk2J}`g+6nEfz0Xj%PNFXVV{jzG%sx%MdjSC>GH}4Ga>N2-=4yF z)y@!KMJFvzNlRiZ0uID()4-FOuL`4C=2O4?dL@StrlQKf;OZC+pjb_OSwkQ~xAcyS zu-?yQ`%~v2o(~||M8VEK3Ym%*fSwn1&}NoGvKq;EvJDz_P(`8xlFlak3KuKmB!RG( zaXH9uAJ&$MSSZhr)%Qe>;zzBI7;PTDC3|NPo%UdFgCTwp|l_Ms()laLk-SB*tuYOXUp{926Rp#Tn64*SzB{ zTag04<}MdSfN+zm8-hn<&ptFO~>q)#8<5%Ip;P@o+_Vq3JTkx8|1#G-nsKoJfB zC!uLEmWFwe3+WA3n|h)`L9P2O!TSU89_{Q=oD=pp{TIbGjE`wTT35GHl{wwzF^=;v0j)I%#=T+k z8{(|VHrj7m#pU|6S+?L2$u5el26mr!3BO8ZCSRPi(E<)GVDcDkdE5U`U?th?`ot;2 z&Gm$Op;r!$-8n$iAwq0({FZ<`q**cTV`{O$J}@I~2DrB^jW=5}dn&YMp-VJXPV+r- zP9K{G)-Nq}E{Z1Y6hu}Pr9o9$^RKMgCd=3%rm=sfWunr`CMz0u=4vzCu{Rf{D%(b$ zMlex`wT;AS-=1X6584t{kgJeZAw;b$i{X5dZxZE??08Vd#%WCUIoSI zKZ~gTAc?sx-vbx@6jPwNliLV_+&X|;zD@eiz)L~?70}Jb<{%6m9wkjPN<4i>3O~Pt$-_3d@YU zdD$lqU&$6Wt*g6=#D+)7m|&exoyo5=K8zU%8Uh6*DVe^N?mRd0<5X+KSFw+-HM8Agf43d;X?5D$DSmaJp zv3)E>`^yw$zBbeM^R4ptIjbKq2l@C-1X;C0ga!d35(l@iz$YeMzAAHJiw~dib2$Nv zlFO=;`h4e0*&qYg$Yj{=%B4>&J#U<2Xm%$ug=Ai^e8U)+*ZzU=XlFHWJA(aY5K{U*KuiuaSgZ{QBHo(Ef{H;piUO_zpFEkoW1IVZ13Lrf9w) zOgY?j`C3c@z;Cgoli@q~)TK0EFh+jM2mcpof%+d~tIP=u!JG42Bkl>};`~aPTR7XK zqio2<#rCdB78HckUpHT$wd5>)aLof*T#~vp#kYlGU)`Bs&02CfRUd->yE#R`%S+I8 z?+oy2&sp$IKjt`Me2Km@Z8YTdn}-xm^{s4p@{3}Z4O4U<6r*DJvhO`O0`XbO>@+|v zp#cj;Jpis$+l_u9wVIGlJ&HV*rBrlj_mPAUooi%f+yRt?k>T$lCKnit84|ypjzoFY z(QO9TWC@t@V@lN0bSdq)vRdR+WgIVHZ}=Ca{3qV9Vi@P|x`qt=#Zk@pwU;!XVGD#(jTTW@eZ&{5kfoLK$K z*P$cHjQLAurVd5Bi;N))m^?=XTl#G5Uhw`Y_Tu|#-)F7vbGTRu>b(Z;i@L>6LC{N{ znQrwRE<~BfJqh5|;{D~I82E7rvWVDwCEn66=+XKZ$W23|3wjBop?hy+%WXX;{hN6I zD|&WIyKAcakZ_+aB}sD_`pCKso7h)*ILoNQMK8U~VSrekq?%qTceXQ){fPRyU)RXc z=&y0p+R(W)|mWZOYr28ILlzgNp!9X?D+AFw& zKKLx=W=PRWl%g-4qJ_I{oP??db(ryLG1Y>uF0{f$Tk@TQghQMr%!k1o*GOTsISrJ6 zh8ZGKbQU_T=Ng-f^I?$f_}hAklkJH zPIqfIe6Q~t%9!$QmR89v)UfRGH0N5Hz0dk5c3Ck-WsrZNSEr?@8jTbCKnT+~=HFjU zim-WUD>>Q;C5_i{)O>wr6yM<@1*;IHb9U)Suc#5j959N_#hWl&6z>Bo_#j6MZxlJ> zGS{&JNi9VQKHVPP4nB*ow>$oPo^I|&S9i1F_T_g=bbk(?>nCyi@_{8K;22`lI^UM_ zeS1APc?7-Od9FwZ%kL9^_Jq+U6NJER_Be9MmIE9a_8l7y#r;D(sIi|S0B9ZOzm>I3 zp6QOXmbK)+OV~PN$0sHv4!ld4&O6L89?1KmzTf;o4u8)27F2iII&*~RLCXnA=O zJ*YmMj>OB$OJ=>Y+iiy$bFA~G_^hpN#bX@CrrZ7Xs}KgDr0=g9wA{}{dc&W9PMe%X z&84#$e0)1pFPixj^bs6YwtqS@;U%_XlS2(9Sh5jY4i@QOwRKbxKJ?*#o4}&;AWV&Y z26w9#XPT;XE_V|mhr) zC^~7W?4SxISq@tV>4@vgK_V*=tr9smE3?;DkW?M2J?s@?7ymAozFm~OVo%-k=-kkoF{KJk@lYWZ&E z3H3D5@W+at@f8c5I}ef+f7G6XRQ!V>%d!y;weH(Bjne>J?wizvv{orvtBCj4m{Aa{ zkn&WFDryoLx9ZC&_Ca0U&FxL=^R(w3;mu%9K^E23RiM~{E#yTEFv#D(?W0)`PvGEx zwia3nrljjKXA*2EHQfZJL)$0#Pf(X{%3D9)95(f$5ro7yz%Or67G^6JmU{?^PN~06 znxuR_%O;NVV7IYEsa6xMa6tN$fy=&#q}zlLn?R$vOBP=G+^ZHzpH8tadBiH|pFW{D z`=fZaP2Hkl2lBiJD4%$&zgOu%%B2QbUsYKHOLWNx+uSizg3OLS)KG;EX8!w8<^xg6 zJ)$^c-GnDf%>)b{*%LNem3FH{cEYAAXWQq5w5wfb=xp3OV8-Fyj}~?yAs#~7hwpVS z(3vv3x)->7dv2c!r$HEDk&H8&ivM5Y}0BbYC8r%K+F7r~NZgP~(& z`!d9I0K$0c@@T|qWmE{W3BtpM%hU=DY5}o<3E~;N?W#kq2iwn`K+s&&S3Exag>Dkg zGoQO{3ivhe`8C};@BbNm?~9pD?}V2?PG3`pZ^wdzPI1rmudD)qoCR&3@ETRJFOY>6XdGzfKG(>&MbnfJ|oVO;`Y0zfZ@K9AMe!ca{>VS+_dUd{uNMGD zf5ht+{o$V>gW(?b|7&v`I$h;-94ZJql}YIX=mln?-=^LlbZ5}tE$4*PbCcTZb(%$G zgS0P}8azIDvpLa|;TP9YzN39~d7oZ_8?@eP+(61|B>m5qweAo*uKz~?!w*{SeD4o_ z|4$4axwjSNOl9pi_D*l0F{$tKDU9*!s|8UWP5kpqQ)lYvr^2=gqTy;uaAP$#J$CBx zV%vg3v{s_pkU|_$_0b@LOLJIBiH59o1#_WJd;T zN7sS(=bHjQ(=s|6mE}(k4O^^&Gxm{)AxMZ5els$3^4?dHSw=)xsd3HkNCMnsgmhM! zuGl5~&(4b*@nS+=>_Av<9%5!v;XW42=Y_%1M98_{2z`NMmfiuJp=>bb-s&!$jzwXn z^$d;@?P;q2U%3w&S`29d~ZYuRXI(K(6)(V={h{ShhNrw~@RmPDb zIj0YXmi##cWzjM>C7hdmj}R3LWz_OMLHP3BcjfP%w?OVmv|5eJ1kG*qpxn`Kfi2T- zaFH*nQvIk58Z1fnX+yL*OL`%MOB4v`1$h)P<6K1lBIJlkOhYw{Es>w z;6u^J3f$bQ0Unzoc$dyYveRH8zo+O3YS+GvZEUuy*vG^G@Ni~(6X+Y|tOX@D>Kipb zG|hX^?}&j?Iu~Pn+adH_b&~F4chuM0MmO z>#+RBC^OWYI9+l7dhP}ezee!^?igrKPey+p^WGKZ@rA9`>j>zT7&3Jt5Y(0bjs06o z7x9nfwn`FH2W7EVASp9cnFW!7o97jO99u*LmH|1KkY2m`dLcI}PK-l+Mcd?FgnE&K zCQ*y|Z~rqcn)v~uah1YY?vP2k-duulg1iGt!99@_2H9_m7^i>2^mW6hR1CEd?jnWY z!0&{=b5LMSl}wJ=0tfgOCKiZyP469oMlz@1OY-22cM@ScV2gHaa2d{R!5nBt96OY? z`+nq_o;-U;Q3uiz3otv0%k+%_JgP3BzODWkMLA-U!B!#7Rvff_sU29(SMS**V=>qm z3lr<3^4UJp92-T_=x5PlU~S;JCQ=&!lSCEf#klK3h#f)QkNf#aCt39NAoB{3K7bWK za448Sw2NOIF&!pWqGdkgH!qbLiZ8ym)Pjf!e?at^m4HH#z#1iq4bXD;`|j2|%AIG5 zOqp1zsHfd|Y~`G=u8AJCEdNj0Q|zK#6oqR#UaYa#61zZAK@_1nl$4ouhHlGB%Nx8l z%uq^ng?Ke(J6s_yA}PZ(47ZYDH_X*RNWmvDJ#=vh>|6e~cMPD6CvAqc6CEYd@&UT3 zAxgi$vy-fz?+=E%fH>>pG>?y81HjHR{TYJ_vMAY9>f7TRIC`}rQPVQ16{a(tS25alT5fo z+vT%6vG(I|*r&-A&Y)`sLc-+*(D!UXS_vuf;3GezBWs_F>bI(F5_>au^x|JGNe;&i7p zT#5W>^+YRmg-3S4QmikXfZ5LB3r`W4mwrJ;!HWrk@yz{*Zcrtcibxk4gUrn}6%>WR z+K&Cjx4fzX9S3Xp&)Ko+Q51y)(&>i^N{NYfjEVku1QA`aVe&y4$E`RntzqEL+g;9x zD3p?yyV^DzbWJk(;4p^R^l5C)5f4p)U;{$q(r9czC31W(tsLQMaM!toqQkDmkBFQc zOVQqa+6-MHUHQ=#@&+q}F4g#BB>a?Sk^7F%(v#w?S-vjAdo;gu-A||?P^05QN{;I!Bl+W11=^IkD#&UK38P*k z4~|XTD;Mgr3KIv5vu>7S)0a4+Ix^n9l7+5T*vZ^-$qvnPC8+E<&%zZ$775CarRY@QRy96oWLr{BJ0Q6)hJ2QqRpesRlMw%rz%{dM-hSG!7#k12jS>kVT11tUMi}Z zQ$j|4P^ni;)yD=(Ea};~siD~&5 z%R!1WdK!oyA|eNwU#g*20hhr^d}T5O|jaqx+fD0x4Ecyq<2khDA8u#dnN@wQ{mOS*1s(DIer{ZvDShSD2r4}-1}YXqj~y7)Di#2a($+L%70sjlV)R( z?v_2#^BfmG=tN?5vm98eO7mV2x4`=6rgO*j5x<-3Lsf&-soDdS)i|4RscuI%_9oU_ zg7sNiZr1eS^kqLu7fuPAxL-Bo&l+3DAKSDm>%8iB0wpxwH|2t4nsQT)KuHr+2sp?b zg>BRN$S0#XAcKRZd`{_K+k%CGHyp@hrZ@~qS)Fz@_P(j0pFitrv!J4KgP|rMp#$Z%&kSDOKhLo zv(J)ob)du+C-h@+%|^jHAQ{{ryK3dMeH&-QdKuBVQS!%&P&+>V!V-k-x2Yth%41Es zF|-D@s=X7cXxuQ6ScWN+HR~2O&elETvJ3K>O54ok^nNw<+dSooIlAbN31tR%4I?*# z7pgtTeV?T{&qXhdeTBx_Gt`vLuhWw|BTt^9|G*luOWiP)8A+iA<$69SRw}*RPv;g! z6wX7w_FEOLG)ZG_vOjUmN&hboba582tCDKj!5$Kql)((Im)m!zIcc@tD3sg)ooV`XZKc1G?#)VgSG5l z61-wQmbxIZ%-r!5GGS%>o7?mHCJxoIwhTm#7xlUn;073Ebyi!7p}?KB8|Vt z=^7+Uas)!ygVm9YAN{5e&S8jhu4Ot3z-CZ~w^hNnuw$6(~W2br3 zu^7LmVb(sz&ziZtphiFsnHc0SLjJ_oNdnKi+2>}0C_7KU%(*GF1aUQ&_J~cUOX1=#9)W&5P3K*`Z~ArF{cv%(UHC$^j$fgYJRsyP^zcgag7%UF-YGd;lBQ z&(rxZ7^wtDZm>PBDpH>nfog+5K7i9q2yoT6;h7yco%unu2xHoQj}`7QIPaa;ITYv3 z1$oC~BtZ7_)CT;D1CytA_>(ZJuG0ZXw1Mch`fS7K-BSG3vHfbvAc1gq#%*sYQ??vN z;J4{ifS=H|&V4FaV{a}9oq|~P{!2*_I;A}y9d1@qqc6Pg%b9Zno)Ge3vG?*3*cvb7i*+s_baYo1koOfaA{mQ;Ugvhw*?Kg#c1B46p>_PrQ&}~ zB8|@K+B*(`_-reS(&nIY;$f+HL(6OQN-J$9PZFE&|y0J|bj#w>cu5yt+U2$TK zGhBwehV`#(--Fhh4uzjQI|4i zk}#R@taIwmxww@Rl+g#*vn=_Ix?a~GAje39E84myXWFA`>UfB&gIYf+k-D8vHOI5j zJc(yav`a6Ku;!E8s;~Z2%fKEc$ap%0%T=)xs-L_=plRfzWA)w8gwGyVff z{1lPwP0|01x-mJiffr^}6z6{k!3m%bCZlk6*|g-vM5yJYeobYo1(7ws&K^NzP4k|e zwddNxVH2b79!sz`*2O*V=F-oDKX-h;q}raU_wUQtZxPoqxVw+)>g+3$mgZq0y1hJZ z@E=~BCXTSVO~^#A9`VIkUk_hYe?gWfCpC5|A~rgBE1(pjIxeSYxM>`F`nP|v#QKSJ#f6#PvjtjFdqjnh;#55R z!8$%9XEffysOW1e`sNEL5a8S@TcQ8+!wp3R(;DWSAe^C;m7$qn17@gq>CPPy1BKs! zNvm~ybb<9bq0I#90>EGu#El;TA1PF8+c)iM4y+8<)Y(c;&1)kV@h%lN0EChnX3I9O zdLDB+s~y(94C{2dN0x=TfOCK2G{vEu#tB+t|D!aG0a|`~`u(^B0&6m&4lB9IS~d9> zS@y@cm!XVx34UuCVAUA+8cW6hv#r4PbcmB=vG!rk{)@$#Wv(dpJ7fP;A`2oj%SWtk zY&|-~38!>-nETFCgZ8tf|Jr$=>M*1ghxZ^q(S+mROlA){Q%K$5L~FjZwXISId;!Cf zQAGQoXLU>N5xhk0dRd{$XD!%{6})DEdQyAJ?dPLki+_jC%70*uMlkb!_+V+B=Q?7) ztB$|V=xksWCVSLc+Fui{$rIoFKe2Ic1V^5qYEB>2GQ{C{{4!`NXiOxNF%EhUDF_+atwyr>rbqnWzB*bd<3z-qN70QJO5+ScVi_c#Z~>{T zUo8Ul#;Ayonw|fNWxO2!9bE5zm9`Qrj9cxoobPD3Jc(z6|Cs%7T}Do8%m=79v?>H(3?_;^h+DWWEa__J}-0$O3Fj+`8!k|4psCRn|)P&692VoegYu*qGcc>VfA>*aN#J(nj( z$Y`U-bfm+5=`os~XC zTLNIVlyroSB>;FGlm6KYpTazbMhjC%=pLrZ;rV<+p&tqQVepDq-cZ$?36{cB`?rwO zBw`B8b=nFBct?DvPHt9!Q}Wabv{?{V1LdEal*aX73ZA7{hk^HkQR(4+q~=UgFW|fe zDu|NY#VR7Vory>A3jyjXeLgLs96W!l&wN=c3wLLSH6Rxvlp7^ddjx(y z&-VJT0?+q1H>+QP*9V+406V^td0}uNU&ziV5`{<6Uzr>-_o)ny_mOyS%V$Ezx$eLx zs0({3jN|0EOam(|_SD_)@`CI*;$n3GJncg2m0yeG5OP!_`4=8#qp6bjGM*PaF-kbu zZ0d6!1*+LBX<;5&R?eaff$RCt1|>Z%C>!d(S^nbOQR%eXu}Cj`p9F?S?D*}2;u`s^K zOB*!hADt;^2$mW-mX4u&<^u7PIL`E3<9$Xl|HEq9!o7xUbj#&;6C9X8SdIK7$7P+= z)BczX3Qy9r*QbYbEEH3hy79CqH`vcz(^Qo{$c-oYDBN*5N&8?n&gOlLEZqaQMP;}i z)=_Mno%($ono1Y!Xb-%{wVE;otkJCXdU32kjGq5!C^ zRLbho|HQPuuA3};{>x?`*{8d^Qctk+Zpz)H<4rthdqhD`G~eE+yPiq4a^%D;*1Z1^m_R2VQLx6cD#WuRa-dG z)sf}mty5R$`So1p=~4)Pbrz`ZbaSeU)6Bn-BeXlT z2_pmEDH^z1Ig>4VT>)8ki*^RIqH0mI&A|R5z{=B1)5XK{-0Tu$mwxlc`u<#XhNfHG zguq1=NdvKSOjOvMF_;%Z2aAOK?6p*L&6NBU+Z%_l{p+jvtl|M98Z9!*wjFf4^BOx} zDC-r@*_LgzYcyF?GPB{(&8hYNBxE^*L0Z@Px2EAoZVkUbN6J}RzLnJqmBrCd7~hV% zeW&+Z(3HA-ukEWBWliB<-X@Bluj4)Vi83bIzTQSGjGw6Y#d6h(_jSEiz7|#1yUqF4 ztCEg}+9@xeI!415yV^(77J1I}*gKGx&JFz&%ZZOXt&e|f>hbGl$V`M;&Cbd~tBD*n zr@h*g@2^korY%d)uz!1#d85LX>;?eNswN7L_%`zVV#4tLJ)16%*~x|&oML==@srsi z;(4@FG`hs`UHr4{4Z1!z6N5bDSy82#Z3UqjuggYPo(!0BYs*b+O*JJW?11O z9Jes1R)&+cVn%cZ5y|h%juqU!xHmQtK}VA?YOUTGeWxue-3((jDu3Om6!&U$%xH-1aYF0X&A`ItG{c~6;l=X=gd z;9}}D@5xq>dG0$a7T{0R((Cr&95*P%@tW7pB0pG*RUZ4Or_BN%UHg=}hGoVb+{3W; zXT|Uj=ifp*`WXWT!|@w*5js5Jkf3cEnb zFGO&W;~H33E^XIAl~xH~4jGnXyIBv|?cim)}36^}R8Up(8Ixav0p|h0(bWgA9d9JroX?SdAqnu z%J-N_aTv8zB$y8gMy&Mi4YJ}kNxd;Zn6@>WWo5-HtoM0u@jwS(1M z3#|EWNYTjhM=AX3ERl0k9p9g01`1VoNz=CVcS}qC!`Ixq8IbU ziuUzk@^l>>ej|J}0+f^2rKRue)B_^RxXYK43+eYxdl2=|bTC7}kow~1rV_JFw@ZAk zGKhPYKc4pICojjq*YkJ%D1F=Ix9gS5$UW$*n=)lj{2ML?)oww;?oBt(Zf3M0LBV{^Mpr9EzJW0(#-2gz7C?gdHEOiB_U%3faz~8SEaqGKt1Z#ivDf+(7u1$)kmYZX0{%*e zGff^|;*T-6zp98D-HW;Htiz1_h5lFI(ym07LI>qvLl;j0w$~UHo9g;;sIE7!1=CeE z*A?+eR4M>vMy2c%9V5}aKmC7sr7Ly`C463_Z2|(|;h?trw7A%?L?z?OqN-QWtV`1H z#caNW%=)vR-G5{aB8aRx(&XHjlux@<7v9|NAl&WijeuR;Oj|p2Eze zQ~TU-t;3f+1mqj#<+WAjtv6|$2U4k@Px`9hyWad_2X7E?a8uqy<`_F)gw%>RsYKeW z?_VFt8f9C8H8ECi__rY$WK%)zZ9O zYO$VB{o_VjZ%B_5mkmL?Tzn z<8$_oyYoBa+_2B}cH&nz5+T3m!UOP6%kxY5=QD%Pay_uk9m|aPyQPoj!l`Zxu*q*l zeHHoK@y^%#csq*fi}CQm^W6G&RaZ|Nc{^y>8XCt3`bRas!e&$RPT1iGKexOOz^3Ao z=Tx90>bk&jjdazd5$Uk91zk(E7xa8`asS!ehw`;xy}sJD1XgyKBb)#K4D}Wz42DRPcYjDhFOPJ zHm}cljWal0qTX#XnBrevt$kLiru$$Q;I?$x?zAqA27W2xFP|;Fq-$iN(9?D-ZHTA8 zSD;hOFHiC<@jNdaeL+rMFWoR`kb8gRhBO&A+$)W{HnhW;>w_dSKA60e%Vv}PQ7j*> zZ1c}zsp^1)V$!E4>^3;VsvJ^&gYDwFc&94zcubGaOh_oNFEYuVHS%wa(w|I#5wB43Ez{Se7UMr*xIDomzqM2ZBj4rLhU) zXKf?z>Tq=VyK@|$+fAo)U@BK$;!o~J9PMWb;{A>sn$ z&cimR9mUf`0jmp3SNQv)^BAb6ql>6!n5Y}j8`-I~VN zCWe={F@d`2!0)J|v?vOh&qOr)1oS&_U-tITIoQDWgWdk>*X*kU&KwylrP@jr6C2ve z`+Gp|hv|`U_G^^xr22C3wY`14?hXnx5&xDyPg*0$Gy@)&wI|aZeUG~Y5Fk{2k)tFt zS^h7y)LF1D#zP;^oNp7zcA}P;t)k$b5vPx|;%nKXYMsZRSS*;CUwN*dK=S(eqn%QK z3{{y!Ih#GCX+ZGz&lD|0R4?rmr_|}FY~xT-s;v}R5jP0#S?o*!HxrO(1|*y5lQ1{R zNG{Ut9Mgi4X?C*Hr()k+-p2p1lLNwQp78E?K7iaO&9XqGt~KAQgJK zbN4rt45|9f??~Dh|Am(LqX2O%z)Kb6V{7TZgRL=4D+&4rwl8X&7dIz`vw1io_gCJWdhjhTl0Q! z{@NYB{TH-~iAt{4)SoC;*tg0jl}lsYo9`@u!tzk!R6RiA6*pQcsa!P0dFC2tQ`avY zdq>lXyhC=$cdU7Vs6j!Yq2ylvmrR;OeES60hHs>M%ID9BE?I%BkRl(T_wsS@DV3Pq zhg)N=z!CWQxC$E{ujIK)RvL7p53vnD{iXS1xBIQvg+&E83rxUVopEy+A{D#sAzt@F z9vkoTLe9E%z82P)T-XMr{_cPVuH)8p-?5Ay$8~ytiFg_+9{FM_9tKL$y&$J2zixNl zh=>E|g7iNes&ChCJ_x(HyuaSWxWYNHjbC541y{UUwz+^_@&rDe|4jib(4U_=e(C;H z*|yg)V)y-g$z%xnSjPyUiU{hLQj$G3C+a%I>1&(iSn5j3i?h*NI9K|`Z>6k-tt;h$ z(Joe|nyz+x$Y&tBwmJ|Kd z^AK?-IQCLQp;E&NAfP+n<4jdhb!1U5=M<6~(wc6N>wY~|{i-<6fRdi>?_)!9L7BM> z`V8RbH48Q2OKe^8TH8Ai|9(mVrT9BeO+e&#-p;H_WmqLAgKVvhG8peehkioAm`6~b zIR@;<$c?N+A{qK}V9#RnKA-Hp3|%Y(ieR*Yldkat3K@C0oPHx#@Ql;L{(;h zxgQ2tSmg1&G)3)9&M$+hBpAxv$ar3KRkZB1uAhyQY{U!cHeTF(HSD?ef5l-h;m~e# zM|QC^iJqRa1KM}(T9*CxynQHU2y6{BnahR*F?dsD@H=S1&``?Dl|<2XYA#fU*)psC ztzC(HFmWoW)TE{KFrZGgJjod*)CCl2grKrvsYtmGP1oiX#^hWIERN>MtN&KdO!3y^ z22;o4rd?G-4T~9TtDkqfOP;r#_ByxGaYYWjO7NROJLT$@&hekFsG?IbELTthF*S0| zQcd^x;`*tkNfcF_NHE4PA`QH6W zcn$ru)0^Ibnh%ZCLAWxWb|cwnj_%vRwD~`YB@m6nU`jw5Z8*gGW z!HCAQh+hsPLW3oT+)hZJZhyj_OHeN+z`3G*a6bpLsN1e0Gx#k^%N{u+mLr3PTFKpY ztZ5){1WkPL8*A1iAdNvI0WOO=sQ0&%_GNxSAInzh4ETq98LJ4XT|zyJASO215sF>W z55(#W31!hM4?Pid-IMRtzjI`n0!Y>!H0@D(ux*FuGl$UJQcCh1cvVtZT1cLx>OxCW zDvoAbXK}xo@Ky+=fNts(BE4x48Wo38^CcMnwON`7bvI%w_fffSV*rYp2xEdwy;{c< z&pZ4za=*tEHMsLMVVZ@A#-*rO&C3ILnu_)B^{I#`dLFRebD?0bJ@UOd$odv|{4uPu z*+N$ge>V@wmS8PsA~dr{J5ZqQhD62{yOX0Cur_8IE8db-(nr>=t+8{48&6CXIdmXG zP7RQsXOY+Sh{ZCTA;0UzFj=YhDA#;~U7-mjM==a^F?MAXkz!G#j+5fl zitjH3PzlBOE{6XPUQ_(-UFcIro3?}2tpM6U{h~ESF;%FCyWNOU;6^hR&h1dQR_GJ! zQ?LXGkCTBDwGS$|w=~ICs!!Hu!GiFbjmnT!^ji6z(NY0C(&j73aTD_-MA0n#Kk}xL z`>JfGarw+yDaaH8lTW}8vRuy1hmWwSG`xX;b1Mide<_tx~{tU$a0y_?BTdyc&k{#sk0% zZBA%j_Dt?&h#db5%{=6wf6FzG4&9jzT0|j?4#VJ<93InQ+1PDuU&|G)ue=0H{@w;0 z9Dl>6N8^&pjutgc_;*2n^d3O~k|Xadh^{FfWRd=B-k_|Ak~f6YmKIOz_ICSow*oT5 z%gj0~AuS5L#gu$pqT{bMF`+j0f*sv4ks=<+hE2v z?j-O=I8je>YEWP)YpxUK;;3}iwFImIS=B!;sX(F8GnN>BUfQU@qOts#sO?WIRa&vK zq?D}NKZQBE|KJ**dZkwIgJC?6{{z=R&X3n(uq}e%noKGsxu!>^8h#urtTyh-r@_&m z>{n{vP`F*iGi3OdC&7r-D+j}u-Yha>LmqDmpySd|m)POPXB(wDS6pW}lP^QLR0)}P zV?wa2S(iz~%a>hq&Z$_ur0>v9R}bhbwMiB+vW}QhZk0q!9<)yAD*$|dn*H&Z_TdM$ zxZWVAhVHF?hzNx7a_Z+Z(0zmcYec>rnrHy_u&8oyYkO%;6tgTjv{6vHO#R`7i+e9P zRCPip^l>=jypOFCE~0VM-U~H_JePd5A0*B%TnPWi@Pv$t{DeVi9fNZTir3Mybfa;H z9}~W4ey2~b3Fl$ZOg$}aqS37L^i~^GSn8V9kxlgnzhRPA062MRFkMNFFw$+h<2OaJ z%Y{4{^Kb)9^V&7p&}U!6Y~?IZ6h0a-$o5X?+kf8vMuE8TByR(EH`?1s`#}tCi^*GQ zny!{f5q3HeE=pA;pc^wphpKevB>W#;QyACJ`sAmyb!sQf!>8RcVQqn?BGi(jZc%j5 zLBNnu(7{mEnq|CTk+h%^`mfZSRW+hrZHH2m&`l6NCc&lwNj!ex0p8Eze>s@{P zU$hoO>b2vZ`yst~nejll(p;bYeFAFvleo0xJE*b*7id97w<~4{Y{k6_j0S%**$%Hf zpM;?=K|V)bHo~`p`bwPr0cX-i$af5K%vd9AKO?(5H+U}QA(M8*DmR%S2NEnsV@FO(Regt3pHu^79*RqgNeQ4t zhj{2sP+&UWS&?R^Lm1P(bAgcXPinxMida&RV{k5_Q+ft5ua=)dqbHU&NA$uemHn5Fe5t;45Q{quhU* zEnN*YZ|z(aBH8aCcEv#KA=(lBmsz3>?ul`y3GWH9zmpz`OpD=E$fv8#la83ThvZc3G@% ztl1_}8&1O70>Jufp?KP(;#tMGRTVofQ z^gxv*Pui=dqGDNHyZ1SA2IrQ*UV-j;TYlj!%4ybw0&5_TK(*G-?Hb$Idus;T_aQ4w z{!F2p_F(93;b_KTo2=Mh;vxrn+lp-QpvgJAN{y$b<~r`80{k$vDoT;ao#ewyrq&}H z83LZg{$Z*?J)bz`yvfF0qXf8FV;LIcxwxt3vH?T~T3sA(%Q@puYN`X;QVkAy<&q@7 z;}EXAUg_;%M?Jke7AU)3nhI;Un`DVI=l#mCO;CjeQ`ogOsOGBnB#vj_>AW}60MCZr zci8}C_B&L5VtdAKerNeo8MB_O<2YNRE&yk8_&u6{eyF}FW0`>ZOu0K7`-oB&8gUL{YaM1G`2HE z60??-=~!yBooyLcjs}Xs!+?1(cvD%i*G4l7ASohAn~pIbpd%a)JTX*Hd=}xni;OYGhMa zwQVu#jIo`j;p(1xDIS#9Kud-##(Z;ZU{YD07}l=Zla1vyqBV~HZ4wm{114F3gFX?9 zm#Wo1-dXuQ34DRL8+dpo_nGnaN5OufI_z`eA%?g7W-iN{m`a5;O@~Q5hZ-AV1m#C8VcbyW2)juWMB|C}F?q*nVjMGf#3}SIoI5p>-AIB2NY$0pj z*t;y{xo(t9#4g@|aRKG2;`g)uNdwo11zqvlPt3LLdhmQ(LSjEYD z>J>Kdx_SUV_nQ5&zwfKVi}hF8PfF4y_ZQ;1%O9Y5eWAJ{C>6fH3S~13KBY4Md`QB- zTe#;vPq+GdfIqYoN8UQ%3Etkt;N{MsJHgFSU_MT7qu&5bz!Cugl;PS(GVODFiVo;t09|Q{R zmqD5Eqswua0CIl9K;>gvG)zX{KVPip`D(*5Xjo0pSv8Z^ue_rs;k%RX7w^68B7;?X z6+H*)uuy*~YTFg%uk6UEaMzmHr#Ikwm;WS7o$6b~1Wskm#bFuO`?*ZotLMs`WVo_d zC4ZHh3tuG&M* zF*|ghl0g{3SR;g|dZlLS_bSy|Vh#DNPDUNH+^_zSA4-&BndkNtPKs3$c?KFK$_cU>@B_(s^`4&J=9Enk$WZ@hT7RPW&sUSNtx@}*Y#W> zmwPQ6C~PDK8pq0rA*1+;6%&) zk23S*RDVHOrJD7!(QcJ|aoNky9qrtjF$^>=7E8PX{_a)j3864p4;xMc6Q_roW(|Cj z48fKLiL!K{?hyX?SF+t+I6BGOITAxB?c_}D3Zhlmm0LYeneI>$=Rs}3_13e-i=>e^HmVt1UN8H$PW?GZ=~)-NfwxXxFxi?p|53%$ z>O0J`*Sm#1u)+7o3vZkZ(b31t_p{cPN^3yfoR1fPKXMgxaH9I6EjYnVjS5+->v3y zB?KzfZm(}uWMQR0u+3UYhGJp}e+|RtwbxEfMWHrWw*)08I#uYJ{eIKEY{&DhHZ$f* zWzKvix9j*M_xmI`=yUf*pEU|WZM z-razC>v}zCpC^jt1^cE2{rvIk(dWJ}fhah_tA&OBdGx!T_&i>_4FJNkTi=R*R}#Uc zL{XN85f5~lg?B|WAaNIdK6_2$+;qgW=Ycd@Vw#HaVn%etSnn=j3@ePMuIAntu7%@P zs;@p5hin>sDp5Aa?pf0Jf3hXH+}Olc-~+MyH0C@^r{V3bv3;o*|EezU@{ zRMt+UqYT_YmaDE0XDTcGpfk_|U-o$~c2D5k5gJzBRfV+v=_cMqJ^DV}KBb38*Id<| zjDI}-#MFQ`{sSR(a*i4ak&TXTOkGhNaoA=m1rA*$wu;P?>JiB%#92Ju!?*ZL>FDhrPYBjQZffa<* zP$0l_p)7r(imc9y%-FA$cCAWthWOm&UwAz}b_B@}mkQs+Vp4HY4=OQhU&c?5jN1R~ z&4es^`u1GQKyh%YWQ=QH+mVxmifhp`ZXJ8(>K;eT9sTCl;^5Am%JNwcsqP5q_4MXU z!P`6He-B8ghd+YZq?(YlS_^xceGWz7@%nwQR+3hT4PL-aV|(iePMcTGfknNdUMakahpIi<_=0{^OI|BDZ{iVa z6%tm9+Qz(z@}Q1R_mkwIsoauxWo`EH+`J^*dn;>2!OhZ#F8FS@0FG3^$~bg0HDK;~ z+fL-$yW9Pi4X~Q?|H#5H3Aw&VOjD>6IZxSzaC5vD988``7EfE{x_b1S=7S(Yd!fte z>$+D$u9b{ZxSr7_zw7==*vY&|{Zpa~#e-y~M6*cIa$2kBx#+XZWC3bRxVL1?oc-p) zz1w=c0NS;A<@DCRav#O^Q46dcjd2#(a@^ES-(LIcNy%a>j3(g9h<2G_mh^sJ{QQfF z%JiwNIP*IsPhqGN=)uhStfi$XLM{Xcd+MlNNR-9-L z7XyVnqj!Ih!Dl?Dx0iji)r#%vc3_b-Bg5iPG$t<2|2wa*7jX2t>{bJa+uEk|+eN73 zE?EVRKTvx5c(~iVG9|3C4BK^7#2_zp;{~cZ!6{b|$e{pc_{tX8!S*iuN)}dAN!N`uj{@qYJRar-ZUvAcd z?{&Q8LD~rWQci+i?0h|s7*Y7zFl}y^2pjfRBDuWb6{jg}GE&t6r(zs$k-M#I~hM0O6>_O(^V1t+N=n;ec5%XEyOqA<=N@nD7u!V@?rak;wuOy=BfM zUmu=$2H~)3m`^Dq>_CHXJ!pN*9`F!mwcM*Yo{bXe7 z82ofM;LgY%J=V?_Ep2ylK0l}KnlOLg*v2N&me#;CB5DHZu^K}_*Jtd6CF(L&l*{H1 zhgIw{!7{hDUBwvhZkeQylB&n=&$<@9H9IVEq_Sv`nT!PauVBDKehq&g>099W9YT+I zgJ6l5orokMN{u{ociHS#^-1Wa+Ny@zF3|=ix(}{zn9SXSF!QY?TK|DP)^jFZZg{x8 z+Pl-&>vnyyQE~BEdJodoZ867^Gxwav=kscA_GLINt;7@F7UTi4%XJ;v`Yc}eLtpxu z`M8_W#NkEdJiGDFrlN08l3?ALf~al~k$PTd>=a#8GG zYqu>j|LPf-BHgNBp)T;~LWrZ4%P8-UoTyg0w{xH$+0vzJlh9Coi&cmm*Gs0Vqi=tU z)V%ICNZ32`aUUB};Y?=oclh#1sH@s=5d}hDmpSYJzJ^U+HQf?X@73qs4gtrSFi1BHXf%4xB8!K zLFZcJ=^x|I+tu1ecZSbnEHw98j@dWzW{b+B9GyCv_{6=n1?OmgE8*Zv}{HOhHwzeLWO2ukpcSzwv#Kevo4BO|S7@IGt`-k~DI<6g(C$dRRv# z;{!wPYsD`&YqR;<#)(@Segl+$8Kv8J7Vc-3?v1xx3xnO)q9nBs?q$uPBBu{#kpClQ$ogJ)WFDe{`!#9X%cs?J&=pWk+Kg(HHWyRsEENwj8DX z_zVCg!0I)TouWD4E^)23{)$nDFFcpIuYV+E*JXWI2scmwjB*B6Y?7C@hLfiJJbrOx zCCP z0a;-o9S_;mz~`^X#jVNmtv1)5J)JHhCrrtF8n| zocIu%oGKpaMT$L>CMhSB%I4TFU68Jugr^W6sX7t$OKo`0Hm&`tmzAwAzE%U4N*0aj z{R?pA8SY&0gwUt{qd}CW>ihkUM;Gp`jSfl?3qCxlICFMR z&}We?#I?w_+Lum@Y`wdDwIs}IskE87Gx3(zEn20Dy=H&PbJU|olvD4f;G{=1Ri)8= zCXx<3mEBhrhACS%SQxpxpwwr`Ono!ImIk~|^cQ=&?Jx@}JTtl!G=kR)*;F}cdnV~d zNddhjo9!NvN%*C1xW+0Md~$Ahs3ra>asDR!2eh}_G|GerwR6o&^T0ZV3qxtXXbI52C=S5 z)W69D%zFI?t|?LTUKF6ni^MIfBDn(UDx>>ZFgg&lUvnoZvPK0$;r-ck$nwI!DBP(l zDf!uaDXg;OJh7u?hrN<)8Nwl3$sgh8DGPK1iTTmEa7kc?x4)AdIR3%EXem%`L_b@A zPDLrQeK)L4=$!BBLgzt8; z?S^>4nQTN%iczY51Xf#wa3tv1-ruSD2L!SM>6py2Wd!frfRG>5Oa-3@(0{0z1Eui_ zMSC)MulRJvAJoiD38kV4IX@b4rNb&(A@0c$oTUN~TMASVIUNX!&jw9pZjhJ4m4d=* z4s`;XqHwXO3tCCLLbhQiBYc$;y}ZcH4&VylAy5yAxP6#{#pkj6)vvb_B2md#^?DF` z2C+D`OVo9k=t%WI8JH94F4n3LRQ~jIrjbEvZ0uJsUvaG=hNLv$5VZDXvF;;M2mO>z zci%CvO`Mk|UVgn%qD2kiu;dlI8`Hd}B9+uxrQAwJFIvi236XwhGrQ#G6U%H1>c}c`8icVG3d!M1-~; zHsuu!G;6_=d(B!}KO~ACx>+!oJn$3oNR$&DF=30Jjok(!+%yCq#68Lgu{u;XERW?* z#p3gVUXmjLeXp5GM-i|1XdP#1`3hjA-JDX5(>lQ$hQSM9E$LZo=wTEgO&l&(%JQ3B zY~cIK1PB8&0hf-gA+kw&jQu)lB0?@*!ERRgD3%&VFz#ft6CP>5HfgW%3}GpNlCVkU z?V%|E%Br}+jgF$Qw6%;`Ua9TS#5Hzf4?VD@bRehKe4$MMr;l<`w0BUtqdhjIz033v zCmG5e(Vl7i-LED^8wo{I{Hw|vSvoONIzhJcZzeANj0kYhXO*68H9G_>@sSg#n!6FD)&iN zFbMR}PHeh1!WTqd{%H@Z!<9jUoTtf#@c#1Do7x(;u23osH}-BggpkHr`Uf>rreh=! z)9Wt&KH$U*Q}`AByO(E#s>LeTI)55=et3Sw9Pt#L=jlsX_-RhX)q<)rZ!xhZ{M=5z zM)_0%&(BrOUI-sMx8!ZzVHCXs>*pUA7Z?UrM0mI4peAoQ4~E+WgUkqeIO6J*%w;7_ zodf4oYI#NU#3cQtLZn1P21j7PmcmRawYgPb6mROY?62csUQR-IWS`Jaf$N(0RM9@@ ze;xg-kJXSJ0Tt+6!_GPJH6++fuu)Mo6%=vERDYcF;{=B`K`1_i&A}2vBh(yg&5-L_ z>T%RKk#G15-Qf!M7ww-cf6Wp5a>uww>iLMZ2aZieb8&ufD)%s~|Byy;3pqtd37%_F zO)JuixR8RBOd5cq88kZsy6~G8@mDLAk@UnyJTlEukxqBbKu4`49JLb^;Eagz{Lo$q zeIhm^?TCg2=u#Cyc#CqPqLA2D=9jZ|8W!X8pv>uZ|FLK*7caS>_0!R4^6!J|z`m_b z=DQhlK}PN)S(;tHRov)u2o(g4qHIl7P=-3AIMPMEX=aXFDyN~8^JT;&N!}zXG^laX zB7J#kk)1O)SP=~ts+`bI8PDv`drPnZ!8H*RTA~bC6-j?Y<@mka?sgsXQeqZnM&Cu& z+;Mh>8oJ##bP&a4m1L!i{QJ?OD^lW#g0{EJ-Z8&EQYB+1(7)gagDkuGfOJHCl%-Ls}e5INKFKs+$Ud)8F2iL ztrdaK4=u+*YatD-T#km*Ojko85v$VWgZz6HE&7)?M_Agb5jG^z4A}i^;Vvs>5}i=s zxF2+f3U_K@oH|`Et|h-BZcF|`G{LE{COq)!ufov$jAaXhd~Ug88%V@>1w^Y%i!&dS%k{|b?QE{41DM>z%z^hA!gGJkd#=ho;s&g%zO?q7rgX*l2BUU|)-rkvXabzv<9m%z?msZd}f1VDDO3 zGO1;|kc;0rSL%I=J+_*}MGFX544O>Z913#$E$nA_)_YNJTkR;^Ot3gHL4&$AQ&Ab5{33)#t&LjmcLtK-ZQZjpFX zaj7(%EMHcI(Z3cDfr}NbRuo42y~7OXyXJ#*KX62XynwNuk^Hb zY)SmHAe*x5$Hda{kD(gUkkakH-L%VYR^A%hX_p{fqK#Cf9l6~mgPcy9(ai?IfT)Zns= ztP|{7u~%=A(}YJVhBD35~wcy31`tsE?KJMRR| zRbAQqN&|s^-Q@zJM{*P3@=<)1aPR}XYwrVY(mjYyGG>vPeIT}ZE=c`~cPd|RdVZ;?ty=N*4%~*Se&@xO z0eV9UJGxNMjJVF)4*Au!3Q2QdyAiMLKfdvpp>HR8ucZH;j4oc7jlxd}!bznRX(e2B zHz&4P)&J0kT9o&C*gyBW(_^H0Vh1aG^)&TkkHMyU@+-gO%UfbHG95^98zqI9AAQnJ z06s+d6pAvtDUNe&zklb-nG%=c)jZK`wogAu-_xpv=zhNtr=^S!tsVD$mr*8FO7l+o ztT@l|<%rM1vWqCTP!2=gh^TaXVS2UTha$QUh3nUa?abZaJnk3{t^6|P`ton?>)WXY z+}_@8uYTIQ-~Ym55YkzJ7^r6Gi39?dxvUGX(+B^Fbl*uc{fp{Qh0l$qRF!=KXFzq? zwXhZ)P+mDigW^MM@(@O_pI+uNa>UPxMsa@IJj^D!s^HEhGtEdNM(~2hG7b)FLx3Bi z<~K}R)X@?mNw_?aD1ETk^&#=aAkK2OxWv`YTw_f3kNNi^3x1Bb+k|eJFH(zSk@As^{VL6fSRwg0>hqg4^ash)pvbCIi_WQT zT*VfQ=BaFXIf~;wh_SdD&dXYL10p&N=Ja6h@e=%ezZZ zqa<)Ok4x%Ow)R@?T#P#}r)iowFR?qvb6drDbE0=RZD~|3D^6lq4~nAcW7csDsJRi` zt7IPTZ(_0VVoHYoj|B+IAoqMXRL(5*C&c~|nnyIOF@Js^2;lkjCS<{uCol+5h49Er zVyXwY$HCN7M`)U;HZmFyC22$&vhzY6(C5{mgS2pezbB)RvEL*~=R___gXqomfjpC6 zrIIUl?Ju0eCjXo!_f8EV9`IoxKK9Tp>v5;VlgVS!P;akOo?gSBcR|&S8Gr6xp=7kE zgFg?&C{E$vLYcooT)hp@;C)JUccQb6I6S#3vQpPolDDQmG6p9F!r?6-oF^z?M)pNi z?}wa%c=A#cSVKn%<@K^X@SMzO7uRs(BAY8YVpi)pCnwOL4r96(+v4s$x-x#_b@5;r z^CZNzr$$XVN7xDRLjJTJ#2c)FbaxY872qM2RD{^jg&tx%t3mAqp0ZswDPl!$tda($ zvB)abM1u_ltvz=R8yE}YimtXYF&nAD^9DFJb7wfB-F@40i=TnhN}Zy4J; z29bQMEny%w@Q+NvC;1*d=Hp|mX0a9;N0F@7PkR^bPORGJai83@ z=)q5S8zm*>+NL}qzDrZN6|;pi%XzZAICd`$RB+QST@%TyFvGZy8D?$4;ATeOY;yNv zs=PkgpYq5vtcDNz=9rKyA(9oXciqQ5fDFxxJ>@Ykt3&=!@vC*6_lmuJ&Aaf(xZJ-1 z8!Iw}8-cQX8IYR^5+dUF$n$1XD0y^Jp9N>ATYotKT=zYj(XB}Hu&~i=sa}xLKE`an zwvuiTy1i26yo&8;JQOx3*`+9tUp~2bWEayk$tIb?Ys3!NnpvjzrU#IbZ8%AeMGDnj zebMGlc=;+kT;i^@-BYa|Nb<-mK|;qr(FnAD*MC@T@w5LL>na-DL)>hM2opp4g<|uFm@oC4ix8tXC#wUnc?7olD8u0!< z%|h&dHO3S?-a+u;%556ngIny3=7_a~L>3$w&@vlX=#x$z5X6*_k>UIF zhD}nWi;vzr11}W9>0;Ix#_AZo)*IYbNI0xj<03iX9;FKO;4*7i$8jR3TpbcN1O3dv z=^;mlD)&uFSkX25FSExb8^EnDfGexB0$iL~^)AB^3BtR^IkS&23n6&IxAi%wyUGW$ zqa%b$OIQ-9`9s@1#5WJx59A8Rr@(*1UxY6m*fOKO;ezp;RR@$ZbNBSA4m65)ehb?MB$080iMJD}FQ6yFJ z3KZTBQ?yVLkB9+vT0Pv4DRz9F+MRnNa26f897TnKR9I;cby0}!9>05|1-~$m21+6( z=}sfy8vKQP5Ksh?$xL~L)F|s0)#Npg?gNRBZJ8@4DuvUe$(*4^sS!U$Txsy|-B-6J za16z^?TR%tYWav{E!nL%@D7&_uiC6<&IU<4vV7_-dVw#OtWKjWyf(nOCx`$U@w#r9 z&Jih0&m~8ovgRiHi+eiuAk?^0d$JEc9l~MQMTy)us@N#3Vp%bV%}8~Eg$tu(NhNGp zTjIbgd5m2T{U)eBZDvq0TV}0C(>vaBwI_-XTU>H)B24Y%k%M=%-4grRS)`dZFC0^A z25j&4+~c5AnaOv_H!YGd;r`|gdt35cbcKTuFRc*j!U}Vl_ws0kiP@#RiuTlTv%35^ z+E=WQ?L2I_*PZe}bS}0hk{sizCqeuN|3(y(BWs^$e6!>aywb zSo!#YTZOn+H@+{NCUZpfdYOXWGh$e`+v+(4|wI-n!JJyX-p z8>(Nc19X^GGs^mtg0|+d%g|uLq;c%xd#UEtK)UH7{m*L$1IwO(ag8bA<*S8RXXK2q zNZ$e9_JGxUcCd zWt1c*os*r4}Gp_WHJol&s?r{<8PVf;Qs&_l#yWcvo~?y|&`&1<2#fHZNWEWyFmZHq|y@ z{qRC7P{~y_Qv9IeWM0KTj#>O3-l=m3leojmmj%5MzXz;WlgZx@qlvoWq@f_HE| zZ@ABd={{G-al}58u!xwo+!MB$n!LYj?_6>22*|p@yp%2gYftzqCE5PoVo48b;n$F2%+0sOe*QE*$Y=WUV8j!Ni zS0*X&PE78#xNKozFM)f`);K4MzT9sAkNYRsIhr|6m3v&cYFOxkPkli!f$xVs7Hcnf8P4VWc!QgzEC`P<7gYKei>oeuKy#`F7NtL)!RG<;P+X`#jNmK- zvpLnF`6*!FUdEjBWk$=TLBBmrBV$U$Jf*n6q=01UMLVJBT8Ppt^@ttp?`1FR^PDt$ zW@5*%61N2?DDeU~1A3*&b&I(txZZ^VLb5VuV@R#B51qoaa(IV&y4AQ3sTtlx;h5N> za1nsS0&}@4^>^13d|9pJi=7LWzb$B+y$)&Fn@@Evi|$(BS5#~An{ED-YN)z{ME7W`-?Gn(tr@Io2a2!xbe^mnJ6 z8?dmGfRYXDSPjY3)*^bQwHT0VX8aoL(Ih}xXn`Lu;nPI|r3`KEcUN*Oe)h$B31i`| zfJ)#GrZ*tbU$hC#GYsqc-%p_vzkZ)>FXIWugSaYBMwAuWq$80MMCVU}IwpvwIQ|7W z*Gc(zcdrRwVw|*LlDKj<@wwbv2w~SsdEj!mb~G|7PX-mqO8jC9|XL(VMeP0zGQ z@8^~3e_QU)T#LoPkj%LDz=8Mf;9lt~L=e%;>D;mp+TA6XC*4J5;JxSf%II@Q1@@C& zfq5WL^0pgYCY&|roe;h^Div=M!!lAP=MCGy3k; zw8x_+9p`G-ELQO3UwY z*m#6WhFdS3EER`B&Wkmt9L5EGicjnwlBs#MN1yDHT}Gu>5Qnhp%>NyIehUS`iL97A zqQtTvcv~eNj7f7 zT!|h|c`d->JBV;nnU}*bd)!&`PC873iV?v`tT)u>9blW{ zBtsW*IKH6e{RI%Hx}&w?UthZBZ+v|zsqBO$}`gbdeM_Ok#-Bj;xYlbce+D! zis7z>iq%A-(4LzYHsQD6*~-GEK0c`HzW}Z?GOLQ5d6{n*+(ML0abfO5{?G$d?C6B| zsQ8xSVeKaU?0thy2`Ynp@@)B56W^R;m4YvkwA#$2jUd4bzIv5gJ+iZoPfFKQAN;`W-*}d1=TCQc|v$Oc!Vs8y;h%VD5oy zVvu&zSm2gBDc2l@D1oG)gpG!3zoApeRy=@dZy#`oeN^g*b^Ezd#S z2-!i36-0$!-`J4b_`9~9S)di2PaFDxQ8AImlfkg{Sojm}l&K6U(@N7lY$(Wk8ZB1E zu2-K39VfIYn4a6sE`B_eh~rj_daW_^Q-}DiK1>zH}k4SdBW*o1$;{+vtUqK39WS%v~$ea5$P~{aO zcBZhZPGB)@k!QD-uVFl*3AOk?CRs5+@$>BFMb+o0Ib@p52NlNs%a&}eO8nHL6=05H zA3%kD_;}2$`-b$#l}bFnT0u_vo84ChpZsO<_MvZD>(+F!Q2Wa@7qM6R$q=$tc1^$M4w%3ui42wf@nKm(Udy*1(3$R9zR<^f z;jb&tLsh?~JyK&_3t*j+ju#Gb`*}`V>`dZ>`fHnshUCJF4?gFTXz>*-F0J*jva($B z#7aw29wEbuR-a05$pg%%AyooErt1H2Bj*e2C|+Wrlc0^M#HeZ9L93zm|BQz+jrjwS;XTneWj3{LpyBufb?#T$sH?e1S862M#$1*HB%-`0dZ zgNZ+~M>LNX^{pz8wgrd4la|pnYOMy-&A}MZg3?7$k+cvd7(d@a*V^v8|FC}J-NJh z0ujv!D5JQeuB{zmyoj%f)lBP-W5$i2HIEI4+5LSv6XLXv_eGNwHB)fpc$B&`HV#R> zmpI@RtRR^<^LQia6@P%U(|jHwnnH={oaUAKn!nOFl4w zGxm0Ra&ohS+w1pN5TDZ4eh;a5`UYXnv(qnpZB@RD08h2-9cF9tziNHo3(yK&=4`?P zAalHU0oDL!n@*??;Xc5{0^#jxBB1C`MJB{SclV*G4=hwq*=lsn2D)bT_foMQM~$i2 zLdBeFm)QzAYPM)cR=*x}?-!-9H}Kn*GzA2aVl@%BUY-ZAcfy1xRpyAcb%T6jvC;j% z(S>eIsjBO(RyinJYtl4N>VHb`BsY(bOQ2ocnK>Q_0S}4$gBdqXJLA!7QRTh8vSF)S zr^${7!pY`J67AsUpFo6e*!(+R{v_WwUn<3+l%^W!Q%XYgCxB;s0$!VAfUnQ(>%{+{ zQlPpgxNg5x^Y8vL9rINbZ(pdG@WV!MbD|aKnfJ2%&3ltI*Z34i__B7*j}v*Yx9|T> zHF5N#T8FL#pe^I@-##i|^#X6O;*W_hs|hjtjnoD>;%c_x{bu7rmF657v-<+7+(!_i zqHRQ0wn!!?Xw-peleigBP*#dH@Q6N{Jj1(_@nhj90-Jt07oV>My{+-ehu`XLiK-e~ zS-9$Z9m_8B!_($7IQ=uSfm7u04Y!!i1o-H2xKXUXXu8cf(8vC+s$oI;;oiDLG@hUQ zfHN;2#2BZ;tvZ?&S1YT?%rGZx!0>4gNYSG>Jh_!2xih1XL92(vJ?;wfg`b~vjQ{K{ z7P(>2(4Z_v!L|PFdji}zOtLvjluh@$^}-W&%OHQ zz)SqaeK!Lp;=fn*4PlFfVBHoDsgY+lhTsF6NLd3|TUvS@=AT})U-DXXPiYyF@e77# zN~jvl;}YY}wEQ#nSo#jUpt8}rBiuf<7gK2z8qXBu3lBVV4jks;Cs;O4NWv(@x(d+z=wFew|uPFFupgDOITI_MHp z#h#Xpi_HUcLy9F%$l5NJG^bypMWA8Qr|Opk58Wtun`j?PM{_Oyqyt(?L1oIx3zw7&*_kICaWPHEE3R}N}5CReH_-z@b^+p?y^LecuRjX z({^;0(&Br)Jv($AAmzu3%Xj!evf286cx0!UE9g?0^hpK@3=bdwNszo_58|S7EwmGH_UiT9+pZ|B}Gac2My2!ym;AoaXi z_d)DeG$A7v90W`7TQH_U%WXPD?$rywTL6Ltx{=}n+X8RIUQIP3A$42ixcDT*-QeVI z{p}ESUeN2-bZz}-lU63^mJrF;z8u5sH!tM__FkLoe=pDA@=fa3JcSp_| zgGpPu`DIz;A8;O)%$}h^@*9#Aw``wxkOiIibSpIKMOA$(0OhJssK;BY>&~OviJ~}h z8yiwa$=<5DqUzvuO8Zjb@ykd#bvIjv%F}hA(APU0jJh<}K^7BKT#BV?{ja~^@TvEw zL-?K{k=POp<0Li9?7h`j&Y$8{U8w^8Pg;NoTk5{=%EW~F7DJ4eujht*gOVRAkxWqq;FFYb7|fFdU~A!c?l-LrJ)9;fy9X3b!d zWp(jy@M%X&%m#K{e{!kh5L&8n;I*(6frwMc^nN2=LLk?Z1TA8{VWGt$QJz4-xPRGO zqL-<^RcAy#^?|Lx9j_Q@VVbF*>fTQSXMFPt3JH{c*_*lXbY_15G~x zXSWqIfINh-;4;Kd0DtZd768DKnBWR$laTVud~&1LE0^%WnLSX{D)EFqJ-1B6DQ2<8 z50;9bp39reSv``SjWLaljcp{>%)H+1hkkTqpBBk==78iuhk)zX97oAU%xWqRk@jiMGCEjRdKey6OYv)ydq>D zQ~nyAn|tEOzD83&RCHnIId4VHbu5>c5-&CGcRHB5=9F0$d|>#ubJuk|`M(&Q61)a) z=Zm-nCy&NJrsOp_z1p_hw)^mGpggVWsMf_Xr4pync2T(sDUm|}!8d%jjf0+$aE(gs}5_0BD>mp=y? zM1-h3U;HNd3VSf_wogpx3OdbMvng=$gms%a_C7=|75vpx9>LOKc^!EEevvt&tw_3U z>>uX+q?Auh5Hunae;@rsHN4YhSxpACi~+t>!~vtGu8-7hkeni26?n+Tk(OUg=3OOa*?Ax(g{BmD+pj+(S zjH_~O{hM0# zz zj?Mp(_D)TfHBh@{R#w_JDs9_#rES}`ZQH1{ZQHhOTf6i9PIvT)zUq#?*}q`NT64|k znPU_<)RjvrEtjaSgy1$=b&(ZQAE7 zxTjverwlxpUe)1=b)K?*+X&*8fA(XUn7?ZO5?{NQPj#|rZk9Q^TCu3OShN}&W{adA z<55go&#u#wi}LsE$xvKDX##r+cCLJKm>M>=t~t78q~C0Gr^6K4kkuY+yUd({qMCrI zq<2o`!BB~RvLM}SR?nq66NpYkw>L!Zgz^W8QJ=dNH2 zH+U|XiOAE{{JoZE+xXiwvMET^H5Ym?!i7I77HunUm8#>F7u4k@&0!g7QKzk#8F>9l zMx$&F%Be2o6WJE08Qzh1XHDutQ$6!q=?$oGz7i;r7l8G~%8 zj0z2w>(bVv8us=gw0t3=Gee@)K7uuBW@?;(y=>+)znEe5?K9P%S*RS={4v~JV)*WGF19n=L zgT&N9x{GcslCmm&)uf{;zjKy=cFjEW`xmHbeiaepz!~hGjp+OchpN4`Gbv99%h*%F z+V=43Z;L&1#T#mEfd&gj)TPaHag}VTtxBC42hRib#W6d@WIC2-Il|1>@s1_InTjup;M5w#6RU}WoR z-@SSkD`&h8W8}tfPZwIxF7Yj zv0lcfg2N#-(2@qRns>h*YZ*)PKC&*p?q@u%M5UptLei)am4JBAxctum+NTm4#(StP zIA zN`xgt@cQB%1@pqkcTy*CbDwHG$6eIZlRpe`o@yOUFP+Hji#>%wcRJb0SlP9D?$#Xi zXC$H1QR-kf{2OiS@vn*LXJ=KN0~oMUQW_>l6#h*hT3z#Y8+5{OMf7fbcacc^mF7KW zv8tMgQmAbO9GSdU?I(`qsiwBzdf5<<20pg8AnKJPnkdI%gckbj`im<&s5Li8>jFAddcl`l}w!%jMRPibqMUXOG^n`6Rm zA?Oc;YhmZT5q5dcUuxSDJKsrOw4Vrr+vg_T=HhVcfEqUWf=-XdU{&125`Cs)gv4|? zHDtZDV16|R>(w&xVxZy3H1o5E2K;mWqo;7dF&e!>D$3^)ZzqVb%e6Y zUG-V~P=?eUNmM{`^kH)RGDT;PFO#o(u$Ba%d;T0Wg#_&Iu~)A6a^k??pYv@^Jaila zp3#@G^S`$6`@f8mJD;GAdIZdy0<8T9%(*l`I8JQvX)mm?s1d}}u z>z)^VD}CgDk-Ude%MP6l2!26>;hgdfJhHa^b9eJ{(eNfV^GBzSmnR})h&Fkzig-Za zQxkA!T!J|^~QU!~9|=6blWNg%8e9CLQ@Soj#P_g5xmc zKv|(Ku7p}56`|DF?>{ovP%1qNa&VZ~Uk!x|Qeh{Ioe^mn8T(|=^6idExG%)oy<*;X zfJYc_>t$0>lopy(pB5i0JdD6GU9;zva!gn-pyW@R57O!k=w2V8Y@jw_Nxkt|z# z97#Z*#j%I&E?hq^CJzh|-0f%FGALIqz}MB)(rB&JckGu3Zl7ULCMAT>k2DxQ>C|Hq z!SS_fGc0`AfyhXobCK68`?@LB2VmpyECSk7O_LF-n&skpzu8IZJGVwZ~@;uJ7 z5Z^xe{1Dl1W*b&rWzs}Y}@ z#-8`q=6@bAsxgdGPQAwOJ9>c9YQT@p63~@fG7MOXXm<0mSf;BI@`XUXsPK4g+2tG8 zNn!3LH?y+_mlRujaY7qH9Ke3jC`Vs8Swfe#JM3s93fu^&K32YlnQ+aQG?BX~9QTS# z=Uj@2k3tgZh$j4}@{-MF4gvYbStR9>*!ZU-&zx0*tX{iCDhkKpT$POb{0KNV8aNAC z)3a1ZGv{ygJPMU)!1{M>=0^b@H-;KOI@dZ2prlgLQSrS}3g`q_`2q;OZ*=emzB@+C zH7kV3ca|-AR)&5%yE}VUI#<+8>&sV>s;%UyZ@$Vzn(>7;cd~Wl){1TZpfT zwl|R=)7RHrt-zJb!awD*c{cX}Qip(O_W+@RfdYX5eBMRh?$i>VZ(Ent-JZ4Lc;EZO z*Y!Eyp77b8HCy^?+h*?5nqkxwuCjnop;C-V{Uxf;=?`JzE~9jwrV@WfU!)?#U}`l zenqDX!D_hxT-90tJh(Wpc-MgIBZe=&)K~LzlID9K=XN7i222q+Q*$@1_`pScY?-X- z>msyQHdR^Qu!&)le)+O=mT4U}R!6j=2282)`krGt%ZdH*5@8*n{u5DmOc(}5hjt(T zvb-wD<}uc5^{||K<`{LiZR+KHw>IrMC(OU|dS^foJC$^MZDLt-vAZ2+qTH4+9m`vmFq0Wbgs3-fEqcq@jjWN(sm2bquL#K;K?19^meA*X-8r`{$i1G;Jnd%e_}54p~jvmR%ndH9-!R zdL9L~@)%*ilURMyIoTT`0sMh)|3Nl zIBz{d%cZZAH*m7k0!TN?pZKUF;uxf3&+myycyy5rCfm@gq6YPhxSnjSIQsFn6I~Gz z{O`zJ&+k`Vcl_@PpERAPA>YN^Ptr40+-P$W7@q zmKLFJ(FbxG4i4|P2+!^)wcAf}qsG{)`^VfKF*v)Qq2v4f^(eOHQ@!;ZL+TkY*Vpj1 z{=LJ8#e3-6nmV#UVCs=SmdR{%n2dDBeGskT%_!l>`13fGas;y&kVT&_3W{cj9&?k) z>E!sLbC|jh8uE(WJeHTQ4HAZYlZwFRjhv{Uf}j|vk@!PgnJ4ae?TL4b`A3*hW+4Eb zV~RbWwDN{pZ|7!Quu4^)fo8<6PD$QQSMC;^C6pFY;CQ$4kGUYZ`-`ggTmDDgwNU%` zHX)H^1O?HJkKP?j=7W3>9h^w6Y)6E40;cpS>V)^q#u|aIJ}5^Glwl5O;^N1G@1sYN z9)2XdWP}ceeJ_~Gn$*Fgg^WknP3OeK#2;xlEGAhP0f&@-K_LVT&n0p|lBH0^dG=k8 zBo+lIeph?EY#t&E;!&sW__P>5Yy6I3?hkz|yVu{}4+6~RuKt{ZegX9T%;W?y5dEoq zi^o(4kV7c0E~DoC&{!jf!o7zqq_|@%H`BUZKF(fu=L4As__|%L1$oJ~=@0#Jen^&% zT{~OyzG7p_sZr`#+wb_)IO0Gl1p>{Q0gwy_xrt_XSrdaimRaCdaamLI*u&vw#NzdY zh73+Aq|_Ab`B^0b!qCo(f?8NsG=9-Tx4%GS>eA$=y)4hMV*Vwr(5R;SxFjng(5)B4 z!>)=-$$0QU$_JJjzf7D2U8T&Jg>pafL1CWv0#4o0L8qV@IwW$YN25ET_-?_!_2A{a zDS(!=o)l=s2c^LbsZIYKhR(mr3E1px0)Gl*fKXGp@%?-iN)Ot6FzS2C%JI_;*#Ufj zfaT!1E?vFvUB(?)5eav>X~vSa=^2QK5&g;xvJT3#2jgvN3j8R^4W8_u^y$zfcO#dr zEvkIz2&*DJL$`c61^hUGauOd|2?WL>okX|Sa==kNvcV!ZABO6;pm{u)cb2G$fh9k`+t!O`myeRi7r#zV6 z;33L+&@1j_Qo2Qk4z*rxS*8fe;jquR`+aMVf=QLttLYdRt=}#;?!Cw|eh>w|WnJKZ z!#^?2weV!*9hUlO3yMY;crQ2MdpTVK%6oJejNO)>?Pkbxh~cnvJEb8_VbofAlL!0K zOipPasJVOy=BU18r@5G)z=T!4XCzNFE}|pKhU=4(VzN)ahZGd(JTJ2jgLCmWpK^kEL8XeT z)SFl)=O>8fdOJ}&Z3CRIbf2Mf7=B@*il0{8d_djeEfM6oz0ZI!^;V63?G)DExUD|{ zJ~OT+_W#&c7aSVmyX{lH>736ne(W@aeDiU;J?{eC>13zidGbJ`7pRWN|0oTS!>BV7 z;`VWKO_SDE*r4pVe{MlDKEA|$3fS_03x|9!sB^uiPz~gh)1xWN(n*^#6TWd8lYlI~FKfirm-^f>@;l{mTI-{9qg0{@oTV0yPBR z6X|i?=7}C7)VnbqAlJ{}zW`5oVYsVx4!@Ac6d z0o_0J+N68ZIJ)6&@b!Cqd%-cj7+4zB}q&qY%%5aoX zk|SCIIfVj>eV5Ao>VYUnp zncS78Z75D6`+l!9?doO75pYB2Xjxpbdwk%*w#_K1e895?c!&>3n;9Aj!Hwf-5p;## z?XLvM7n5sGu5GwC9f%s{v;<35GI>Fm>VG)-?uy@V#ni26i-aio5N$?s$8Bn+3q`^6 z=Z4i+6n1Tctt~=kjH;i>D<{wy6VfR4adNbO4v0ZT!Y0i^>;Oq&b)aih(P6afcEUb3lYfr>8La9d(cZ%XBjY1y>y&ThLvytQ;7mGKE_`>+? zEr;r|^~Tj?`k-t=zQ23%8y`z6Q@eE1W<1@~FUO+P+@}Nj&m(0Q7~wz<`teX1 z*@^D2d7_kQuELyxg6jA#-M9pG`CnjOTO6E@FcNuiF7dA?(C_7xufm9ZzDy1Il<5jU z?&%sLFT}%u=ZvcoXI$#gVDqdt6fV|)D|E0G=JwBC*P%@C_<_9F(8z17l9fJ z^FBaQ1=L=gEfdYp0hYZ5sJr@9{B;h!VIOno(Ii!(eJ1lw^=$xa3%1fe!xjvjtC1kl z7u_sUtegwy^&?fZTFW?z!O9{j`qa|L`eAvI`WF1rW$aFeys9L=5Ho5(-nyfIIWsfY z`CXl`HXeP8FXJrFR9#PzrbR$j>QN_IpMAZ$t(O-$FrqU*?Oq9u{w7U0 zT1)*d23{_wv_DLVG>>ka1pj~}@$%>0NT}R*|N8Ly!h$Q4u$TQKxi?URbncSPWo~)B2Zbg4 zE|CpjFwO$IoIuar0U1}_j9u^>E|kr!%?5(e%6AzJHrZ}q1~Hb--9t*>R}%sYS&z#* zu8*@EeOx-fN;;YZ;x_Hv^%Q?cc?3gar#M(;A`;^#1pW*oK`tGYYBn8<1XfLC z?)m2qF(1G>XfDcvdST~5%8P%^_W=I$$1`ZTe_%#DAuz&LLKx>IzL!=ZQ-jqT2t?j% z;?KqCn=CXB5398@V+YZpJKH5KI1FvEouTFSmO&Kvd}T~7sn5y>A|1zMMMBWmCDj8g zkWkZJOUugRz{9vkVfYtu!~}WE%ya7j9S29vGIHrQ(=uiQEqN-LB*i_?#C<75yrjp8 za<~!yz5sFICuQhLI1qQC>!4L=Jrziwy>1b&QCA2LK4{A;CnRC=n4)B|DQzth3}`vO zi`ivQX{iT~%pD9H{&8Sqj)0YU-x5(3o2wajbDJ_BIxzANOXi`MhsAw2qH6jATWl8n zolV2DuOo3p^DD(7v3Q}thjv~?%nns4-*oFI->En)QL3v=TqIp z7g+9+{S7@EGdc(KO8;8b$fh!SC`4%~?0 zj8Gr*%|So95ysqZB8c21M~m0lTyWI&nIM&Qo;X^^qK*#oLi*5cRyaERVD*(>Uiu>N{PXi8X{>TI11;H1E#@Fh z_F`T`7VpfTppYd+3qn*T5yl^ygXOgj?yTlOWxA+%d9*Mj3e>*bmA%-Mo>rn?LnYN) z{>4mErgC~>J!ZCG?#WRmW=7^nZnvaj=pNF(8mBVt%Vcg0Qm_CeJN(xzhZYmm)Q-@5 zUk3-05bv)JUL@KFIELq65f^qHF>7^w*`U>)N0qP>9lKKnOTiQe$O=Ua6Gy1wKtTwc z>B169lS^@W9Z3`wO^LbPGNr^ehJ8>TORYks+Y{Jti0&(8-kZOKLCjgvz5H@%@!ZiM zqa%a6*s)HSb6YDtVcNO+)0Aody|7P1i6*$tY% z)KVI-Pd}+)J#81Z;rBn|SoJANqbCB0&GR*~{t?I=hCZfj)q2UAbN`bwKWK~FvGiM_ z=9r%+^x@}|M-VK_w=9r>_P~R+HA8|3feNB&!7Dv%lb9RUDc(GXiLwtcFF}5&A5+yp z4(>MM)3R8O3fL@wxf$LpkHL3LNNQl1@c)aU#p-X()+Q)L_rnUIfDiPyBcA5N3;(TK zo2SqLj1{6ES|g9AEe$bu|GQb(ny%PgLr41GyABsj(X)1Z z`0?uLQGwITN#M!v*tq1hRI6h>-CB1g$X}8{0T@&)Yn}Q(OM|Mu`b8@Z_7smR1GU@g zhi~|O@;h+v)%Dd>!b z3>_nblyt#&zf89&SiaT@D&j z((hGWokXJb1B)Ztv23s2@ued^qgdtGW5{Hc@19?&NIKVZ1fnw-!AzCb*)?h zw%?_WUAS$^Rqce{ZU69HifcSyKaEUolvm0^C8Vh9ttn);N%Z7Fpw+yH3AX9h% zmLq*=5sN@XCRii+OO+vb_^a3|1U9v)9zw|1iFQD7yR>FzCP8O^dy^(0AlOSxL;XE-vN z|69NK67h}kvZA%j;*sqUYEuxvG>m`w61H$T1)< zjQh_l-jx#ViE8wPJUm`L_p(D^gG=MVGr$&y82&5bmT-P1o-m~a4F zP4Unw-SPf2dmPNSd=0!m*)0h@n3Qy(i)kXix@Td!&ln|T}j=}MQe ze+2%@QHmUY8%q<#MvOZS&q`;I8%&@JBHtAX!IgkRJz`#{lp0%QSuC`UeG)IxOiG(` zcprCT>eGlDp98DqunFy#whMJKvKX3{)J@04ZmRE4P0%kx%Z9fQm%w7TTbb9$qK5aBb7s(JKWkQMwhO3wV4(}x zv|2RJ$;Lg|>#$-}F0tUHdcMraZM(P%TswMEnn~!o%_?iUvSN-K;4rUf5P;89J_)Si zx^fsB&RdfoV5zl|Y|Nl781@*DVVGIgLc^w3RxY*38{6%vjz{^f`R?)9Lq-W7=%zBWRq{KQwMy-h^2w2yK=8&!!_ zrgmNYrE+nbn%hzO-DFm2Ho}2Dn7J{2Pst~0BN8dnCxmQhvk8lfAT{` z-kiL;<6h$z3uNK2$TA1|Bxt`-zAqC_LqPcur_=znJE0!!{SG>4g(R3d-j*;!2jdnU ztgYIW1%(e64_6&_c6N8A&*;Qq8elS=dXRK_AJN^5)`v&tIQBvv0qcg1`&x-#$lCcR zO^UaA$VgmeCG78O>g`-rz2($T z&I87oWGc!RGdj1ow;&W72RXuIBDHlC6~aeyhlBJk!XT7dW3gpB-W0>Ff^e9?o)%Z0;x z>icr%h5IF|{LB2U+`*iUEAB|J+{|lXU%7_FK<*0-UVfws@C=F}`1q~xh9Fi%q#a!8 zAwX3)9pz((!!azz8Df}SUV0&`j7F^dnxw6PD#FHJR4r8)_fHD~7ecX4OPK4H0`Hnd5}3Ai|lJG zv$?yD=$sz_GFHL*qGlnG*Y0cd4*>a-GAGRIjnTqdMXnIY8dbn(DN8Un@%@DoxyIoxEgNP_j6*4td9n(5 zi%Cba!4>Ff^qXypsTjag!EInw%wFuH&9Z?K*8BJaxni%%Wo=Q3X6Qi5FY=x=tNS z@m^>%y_EdY)}Phk_bfq411s9h0$}M4H8UyY7RA^7ozH*&Z z=!e!w+7xpf7S;Ntk6f7tw;I-pLE-j6m(z8vbXR3E6UiMgC$V@bx22v{B+ z@bzLP_Xk}_a-zkU%6H3-zXFw6w1RuaNq-Rrb0%7I#_EC~jvg(PC99d<@p=7zWKpSq zo40}x@QvmDdRpQ`V-@m&+9?5f3C`-dXyNoGz0H_vbAZw z$X(o^3vBOl=jt~9eGt7Ql7&w<-{d*8*S#>?T&=Y=K;$0prfn+2KP0=($m$^p)FayqsE8K(dFz^#4c7tg#~0jvS5sapi=_V+!$C%R%NfE5Rit#^&j zbJ`l%u$H~MbN&I_-o2$?OU+%NsnjR7WA67ChK9x~5)UICuQC=TJY|{GFZr7hn)=3+ zt3SU|u9>ZO%x#*EUI}2ebD$G5*PUxyh<=5W z7Lx`Nzv{iV<0L`fni$Jx>rc{_>97@!QISQawrsvBK7fDPzPfbsaq zO@z6n0M}Hq4p|u$)9DWYxus(jvyGJoo-2KCX6|n`y1Pb^09Nei2C@@szm|ZhMQ^ig zTcTq%_mxZHjmdqkk-+@{tAJhlC!3c_EWqvg@pM<R+|Z?D86`_0#2pz z1l)XD;r8}w%Fm)k7%q=2=2?26D;0ANm=q2 z8hUpXQoMwTDeEcAkSa0@x^$m1_MdFhQM5tpAO3NA4Bpuf{}{b!fuPCDIKWMY<4kxA zHkybCzjV+#welml1}v#e2PdgIlNuth^hj~ubh3Kf(*u}bvc1{m0)wvGf1h=#-OwQj zU6A@FZj111hg*8xbb+PTwD@Yl(VgJE_Am@Bidw>_0heK)4Y7EMQoIznvtksHs`Y#X zkuFV{!T}WUnLGr8>{pzXUrtOd?ofWWz%puVymzqoc0QAR$fpUOwR~2L1v6V*)=~iw|8?7H~04iQ8hAu9=8(LeqwK5g%z)y$CxfTsU(X12P)K zoyEf!z?H7(2J7}KDtWD6AboqXPOk|h^(sUzGtyjQ{%=8W)*qcR7c^=W>@d4U*hQXq z-TK$3OX#}?OG^dWJ`C%t49B{<$0*_u{->w=C2v zw64MbsKB+}4q+S$klTKbsVgTvlPM>m@nSVdWL&toB(vbB*qm*4D_FFLp5Wz7z4{I_ zIfHFx1K?x8q`!S%?EsE+Gc|pHm9~`~2Gm*=pKAa=$Jy%3_t$VVUvX|Q9vO3l#K~2X z#F%O-5VYd3el0~;-}-H|&FI})S#yDfXoaLwu6EW~#7Mi!KchPJ9I&Y104`Un;kMl! zUC*zT)&D!Q-o}`C2J9_(e9!DP^zG^c{;$XyhiYT*yg*)GOO!#$c*@L9x(4HgN4+i8 zFrG)|HED*FVy*K$$GyKex zGd>#dp_K;QSkn=jprOkr!w32sG3 z-nmyHPQ_bA?oOG(=#49Zu>RtWuEH!0trlf0@C<7JxY~-({| z=s(3Lr0tS~L6w7hhguSuV}26A_-Uj*o43)6c;6xtiM0Pj;KIsdHUegfrQ?)AoeYVQ zKtsa|^}F(;D5nT%4HKVKJrRi=Mj{A0=iNjCCW=2*m7E_h9NI=SScihl%Ixmi3t_E+HiTbayZG<W@bWXA!uGDZz+# z!SM}Qm!iB6#_FyPYk$8T)}A_B!1f}d$=_tA4|CwCQp8vmGQ+H?Bu=vH=8Pa;4YqYh zQPF+oNEA|Td2CHXKedzxWu-rNV7|Tv0DJ!vX5Vp2V%5Bib~G>(Oa3D;W$erY9-pR0 zF8*}`ZjMx2x$Av&votebTk?G>F;o#LxsX;DbZC{bU5g+goHyuq@OruU(rGADDpxq( zx=JLQBTiDL#oRVThQ15Nre1R#@`xLjFz+unRYIGbFqJO;$ha_M`Yg7; zjQucF2m(_8^5C^t0#caQGkAf&!r0!A&s-k)~ExAyQig>3H41( z=tYcXCvhmv1@qpHf^35bK{UgLK2?LT#~-@2sb==$>5J^TdizsEQ&I5RI}AbfM>4i9 zyZrXEiZz08j3dPcs!-CV+)2kUCF5Xi$OBIK`jmP_RCUDctRvX<%DN2 zViU-o+I&PnCyf9}-9QPVAa~wP;mU`^H$MQwf41-j9#!0J*z%yk1hM^xevIIxU}`Xu z&&|TRXQ1Rfe?y6>KM8UG(G*|eOah|EBrr`tWrm_DRcLEn!LkxrSd;C%_ezPvuHQ1q zBZcDMK%KJzOAD0|#0<}YhfL;Rl&8+UM0p1}NE!2=30%q0p7zHCt^$!lA=}sF59aPQWuus)`cIL2t zp)INOTS5g_BkVlYVN6A+X+!k_=^Zgs_i~Bml<3PoE9QfVa#aLijFp6-M*Qb`f%@Kb zFXl?=bY2&&oGJvTIz=_2qPSa7pHB6!G|u1TYr&Xc50rCvsUI1*dPA9bxLO`Htb0>Q z(=7qd{{kT2^UbG7|0e@af)*5-@DN9*uOl33tyz#b5+@A??HdYSMh*!>8RgI;lACD$ zGuEwuDd097V4P5)Gss0#8G!0C*dcq0OC*1H)Wu_crC#I``b;YyNr%BU>SnGI&=AY8 zG?LXhA0|N%u^{#=pY2RN6KtmhOf;&F#MpY5+@keITXxhq(sGG}yr%LGvrHnD-07~- z?>-Azam`=Izy^b`?kMI6yuuC%B?$Yvv?;8>S>lo+LSYycwi+_Li4(HA#M4;4>7L7s zMQW^2aVf#H__HuE!A=Tid8qoB*b>78@ly6 zHd1Gh8BwJ2d8LEs1*mY)e|w@L#c@cJ)x{0sBfV3L@{#NhJqMV9fE)Dw*rW7X6iR$B zwS7LH&9ywak7|V((5{m$C)g`N(_9uIdkdPeNik7H@{V+S#p^pE!0Ih?jtuPnPv}>^ zSpg5Jlz9g^uiW{=lqi^hvIrMDWXP7Wb}^Y&VHkiW#UM@}e3Yc7Nr0$UNv6=6wVa+U z|D#TqDhjrlntHDVV*jA9!8r&d8D5`Pn}8CbQiFt|k0lWWQp0+MlVH0e9B@*GqM%i$ zt(WA0X17d-)1yEY>38N-fFfOY^^CvSRG;J4%&)FXIfozUO8GRnyHX_v;`)CqI4m zL8C~n?5M5CglnX*Y5k|F}`1HU0Sy02y(Ra>I;@^vo7G z)dg7kmle%_0OV2f(D)?5@`Lr{GDLE0mnd0UIU3kCV#%OM9Tpb}7LG!K?c@PcIah2( zsUoyS*e37a!eP(WQT&%y92qx*YE`TL{L4EfFtl1Q%|(@es5XE!R+B2g={b=Bm@KTB~r_1<{7aq^29k>Ly9f?Pd|K- zEp$Gos0K*_XNvI5ML*mLEL{0=W~nKC0@M>7kg|YEr*BErnsNtUl@$FQ6-70RTbtRI z4hFWX*p-$!BI%&CLwBkaFpNJCKzgGtJwvzQqOtZ+xj}ivr&MN0^&Inpa%O!*2?Cz9 z?wavNp1JtY_uvmYzO{_>%ZbjIi9&WmPz9G@qAEShlBc>hsbj7PiKqkB0Vb>X$sfx5 zt57FY@rp@2n z8JNAGURgNUCfhCRNDNjM+xlmtAzNo3!`PQS8AEZCi~K<=PH|!2o`H;Qm84#JxKERR zoABZNFBPN!4$g{Ibq%r!GxA9m25KirT?fxZDcl1{M1Xnvf+#p9$6Fl9`*Fo&|G2EE z_74Hs&ONUNhWU|$h}wT3K>)=^Er9<|dyF|4m@Ws+q{!-4;_bo9`!LPKJ_PI^bG-V^ zRN;Mh*=^A^2-bRp)gZMg^U6pmfAs%_Kt|=kbchFd+WGjpp?v@}uK{GcjSbW*g%|2* zfQQSq+G~I>z1RCLz!x9B2rGW=np+T9W0t*3LB~*jJr~#$Hy(}{+5SHdICk>t!JRAJ z=`vMQ${&1e=;MNr4F++40}MKUh!OLf3XT z@gI>1qm!5rOZ;4ec?XyeP6U#}Kz65qHw;``H;N*6YW&T;0jiOOO3~wS-~h3*gqyO>PFj$<65%2IxpN z74r<=d+$fo-2vykhqu}Nw*Nf1*!~$p8xaCh5G;fPQ>nv(>79ihAn2|$Q|NEw7VQta zPm~9Q0!04NLF8Lzp^37G-wQ8Bz#W0mzz3HNDt*V&E$n6-0khkVkZ98VFluy3|34PJ zU~m39jU;n7hZ#RFN10N37AX-`I4uSJIZ43aOjV~AD_STsy{h)UxZ3P_-n(++(v|pq z=fXNG0CfGaB&0X$xg?@D$Y$R6I=Vr?!?4r&7wJC@c!hLZ==4#te5`}cbP=Tub%`D* zn~?9%hlz%V5?n66qgiAPkqnGHdwx3b)As9@)q zATJ?==*Y3*A;kT%-9pERg=w~UXBE1k>@Tl}bVoaW&gJ=9J~V~(RX7z17|s;6W*SLr zH-E7@QJD-Qm#1}O5HR2-Z0#m(g2vTb^w59UWSUq8_**+)& z*FwKqbK_)5Hu|yeo{`vkx^nBTobtN9mEzZujw^g<=9*tqqUHhZX{}Q9wGa0p~S!n z9;TYHaFH&5DSXd*dpP^hQ$~CgyR-wcma)4 znB|mTpO1gNs=JiyG~YNjBlFtS1k60CGb&{|Ha+YJ&A>M#yDP%OPJac?jnk4T%H}HW z$Nf|q6-~4!%k;F&Ktil>eX2P>a%uJL?QK0G_TA!EQVl>7j(aLkhTsxEM$sw1i z3Y~0p%Ah|rAt0M^hb-nY?Z)cUM9v)Oa*gZs^D2){BKDZ0CY*%H${d4q`=cnsK!jBA zLFO#Q1x{$_N~cL+k$$ zf#X6S?aT8zT`>58%9w8()M+~Wh`_}k+?DN6;S$I7|9%+AtE+dNh3Iou3-rv-iQv&0 z;0R>+8Ldv8`WAY{7ntCgbof*_eU3Q}H=yuHfhaNR1ziy6_mndHyGZ%&xKCv4eaChC z;~o*_Ys^R7n7LR%;Cshi1z-N)=qO95P%_oijh7H5!!aRQUid-=hX9MiDU@@GWVsIRy%5fh8E;Yg1$YU|nMwCj<`k!?1`Ft2vJ~ga8Azx9;Pn*=UCtwBpyfN=&?ly$fU#}EnZOTXw z=a}1`gNvo}u^l0%JwPT{N!h8=yX0o8U)o==(uvX~m&l0Zm?V>972N|jJhA0(xe;w?PzGBR zl|wCynu;>Y*jU}7CHg?Wh4>9Vj)2d^%=t@?^7{$ABAO}niBEi$6Kdz-_sgzXPY&Qo zj~<8jhgdPvP-@S|5cvGeqB*Iru})9^yv@K2GkLW5#G=??L6uiYso6mCKI0?w0pA zmpiS#HfqUPcszZr@TG{Zml<(Wy}YaId}>n{!`_%mlPQdg=pSAQAhme zRCk<`fiPROCyDubb=BfmVR@H%CBrsrPEo_((UWXG*snbS+u1T@9eT=;|E=%x-n>-& z1M03-(hm)e-mIuzfKOM{kQ?~xd?&7}srf_^g#_3BJbxNYs<@JE^#8KvaMyQs{0LKd9IQSe0m2v%N6H)GG8|rlU0WaVBQ*ANX`u|gH z;Ze?toT@jtJ{|T)&2_?>;xt;ht8#38ubX#-_4`i1U4(Hf@o_7!e#|%dY^8!0gmMAd zPOVPf>Pmq5$&tO}ge2#C{X0^Y_l?|Ff3JZC*j|`OfEr_85+0vwH9|Nv$d|agxMYRi z)Q>4;m%ml6{vX`EV{qkd`(PW}cE`4D+qRu_(6MdXwr$%sI_y{-W1l?l{Ljog=S^lXX^Q*(u zWIp?BZ9CbLO;0rDF*L_Gx2A)3CfnKu&Z3>kYEdk8J92E6Ez1GKh^_$VjPkVH&b9JQ zXEw;zrTo>w1bX{&k8k&fC%igM0K*rR%^8Q-S)HQkGySt4TN>JK(6o$OEx_~t=2?&c zJ_3mIMXhqQ?L2pwnqFpB0Pq1^L#_(r3QY90$J!!NJmjMOY7P@{cq<_Z0wS#I?#78Y zq~_gzimOS_>KMj6DHJkew3(fUACN&|RoJcj4OG*e zY$$oC^Nb$ty4GqgcsKHnpxaWT?b=TWd6&RG^vtuwU_43*El$k10uj)5XYzD&T(M{Q*EN+8+ zGMdqjIo>k$BCEUALZh)VCPO_hOEU6$pW(7 zADZ_djt^LV;6L=x7newyD6ta)5RNkz;z*oa++Y6Alb8|%$OSbOtAOkT`w!eD&Kw)F(RE5=#&oz4l$SK-AK-<|pmH3-na1`G$#wh9!N8wL)PeDPI`I&A zOtP3zEBuzM2kUD7H|TKuC#f=dss%m)19f@7NT@#8(wbY&Ir6WHTdFQr3~63B@grxS zZD~^~Okv1D7i+WE2=twmN{910yr9*22%K2_{}%#tY@vL!1+p56VHL1)uceQ!yAW^y3zFV_w~Gx4N!}ol+(;0?Ei(eU$0k@Z20i zprVKMhrt4;xHBWbB?M$Dg6PgNScvvS`&Dyr*)%hB*8$ zV#=JwAt7Q<_`+st;Jv0j>)0c0~prWRlG zbk`HAjQw;?tghm*GQqFd5gnrBDl2b;25apA@1e%`ab@i zxr?joWyKYq-z!q^^LdO5nTaDEg;Q<#?&pPSPrEsB03{8U1u9zxUFT8wM(3mU^?ww^~T|}#+|~dHtZ(s@zaV5 z4e})dyyiLK_m(9@KZE^0As7j@R6DIOxn)}aMDE|VNE=d_LZxAq-v#DC57i%1F18;p zPcI(~b-S6YxTdjjI0o@^d08xQrQZ`ZNJvQ7t+v40s!gIJwr0@%Hx7>NM}_mNG@+f{ zZv4IGfVW>WgJamTaI+hpvUMen+%;-)nA$DMPCay+@BLAyuGi;6 zk}zIVru>zFs&H$ObyIx?OiS0)7H$`cv900US=JMnNu-etc?i)FQXb z%@V(ATA0J0R1v|g!I(l2Gn?ngA)m4WJH3w1Fo}EOyUyMGUkxp`!LDR(7wSJ<`&Ux( zJG!yq9$jkz>Vn~&tI216;WhNtp8tt}!Tly+Y;+I)oq##}8W{S#@!@sr0^wj@HITCu@^V;@s|XmgT*aIkc`tI?&q&CspgeUsJEyf6~}O4FkvpeWcq^@wrOCKlYknFm!7nNg+tmGJTMsHU(7tpx0avJ!DFF^dbZ zi)ZoLz05+qW!Wx;N1>w$35co(tum{hgBqZdstgFyYjPODLd*A5XDsF#$9#MNNalJv zU9QoXdxAf+%y*c$24RU(-Nx+`z8rGNubR$q)>)RG&N{l0d^DXzVZ~EP`!-^+?y{hk ztkFqTfp^u8!qIgf55qC1E73Xv{k|rRL!kv#W);;?Fi5#pUe0ii8XIr#ZoMB*>F<9y zN@&;T(%$XD-a96sJ4SqP;Nh(t_Z5{v{(m2V0srqIFfsoLfwAoeKuQFW1N@%;HzP0_ zd;bN2Ir%RL%);%T{~Ce0|Nj*N)BA4-48ea!V6FfIO+jG~0R7w?@t+QJJnC#hE3e{Q z>JJ(MeJUgIdpE-2gXMdDk&#wh5RY;zNa``2n(vuSfl16w7s?`h)_U65dQz4snPQb< zS>kuctVGMy$gyKxHy&k_aZdNO{0$-f(uE98qi-5Z`s7&p20wd!d#dQuFPy|-SA*fr zapy%Pf@HOlgMxNVSnq3TPjen-dme6^~>0n9rVMr{vHe$R_ zrW&u08Z$nQ8!tQ@;0MuhmF{!0C;0HyY2&O*Igq#16K8o$iY8EQN4opsY3M%+Wr}Hk zd+p95ymnNE=ViY1VZXhH0frvFxdzMw*JM`xn*_ISQ%kb6$gHWm5O{Kq7P&0wF5)7o zq1M3r<0Ay*JtS@50I!n^X@p&AYZ^=!v~;+Lnwu>476EnBlHY>Rb@6-pe1G5`n^8zyXqT5RDE; zWDZ6Vz7vNrv-W(hs4&h;$M$q$m0?q|DOMs)^Bt7(3_6JJ{;gb|SXm|0IAyO%%t#H9 zzEE8Jlb1qh+L9Oyf^VWWzo@*Qd6!8A<^!>iB|@Au`DdFhSGOsIR%)%x(%D72bcuW$qFYRrS_(4)xKBnlVOQ2pkC*&^!ml&z`adzP41Oy2dLp{crwI;FcqT*y1|L2)kw4|xt6Zb8nYrmf%jF~ z|0;lEfv>v_8=SWezRS)o10At=OnfU3kQ~3s3XLCUv{Bfvmp zh`_`$WV90B6qwW2-HHUTNqjVt&jSe`c}5oOA2n6X&ucC;%`;W zm-Nx9YQS;n($%49$oYf}>er@ca;8)7gy6wGQZ7HF5*l(enyVzBGzeQy{1)+a z6vQEZfjRaorpqE-Z~&pg3=U0wjHfp=7K{C9P?j{AAPWN<7zJBH0ryV?yk9&&)lMkU zr-V81HX3}2RHG-{O4| zmq!V5k}z6X++`8>g=Q}7?{YBl_$E?Q$3vRB4#Cdc_$iFFN*P6cEg{Qna*h1?feDDf zmE8DN5OT2LaNK*!#qS9C5vc;vaGx%%-8Zq792ttlq`udj{XKWj>y^xA^JjUkHj`|y zMI)qx)#J~{#)Ca8UKFAkWM)r{7D;Add5)9oG7l66mK$+#D5JF)l^<>ypZtk;XSI{M zh{}pBS170y!5~V5hmr1!jhVnHOWhPdOpN(XFC{{DF#a5B1q5b3Ab3FOzapOWWa0Gs z&m8LFA-)4?M7LR?n(VL~Lwj9{;w^aIKGj?u!zb3~mH6-vAx?j_Yi!s+8UueHv}%XO z50rBDL)79b_tVA$u{88$l(=yd?wrlb>FgTgP)u1yN#=;N6isbXWZfESmG=C#>;g5R zMt%AixON7H;|{2Ulaf}~aNuMVGa(}~DzqAXGRi^LL@a21yG<_Az1|40ITfnncH^#o z%46F7Y6DK+9_0?8qjxmZeXkG`B^)i_r87~gOyjL9wqMDt^6XcPf&^?V(Y#zV7f}fm z)(7lBQopRDHJ!HC9)XE8>_RZj9+lWhFe-nS;D^)5Tk4=;6xrl>HdN);9nC~wsr_L3 zjRXtb5@LgKP_w3&?~Inwll+$4+;m4*qcaQ+t+1rXe3uPw`8SgWMV&d9drJs0eW3%1 z4(=UsUU_ zcLcmHEH}nNahgRv;06MhR7BDsQ{Ecm5h{lM9(IOefRbpASg2J}(?;=NP`QI{|3tt;2y){!n2&SLF`9?&w$b!`?D9cI|A~MXoVG|7 zC{H;{&HRd0veQCQNdbw@nNfwhp7j4gX1D&S=AS$vzGb*9CD%tBuGYGsF1xq~Mt=pC z39_y&EmvKapoBm|6{0%ULZd>W$bN~Hr<<>h(_CDwh`=w%+hC8HTe6pn;Ink6!{bG7(Q4?!5_BApl zq06kLa9zAzmWaz1#|t zTy-X#`8+?qn!c=@L>g{Tbnv1lCJE*;PJHn)Dg197w7B^vF5Z*ytZrHMT;6<{#}q2O}6O_{j{tyj&Br&?`llYZsOW9paqH&lyTlWI^1lr1-` z4@&G6|LlOnCU@14f&9^*G*`}T4XNf9PSs)ITJzL3qH-&eA`|~bbA-hycL9q2_7)J* z-Xlxg{;OD~{>GAiT^6cRDD#%M%yA85!7TgoAOW%Jyxrp!nO??Ga4BoDS6SI=3xxK z?*$F8E&n)+Yl4K41yO#~g9Qk>;4i_!^*U*%N7Z*x{2se`jL}IQLB=@!$k{CW9O3jX7Y<_kg023<^B{h*03Y z32c}=Y;ii-4V<%C(ZWmvVYnS|2GS*@_GCUH20ZV7eBILC0{UY98Sd4YTE~Aub6(NP zJ^gV{9R*w=UfcQ{QrNxi1eC8axVC@alL&yeH zCrnGvqGNPVGnX*ceZPc!cAslEKG#&el)cL^QNfJT-Y&|8Rs@26JS`T{I#d{7I+MMo?-t!!i}DF zeIMQq{9++uJ^`1vME^KX@vq22FLbt+d)|7Ng!~<`pX0tAYR}|W{6!)09l_ot|G154 zk1BC9CEUaISR(_Piesss`4CaLgmEln*EordaCDpSmO3&uc2#==26VxG(9*82+ZTL5|Zkp4b&Vy z+#X377F>tvW;D^?ku~w+r90kka)N|8zPNb87^UX~?3Q92F54TRD8y)=J(TZ^h?fCMOoZ~;w@&!@VK4DrWmRA;I#H`gmGB)sz2ApUeSMh zO`y0PNPG;HRXTL$9tn`6$Q^jacz6v=UHFrW#5lSf@_f9Gj4<@aLC6fcnG`zpQ@UW7 zLrLRP+gIh>bMP!qx|_q7Ca+dA}bTz z`Q+|+$z%U74Fr%>blT?InJOl6)BLbh7YBH`Dr7sBUK}<0cbFm!wda#^3s{OW_mC>L zzqavB2^u`{LSdp2V&2qlE4_UHSrnr|`;c)n|L-m_nBH_^m9e#;#%tjoX!wDl2Rza? zye8}C_Xn(@4CBIR56D>_!G4XKA{sdD16Q?AohQL37m+x zA96M$p$qc~aa7NXa7Cy-1>J)R^@kY3slt6`apTej?2_6$= z@?PVO=wT-v%18-hOG#ff$U=2wy1cu`-PP6AKD&5eTnVJvx&y=(H-X8tbRP*&Oz5j@ zkSE8H67TzqbTUz0iH^#VoI5GDVxm)ZiLCjKBTx_403s4{Rnp3{d^8mBYjN(t&OrKG zRK%j+@d09Z6R5udQXnK2QfA-ac)W5Sme&wDLT(DYH`yNrbKCzv=D-tAp3UR*o zqfAN>TZm>&#kLK2qYz)Q8td@q5S3^$;EO^; z=~>~b42dPl>z1n>dq|>QQ&iQA zDN{g`cBW3!<09ivko}k-$uax9+KsZ{_W(y$7!(z>q8fZbVXd~6f*x&%O~#Qd|K(3< zS|Qw~z6qymDuZ1FbwPdmu?Kkcb4`UD8E=lP2qOP9A88UvUX=R7$N>U#X}yB@ZOwX_ zIVabL@Q*CC$#<`3+PGJzv#ZR|xL_%KC)Yh#ZWwMXgEvxkK z`Sq*Y4nc@l(nqG%E9dmAHKbGnTcO1{E5)2LiyZn0QXdc@-k_VgWQD|t(da1R&^%O3 zZ_0tk`@l@A`*6v8!iuH`4RWLvUPJ_IE;QYV|7yWqDvg3A{&Thw6>K_lKq0w`fO(ftL86I} z)G{6~9<*?sB^qeP8RzrEgLh&sX7}F@4t`EZqv=0bQ&S`(MfBSzN9Y~J z#3kh!xrc}9MOtCqEi>jahWygvvS(U7PFbYwAO+)f0OVOcY9 zR630;Yt)FD6GierapxqNs)L9SiX#1F1a&!5!UGpe?5|*~2z4PCivGqKAWbsCWdCGM z*k%e&yH0eH#hr6bI+sK3QVDJhcPpyKw<)1aEX}QH)d}k#`75zB@Mn>8Zgpo5AbG|e zv+t3ze%M7uxRW~p9R_q(WLpqjCEj;OTbRZRO%kFFufnL-y;d$sscthsoHZ-7wzZO< zJ9IN#Ct2t?kg$pH^2>aSEg^qB;BRbGVWtc-$fsm$WKT8Y&?t?UQ{T@Jog0W>o~4s= zY<{;BK+$Qk3sM=?z#=2f!xV-_%r91GvX@Y5nUnL$ zGT(DU@5|3q6~#awQ_v{y52L5p+KL5LD851a9XAV8Kw8tip}<~Cl^_@MFh*N z1J&_`D&aLZSzxC}rLap_^7_vfYY6dA)bGZ$+#E}|`YS||PFBC{smzXd2i8F7b zz7QeB)N9?3KdRv8aKW!*XEb}y(;Ey?j7KJ zSNRLQPKI?8f|lv`^-v(?Uv|GLWvmp(D|JJh&oo6%&LI>wJ=HN*TPS#LrKk!=MLk#U zD%XpapNj1^Qh~t_c4F|(#t;&V|6zT7O=rW7g8|MU%Q5P@(`!1WQn*kLr?m}6jxeYI z`H@ugra>+>Lh!dkR=>xlo_M1v=deGx92RvedwC!RG89Toi{R{a>npbJLK1JD7K`P9 z9GQLLzNb@ZG9 zW!#{yRDO>!bV#94-OB?J${Cf=FU}~=+qZNWV6|wXS5?Yp?-e4e7HFI@o)X+dV$x7^ zl6p1ZcM~U$D+`jVDYD;xc@Ig=)wY8Gu!#0&sv?;C9?pMHMHqpC=IC^3LI#RqLkvR=T1W6emaB z1GoRssgCS0;siKj*%P8GUCS6orJ=jE|G1se$;1$B6(q*_Jo7~=b#Z-P3&;+44G0RL zB{#0;q$0D^a0Z$sLn}C-Cq$TzJV+#le9@hx7ej0GH-~Yt@kp$qj4sE7D3bb(7NM_E z`HQ{vPpbA`udmuZP{QAvhc!S5#sTK=b_NZ+$jjs^7Z0YaL&H;@Zj6~)+nfN7ct&d5@9P7DkAC@;dh=@^kA{9DGcbnnc= zU$&xx>x|`|IW)^UO<9XF+I=ZX#W{H`3?Xm=@8zVqVKn>jY9cRX9Wj-k3uxw!gO*q+ zW$|Yce!*PcCV-pcPe9b0!Kn#G87-=BE+lL!1`-FYLGaa>f(No7$C*LpRL;ZCM%5Gcn5ld7W5zoBp ziD#5#27^^RlMjg&m1puaZ?oh-%Cn8XkDmvNthDJ$d(b9?JnRm$9$%evdZ%MAaVcS( zTvYqrSKCxOx9DvfvK5x2l!de>KE_v4_qYaTyd|I(zBqsq?A~>G{42$X_{B|J*N=x>S^Ob z_eWgxf9X}m@RXfR4u#56cSVWY+PB;r_GhA|=2cAxoH8NOzp~cFtZh0T6NS-=K0Fpu zwX*Aaj3>hoaZD8=Z*A^a_uJ%#oDOK-tL6(~u)}LOxSkFKKly61=32Gi$M3Vc)iP*R z9wzVAZg+fM=E3ZHNYSxPO9MOHtJ?SSRrsV%o4eF^4(^ux8c*807unox)X>K z=WBU@pK^3R#5A|6CGIeJvz99kS%HXXBeM&wiX1b2GyyN3Z<284{) z8l2fbxQWhGtZdMXA)Ur&!13RH07Dg{L;;KumSmMFX=PULGj^zoTy$(W=9ss(U%-6} zD7ZfXGi28tG!&#GUKR8ce&R_gjCH7M&UOJ^=|rPVg_5l}%dJ=(=6A!GCd+?>FjyKd zP*5R-YOe!p8k;?Ww6r>x*uEQKjiUVcY+4S;;!Kj^)2 zhI%giGn3)t8v)ynv9VIk9U*HQr+?5VantjEYyer}xpeVE#*!#hxSlj_ByaZ1+fWWQoG zXlu%&T0_&qs-JNYF-A**?jxdG1%mdUlWNIyN7&(kw3zllj$SMkKI&9k1_DG6cK>C4 z{#P+dEbr-3ASI3HPbLEloPbKav-6x16wJ_Z)T55OK335Ja?2k1L^MLif ztrx$1o}K#)Ld;XYor)K~OWqs46F={G11f;cTt{?beI1NUUr7Z^LnH6@M7N4GUSp(> z0={@9Dyx3M7zQ4ho!i|e+*|N={5QOodSu$h=^&HJb*24%>^GFreySW(s6p*5Fd0c^ zo&BqxG3?`H(n%TVKiRetOW^Y9&;*D9d?wNwCIHq}^=AM!ZWgCM0RqEwy8u=>u{B0u z6^@sj-F8@RaQoSV=PpU*c*;h`<4Hhia~EV+=34rgBkV;UdrNQy3M(Q9xOp zNu`XHUuG?rHCA#T4#uhfozI;XG#V;|&Y3aUa>nfAIZ`9#=3Hf~ZbSaudo6FoWQ0m2 zF0D2@7}_q5%411|Sbcgcd;(2K;O4rQ>Mu&S$aMCryShd$_aRD?HP+Z1;#)az0@^8% zWx7D{*bi`T<^rj8w@d%dpszh&KF{a#&$=={9&nc1*{40#+)fkUnZ_lisKY+J$B&x6 zl6watz|#hW`J*rs%-Cb-HTh4b_ntqi6h~!+r@qbn-HjqJ)u%}MW+PnX^fGj;HY-U_ zl`+;+8~bYxB$iAgtku?^g%=3Z1eg^$qBzQ~do%%9Q?c?hc2_2>WtbjaQwAX5su=SR z$&4J)#!vTMLC-g{PjS2Wx!HNweO`}c&w!s_PoE4ov$Jt~KYDoHYjvP};dlO|)8oS) znj}Nc5tCy)>Rw!D!hdk0aLMrEgkqx}nDx!py9B%}p3acAEkZ#YW!3&F{qSbkuub9}&_(!60~b!LI)|eJ&Hbo++XNGxd|uBxCagzN zbIPzT4g<~PogJCGq#r-qag=AH?}D+=zCz7Wty7pqh^=vJaW6Q~i0f<+r|~(}$Wk9; z2suywagm=hNn>L-$1#z_)ZCcX3g?g0-{x6;*st$-Ij>ugG&x5MvVvk&u-cpe6F$MQ zUc;|vx8ee}bUJ66SxT$J=w+*aRmyZ-;Yx+8t-(`0EYtE!&06ECn80vlQ=!GGrOH}W z-A3X#^w|Df$zsIeTAtGHg|1Z?JkYPTXP)_U%Pv&f!fg{irPsa>pBjJ1NI=yyvyI5Q z%okZ-ZG*G4Q8@+6-fM9Vb=dB)SeU3adY8CgN))kPN+xR3eu%fKRdNAMML=OZJ=wcO zn}|L$Gwkt6Ow?&XS3pOaUyHZTll5gtVfJXf<50KS8x)Z_Y^zXr$Lzbxh~`2wpq6%~ z1`?>|s_48j+vLHj>cQp*+HwOB0D1Jis(B4FOBS~>W%P?Qj6@BK&)yvv!{+2p*K3a}L;LQD}yr!0#^?Iav)YYUIp(p`)5&T9=)O_HTX_vPgCIQ`oRqsfH>9ilL|5({L8?r zYpY8Q5ZANQBLRpDd!ajytE!YihFYTe5TjLUc;%_0U6eykS=Iij_!QSmp-$H1aN)gp ziR}~TOBu8FH>SN>b+A#u;rYn&JNG0_;q#?SE91;t)&)engNdM_9<|AvfBIxOBc%0) z;XX$j(Py1g-MGH(KhT2$ks{%7H$qnwv-P%i+E>uH^j19(q?fn5i;q{HO4gxV!0^#r z@v;UpD=HJKThSc4r#@9z!`*uM!(eZF;zMg-8@xUtQOKA zbjk!hufjtANSIW5RUXubq6z{1I)jg2FJ*hJvnsmC9E$DXVy4cPcCVUOMh-wBL=L z!LqLpUbH^8k6JZ#EyVdLHx55x%w32toslAc?I7F$iZNoh{s1gz`82shtFfm{Z@_I5Y7s*VBIb^_Ocuot##KlJJQF~kq+jYl(pe<&E`=6*-B*5BD}(@dyT z)V)zCGy@PR926+xE-hPqDp|FbfXp{`^c?k!!AbVKNN^NbMD}%^BUrndiPYw~_{y6x z?xP@Gi#`swo8Chro)fL>MEJ(s+wAHgKFb={!c9Bd_?Sip!K#~XUoMH}FLOE)b^mRi6fi$%<6a?bMUdK ziM5AkL$}-JdOh-!{mui+hH25hh@_$7>)*F3Xiee(h}Q%8=h7sHvtGv6PLAiKIxjnc zj=8R@P@Uf$xSc)pdLB_;=D{6g$G2rLQ#E=Dg%7TPv(e1Q(OTNFsb=0Ybh^xosTR1H zJMt{jh``N1PWu3fud!|<+LFrMC+#VJ&Go77Az^DD_+JmKxqowQ>%RLckAi|T&HkgO zR->TgU~{6dos2Wyut%r_jlX_?*Rj6(E=WM@KP;1+o+_hoGt8bnB!``v(BD5$>aEWf zLvw`%>QA^Z*Ee1z@2`q+d}-Yb0XBT6F8sdIN0Ka)Z=-8CM+@Hp7Cjc|pLNgO&zgW| zAzyqa*@tX1XbhyVW3shx<=g;kUT*d{pO`k zeonzTm#Iud^=EY-UUnkmWbk2JVK*#5>=xhe_PH7su!1;RRQnP4rJ_r=Q&qKH>{>ad)mWmc)eh&VuUfZMx>9`qV zg{`v+ZF%0!J8!g~mrU7P>d}+dBE`r$;*Pq{CL@&|pTm3h!BLPP;SSF@_fNlcA9TxZ zD%5`Vf6zU`mIE(X6=Q!@gnT=EB2!bqz5eG&w3IOdwn052w2;5GF++RlpnzjmA;=z0y@D zkd4>NwJpHM>(>T&x-~g8?Q))WeIj*fl=nY4^ppVLlNG@;8#48!<{RnYhK8`HU!|u1 zWsqX^+gvL4IN(Gi{EU`+K}Qv3Pb@Y=Uu~P*R3UOl63XycR}tHg`6td5pawq4Ah*-s z;CNqpjIty^$#~7lxIUnIB39L*6s70{CN@a|Av6Va(vGt$Tz^D7({Ug2v9DFSK@+9L zt_{7@L9a0vEU6`<4zxFPx7>Z~sd7Wo^dq{PoPfdY*MYOmm?U!;wC+<=bjVTiF=Iai z6_-(;0?BJNYr4*Pwb8Nk%|;m|MGpueB;bAIr!C{tL$=(a?u=lbdNpr7p}<`u34UCJgN{yv}I{HO;Vhd{HSXDPf}8NFxFTL(|%3kR9{At zOlSj%2uE)5{H9@z6=7zx1^i*}d7lN`YrZ|0UwMdL41qnjBk?`%1ik{>TSZ+U-UMNDtIu%U!_c-KK0 zfhG?4a1s?3C4Gs-k)r~U3aFOi9LAYA6U1~{;s*b|{o^&+@To>Xx?Y?r(q?Gmue|GF zh%fQ1giSbBuV|h*Z0{_YGB}I*(dj4pFq7kVb)Tnt`%Q6II1kg_7@KXDMEPp;4$Z#2IiQ(@wteWB=v zgA9nq7)dpy*%1!JlTb!~#E&@a)1+L*(sx$<>3?tU$~YKUYJ~@0-yN%!Pv^IFV`bj< z@ztifgKlN}^>xE{aix9%czp7Vd)o{8;1*qxTEl>1(bZi zfXm`-C?G9GC(vYyDI+0O4!TQ+03lPInvRxbP*IY3k5Pp})*0x{3rin^*2-#)@C$87 zUPC#_fnzrWn-Pj2vhWAAVJXuK_>Ba~q)P2)lAVjoMD2P|8ncKG0ekhfwY6<+dEFJZ zeIATFsUyKrmAF_$`2uLyz4%S{o;yc0b2#{J1td$9olYF}Ig@C3_JTdR;xx-o%u*Eo_W}n7p8j zl_C@qmZA$}s#$|rqD{K6)!|j5)@Rvd)2C;yU|uj)U$&{6#;ms1P#PB&4hfb;iYEktC&VKmchqiuv%QRm0Iw+tV7T1r3j zK6!=!&z-lm=S)OUP5azOMSvfAhHB|3udUCgDBiO#UVx=I+#+F?k{g zjYafWx=mVloY&q`>@ORVE}#6mt}Q8H}jpivQk=FbB?Lnb->zf&j#-xfh8|Q^XcHxo*F9%Kui`_is-os4( z2Bcf6#~y=8>Zl^CeUL{Z zpw7n;Rg_+F>pjgZcBHnaeWg`ZztjHR15*-*6vkwc*kdSID?+T#1;$k4s%Hb22S4CG zK4V>V{i8N!GnlW5#QzclZm6j`tjp-hxQ)yovb7RTMg&?_e!w^RW^;c_y`VAnLp1RF zoPqOTN?I8In+fE@4HW|QKcZkNk*hL5KeQ{Utj&YK3c|<6k03eA6UUC-_3kIbZ(H3s zUq42E!hP?h8#KMwjwvzH#}$=eMWiH0b-SB9k-NEsWMT4nG31XG5sgMvk)ZXmTX{Q@ z_hokn1XUts|6N3J^{WZtzIm`1qU-g(%WnX@Hkry+y9)XD1!oB+qX=Lwkb zKx4J%{~E6_+C?P@Y&4&B43YJyCIS_)lT?0S)v(dj?5Z8G)&t6AN{Esxe!GxlL}Bp+ zhs~BP$Ta;e2Zw^ZFF~InD#mV>MdqP&U_IVslNmzz?KOR_z{@vOHR(W*7`uPieRB4j z+Pn8!uX)R+xaS4aNe@+gEQoh#;-0Vu!bQl2GNel)LA%!z6CIvHW|RZ2({OP_SGs?@ zynI<^Iz`)s>OMGERcBEhm9!-#8tk?0|8h&J=j`KP5%b_>+-7nD8)U(Qe#g6~{Vsx4 z+;6t=7&OzLa|dSfRSHKbxfc?>`%v$dDdV8K*AZ(`)PpidpGn3Pt@oUBpQ7AJlhZzD ziTJ3KU?7+$RyxU(|4q!;_M~0bRVEqGdUBU=#7T9_j5AYFoiGx$`E~$Rn*^!yyHDLS zaeyoE90e~ZmQMg-+XU!W5-Y#4#P*$_DjrVz=#w6$PHO@cRx(TK%(A`Jld#4B9iwt5 zCHwnA--443SAfz-ZrejmSIK`YR{IFN}E6@<4;b4Nkmh-4HDH|ZP|?j3Pl2p zfL&JRAp5GI3cFI{P_rO1okR=t!bw05okSaN&f@fRk_;x>F)VqdTA9_}^@ssy zNxWu1UM2UX zIv(qzXtdk3X>kvjublnNQm5bRcviWsEB~OxJph6+5oxh&T91?&tyw3K65%cLA@?|xa@4jf zGgOyxT$kemNaa5f#$0GeOC;CF&np==c{5W{qaGR?Nun)XUV8}N$0V#NZbqX}cH(*c3H~p|S<#{``!qsX4DyJ;g{!|GTW? zx|UN;J6Lv_kCU+lXR@eKpj(Nn^zf6a4XQCakPjyoj*3z{TGJ$ zI7ghR`*J>B z6D$NIqISA=i=jcD)wM`p`gz{GBn_J*E79p2jN2PN{jVPK(!{BCe0Oh~rB(YdAATjy z&>f1H)GL7Y+Tt^xSZmi}!aJ_{|r}0ZtSz9sAUy;YO3zAhk2H^hfe@*ZMd}`mC{9avGdJXxmQc`{R%u zM|vFAq`x!#l3 z+rk9PV)6UdML-O?+ei|Am2~vbr-$%BKi>>=={7RXpsF>n?LAUX<(4k~cqt$_FeaGA zz_K>3U5pB-^(d0nvr5GX6w&XhVN5OYz~bT|fG!syLyx7r-GM~TYmu)dIc>ZYzBcem zjfyJj%XuJds>SSH-B7L0)^$7Oa&o)Ryy`~jYJq;)45HPbt8-1VJ1$36#YrMlnH_+1 zn>(XAhM>;)dI_aP30oV@uN=cmjPFCv9q{rq_Y~ZOBdy)(WmP@x45`GvPK2JnTMJoM z&g+yipgm-9GbE6=$@%=BJ9Ow(MVR8s@~vHWRyG<2@a{DASwGBUx%$?2m#_g*uOoq) zAc9K&irJ=ClqxE$x3rj;I)E6=O&1DYO$4%-_qRKhocim(kw6J-9U?^v)U<=YqVEgV z$p{X+(26Tb2Aj8E0=beOTT9{3;iw{=+c5oBr{mQRgM0Uuqg?O$xeX~HEu4QTFo*6s zljUK?)tdIu8Q_tVduN_9fb@mu7(M7#38>vMA;rFgMRKiU=Wz(efM-=0y8 zw3wHWKk_W16xQMfK{JWU>S5cksvASh-RXR-;jN0`7}DM;+ta&H)LVAz(5{2`Z9P0> zhGin_e{OBBRc2GQpiOBxd3Qsl{Ti55HLin)ZDDTS z;Wq0$RbHWf+}>WHQJR0C(s!%hil?gHiu}{M+4*7mejy^bzbsDSvIvx#*UoZ6yKdvk zYS(tbaKI89~gFqNe7=YxKOL*&|O>voBc{u z!qObLn-m}$xiLE-DrIpiS|bBLR10;V!|P!^3PN_;_8uHC=e^?lg_L-bA?l;?CR@#^FN# z@!gV6;PohDD@yYE8$s$#R@)oig;zLs0Y!S&RZAiKwmli2QdzF{UG7Ac`j~09A7J^7 zdP=$AL6}Ey?&vU5PHBc5e^F69u=b($?1C^mWMu&@O<0;gpE?;%7(c_DuR;cb+6zhz zAM>)_f1#cV=I=sQ+7Iq#`jnn=lq@s46krTH?AE5U4|AZI@VXVPe&PMAotyuL;uXY# zD@{_V`wl}!`tNNASL|+f9!X^U3|pZjUOftHBC(}rIulGt>`Rs=mI*^p53Pt-lvCHY zRtT~c>Uw6Kzsx^2C5?xYAuLsLF4_w06@RQ6v`#GfBb8E@{6dWSgR(!Ckz~g@H2N$v z{a0Sc>Z0Kt4JyA^M3n>M>{AQ#Siq^tI2qtP*e>NnqW74^kMMW;oy#0im6saz>rrY9 zXR$?d)KqNf>%QoaSatZjFYN4Q0&AxVu#Mnv0;V= z)-q9vU*9h;+Xa{k6R%-=SP=hEvYD5`A8`Ztb>CF>toP`g#dG2V45wIM z@MV+;jZc%Sp;MgOfy+siW}d4%g}NuhkRH@)yM!yYr#=8M;4A5XZ6 zEzFZrVJ&&1;9DA78+xo=xb*mF8=3bVyQ{+x5swVSA{u%$fm=!F+N)9Sv<=RE5Iw9= z$u}5eV8&+o_aCUnL;#l2q3UZCovBJy19y@0Mp2E4!sM|=xO5BJ{BMfoJKfSQEDfno z>zY$Ce?lSiM*o9_HGX1YM4wpLtc_K&QaObIl7JB&DlhrMrPDI#siiUEt(A>-JF%*@ zY`sEhN7^Br4?;qU+@fAL9}gEF=LSXR7cn%Ov;n&83jFL2Kn!pu(f-CHe;#;uN?ot^ z8xAA0YzYspPx|5YWAB$C-tY}htT>MwNk>IQmb)hxl*FVA78uvKOHYICD&n1#!#5p= zbj}L>MuSD?vIMFpr9U0&-5G34t9ii60U2rZl#w}@=!W;_h{VFBN~@>KIa3Q{ofV~@+;N@JDZ{n3=(dRe==!q9#Gh;=X~ z*l=Psp>z5wT99aipg9ogTg=px+oA@Qm+@D>mO{b2TfFEh6mu1pqUNKe7D$B|%C0Gi zGEwN78GHDluw?r2i0$(z2&{`cTUTO)74W*%kz{u&ACE&sk^+1%PjA9Z zqATc*v&as)p%PpwZGTPipskiB2PbA_s1dL)z1RQhVLoF!O}D29P^s*bQ~yDA2;M`_ z%7c-XXc_xgNA+r9W0y4j$ASSai$hL9l zFA(Ic_eJGF?r&V6d8|eSH>vDfAjIss?Km|G?Ekc!`BpA@QK&Vr54xo0|8P@8Ki}N? zp#?_xqN!8}AT)L|ST6rFw8{79QueWX7#qVCK&t?7Xx0Qc*t|YJ?7m+Ly;-{eEVB(# zgJ2t;-4u!uwm-CBe(E`fs7EINcDVqo!q-03#Y-M~5!DwrngLI|+`2f^R|s2IDOsJNgtBK5u2E(LLKt+kml=5W?2tOHjOhOH#+N}pwcbfHE6B2_6RKl8G;Y;xTs57 zz>lhLE#e;@A03*2%jL=Zw@3PJ8zO-DV|SX^=6mg#2ha)Y`HvIENaJPv>4aVWF|X>p zt(*F8ho0JUuy)V7dACsylR@Mg1aAv>j<@m-C!l`xrfSm!GL6Q)gk3nzSGKG9%=$t5 z357xES~}-1cr7X>9G_a_Sw=B8w)qI$x080jpi|=eP!w5#TV$ufda}m zl)srwbPjoVd=X8QJ~xt5HU;d`-PajddZj3n+l_b*0kp2Oyy@BY94dTXT zdxv%}(*f4R08p0#j$7q0;0k-Mn`#fljh>BVH5K6e{0{Gyk7u|#AHGyUb`f}Id!Kqc zW-htrwiG~XSLwl`hKFC(UMtYVFM8(t@C1HhmV9El1tabrPPfiFQmvKztj)Zwl}SO` zt}J5+coNT~?X|#y3ma7}fz+9LHivf_Qn@*Wf7U~5k(AjZw74&JXkd9SR3yo^i6;l% zSkW8TpVWiTaK5fH`;{gUP|Ct6(cW(Dp(Tq4?nI3YlMltBWS0C8kB-P|xO7s`d@}0^ z=4Lt{Rax{@ZwqpG6)io+^Fg?3M*+x*luh|^T>&aPD>(t1ULC*-{mrE6CK%P)wE zr0mz~QUdGejc%>>>!G{)>D^dS2D_<2X^E~PaP{HezJCVqpe_5B+6aN?9)sZ)n3c2& zUY_n1RCwu{N#O%USO}Ofq%O{5kq(pH-S;na-GFz72=W<b>5U#qN zM4ZmMogf?)-p)^v-)wS6h)&J{969&sm?NH)WAoKQdavgq0fYW)*ZHwCw5(H4Hqkz$ zV0d*GY@2IRZ^@eg@{rf+4ZN2G+kJ+EOTc;O#-y_`z=r3NCkudZb2=aH!8O@c1ywf1 zMpcv6?foWzA@tD#PYC^P?Si_$dkiwqrX!@zw$ih*Qto+Vv6l@Y}wE0IxDRD~7tro!g+g3-g2G$W>d~ty8&R1qxpqe`gwlK3tAcvm_&Cy84-I0W8ha(wLhwn6iphcy`)2RRJT^^&#h z*Z>q{C_%$NjqCLkveq=Q&OZ?;Nz8YN`$-tFBlpWt?+1e@ldY)qU!!j$=9QsjpVDX5%MLa)yF_O|<_j z3W3;pe)y5xz5i$-54=PJjw8X*z=$qr@Jr)W|DJt%Ie(EQ`Mg)cq+UMb@+8m75@O4M~8tE?O=jbwH6wFX(WAqT%+$cPv4ol))el@tBxa+(dZorC(8_6NQN_m}j13 z8<{JXlt`OQUzsbG4@8~j9Z$u?xm8u(ehh`GCuSQ_u7L#}q$`d3e=j97UXC?I*%QE< zS5`z+c0;P!icnR#2OAHF7yP5`La4GRP8A0>6K8TJDH(h{&OBaE*?yG?_^1Z#zKLh? zdJQ=&`n=3P$9+#q6jSj~9UWqtgDXn;BjXH=Fa)m%z()J)iGE9nG+f=AtN*4T6mln> zp`215LHsFd>;KowXT#I082UbT{Apg_26*eIP?lbzY&ifDt!*Qp5yz%2uA=DTB(S)Q zCPgB;4}y{=ZT@{q9FZwa-0zRHz0j;eq9G6nGl%$SJd@5vf-;W1mP+RTiDzPZGJO?p zPA|F#UySuXfiUq;Agm3HU186)R^xXsnStAu%@eW`q?)8BvQfW1(T%yS`5AB=iA)0N zz?U4XEH38vHmm(75XNH576p6M%xsuMFPYU$DMtbX!cxV?IK;|t$$pAh`Le4v%<=yV zge6Ml;m^6MpLzfI7YH-FKwtp^VK}?>21rFfAS~8V(fH?UE+@+ca2)BPpgq?(5Hul@ zm{~MyF)7wphJ+)8rdUB+6WhU3tD;royTkb}=pl^U9;kM*iF#oqc9~s`)<5R539%-* zJ%&5{QLxMr#hQe?e(Iz(Bt#Xl)-+Qh6cU5>Ba91Ss!6sB+$01P-6sS67P}z6D!!e> zfZ1lWNjQ$hFR?f`;b8<31X~$vv3+um@q>|T>}>rBe0d+2NdSLH4AGqy0ygkyE2UGF zv5GH{Q1J3?#)Pd3J1FAzbqJ^NT95w5bqRv7!oP97c2T`d%aF{sAZewWkL=L?~*^3 z?Mtj57TSEh_YP7#?(y#8E(i_+E&?T;XS@aiD8bPsLfe_6-X#7~dZ4i$(>W}lNx?6T zmG`S?>E+g>z!Z@o6bZTzzAExSPjsMt`x{lZoajJ=L9w4p@z8+8oA~u>0il1j{D;^I zRRYP248gwt^mHBGD1>y?FzwbvWg>Vvwifhz} z+p|QtmIp1=^hQ5jVRag!QgwOK$gR4fAFms9NZsv74Z<NgG>~Da*HyyK846irnYn9lwI4xO1*UVc5CUFy{{+5I zb)k#eag^t)xsBu&d8%q+diX1eiQ70{Al^V)%-%ds*tRiTdOKd1DOeR^z&(hOajUCC z{*Wj%9b<5j@awdsGE}%?yx{T)y#P{Ca=-YE#p-14(}N0$Tb3*#Df23XXsHRP>HFGs zdD18;0iRv0A3kcdo4FCGw3tw(*{D^s#;U?7yGC;#h@0{_{mC9p1)XYEi9tQ6c&(CU ze`$}G4lbho0=nyrD!vud&s;p+Wxv3KK;wvBPsd6{I%ljq84)ic)dewW_I^RZv443m z)@c^tG?M&FVqJZhhi0?5I5OU?Ji~S5=QNTz@*-mv@8>kq^l#K3G{9-3K673OQv8L? z_G$$bbbc`Y#`u$d$r_LsGf6jBJx4g$8}yOXa_)Q9AfL+sdVs0Ch#}~XO-L5k6hU@j z1xPp6uUJ?@FG_q>niC{!oq3F4Ie!JgOp*9saW$$r{jG)eDgKYZ6F=Dimy^k9GwFG(HR4ErY7aXZG#PcCX_m@FomxRw^ z3c{#rb~^V(V~v0^tP9ZvEKNKZM7KJ;y5~2hMsd-=@ShbCTFK=i1&#glC)4rR(q$L1Vdc$2296Pzz-^t4B4X%aluBG4+Dx(V~zlMEI# zQubw%zLcg+N&fAuTVfJ8&fUKkz}REY~Z!StSjLXF9?2P_FytlzeqoUF!55~ zxCTLsot4J_0AaEUKp>1>vS3`7**e~W@jv58m#oUk;>2cJ7O%L7gsaUAV6F8FX5Peu zR0HrNb>6s4Ew;6PN&&BG~55}>--Zq4AyKggS zLK^|YcWpKf(Zt`z^%YU9yJv>dKo3Y7Wc`kNTl2F;Oxt&wBNqynaJ4DqubI|WaGmjB zGH)g<t^((g{&pnZ z7jRp_XgCJaH2<`(x@+lItP<-Jj9jCLHQY{A`#L&t{{_a*V_DdCIhHopQ_NwMOi>=r zmFOn2R|iHo1*$GujJLv~Z(>ZVMf$g}1VtMd=Q4;>;JWn0ynO0kh zTM)di@V+p@0&>aPdjO=2O?)4RHvsA5Uj*_HH$-QI9s^T@yrJaHyr=9#kA7)+#^b)gA4Lug`AcI#rzl81*;lo?a)dLoy64U#%*)QkDD*Y z&s;puQ;maitw$)s<);6HV^ZGV9+SP!bLQ73B>{e3T6bp#tStu!NntB}Yq*W_p_0AG zX+;CsTHaHe_w(edvg$1xch`_;KXH2%D};{yeGD1`z%?=vqMd+J7)$WdckuORIDf2WKQ;3;>!*ldlJL-AFl~~1#Q7sMerlUSH|!|+PuyP>zX6X z%CKRT`3DL+m1zx|+z(QSfo900P+9xm5~>IN`wseaw`FDp7X|JwDTM9Sy*ON$4u9fj z?M%NF%kg>!)`l0Jqo_$@uJSA$A1Uh6izH!baA`b9*jTwUnj46Za#*I6hqf)^h(cl!8K6%8<>H7 z(3E}IQS+=Zk&WuEkQ3IXusX?5WV4c4=Rj>DtW2TSTEO*qw5RzAg~|Q#UC^_oUyEs{ zhJ7mjeL$8TR87#IH4`Zt=tJAHK0ZFNLt#So-C8zwSn3}r>|A#BKhaZf|0fjoPxQ1A z(O!9Mu9C9agKqa{8zxQ5SXKD zGx4G@`{7X3_9fHAv=KUT8!9sO&W3`ED2tY${QxQ~B`UKX%{N(?TV@j2v!k&wP+}Ck zyaaRRisu#LFVq!fRM3mjqg2`fNLP_z+P{bv^u+v2W2-61N0_5?HK8wvNs*k;UwC2N zoMq|}3xay}br^l|amS{jnyH-2c|$yqm@zEBjp0SGmNU^d1IwbpKTWZhR76DXJ^Bsy z2??c!lay}5j>FWGk{Ax0m>Af82CISDr<4CnP*id%#R0aB6&>iMx^6O-xjFF(g*}#f z+US)@E<&AwtR*h6bC+s2&>kmnrb?7)p8Ys8ki+UQcUvS#qyzU6&*1Yd+ssh*r2QIM-lGXQGjnuN=x(o+x!O_6*Slw2 zpGDBx*5fjHvwgpIB<_Ab8mY+eLUTU?ryp(4bY+o}y^~f8xEbrLZlFlq7hnK%0wG_y z@grCJqXA*v0#-j-DUC_*Omz)r2i}#H8oH|2K#@HXOOK|Es(fi32*|M(TIJ6F~sna!^?Mh5k{h*pkq&o$&zOD4V8L60O%9M8!}W?pQRQAzm#oA zxSl0-w)~UPNLGE_|CGWw+5d+W=CjNqJn?TpL#YIQpM2)ekI&25-?Do|SPKcaSk7c@ zH0~rwXr~2}Jg0CDpX*u;GZu?ry>Ezth;-#h_Uy}^#&<&!kBtG-$&PKrobDD-nhLjN zBH#zI#Kbd1L8F-&8n0ayInNma_h(zOF6LK^`|89n2{i&Et+1a@CHkVdiWAY$1W7I{ z*UoK)I=#~u9edh@adFCkXXQhC-+bSAf`d8vOQ?}fCTBRLj{k~iE*rV|5@pn(ECS5o zb(hNq7!d4;s-Icfe=Z-WUAYToqTg|2>&}qG!lDtk|Avn2%OIDTlx3moq(EvlT8V|} z*PmX}L<#DW?qRZ6jPzg2gS%W@law_In#yqQO`!5&6qKUt%i59U zLN20*mHJJafUx`9#6^P{`w*^gL<4o{R}`dOBVUeN_Ad_jBO91`=S)SYg-6Xax_NEb z-v-Hy(ExNvVQ69eBgW0j<3sLAKNPL7HPdnq(n;re>b zpdWK@5W&b%6BSr-myLRUdROAD!WU3Q$^AdJG0kWRcR9h_R7*|ik;?ZyH=AmkqhEiq z8wqTXUvPPsfyBl%s^rP7^=%e2JCA)jVI`gBl|Uy9?@EGd-{CPNYjgF!X-#q;qWotn zz#;L}l4p6!cYmHv%L*98fJk9Dr+x-8GB1Q4k`5NB$Fy0gmZyqd(B**vrJeIW=EDug zvURv>MKGIfZa%ZoY&hm1yRF|ndh3w1k#tQJCV%xrd?f7-cHqvg6qRRi4yQq@*)=19&Uc!+ALSZUZ*szdCE>gXvr(W)(TNks?ngBMM%*e@HDNrqC6e=PS47{~da?bm>Yt-fm7_?!u#j zT+)yW8qFj!ql2YP}fA3w~6nd6OtK5C? zg2q4%j2E3!#6)rH>7ZnkS~=k^Px_jwi%0;;_u4I*n%Fj34ePg5)T%ERw3ezTz+c@J zw|Z3wnrc;G>S|>skE7SUqa7{q;j4xa8=U%en3m;1{C%(uPDnW(weB3q5#4c~3BlUQ zc4<3fNZP%v4q9O1*|!e5bX8}s0?cIV?W|KmH1n+<9Ma0dxP1czBiwIYmFW;@iDUYQ zwfu*9|4+%w`}yrNORt9c>4td@MOU*nZINQYne|D6`f@b zfQ&7;nhez1`P=X@&J+dYhcFfW#osj8A&9Upx+{BP-!yHu5}b{BSVdED;)TQ3 z){WG>5uj3DZmYSeNv;0Z?xn_cc4erpJ;W2wLTyQT!OUM&jRq%_mwnD#Jsn#01-Xis ze!~=^+*AcUN1~;t*8#=>YMgsJ?7{wde=jPdxcs)FZE=scg*dq#f0Y&?jj`373;Skr zzuW9*Z`R?u-**5isX=077p2NgYGzIDca6D~3=Q_c`N#QD`UpyJXUda4__28qHYaLB ze?)2zAvD}ZnDnB3rU+%wLs^&xDtQ+jMUGD{Z0(Z^n@3X?4ISJHyD^>|WluNUeNE!yHqtYlsew>@5>xeMmEMKG~vsKtNCQe01 zBaB8x$LOLV)S1`Oc_Yy%MTgI&-jA$=dA4PbFP3Q_T3lYbt{wXZkDinZLwd?vI#S$epXv|Lg2!e7l14b{JcfeSOuvtnq3X zTS+5->F!JBq9*UB7S^R2r^}CQ$p`tA;5A&j$1@@Wu{x^uoBK4CnvcO~869x|xs21PfKp>cIUV^KM>rP@>QC zD`C7}qyWKZ_qJNkLFf)8C(`S>(@Nh5Zun>kBmVvxG$nEGu`vIkiPi79inDYPnv_2G z%Lt<0qyILzJ7i>x{ItwuUGx~2z07rv#{1dm{b}#r=k5M2C4mj6@B^o$Q}L|_PuICj z*8|x6EnmJS)JM4yjM?xL-S5R+)zP}2T%VJW6yEfUq+a@(Z#NMhVNhk5w64w@K$OGP zHz{Q*|H+0v7f;g7BTIwLm{iM+YvHo{!oI&9;9BsJR9KowA4EB$%VZ(@>VVmw9h?uXMF$OO=Pf z8~!Ee?X#!e%A7=&cFHr&q8s?*H`AzLcvtA(c;WGu zu`mY`P9)2<_gB9Fdxw2Y=_UaU+xaUPdk00lefr$ngv;yenX6rBj!Fyggw;8?AZ$}< z+g09*R^8uD9uCf*idZ6-R59$jx)D$B5pcRRL!XVi%us7bU#k?(O0cJ>(NIX2D$&+y z!y9#Lf7PzPkuNI*a4Y}hAN!()xPWV*sh{J^q_0qasmGK5F8P>(av@1hbA02tMbtWJiL+H{!@#9(SgN0 zB-NxfspjypFICmx=le1SB{mN4&ESG}jYX0Gq0y7xI<0NC z!W=hNa#!7w_G``K7W2}E+p=jiJB+lp!-adgjr-*?M3t^!*K%P+?z&KW_kzXTRDdJI zQuF`UG0d>*}XoIR8LmvIr;=DKqTAp*~gv-i(z5%a!7YB$xA~$-MZY&>$ zHm0Vp|9=kiZw{lH1n^~=&2TI0XvZ+xCZgT~d#-M7R7(PWU2*%21MUdAqd5yiIW%H4 z1Q!h_>53`s4s{UP`Wr76wB0;gE(Rf&!LVw-tn8wcBG2kf5ecgcQz3Mx6a!NY*j0iI9Ufe70*z z#C$s+6z68Vfm%)D*5DX@@*JnI$m5Nt=W_9l53Q8<%f&FD`Ju_;&Bx{S`r}Z*TQ2}m zZC|}vi&mrRebHUy-?Hxkus$EepPLel0ZtY6^WZR%KBpn;6PkiS;>=N3!0DX^18?WC zsLKo*+Jx_~k88_Ze_5zdp{JX0>(eBzE&pJeDjdGK4-H?7$C8s3Ck2H;j*nEv}EPOAU;Va{Nd!ooD>_;ugs^5wS1W z;?ZB>mnW6fKPb|<)@=}YzDaM==y3)|C8G&=!%1C ze4puoSiUNuP{Hn1AjIs-334y9gt!>RTJzAT-%2popq+w!qL4m2dL${BFii^#yXV8{ zZUd(k!)&=XyoNXge=El#EQC!!s3Lh~F)ZRnbt*G-)|@sk z*8Ecp%X}v84ozs+cSF=o?IM;iF-!}hF~}vWi+cW#7^b{JN&(C~3~&c{o=j`sTE4 zE1Tu%H`8-Iu9DX){DtN{>Vk}{+iEnz$Z-_4TYvd@Z*WcZ5UCbQ%rk7_i{+BynKRiM zi{)~^QRaB1GBB_nRF?E!BB7XwS;v>^V*$ai;zS4zMVZk$!YQ)h0Pd2~0)mnoBKC?-sS?l>j8$IQP6OSAPC1d_Q z%bpcSV8R>+7Vvv7xFDZm2ndGFLBClp{}v=h9wpdKq4DeI9-o}g-A0?!j~vF8W%{3B zSnMYl<^*AzH+=oGo-s(K_oip_inIu}D)EVAICNilZ*FV$FC0jnOgzyzs2sR5F+NkD z)!ru?+KP{r&~RC{l#^imq$^ zzH9yqh7GOZTL8f@wn5WABI!>s?6_dG|3S#fVl5~_VO_#SXzbe;GAuE(NbX8%LeO+^ zdw31m{FVmhgT*FwhkEvdm1WduIzc}aXPGq9Fd|#v!!rATxePMG2|>T1em`t1GjzE| zK~Jz+Nqq@nb;5P^tO)t|$o+WZGL%ZH?J_rUetGYOh_Ho0P>^yE53!Fz8|n-k`--mw zYo$693cdMy{9OS(uY1Zw_ub-7WfxZt< z&^?~;91*VzjDKB+N;=DX4gae2jV2z^#T@%Np+@O}=GUy=K^b-O_o@`Nu=1uZK?6Dr zX&F4}pbNo=LU+_OH)_Nh9F=;?Lt$F^Q6Yshb7D^l1bA^FaJ}5Om2z&TMBr_weSR(`{QN*}9{kS~~^lMd^a)Th; z>uo0YAqr*BXZ5V6Yns{iKG&3^fs{xbBknBOKg&aDCusyBkPP4R*u9MK2Q-H2q@l%> z^ue*D1R>eFPRd{kP6=Dh>?+$p_LKGl2f4BQ1izBwd{a$VT!BtSMI<&D$rlf}B8*Sq z3_=80)J7t0@heba3+}1wtlHZP(enBzu$O}ZqtST6LZf^TbB&W5P?>G@tv=ph}uS7kTiQVAVSlRf%xo_QzqWSy5&cjF@3Uq)B>GYHru`+!fp5 zjr0}J!%)&dw_YOSSs;mZHcNjYpqGL~76YwdO>3|j9*D^7!~hOHXouR<4w5&K>>if0K_(9?!!TPsI^Qa-(g8dX{Aq;Q=P0d4Fh zzQJ6o9G{$ArZROMB?%zuOO4)}B?{ZJK>OWG+Gzq?2_1e9ZEV!z<(4}sg2+Y}St$gc zlT?C?T1XO9w_xB;B>f{SZf~I}Q|S7OSRQ&7^Z2_{l;0&Mc7bytl#Ns8Zlhu{kz z#rNCUVae3!V9lAtRqOhu{6xDt>k!bJl2n8F5e;>{3T}lylP|G9ix)$c{9fBQ3HHi< zT;)~ru9)--@^sXFftr9CLk|LEcQ&6#uZp4v^6|Z)sW&;0x`{MtCL0&aWprY%{&ub2 zJrbWZI(&LkH$$#G2$f+3r)(X%$$IYs*!zq)$Oo~->vxOSaknnL6V{3&QlLTy33k$o zEs@>-ToTZmXh*3no&?TBQ-GTAj{+NWthqcB!N|HF#;Zj!Y}=nmiOAmO>*~ zxFi@?6MsFw4FBlMq4RB~ca~0`v9cVS_Stp4^&Pvl1_T*I_rDA>qL|C@DnEe@0xXC0 z;*ElJIB?FHkr?u69rCoh^NM$%mvM-ibOhS(i%N*i~pp&*v4|BRN5%D-gai!_}gn) zf~MTllA~rucHzdOSkkfIT5w*eQs7gfYi#9gXVj^bagF%+~9Uz$yoNst~sE2@<$qr{4uQH4+E zU;Y;%(^`7aA`oo2Gz9-?IyNInKBzeW4_@b;%|PFx{!7AKU~v)j?tdU;3OEVKFUeWb zH^z*h-Cm8}@j)Pb2{f{nIYw;rJhApx^a47xOk5enax<>L%w|-Ib;crHmB^=-6$hBv zkODIsOZ$flFBrFVU}l5KgX#G0%S zYUi4_qU($`Q*@`uVWqQkve6JV=%*+)NyCb`uN3;f41QGP-&;J?3>KaITri>cc%M_) zADDM^Y-Jc0r)YGcv={LD5}%m1&yXr`{p~!v4_$06(aHI=)R8%E#h&6`VkJ?0)+kH7 zIU-oxG%nS^r>g2)oAz(}Q*%a@E0h+_A1exgywvjX)MyLXI%vDbUvqsB-i3%xH*AUM zl~;&NN%?0(^XY=jg@-qb6WJ)VcQ7Mtb=`HqTmk69Ni z{H@EHM7|e=XJq@wTH!d$sFo)0$cFq6@QTQ`d75%;MhZPMA6a@rRm>&q6H!O=*3L@!^!?Q8QUG5~Pn zFHi!w;&45!NIIT**Fpo_?%8(Z0+_J9A^@!^_yyQ;%NN{&M4B_qd}_Le61*9p9=Is@ z@nP)gzU>o*l4rH|yG)GGj|hF8`Y~_(qTBg#Plw63a~UyEdkgg&lJTh35Pab6`|2DN zt*aWxd^I)^SzpE>4?pC~O1_M*&p-v4k<#Rf7<{b3h;0iH@GU?7+;wLXg`w)_?2l3p z3HI|bZU_L;$U?~atZUGMo4KLd)+%)yDgI*05i5JRyBk0BzT3|4j9H%foc`^^iu$n$ z$RElNaQ;~86pQKJ_i0JlQLhTY7nJW;&9^ zl?Pfk2G2uxhzn)ijx?rH@potAR?sLOWpx~x4h!LMD@m}d5&9Uz^}F-p#bWcu)`t!L zMh}i>Ax1J2IKZ}Ti2PMb7}0vEU% zb|GL_v7gR3WhWU&OT%|?<(lzO^Z;6q5TnEql`CBn)9v9kjGTnkey#81FA|jM$GjL^M}vGD6SPav*`@wg zYuZrYfakrnWnFY=Vu#9;^qq!Wd@t&IeCqg&bFmXG=FiKH0+cF4EPHD~Wd7uI@j1kfT&x0rwOco&QEz(IZaI_k<{ zV3ajuLqsaXf@A6x;p&JgogtMw# z?Rbm*Vy!^vSb&5MCM0lwCv^KLA>7v?AZBHDt7{1Aby)MkC$ z9hIF3v6$WSWkgB%E$BT~-e$Hc_-*<2&PzTsyZKQVxx|dF6)`e{3JYuidST1uC}B?{ z@)U-PY$Ug<#0448LxzsF<=I1=4{j;IL_~XJuNY1wQ}($O#^X8#tc-;bPyfTf1nn45|r_g zG`o*f5(hS!q)fj!5<=VGZJRs#NoG}}&r`TU>aElcI1=uO`b}0X18Cj50+#SLEqAgd z_GHq<0{hLnj8GRAVkF@3-_I*o7^2f<4Av;72lA=l{|`J@xb- zO7@-Or36BdS4}IN=|=xUacL@sF>K~E#&N1=wopusdd4%DP zeu8w`gQx(S+Dvsx0~5c%mt>mr>iXZ_3$CAWjf_=l2?L=;oogO{uxhW+k{+(WoXKge z)Z?DCJBn8F6ZF6TkQqN@<0(x~M2g-rW>DJIe4@{7N4G}OK``2~Lg0RFL$;)bBT zFp}EkJfjweU1yUh9&JMn;AyRlXj_AxN)ALbFjR6&*01G==yjFrFM_}zx)%*jo z%&(|bJ1U&y(3Myz1iq_5W33Q%SX*^g`UuPbrkD}mQt-C6AbI+K(i*1U2>kha=fx~4 zeV5;Pcq;pH@BY(dwzkwNaha1Si~O(kKJNrbtfv#g?l9y3l{F_zJl3GHW}zGx9`=~Sy0^s24@zl= zF!%9^|BnKwthq^quBT!zMq65{?m-rw<+GS>a9X+4HLfvrU4H9UtHVF%9zLBv>|m!TyZ|^Gp=27W?QzRzuB!L#;+Q7|LVs=A#*sdJ% zBDS}wG786`FRF^>z$-t5RDRlktmdgy=s`V4(?2YESbBwB@UpVD2RHZ#MPiMS8GH#v z`k2((P$>+kQt)~C5J9a1Niv#C`b0Ro}SsFsc4tL&`KyK7xGoVk+psdT=!L-VWGjH5f}-W98}Ei2+s z=h>{u!Zawm1)u7MEyk2}Tvqc}HVy!LIz!%|n@E=yIv#@zN6{)&pQfE`y323C)9y@{ zLFAg?vUIVy&jG%xdRFO&+FdRM_x$^n5zPAe7T>1-DVs(WMCWr+dBGM&L6`9Fjp%cK zdVO?4!O?=GpMPj~yNCol*BgO+e)serZ+?D)v05B8{cflXGmoRQA2NX)g3d-pB`Fvw z4-D~iglJ{mbs5RA>z=rIJmI+!MfW_{eo|896)~ zrB|VXFh}&q+8Lfx+t+$`rL?ayE(v!%%XM_&H1l9^b!(k^hqh(k?8*+yZvd&EQ4M&7 zxi)65X1j2oGq@`#su^QBIJjLKv1U@xKhS)r80om1(md$8Vj*m1ZlUG_MKupy4oo}5 z-|U29{H1C+M<%;yR1c|n43(K`|26>XuD_jJmP8vKvO%~>fufp>`7g<>(M&>NNgaQp zV`6#+8+^QeYi+Mv{bYwcZIq#*Sg0ddT=U_>MsbHsL+{1monbw+iK+u=O;!bp3v1!X z4c}UZ;-QL1M%p+rEW^#ynKEFNDTlvLiS@J_Y;Wl$j+n;RIQEyy#iIng+u!p-N!{$QbF)gik-rnJJDQTAZBD| zZwT}|JxG=-o5_7%cf{h|I+;QK_CHHzteof2Avp`u1dc58(uVIq3&JFFqnI5&OJaEi{fzu+9Z2er_s5HxK9*jX zrrzvz$+F7KMUl{V|FXBObgrLj2f!7)i-t>o=-BLY|2E*p(AC*G&_dCHMEl8SK~9{G z8*~N`G1$bWcUur)Q{B`{K=>E247dUJJ?PkXkRQ`iv&Ko*=Y%F-VI}&9xr6{`488=Z z>Fo7qR0E0OF+5#n@%UP(%-7F=!EXwRYQndoQC?Jtnou=IgQtIf&(hNLn+>}>?XW+s z!vEDc4aJdZCL6*=R0M8k&cIi_w!zRv!~r^Ka@#^Mz?&iEt7i8}UV>b5%hQ9c*l>WX zX6R9R-;K(HudweB0064|6iQd7p2KaH+q>IW-XB;0&{0ze2TeF;6?#A`C*!>5Kg`Bw zu!gSYw;^h|yxSF-tAGC+Qq11V!?8#~>P_IoO4sAGc0%Wj(uMfCOO4VR@q zW5%k!Ot6N9_ueW&jhVAnwsz0?SD3&PBFD$`p2l*`ra2Gj7#bnn!YM)YYWDdpHdFu~B!|<=#@`Pg#cjnay0NRF5I!Oo=ZHJV$1&Eqa|@r&O$)F`l$Jh@At< zJ}G^W5L<=eXR{)2n^0jS-mm3W1q%x$p9h=aQj1|v*js+3u7|1W0F%;*D~4sG0rMV6 zXa5LST{T~V89`R-`)QV|MY3ew)lXSw{eNYdQ6D?X7UEQW=~6A8DJB`1Dht&+ftH4yA*|5_DsBC)hK$nA7E@@5J5RAX%P#1(dvuzr*rMWYVs)9% zMN!E!bR&}$HjEB`G2v7_VdG^Ce{jTSZJme_TgnppczxXNUxeo9b@;vPOw2Gc3b^A5 zPjCHF0N&hQ$z+gp1rf$_`)XB8w9V=J^LT#TuKc>Nj30;x0W%~pAYeus1kBvD{fM{j zg-n6zZ!jFRkPVEAaw~UaZO?$8OW!|LJa4gM%|VQL9jt{a!>*FNo*MkSlbbWr^)Evw zG{=H;kMApd#oXPWk)eE2GV>IApNFU7(b|gB>d17!4<*V>0#fg6ERZmhsPjwbJJKpy zIEa|BkCp#;y{(X|Q`~tjzJqM(yqx@=K5>_dkB?7hy9L);X%!mRG>sT=W^UKKmQ%l0 z71-M4DbQ&Ny_W?tW{4{Ky`Dsd)M=TzCb8sE%37{6jU==^H%n8gn(c~?8>tt)Mn)IB z?s7yW5Iu`~0v4eskr(I<6FfU~avL-kU#Iy5T4l~pWR2yCwN+*yo$|CrtrhJSi}oUB z296Qh+%^OjF@t9+50IxXyZ;sowy?DAT5OeAtf^EYUQ6mx=r!Th!)up`FRM7&YhWxv z>!g9A8hV4`D&r4raFWq->N#NCp{t?p+;-EJ|Ie+7^R8jErMnN^6X4r1XnXkr*wzHx z&-nMm@5fOGTs6lwBl)%VURyl(<^a5asHv(YVc**azE5_=P<>P#+;T21^CmqIE&2})JO)_bD8@_E_u2yr5zg)nV znhV791-qqr0LPaAsQR=IRUgADH$Am2YqqfC5xOz=<>aMtGmf^8=J66FG)>K@eshK-DEs=xe;_~E z@>u_-+R2dsBtQF2RDlWs`hW5$FP>xjIy#27_PgGmHyA$h(n2cnpk}3%kIZp zQlb=TSsTR15moF~@@;`-YJ%LVaeQHIchx8b++MlXi7`1STa*<(rhy)Mj=PjA068>T zqT>Dw#FhKV9|%;+9h_9U9Ga+5GowZMQ>nH|Pxo9w-_KT8JB)eX5F^%H&SFMTvSDi0 zDc4j#1U$1Jrp$(Vv$Gl%rVi4M^GGD_4NSytGulwdJ^;P-UXU=V0qz7`MhNQvSa&e2 zRU&ly1pIB8`k-=Y&5LPZWa2Ze-p@7mYa6d`E}j)n@=frM8CK4U-^#;%{d`IEG@S+!9%MV@|&SpE8Dg!ke0D+@l3KF=SMh%4YYj$fLF zpnpizDe8mk3(hM0oIFYS^%agE<-u;9it5$kEwIMK=#R_=YttuC|6YMT$X!$Q(-PWn zOndhd{OH2eiKf^^Zg^GZTGjC%K_4M<8v%c0+DWa|d|U~z$oTgsqkA4VE_F^gWz5I# z7EFd*_-62&m;4i%8P1Z7kyBjNSN%k0dJ8>E*4WC*gR5O=j~Q(( z(7l`Kh;1XzSi|m9gy-b)E^V#ImZu6Rja7&>9Wd4{Q>tD5J5Dxe)RXm(Vo&Gxs;c9| z`W)ymIP2+@eE?`%!7h6Q?nzv{=Xl3HN^Cs>ecz$>Q`InNYibE9nQ1U6T;qjR7X!ty z?lW(u6Oz2SFjw}1rZmz~Yq}Jx5=@TT@=d5K;$@)ZXBVB*2Q~vt&hceV?XcoR!v~J> z{%yGlm(ymMUl)7{M~8>KtC@z=&-CN0xQ+J&zr`OdjO*eP05~c+G4d*ulDuQ}JoSzU zjFG&n^QC4X4fxOJyffpa$561M;v?bLc7&32(X@x|1nGGo^;t1|WTlL>JYD;HgWqy! zQFu;_lwmQpU}2hmAYrQNrS?Z5)_A4?``{Qo6NhY`7dW19yVxFqBV&13UG4BkH*!N7 zoxmp~iP*=CH~;(n?d`P}f5tAMqUNR}f|B&yQ=dHivX`=u8`-`cYbIbs=_SoK4vI=vGwZ^5>iTvs^4^ccqZmpyLwSi=+MrS zV><%g`KqJ_GykJzq}||zGlW#tNcQDYbwJci_7aAIQK2A+nz6;q4yBO4j8awfDSVTM zNSm<0sf!wj7} zEZyqt^%H|02io>2O&Zg;(WupEXy4ToQpa$gj&e z$izuuDC9tK;=a&mO?uYT+?G`ntWyg<#c7~G{~VcGCMcT&J8nx>?0tO@ctEx8|Gvd{ zRsYu%phu~B^uXZho;f6{YdKs4&-h)Cjb*Igvp-ZZTdchJ`NxYlLL@KZJkif~kBjXB z%U%s}Y4#~b%_wrxA4WZnF7Gt=?PT)B&EgA|4KjrMi7EMBEAxc>L8SQskCr3PvSm&2 zm|YUG#{?}hR+G~6aW%1!ku>?~fk|`inPf0vVD5-#X~HuQMI)@g*YS~K%MPr#2U;DaAI5>e*}ExX_I^xN-tD;+}93r!LtMi-7$ zJt&S9jDsMjYzdej6xkRn!aAbSRk6mV@lmoT1%gGK#*iYCI6W{%PkW|!2p#P6%C*DR z(H&xXDX&Twf0bNmp&a_X`UY@3`_h^fODiXb+1pIgx{I+5msg8V;Jf;zYwnhAbar-a6+y2SqfXCmy@yi zydc-*NWfoh$2BOTP>*bXn+LuJu+ilm&`4C=F&K&21+XU7?-4VOBZt#5l|al)S#Yzp z>5u4p2+0Z_&pe7iqGsP{y_9PPRk}jaLuA#bEi2L~M6Wq8m2%@;QsEf(LhOY##22{( zAPk&^CYj1C;@OkMwQVQT&jx19%yl_8gTTzsBbq(v#nXs$oy<;R0B;%(JJV7Mm@iQ+8WjW)7HERz892ZY_f5wfyY_j2Hq6Q zg1i}>gcPEac1fI;$4V(;KhX~Xiw{VI$@+dr@k$u{wbsS5cumUxX={v}=m$V;jVMv_ zZ|w(=H^V{(^>;j`v#6cq1~%J8YzCV#7#ZDk(nRW%$b_MWXh=YV?4JcW&exfbJj$({ zokVW7DU2VlkQPS!J?8AkY+QH$Si(>BO&9(Y6#J2+*zl>e`8G`E64ks6=ym*h8+M34 zOxVeUbz}1REx`tX2@@7tFWKwKYr&1xddVnL_1V@Wt4EYQ`UdyvvCGQg%N|||$Mio? zE#4&)8*tQ`DJP-T2#OIp*A+Eflkjm8BcS*M-wMpCC6Le9gHdAzg=9a(F&G$$#W?66 zDep;A1Vi==3{+6Ypisv-=Y|W?uLY97DVaj-Kt`%bReGV=*H^=;Eu&oV|8b8J9XNWL zu(X~cZQ!YQ57+YPX%-qBi^R?JEibWiyj3Ej2&I}tUJ2zo)6j(7DTWgNQA=lLaFevi z5La2WUp~}u3eQkgPRcq_=iEHqGXfR3P#M=qN&(Ryq@OY9T2Pp&}H-=#xW0pY*JqR?+^3>O_DT7AYf%!rDUptwfvGp>QP zfS>C~uZjNzXZ{vbTK1aT>Piz^FsD)c{u}=zOpTN7`#=WVVy7|hE>W&lD%R~t>uxT*Pv;;WH&WT-x%Ky zIuOr~+g<4z6WVh$MbZ_e(z1k%TI{xmmS5KFZ{{(R_ET#Y1kPMFeS$L>AaEv7DBe$%FqcjKPbg%A=C?l-cry z?yseUZ+p@NqpRCsVHadVWL%ppzk^HSl6n#gePJaO2^TrlD-} z!X}PQ#ud-b6n`^12v2Xt5RA{#B3s0Na`hG*R<*wVThA|T-fzU#J`e6V0F!|md~m!9 zdjHFPT;}Qizu=6Ka7Hg%9z)_h9?<(ou5(JztNpx`#y|EZftiq zpMn0-0oVIJC4?~v)~}Dm*OV>PK$ZG+3xQoSB(9S+r;+s(hDLWED_tQ_NjO_95CqP! z=k^yAI4uP}1=5E}gTNUZ*5k;D7*?hB5@`-Ev5xwK%g>+jM(Je}M%rZ1(a z80iXlLT!zs3I4vPg5Lg|9owN!Xt>k_j+^3gxBKD3#U~y|H>1tSeuudBO7el6?CdN3v-ko3m(sgmZBTr<0WG>9pzb|Uq_^b)Am4U z2!zGPx`&uh zF=K*vF+lz=rQ4L{XW^H|Bqg*nYxKlE$y4Fdz15^r6e>J7SxNaDE!;ZGc$l;eRyQd& zKpxZTs398FTF5wR38d}Wsvn$a*rRWh;JFQ#)^Wc^+44`p#VZylD6B|&YWm?-?f^pe zgVL|?>(T;N#!A9lvk6k2vJ~4*Rf`I>$lK=e1C|Jh=OL}oN|gu{#bc&sq0al%xC%39 z@}46UTDEi2`;3cbzjn#$O|Zuu`AJ$ImQ=BHMha98t;Fs9a(@^OG?*Q)}>jJE^SXuk73||!htDxWDsL0wik?7s^f)Xp(6bA&v|(_ zETeYPjxQ;OLP;C3fL|&DRVsZP?Zj;c`GgLY<$P+PA!`iM8w;2`hrwuQP1cNXKx9SD z^G_2;J(tIT1C9p)zPre)hC_jDMA5q|R~d%JpAI3tzHT*JUv8sh=7_Cab<}@YXK3-9 z-~#<6$q^$ET>1j^0zFaw^kj3c1B!}6TLxmfBXJ|2*B8!`TR3@5iL1#=$C$b)p#+7@ z9o~s9^YG#)jjf*nlO1EmHwOF@)O7ZAWp=DUt3>m58tP>&;?}4HTi#nkRf6Oc{q9s| zh1ygkE|*&H;bit@$gCo6>CDjmqIOZExF-}WlQy}c6jrenB6+|d`cuqlE&BfaL-qRZ zKI$hn!|4GhNy}_MH81&Xce~jD&Q>1uK$Pihq;iegf9ZxN?V3vY16_73o1J@jcw~rf zF1e~TEIKEOxvnq8fOnb#9KmR9Xr%VS zl9|ehqiNJ`LrHGhl?S7S${Y_n!E@B-RuCB=aj$!vzvvLd^JzLMKASuE0Uxz5f&RD? z8ahSg`25hZS*~dOjecdj{@M71jeGvtQJpWqtuZ8}=wTaoLK2h;i}W1Ysv2^l$$*ZS zaHJ8@Ou6ojLnXlD^6L!_PVigkj$XBdA93FGjO4rX80ef}2;%ls(;grdu@p!YxZzZe zpSXTeeowUABsTTc?3c4YmFoQ0!^`6!;d<2M>A&CoFN5>0iQ{t4y=xphaPlBQ(9EuQ zxkz7k7xSu60NoVy6!+NI0oXq6>`!Rse^MB76<;qe*UuEjxDFI6K`vyl zOv~NrEQ0|@k|Izb^vcE|M5QZz3q)?E@H7ohpi&U^o%xfQd-$i8k!cWkBSxrOt@Pyx zL8taX$=~PR)F^5Y?Q#G5`r5$d`QKCCzpJjTt;oOB1&REtc?v`lp$diPCl(fixb1|H zRT8m$ReM?Cm(B4*Y9-^FTGzs+v58BP36<%{`#Pv1l+2L_sn32q*6wH8_71=%B>QcI z&RK+hy!Bl~Fo&%xK8TR^rM7QT&lFYIMBd=2VK2$9$vZ(k3A(nUx-JYH(tM+yd@1UV z)?ARZ&G^A@-aC2e@ZiHIGS9VF(VfCz?)7V5+v1y7|Gowf49d3&zXi6*EKrj-cy3r8?X>t z1Rpm7N^_h86|tL`BJJ4lDqzJM*un1sHQOz-V_h)~oUFK|?)8rZiLJHw@;wFE4#{Z` z^%C17Y~^IPs+08y92CH)DZ@z*0e1_#kItVfiF>PdzY}_AEtkrM=EbHf|gmd zX1S0A92_|2KoxjIwlrcg8u_g_qLl2RMcW)V7JQAn-Ym>gO;#Miud)Yh@N)NEyE5<9 z`GZDA-CgETZS3>z8&KIiImcK$gCY-VKt13Wf0y4rCJ49S zmmCanJ$i}KJ}mD%aZzx9oSA5Xle{Doiv3gTUFa-9b%SH^Im_nO3LUU{Txv#P<^66_ z`!cVuDrHZ2R9FHBB275p8oP;y^jK|xr@-c4?K-J}We%y>p-dN%Vy31Km3Sc+%~1cO zC9pi(!yF+j!Qu@$@_^B|V$!XisTmL21$zpb%zFFBnxwh$U&(ZkVBW-{sdby~w+0nP z!Oj%6f9_M0e)RfD7IRpti+r4WzVa7OPOW*9!l=}z=zJ5GPlEqx6l`_Pb1!vEH~0`t zkg)$lf1U>>wxbXWYyF)sKPTi&sIHuXp$;}i%-sH-Iv06j%LUlO<%FI?|8f7mD_6I433t zifPVv*Hx9lx>`wDVU99F#dZ?!?W}MV^wS7O%so;2rq!jY4A7jgRN=GjF$>wq>uo0L z$UWM%GRN!;uj@Dk^5r#PtrA+ns%%zvIb$731&-9uK&%?ZU7Uc zJwKRgGe){O$31tQ+)h1C060|;b0Ip!xWE3ivMCfYxjkZKaW#W z@;V+&O)AcfRhv7G_8Ac4hugPfVsF1Cv<3h4V#92q#Z+iIFcHpr@K zAR+!`(_I(87b_@*dHBDO!ldwsjUIBDyTxwy{rNH{?x4>!^2KU+?_kv}0Y#l}`aSlb zY4>Y}yrxQYIFLeJTlN3%VE!+4Fr)t;9Zc@V>_*$5(b#GA)6WVsu4UJ_g^~$xW1WUk z*Vb4hbH~ZbIP5QN77qJwg8B+ zVCD4u;atDH${efRfR_sUJIVC^0$6QJ$j;|N&-1T?b9J@*=`7jXTjw>2O@?ypyzi1m z91j)lq?n4&P|$cgiOVkp^gTJ)@za-)fao|l(*E>ikpBuw`iLNgIJL6z7MPslp7!X* zJ_v3u>kXi?Xvyp2f##GZ;0iC>;0)Y#asmO%l%3dzY|Amt4A$_B*tL)-uZcX(Sw@?gQaBM$ATjkyS8*sL=TiK#HGtZ@(&gO_oVTvlgYmgh(3I(-mG%!NZNpmw zYJE`g2Gq;QfQzX2`wRHpHroPu@S_QR>p<}W(y`(hMe?$IFJM(qbJbt!8p=XmmhPc; zw{}x3e!$oWBlY31{4Z0*@iG}iU)GY^3rgTB-g7>%Hrom?&NdP+dv51M+!-1qvv)u- zr6#o&FirYmowfu?l;Ezd!E1TuJGUXi8=F*SO=8U|lqaJm|BRh!xn;YM$<_%m2R-;^ zllx#Z2F4nxZsB2<%kVSJ+_SJ<$B)O8{_v!Xx2%(`#r=b;?ET4i5L}&tqd-Drv*iPe zfdU_*hkp2mQ3{iCoL+x=?V#O==+DtFg~3qY9dAw~`o5nDZB9`e0_XW25NUi|Vwb zByApyCjW(lNUw$CZ@EcG@1N-f;|E5rG{1o@RH<0-?`x6FEwV~(c5V7vo7JleO1$2# z@9itc&kJyOx#e|aA9&^kE-K~9>=fFRyC;f`%U0$>$L(DQjzwU&T4l6DIya8A6e6{) z@+Q4S5Byzf&^bLXhLQL%^9S-mmXgZU{>vUO1&0{hW^g%yv6ZkVFvGkfgC+{0x8o;6 zOk6(i&ojocanwUBhcT_RIMMzSvjbAuB2;j#K54E)_>;AD0XCpfrL2k~oyY(irZhG$ z8NMHu6$9X5_a-+}x4&WQ3K!0?rBnJ&ajf_8293%HOQh_hZzO3zVC0E)0(yEG@j%X2 zC!d>#IOmSqasr;^LeiYqaU9Uz7pP4}IbHfL;GNg8`0wiG_*(m;%KODP>y3yWmcisK zyQ+00OzeMgnL(LOmH%*=K9PCDhc550A;=v=%R{7$v}!TojJm=1)NLrI=f}`PzxO2b zWpT4zJ=%+N!H{sgFedc2TuMWi3wn#rXSJwq+L%alPWRMEWm=HAkqeb+#}sg9bUU8p z{v{btW3Fz8%teh@BP7x?6@SUIX z;>uL{&AJ2`?!T3#MH~4sXWCRgVxjN<$R@M-mXq1e+;FeD6#>O#_n>ZKFF_x;v?cKF zcI#C%)AegrdjRQtN37`9RdBBv{ojYzt(!T{@GfX#3EYvK_bejY>T_Z#kGBTqF6%pB zojcI)Pp_rsOC_SVhU@sMyz4Af*CCzc!kDSEZYi|mwzH*o)9m=FFbHxHRoMud7zkVnLW`fa3=Ss9mU-EL)GmLfYc{x4 zsHq$)ts=t>Gm9qlI^6K;ua)sGBg0C3Ti+!@7Q0bSo4WYVmXT7mJ_6Y?I~WyKgnzmB z)O5ZYqLt?&ET=YOm(aboWZDkPV{Dy1wvUrxXNg`QCQa;mW~-hXo9;o*v)SLu>YU+) z*7(7+;xn(Vty%A##9ZEgr*HNIlCr~IRkrW8?e-DNOT&p$DFj=e<__Z7a3~9GSy>Og zKd!Odr-;|%UeN2FwTYYLk*Z5Xh20u`<=l6t5n@HdE5`(AU8s+sf87BsoGQFn=`qbax>Hd0VPdTN}?_JF&SE~`v-pxz?)ni8XQ zl6I17-e&&U%s!4CEfx2BCWb90#67`KD?x`Zf^RAMaIQyjOJmUhV-Ex7V?~1%|DNTg z$v@%j8coimoigB$s~11(OR9>={&{V1)d!)fW}_;Bn_?Y=^l4C?2&mBpW9Cc!PnQw$ zCZ7T6G8Z!=`ud#>mh3e6M6R90OG_I#;&q;c3BP{*D!A|#T2j40zQtVq>hNzI!Tl}4 z?M?N^6UWsM$Av+AU2E7~UXsR@-|@2*;Be`sm)eN!DNFg>f_6$90}XQN0)2yhK#Nhc zMa%vp;dqz?i4A-DwHxPeE5L@iDg~~deN2m}^+>`!oG;6A9L6M)rd80+CDMX^bmG-T zhp832!*HqcZD{rH_tge`uvG#LbPOtYGj;q69t_)z(S@&sBV+w%ZoiTZtrxy!NM9n9 z?Y;)kCggUsL&86}j{92^i>9%ez7oy0Gn8B7zCXS(UbnUU!+P1Zy4p(m=!!)|L|lE` z>ihT0{&5SGY>a>W&(M4X)k0Qo;xVE8! zdy^yW`L}+cn8}{@sK2!;gbo16=Q&#mbidxucy|}*sv-qaKXt`OY;Dw>^MHnydg`QN z^9Tz)XX$*6t6Dw`uevQOyY337@Nd+Lo0FHlHj+1a+RWNm2i7kX=BYll+>4|9_^IaI3GBFZVO8 z725OuW%Y&*ahVhG>G-W#fCCd^&d#r-cP$)NTC1 z_70ED9Zu2@Cj_@TQi44!10FscGVhnzk_{<-U#*Rtr2QF^&=0bl07)+PB&C3X0ed-8 z=8^MrC?R#921Vyp44nqB*^q4liLXsU%7aNRtY_o!!n%L7`p{<{b?fe{AFbd485 zfSvnrp?;8+VA|<`&`Ik^{P!?5et_?wO<8-<6iJZg(&8n--irH=^+IiT*_dnDumk>; z%3I*bUAaFy59Z@}y{{L;lqeAb*I#Hd+W4*YXzUwCl)?pC$zrhfvtbnL=F=jWDvdm) zecX>1>qs2-FS=YJs!|&9`zm|pL!>DU!LjyTRUc~_AK;6Uw3x%MmqIFowDkeB`&~1e zhLj&q*W213Mcyw6`y{SCvya&tUP{yE0u0w|MNP{6wnHYLvI}8^wj7uh`Pr4C_{Dvf<-uRL~ohw{x(kF=P8n z;J3hrZ7WukPg`bm=I~XfQ1YW%hOjyoW+-u@b_z;#?Hp+XqHr;wKNGW6=kP88>pxq@ z?1+RbB=cDGQrO8RV)mmnVzyTTjeSC9qf*n9KBz#OtEnn6{c`P!(2wH?2+25)095E)@xMHH0z3mL-2VOa{d#tAv@?~6)@AubWAr-IZF=sSi?z(uVVO$ zc@))Wz~U9&u*0(vm$#>`6;;+KVgRm9aFT!fjqamA?0HBQ zlO|c7l>$Uzp@LsDj)NKH1v1qZ)NAib5gfpWm7I7gK!z|o&B5x+&xnm(`^CYLh~fv zD*kmf`CnhWccpCQDzC5I#i+>raP3r#S$JRAb1%ideY5?hh>a9_B*j)n>ENr(7NnX- zQ0_aSIUuFWdZr}8XQiu!qbp^Bxh7UBmLY$0z)M!INQxi(G<~P{M{;ze?Xf=@j5fTJ z1_v*DDp2@)`=4E(J~wrq+)7%U3JolfE+fi9fWT}l1%v3NooP}_-keIlTJVR#lI?7p zgj`}jYftMa6KXR%G}w-?m^7D%{td>{PZ(9+V$`|diFIJ40cutro-~NOz$YkGR6H+D z14G$9^UP?a70m6}vnxD<<`0e@e_hy)gx04nQ&LO=!O)p?Xqm2W2-;hMaT^=b6a2_t zA~&A&0enGK1++o_naHInL%yYuWdK9Rn>vf2WKzx*F_>hBn-W2zbRR6G;-B-S3>-JB zkJ2185;Nf$+xa>h)cF)b_M(Bu*Y21dT49B~Q^K*h5NIjvSyzQzDx@!bTl z!PU_nksS4_@l6&G$*;|%TVs6NTKmb%Bqv;uoBW|Wlb>uI(VukVy{ZsvHX@VmRAz1| zGVZ%AHuF-hu?3v8*RiM5ebuKAk!jhTWpoQm1PD*z!4KOPlKO#o87K5hfsd>o=M}sm8|<3Az0YC>1n3Byt{o=Q>1DtKE>}HA!3Bkr?G;1?QYkztcG-C zfQ+LQI&oAKkT1i@?T6vDi9)HOlXqIb3P2(nl~L-ju|gseMo|)ct2gl}T+o(^+#;j6 zk68z>nFFkbmxltyFqGtaMlJOwe?vq5P?i#&J{0Q?obswFHP0nPM~6LaWpOl>>j%V> zaYw}K!j62`Jq}UTtgD-T^9LRQy#(Szk+$|R@C1C0|4HhxF~rtBmdC|nwf+w<6X7Jx zHI)ZJg*FvoL&ETPaA~N9SxUU{UMW{OvnG%v|36^nruHR75AT#(bH8D5le9|4x5Q1c zq!k04)2L%-H>!O{cAY|1-P|NYw^r6b1^#=Hrrey1(BH%v*>CV(@jQVs@vvhaMY6iv z{+L-(tfYTrN8bZY<5ELc(zwyv+hfea)-82~F?5f?V`aDHz6TksIcV6+1h7cOn736> zpQEaY?Z{JoFK`45P|8ngig~C?U}@QZiMxmgGmN>75@EZ=nOhgPk%=Pg0J|)V`}&XF zj1Ema{H}4BMaZukH+jY>DR0petA>1bKbE@Q`LEC0*)hu}T#HjCvfsXxJBXe!2VAUOhd-IE%=|n3>z6}bZMaG zb9BjyZt`<gyP#`3A=ZBMXoMiC96SOXga;oMcO&(Iwq8)sb?v{~29kYDxYaUHXeOHeBJ=!@QD) z0Dc9WqpHmwUlUHLyAcO@$TsZG z;XGWhNQizBE%F^+LRJDe-KUg-CD7=S`WWqi>f0wUgI7p_Na5^GBC+d*ey0%kPp zN=Cyqa!F%bR_3&x;|THNLcVnhzYtlLNyQ&?{7i%s7Mb#r#QI)CD$!hZSK>&X&>OhD zySs=s6ox*`H7%N(Y%3J+Tf!U&`W{zCuHF&Gp``&ydmiSVGtDFZclYk?u)e|k_bS0^ zn<#y!!8-BYff&Lh|DsA0r)LRDf(W{K*m+;3YdKx8g9V20E z5wxGe%p^#dDcEbGO@b5j8x?};ROd`C_>J+sBfT`TaM`UbivW<6pvwmzCn1ACrXZi^ zTXQQaw*gB$B42@G9rM=82NGu3JQTHv3RsM4aP>}qHtz<>MfW=+R+laTKyH9FS7y zf|2Rz{(%AJT1$80W0$hls(!=45r5roh%fzIT{^X(xqYrY#>L)8h-Yf@Je7k5ZbBM< z9Oc5Af8DJL&DNgE)j}T+xPrnX7m?IYSJ1|NfJkDvg`1!hq9&Ol6>5~!!27W~tJGrM z+P|VxHLuc_so`{t^o7LA? zGl;0@CtFlhXbw6ZAAkY(thLn=N7heyzpuh(6=#cINGdD8IO2;IFwgp|f zn`B^SQ3xN8M+U)^@asxOYxZ*%9SECE{UT4^CCvVv!hDc*ifI>`5KP}5q9p_w1?LrP zwwt{~7Cfq*En{VOh`o>wU{a;-X01zu?a`&aPAEre(}$2uv#CcC&zird4Ya6@el&{z zklC(H6IA}PCS4N#zy>>NNbX9q;CLau@GTL6hl0GwgFbqNbtGuq4!12oJ$?q;_MjRj z(N@GPd_J(z)wTEsoY<`qpWel&T=iU#`0NP>{8!{aaw4XQErB3XhSS0qD7ylp z1}5r1_9r9c`V?r_YwBl|n%}NjMt19=q}@a|z^Rv@(B=<7c?8!jdUVk-Oi{ZDT1BHJ z;1Pvng(=0?*9)H{JD5cWtlY3+MW;8sq@2CC%|s^m^9D-DA#J z?FjFF$ZuKB=`+qIYmdl0zHpaR+J&ws4?ex!_b{)TUw=d1McwMgTa1DeiSZhoPjPEi znIT!*lrpHZ4KH3c1 z^R?hH%Zy)|@&R-Ut_Pl&0h(Mo)*H1`%I*wiB_J8IEy>j8tE@>}8<`LL%>%`SDS!7U zn8Yqnjop+Y2Z0si0waevL7f37#5T&JmxOLZkE}a5!cO>7U5Yq1#%Lg$MyWX-iqF0B z)kFsGJaAH(05BncGmK-`@}K9GLWr*+q*$75d~Se`u1QhQ6U1uZXF;lWv~N%lky!!$FSAn#4%`J z`rh0+6o|2jJCZv>uhf2oi$uVqPmqc1a&CNsd0;+IVbGtG=1Q54F< zfqU%ZQ_njOj5Ym7r|Sfc-CLq7{fA+$<}A_KYj9G78z~Nz?|&(oNt;hfW{C)m$SIRE z>xlYIh!5 zT9`*$+qy3a^DP32>33foOCBxUCD>U?&%NuZ?sMLDEEw@UYaC&4(QxpETKU4sknEi> zz;VD2 zDn{J_)m(D3OeGd+?pPBm1HVIoZsnoU9Y>HH-SVG)4J6FGhIhIbQD|P@^)_5) z3Eqvr%$GHjsDpQ)f&G2HF3`{)2ztbC0zKl_rw}tRgn4WfAo0~PDeN3lC3G6S%1L%c zOR*1NhBE|28?Jb!sVdji?C?^LwVTxZ%4yNSC&A#uT1R-nA@LC(>c`^kDxOnOeeq-S zO;8^K?9r9+RwWcBwtHV8+@I|Cy$SjdS+ohwZ&xvM1 z$$;RtCCVgHRo`pP^(xMlPLEM=J$j|aI+@!;j#orjzX_DV-rkx>qU|^+>R;LCRlmez z)E=zn-^_?M_YcH3+PLl9PIMfFp)hx0VN zVaUfMV6YYz?atK8E2O!VQZQ8%bA0C*ZXG?UmnQvD==6;z5+-^y)A{z7)00yAB$1-G zId3fJ#6?XAbdhpQE$=~o*hH)>Ue?;)#vkb$&FrE^*H6?tO zmz5YQ$zxE(By%QsO3KLA2}4^n_Ga9|=0XlUrv*teAVdKX>P}AO{KiCmD;D65R*mOr zOlAp(RLNkD5^|Z*qzRNwXmsJ3qs0ym0+o)`99KCR?Rh1;OoWv4Gxqs9nJ3j)yzmC( zwN#;5rj;m72rbXSy_h!xFdV1Hz}8ueNf02N1?T@vihyNN9z$}1AV~A`V49>1LwUp| zlHT`4F2hG@dET^c6G7!S5QA+;X(YZh3G|f0dF-)|RY{M^bL3*4^k`A}2FK*--qSr` zYzo5=A_S89t7Dk4Sb;0h6WK?PI7~sve6AXMn3)|j1ByNc?f-?Qg1pLC$MT>ZphXH7 z*u-24^fcRRbuwq<)iH1Nhu6UqEa`##Swk&HvhSnh(`Sftu?%>aZxDzrj+G0(R#1CU z&8vK_(~Euv&}mUPWp&ZD(u*4RavZ@^f2#%>7>oUDFlNTr2|g!0H{POJHEKsSpbVhG zeAV@rS2VhSC?SrH>GoQQV1mY`CP1yF?Aptq5GaV}YJgqBM z(s-U^P-lljY%z;TKz@Dm`tK6$$<&I&QE~F3*X?B5hD5V5aUufXDlCC=yYQ5dyx{*; zPQ;W+mG5{&w6q%83vd+29SibvAng;ia9gI%nBd@|hoRM*_9;+i>Le3<@*i~;fo~O% zA9hJppD3e{U!>R)HKNM|STb}AsjFG&%$>>4e=mhKyY`uH2%d4>LKZ%I=cfs;{9WQ- z(JGq3YhZ?8%|~u%0^cOZ7bGuz=*jOw5RkHjqI?cXWm-)w)08-)m3i&ww{MS)L*TZS z;l!%3l!?C=f=J{LJE4>UwEas6J%ZdJ;KSrZ;7Hf#nJBhL~K)%7E8N z#kk`b2L64EgLop4^3P!l6N8%5bEZhkoU?=s2HI+vi zD9WW<5S+s@Qf>P@4t`2~okh~}Wk<3Llg4t#}2?4uA?Srt?>7PQ|4 z)4U9*`P8#YZ=AY{zVsE>(xqQOI5<`X%wF<{rWCb~=DHkN`Q6m5Ixh3V)cQDj{n~35 zOOz#S2jq9b$}l#(Wa7TcG7ROZav9*V(89QhT&C#5qr$}#c^`}Nj7LdcbJ=+`%f-%A z*6W(|8P2$BW=WC#r%5In(fA)VhWpON5ZcbYD^2te+%4_9>4Hp32u!Ba zij`!`pnNOT-OdwL;$=WcRMb$?XRaR1*&H~};lVG7o++}m_D7v>9_{~<$Q@@FmX_R3 z`iepp6FjA4@iuXgxtKV|u72aQ364)(T1+2Yhg&sUNPh}_;q;5w=PTTR!1gA?k;$59-MGUOATtjdJZIcWlR&ryj zTNL6#B`S%XqdfeigqUzCF;XqgD4RfRX&R<=nxa@Jvw|w^ICOo&rmSQchB1qSSB1{Y zU<&DA0a6KE+r_B^w_0(m!8)$ItuN<-pw^QbzB0KKGQU#r43}t%+W4}rB8ar#I3p1S zbJ*{vyl2=E+fYgd-Lo4vWF%21gXT;6q5QS`7u)Ij@tL4J8Lq zMOFgYB?!CWteU5;tCA6{#7ans3^$@fb=m$Lp6PR|affQ;*O#(Fw@!rGN2VNwY+Bi6 zC{(Ly^cAh;Q@sFk`eXhD

gTE|_8aC4mzG1knB`es zs-0=t<8|nS!8C+Stha6Hc*Y@u)`nXMj@pGC!bn<|&l zfNOUg%8UZ=dB#}GRhM`+saPHeBG=jg${X>mN|OWfzDY8N{+XKky7@BNv%*xfpH%M@X* zv|-+0;1%s8%VWm@rj}KLU*xgfM7_QXkOs6UXgOEg-?sl4nD&RrXjKv{X1xV|_OtSDzpaUHuR{fi=n? z54j2-(g3_|1-IPU&|)pCfM&5(B6;XFN-9f6{x&c2G(TSuUVzYM#|+#OBxAxaa{NzF z$zXoGbyVcBkd%&GHc&u? zkmt%IhkcS#&fiGm9JqocVE*z3q-Zx$-Ye_v^0LkX%L&om_3>mV0DO_~vIh3@mpVR| z!mCWEPXo>hiYg-&0Mm>-lBO12P5@d7E>$KR!wT3(8ddTu6u<>@8|Eb`**W{D>H~0P zd;v}X-3T5SvuuHXl#I;l3RTAJW0clBxn!Y-9UrNWj8$nl^itI~7qZ!^ z30Q}M<%xloY6!_`Tzr+e5Utv%6W-$9K(R<1J0yQG+IEvK(d1BsAVN?Az)e`+qOcd3 zw@yHALx}HpO;D|#h7LPJd?JWRNs6>CDd>FXFqWC40vcxtnH)j0nkE;_IIKvW|D0?q z11H&fW)gQkdw21a2hdM2+BN^qcEd1yzbm%hvv;&8-iyC?w#3YiRJz@*hg-@}rQ@Iq zR+_0K8Ly;+k94lc78(VUS|+BPQ85+HE?LP`PGiWYp3L<&QeHuXv$)F>r&Re5ta?x9TAu5^0pM#;KdcAbDOxjZ|{!QEUery!C9vmP~Gb$ z)yb(n<$(b~5}ow`Devf#CMl@7qC7=es^3w0U=h$R7>}v`+%gN6%yJWurnS5V4ln%% zL(hyRt0!CLq~tt8kXe0K|3e^Fw&TMkeBpaK&%lc>)9 zn|T7U!(YH5RPavG1tzq{ks6#FBzAE1bKJ2VgY``8D2~at^jg{h52;py4#Ay=S2{gI zvu93J^t4KWA|cJeOd1-$A#`pqDpHwR(j*q@r0Yhk;r0gyH#KJCcoNLT%J2;ZB=8#) z4qr`G&nBH`dP}9?iZa>OeH%aE~$QDxUNf;p; zCS$pbwtUg`C*EM-b&`DJ=WVx^FX+_rC0$y+tV7Eeac8leSz4|vT}PIt8%xuPrQ^ba zRvsx%UOYAV5g-c*W^~f7no_gTDmW=wAb4?xqQjqTPdR~9G()$NIgH6D936z;3xZqH z2sfc*d7P!@M%y^*nl21ktdpM0yy2cRRLnR`>4JxXh~0-bm;3T`c|~ z)?2&Ws#M^R6tT*QYU_zf{9@c294CwJ|L0t6-95-Xyw`kntX%0w&F60{e|UqqOCkG3 zG)2XfmdP5&tm80LpB?A%k^aaP?}Dx~k^`XFUgR0t6h+_C5eK=lY-ZL?>;jqR-1ur5 zKO#0Sss-5wH&Z*oMrjQQ3{ZyvL`>ONI&4MyXIJ)JgG`lM&E!AL5CO?X8L4d6Fe_np%!6>KdSO0Lj{4 z6Gmv7T;e|X+wl>>_$W;MSXF|!ffU$f5=m!u04T{$tRS-X@XGzpP+=?qMv+D(bivRv zu%lz;p0bj8vX=CqtfE#wF@}3V%uF(6vh-mdhvZl}HcJWu4M|-73+6!b+S4dA_^>Ib}>8L}bF! z{L1UNP7a8}ZxRm!Z$Px)Bt8l7a|7K9z*-<|hflGT2^RUH@NHjy-7GX#IS219;~>eB$`08blZ)gk zIWLoV2m!}Q%pzKj(W?<)cx9(|T@FVh6D$wC=HE_u!epBc4|Ipg;v-gyKjl+g4AOX=cN-1K8U zCXYQL12*-ek&sV#4X#Aikb-_H^a-=RMuQlQljamk_2|zqhp(Oc;M;G*{2$8Sn-Aao z`u62%SbeP48DmBMr~PjI=SK%mzU6;S;UUznQz3ZzcG`*X@@^k$tqYkCHq3JIM^N1GSp{nBC z3@};Pez8dfI*Gi*yuauKU0OBXFn&G7N}k4-m)f1k^rvcdLeKm3DIm#IBcTyhsZzFn z{_J_}uu#3gV%q4%16xAf*S$bT$eI#anCapQLpZ&Fyv#T{-!-bv5vv}7Om3>9K(VIa zWzs3wvtu$@n&OUSOIJSj&d$y}NDhpaDmylp_&!li+Ey5@vXl%rqfH;X!gyUBws+1d z06}1#Wik2u`P=FKr}*9Q{QodxX(7b(s#u*0Z29>=Iv72E(mMYS4)#Xh&j0)P4OE?5 z*t$|*?W3pV2dO6fprYq|5`6yL=B&*+!_I_Q`m>@;vT92F@A%NaQ3ryzwEx|J+|Y3g zYC=S_2ZDt48!~=Iww0?$|D?kmtA}LT?gZcA+p0|Fr}cE2d?fzRcee!CAwPxnVG9|J zxm|yT9J?K z{MoXKF}PQfNNGaFZ85G><;j?HX7}hl9JZGU-hy|kp{@$Gb+zDH0yHkRLt z2W&cJQ8gz1oBUJ)ARepoRAb`5{G+C+-%l+T7|Q>>LF1VC`8j$}6aR-%I64USd=fsl zJd4BsFnPAazb4861N}IHAWUx21|ckM(CRJHy5jlJW|1^6e@ZA&g zo-2O@HHMHr7DTKg7OA}?JF^cm=Gru7E!1Jmd_iP6qq{X4M%|u41$Ve z6Hc~eYW){)PDNYrp(t3i6MDvukDX{=G)#MqvC=I+H!#Qw;3c%{VA)7o4nC`1YUHXm zy$#mFzM?R>}dkO7IrUur=(~SW%=ST1z_glT;xSNpN(!HffvAt`$ z^)1DMueH9-d;o9yR=U|NlrB2vw9w0A&aUNR`%YiiFt0YETZ8!WAO2dM^kVZ!ms#jK z+u$|PO6CtW-s-eU0@KKHnZs;G^9eZr$RyG0axr;rZXD<~X*aw!oJVwDEBCqHE$F-A zG`DHb;xAD5L7ps>>-uJzkE7*aJw5{;_`SQz-t_oyb);Q`iB3$y|L;EWKlnG`@PAb% z<_YCVr1Gt;12#7G1+C3kIuH2_?<(V!>J>R*aewu?(PFahP%v3QPkPCe1BCM-Y^U+i-*O5*G?uK2bl@_z@d~!|xsPn3^rMo=J z&a{WGNtOSd&eJ6=`uehS+lklm@8)8>X>pn6KasRfTS@%ZnVM-Bnf!DI7%h#jE&g4y<j1;v#*^Rn!L)28g!-GaS~O;M~c+vwaj*Hed3ewlP;^*e))N4t$ow&Q>{CAo|Ek; z7oxfIVrzXyI!|_peKfN_$hNrG{5ln4j6bKvJ9KJ&4~b%?;w<{~8MbXetX(NjBhoyX zp_SfuA!(e%s}|txj3KgAI?NuIueGsfQ1-yBk#H(A!W{>Xg8o82qt-jqX;=I59Up(k z|6QE+rd6{qlUWd|RH3}7dbqUY1aoriw7RJ1dHazCp<0k$$lCCHB`_^=upnLszh54? zYoPL;@z{IrbPns@?WD~#?ww?;ICs5?9KGMsmU8ucD~!83Y|!!l&))ksw~ZqW;;(=F z6nNy!V=HG-lK&-9ZRXDFWOi~-CgX}TySJ4}#V`p;98)BRpqyxvT;2QomiI}1g>C=@ zNKlky#hIBkU(Cc5*l08wjqXOHyJajZFFrTszx&~BchTP6_|{s`zq7enW$AzGSov;`>mmcOqkR` zp6GPZRxKl7%|u3?jDmHfGcqG#D;u+UbsJRnO4T|->h`$#&1aHCNyw6d9Mx2QGvrow zyu=)jwbSJ@u|aC9VvSe|&q}PhX3&o(EI7PLf?+6jW~NG}q*Vr2_bll_o*=em@+Lh; z&733CPA&Vgbdk`~AND2Y)u*|1qS%DB`iCDqvgv*Moy<{nwL|!~6Pwck(!-I9#5WhmO`eyI={} z-R3?IZRos!>=m%{mI)8lzF&U+S7aM&+y90;vEby_Yz%2hW$fgV5qbJLPdT94sfHw( zRWZg!u&t@RU2QZjW8>m;jJ6(mJ&C=Y@O$hGN|31&A@;~Q2o$jWs zAZ&~LA3S<=Sdst7N0091|D8O}V_hqEixDhexz|{|(P)bDO@vhsd)HCw&{W zcgm8xrf#ZoGMHmd(xR0;h_23SX1z8b+tp?bXGFD(Zrx?)E*WcKU2ou?wPx&5%YC&O ze_}3yvzSJ+H%e;C%hlFM6x5Nbub~$PAvU9K$gGb|Z<;l9JJWF4tc7nYpEY%)M@Wcb zVYq82D@wpLgQ}eYtdJLRE9>^&Jm*nUvDOjG+ zd_kTJo(#xdgX7e2Tn?JfCQHTD=*;$j+T9J!?pAY}uG)8R#f>hhGOXr|6+{+3EV(Hq zhU)kH=V$L_0ZZ7Tck^YB^m>9xpMLsevv&1E)9fnK+EzhnYv`%5 z$c^@&$**aYkj{Rm6|;(napr}tc8`rP4>a%TH035TZaB?&Ri?{{l`DC018T;0wG2Ne z#9JcSLbPSrY>7@PJ7og~D!XMJ0sY}x{XZ}=YmT0hYR{h;ugGHP<3V?Yp)IF&Esg8i$h1azYZGM^Y@@LSy)yeoYH=Zi zEPBr7y!a)Fe%qMw>ZwtiySErnOvm05C?!H&@?brj+O|ZL$>Gxnduns)R(~t8Hd?_J z)?EXUGM94`8fA;8{WL53*N=H0KI>~w*E*fmG>&S2T6yUS75nM%;1T4)95yu4JD08W zRA<9DLw6wNRGxlJ>zq!UH(feqO|QyAAy)Q5drcXQD#e+< z!pxb;O&grpE{q%IavP{y86ch4maG;oTYgn`*Vi~?x=+(4#URm4|5*F3^%|56^>*1s zMqHX(AVPv&E2zbc(UA!w(Jno23D+^LP6MFwVs@rQk#$y)MNa;yqtPsbbeEDsoi(Sc zmg`VG^?ZQ_IjpbzYEv6OE5%>~Nt|9E`8gzRu5VWj;~|wihPC!m<$S+S9$>yIp?bQ< zB{BC6c-5{9b6$ee*Pw~@$fOlUJFGQaY1lBL3?;;r(YToY%l=98NP~?@j4Y0yK9PBr zvAku#*xUx<%>3BM5{}k3xJtgrMnG0O8$><5WFwSnQyWk!@aKt0d&sT2tBU9C}Ro zg;rvQ1G4-Idqe3_v(H+&SjRp0CNRy2n4@kt=t#|pT*+D3lSz8OMSJ|j@30U)t7dfM|r-A^ZeqRf@QGHSOZG#`^rV}`#(f9cA z1- z%Mz&F_w<+O^4rno8}xYxaEoOsANHi$)>%PYaPIMH>%pv|<}aDMbKKC_dH~(@o#6EVOL^TN2?b0&NrMRaxsY_}YcJHu?Tc5`D!RC~L|0s%^ZQgnx1` z;s4YUetk|OOPncgk^f}Lw|61+pWI9R-d|KP6hLbvAZoz7yGlS={rqn{7Tj(K8}-H+d8t~j)K!v(Okb_FWS290tlu2% zWgem$K;4dctHnx)8i{lp@wAfdu7;|a{nv@9*6NjZf8ITXJf+#)7)PCjL8pHoX6XwdQnz zyycgG+V1}ihlA?<@4?~m{r#W2cs_lS2Rc^tt1fCN7HzNB^Hlhjk^?eA>plU*AN|>I z{g=Cb4|RuS5}EyZtD5FPU;IUj?e@2REe-OU-|NF`YCmW6 zw>hia!X1&_irY()3YP%+@@4lAm&IVi^}k}~H+S|eZDQY>XVWjVHXuG+hMH-bKQb_kXqQD3BqJ`I551)Bm^7Ps~sZssv^ zY4z4Kr>!0i`t2?%8l!EBJ#`OEPqy_(mWFm#!lmy+FQ`e?7-Peiwd`4fp>nyjmgghkV-dzYPb~{6B|Bj}Gtm|GRiT)p7zni+DcI z#FqKjc{pK(%&!ht&g{B(r{^!^X3qT|DJLD|{Qk|be_n^vsaX=bxm|zC+2o29bZ9Oa z)Gi$T{3)`Fteo{@Nm-fI3l2F_Jk+>A<99&rlePVMo=xR{8mbfab{nwG{u>_D{J+Qd z`Je9QaoB%m{?9$}!=4dmU>EDnc&2q%LwKkV3fqM2lD$>;50Lg^6R{z6y0zrul?_7g z3igNF*b}xP*6X`%g;eV%ky;L+-gzadRF+3KQOqVR9N{%NuX66(BfIF_+Fb$nFpNaM zH2(Xtv%^(BY(%`0KuF;%UcNX#vtOLQcvim1=gBGmg)_O6=JS}1$nH;bUc45+yh@YT zXUmisbF<{zzg(B?E*!0+=c6NXVz&{)>kcY@r-w=|iQhxj|>hcv&6LL!EoOt2hg znigat0QovHbQUEeQeM@W(T^i?a(sMv>=82Nn4R`cJ5@JO1(AM;VurP{6?uObmdyRL z$+PMH&$FJ91N>I!zr({v!|MJ&9Nh2!ck=wv_y6^hj(@dm<6=Fmn{j*|W3+6KJP@tl zK!HnDgw-m-y+W~h)469ss@5Z|SGE714E^D;Kj;lQE=-_iqRNTRN964;PSfr?E8l0O z4f|NI1g2KCD|0uhNMQhF+Yu2H?FqSI`4u0*KJ<2X&KbN^UqTihtiG{%HuP`3@O~%H zcI5x^8W3&r|KPY*|LN%PqzD1=H-pM`BnMr+TPX8uR7ds z54U+XmH(|yALCgr|Az-BhgJE1_~>5#-_7%>EaykaQRE4X~T36SP$9oWaD$Htc)x5-AvIqHV zJDn~d^Pt+DSu_tE=4i(BdcD<6nKId7JNPm@Lf56UtFlz9);Y?uY_f|!xOF(>17mJi z9%^;Jt8NO~(I%iyg`F9&yokn8AVgU-Ngyilw>jg57(|`z>+UTJf}Ew!JIlER232!M zrH|`vM{rMPpL9OZBA9mGsarK=+zleFwC$CVbYi5TY=A+YCcmavEBAjTq1BO)fYwJL zq->TZoUJ}nr6W>6r&-2b7?p50z)9CQ+BQ-p;PtW6nNM!REgHxz`WKtY%1DF}57HZ! zFP1DS8}j;?fqYGqg2q`ILgw&*#w=eMNA*dROkC(SfnOg{8LhlJ^=!7P#n-#A)qSjw z-d04xW?}%ehSi02n&x4Ym|Lm~6$H8gGSMt5@W}l(Ys~<&J`S_Fur!m&2X-}0(`%IS zbDim7MaB+i5f|)}vq=QSyOt3QLI##~yZ$Ij1?x71ETL?GfpM=XS{;p*mRbe&2FRu1 zEh!@atlw`YG*n)&q`=sqAg0mmR*1ME-X629BDD!VX&CCj*((T~T^S`j{BW-`7u=1o zyB&ejaD!e|0l}^7YzE+su@dwQ(xk`*I;*PyOVFu6+ypZO``bJ%=q)iaA+3)ISIL?- zM?r$vq;GUD-mY`)qFlFkS=4kLrfp(V6V@h}q&b}k0lAJMD8E{|ciWKH$Lxm2QTQ#C zj+Tx$7zv1bS`;*xu4TrpLgpIqmGQ$gpAXuTL)Yz-DN70|iv>#_dTwcDjNF{~ySrL*3;+M}pdaIDJO4eZ-Tym2IJnRMb0<&5k-p56UiGs(J=;}p zx^|*JsC&`7)$)F8dBjT(_qL*Xw&cHd`lB}t^U*H)XpBc0HJ7-3mT^0;JF`6Mcnf@4 zf7M$hI+&<2lxXunA}48AJ=%WoZH4Ub_}NhYs}u+;9Rat<|HI)?RsJ6xKDs~uy_2UQ zvlGnjpW-I9GgPNiYCWB?V#?u=Oc(a;K1^lRabzwLzmP9qI-ZeUy;m=KQBU=;PA(Kz9F=OGq@h)9N&W4FZ8xsa zwt&kDYnS>!9lkXeOV61ZekHoBmx#V`80&ypl3_-F{AxAIw|-ek`qGfNT}?xr((tb| zrb)o^7w2+yTkqVATKlfEBul*OqYJR{Uc<3Uy<78Fm7Yk}Hw&WBs%*9zaCs*Mzpm1$ z8T01)#qyGcpwFu`F@iZ9AXk~p2K8afvNu!cn?`(Bxi@R6G+Vn{ z;bPL79V_?2ujv&G0qCua&5k?aTxS`M3ef7KLTlJ0p!uero0fbgMIOys^3bl*bC!m0 zYMobJ-ll)ecohkX?xmN}EHd}ZI)as5a)~ob4@s{EJGpA@wUJ{`XINN`Xbn9$s8lO! z)|@-GCH3JdcqtHO4vaO>GV;cLg*34`jM>gJtZO;9Pi&MFEWe>~iKKdCIdfL{a=E|K zMOk3iH&{Th?B9CbhJ+Kw+L%Cbs9$n}2$msV^L%WW#`7rw*HN#o1!pH z6UA{Q3dYM3oZY%$6M~HUn!$D#C6lH~V~%cXpAzE^r^AK45{R~%hw6oio1Kk9d9;k? zJ%T|ViX3m>zn2AS-@kvSQ8qT0k#d{8k3!?tYZlW0a=uxU+S>gl=nL7ql>rE;yx9;; z6xQa++{G4ooy}I|l-$W9k8dHmrp%)vwkiNC|5j;75IfZAv|OTtHxpLWKcyLDXt;gK)79=H zX+-cQGRJ?w-G_Vm{bzo*CI6RQ25FQ3!$;Nnua6!*8s5wQyLi5V{C8c~_;YLnx47f6 z4HDcZbG?7oeKwT;H!LrrppxRI$p&b<|1mhK<^Mjs&;NZlk9q&EB|Eo~_o$TXoo-o* zR58VQ8lIL+p|gKib_*vsxlR1t1?f)_L}HffwcIpr;^izzXETW5eBRunA#MJo(ZC(8)cO>e`i>G*MVhm`aD&GOc@ zVI-p(9q8aMmdkEkV4EQ~qaUG`F*A>H^*fU|RYzwlye*P&7A18o^}7wQEaA0diKn4b z6SGqp3$wZ}^BJ#v2SX`p7t(URV1+YC1Y14RS^He>S(IDr3$7Ix(>x1lA(uCPXuerf zUfio{fBt7J{m-AWcot2PG-vDd(W3tij)%t;{qNx5KL7LGJUir^76r=_PKuP^+>#Gd zmXND?6hpnhj0V?q!np74kT=taLj^>rn$M>!j>#lWuP{C@N+$bo5LrYwjAXQ!8t-Wm zdOIXx6EK3wUY4_Q^pS-q*8ZdGlb@4#LDB@i6ey95QoTal_ znvlPrU63%!x#v%!q7VN?^gaLTUwI$?t6oeeeeoalC%;MhC7>%BT+cHyj$+2Whd%$1 zc@O<7dhI>*i&^G9{NLUV`8&;{bk50(XV1CkXL%|NZO@NFM*G+KPKq* zXW@E%w55?;_a=1?~Dt)nrWu%vjmH!X^c zkNW*dR7~era2cQs6L&CE*Y~`g9r8C;kk>3rx!}G9d6A4$?=Sz+>yd(eEaE6BdSS$c zE=GF2@4d^*%OFj78Z%F9MI>igN@y4wclCS1Jens_u^fg%2idR0L_QN#b@CW?oy=c_$LLz(7p)jt)G?+upSnvS8uZ$7ZO&Fpe ziln)kg6gOv`SaV?&tE=2y?Flq)#*>q-#KaEAUt_Hj_HKIGlynnPDBw9Z1dEy>9js&)*oXyf3#))Vo81}z33yQ9@G+{}>3FSNu zB3h`;N3_JU1b`ch;RC3X%OESTZw+1J(K>kDa+Tp{lbpha+cbP=z_vK;CfN3|V7vB_ z?2w#2%&%^&KKM{9{+M5G^uunK+C|`pqO(rZ(C+iE;{#Be_^(EfD1Aovz`|wfAKS1koD(rAEPzh$_c@N36 zJk2r|D%B7=1Ma#xAba?aLz3o17`!=wh5Fk(%30X;9ugzR;x${44^;dt2oEJi^zpU# zuyo(K8X9?@#f+SpGR*UU;m;uRU-U0Eov_l#U)fk_y;AB%$)sT|L`jjSePD+7ur79j z5?-&EGGwLR6~OC78eZ2YZ&H|KA(ymp6(T>)@a`7Kp^z0ciUrBR#a9U0R;2)+rXR=+ zx{_90`ncr9!i3+T))s_Vma9u{K$rE8FZD$gNPEHp(B5%HTvp&+`sA+*!dX%1UuzjF zG4SznpA5y-9VkEJiT*wgVT?(M%jc`Db116t5Hov8aj!a9n=kR>D15n{XJYywODMv1)L z@1K&Pe{kXtdIRzoGV~9J;-9 zG3XrUb3PS2qTJww3QWl}s6TYUpp|L?Ih-P)2l(D~S@s=BtSG$emHUKiwR)HY&d~8Z zJyQe!z#rZV*MIA01No22=NcQpa``_zuIK+eJUYD3|8XbJ4tXXsJ4l;CXkS%1s6GD? zCzSm2^rx3Stjt)j5d7w%!7~=bG-u?7=8@3!xvU47QO;RNqNGSs`_@rJzUP(QLp~s- z?hEtbDwUp@Qci-O1|%ZbeiN3k9K1N-X0WZ(XK8TFa)F$;LmX#L!@uIEVV~d!f zuAT90WZh2-N0>+(UgFLrDfAlaBP>Q3R?gW=v2Wtnm2bTratcx0VmG;9vm2HR%JL$o z!g%6E4tKrwAoV>9$<@L|p`SompG8T?`y7CDVHS+_Pg0p#f&1Q$w?lpqb`uSzV4g<_ zF?KDLgMrsAKdk;)poJXaE*Z@8T+FlSz(Yf6mlL^z;o%bF-VUM3LhY}-I$p4oZ=#Te zgo1}bdp>3~L%e@ZMBjpKC<5Jw-$Qe0piw9WXA)o>u{cy@2w8$7x?<9QY&aMbChd0Y z^2M0U=5Y~aF;n9pV)?KHY*3c7knDwXCg>u-$27|Xf^v9RD6G3L$ARCgzG1hAHAm7s zWI6WuDup7^JESMZo6nO32MlfnwJ7(|`HxF0ju(V-KA%Z%s4<;F6xvSx{K8_%xlzp};(trko{~x}zCqMBcuGzRIZ`k&e(X4~89YM;Ngf zlmq`(2@W_WHysWRjt}GyQmInyh-7~KXKg)|KST*Kqw%8NX3m-IPdLk2 z$dV$WG1n_pYpiN`u&hDOveH#eiYYAwsjq-JPq!YCj{diX1@1UQ ztOCDC=LtYJO84Tu-_Wxtckt-&i7C%mCASkok*IMETZ?1^nD*az+yXK343f)xv8D*q zsZy{n&E{cBx2dT)B;S#sYwSA?uVc-x$(N>@R8VdEbZ9&?z2JSC zMgA<@(SsEo{&k_G$-ce@4bn8DIh}F;y}bUa#%4$qJ(RSVvRpEGHGxd^p1Nzokb0Ax zUU50qGE|DI`4|DNZ$OI5Og@SO=NxIBrP&<9mJFira!q1RQV2r#H6%mOvpi38lFkb; znjB&nfhAG)h<_UKSjzNKp;!?ooE3z|X)@ta$Z#70`MVFDVa4ePuw@@J76|bR=tE06 zL?vvHAtdu?o9qlbmG+qEuWtu*>8wQL(-+UvlElTodi8ynM#0qXM`8jn7G!gc~N2p8KS7g~7F(h!Vwy@yL;ifs)A7+S-E zo9Bg`L#Fn1@?pwyW+0w3UhsY7`VSFjBxdvmCODbLv1g_e0F4OfBc@D{BA>H;OJ5YU z8mIXOnunk*Xdkc^R8uhzFV4m1!-Gfuz#sTS1S>yJ6R{r(1yf@vVMyYeqE3h!poRxK z!b?Fu(L<@3vFUGCSm1DtBc$LV2QsjRZyr%S7zI zyOhP{(DV?+_P}t(h#p9qV^0WkoaO-oG%w_3e-n7yyvX}RRIgE)S6I5W3QZ2k%2(BtpGNae5JV6MRlr)NVueQA- zI*#eJN^?Pv#xQTV_eDD)1rm2pe-65|T4Y?c)u(49;g()JmKYeX#lban;n;oH91SFr)h2R7{W`{2*8P5x`$w;NVA-93opjxTdmG?mKr&+;s} z?sq}MOP!=jM6yU2)hsW#uQqXHDMF3LAc9TF3rF%4oiRuOxh~W=wOEznSj>m-DJI5a zf0zc?PeHGY#^ll_n&i<-t@b5hwTbKxny5apADbSis%d*r?VtacGSH_DUR0{#{3?$4 zw5qN3fEU{JE%8^ve8oAYwE5V%VH|*0h1eauCt*c8| z)ZLrX6U+UwvLduraNVA2I1N_DEo@bG;xLDYW zqpO_ei{90oFRtKZNumJ9b3;!XEXtgSl0wXkusX*zf3Vi>#<}Jo8ssVG zCH<9ISm#v<1oEE=Ms-K-rFl7FT%cxldY*~eEmVv1G{k}Yn0{b4EZ8+~Op zQGg}kaA{dH3VMrjyc=UH0q=5>%x70D2R4xfxm9Es?(+ay?DEhVC`V%dw5USm<pUZ#Tm=FUu4UUf?Ljik4_G%_dkwL27~+a-@ABr$ZHu` z1J^yJ0novd$M;ejcrRL~l3XaaX0TCyi?}OiGw~;Q1fPZ^a8X9W|0+-G{J%wo zI@S|#u2?}U^&-lcFMWYpv|QTAJ?nqNQ%ZYCG|S?}e>u(%%Yi-6$JS)@lt3jTyKJ>? zU{(Nhok=#+N`vr?llKSBbTf$tIKCHs&saWzBf<}PR4^0q621)FShhkM0jj!92H+Z< zP2%c^5F|(k58&FTs?#7E$|yk0 z=3w?hzI-u{SB+#3?^4mNupsBJgM#iOjMO{~h=FO)&Z{?2tX3fu%xAm5iiYX;NifY*4sHM{H)QNFF>VY~(PcIwDcs`m}!h z9q(zad;OUAXqHtn z(=^P2%su9AXbA+L)Hj-Qye+e2^{z14Xb;^9wGgFsh-EFCThXdg`OCe1*zq3?#QTo- zKwMZ7`7X#;!Oo%Z(71er1f(DKeX=+a_pzVi3Jv2WRvYlw)=XZYw1YJXgz&px|wJ# zsncU4Snsz(;Uu_2IbBBzb%61b9kdL1r@DD6P!L_*q+{orEpCIM(WQUQ7Nnzs zqqYT9!E2X6HaXPzw=o57@$c^MPdx`M0^IngOukuHrN2Ru`&*s=Rbo>(DWn!9ox2(j&0lS*hYu7tKawc?R~yI&c(SqH}wxxjZtf@ zXFhY{o}0H{4Jxl&oW@%Wv%2Jcmq^)ySGm6GZXDnrd{YW-H)px$VC$EoN0xg7jz!ZS z%~T2U4h?217{6(}IeL3=IHvZFCbB6jGxl`)!M0hvnit@3x3*-%((znpAR@)_UtkLz zN;Wo%xA^}??e8+U#=Kb7>p8e;b#9a)JIc_(JD#eHqAwL=rvco7}8$RKn_D zm2W7NW2+4~YuVkhD~YYbvA~Ak--Y<5HF8X)cyRZ?;=q=hHjzw(UDl3aW6(5AcYjkr zwuk%3{Eb0ZmQi)j))~hOc4PxEajK)Wi$9!`M{kV4@v{;eI8&0ny&8H+-#U38YP=H{ zP*iwf$~t6Gaw2R*S@UJs&Y_Ijvc?Qqzi!#QS`7U$elaefD^RmJ7FSU7o%Q#Eb*{e# zo?)@fCskuRNUnu8yn-lh>Msa(eZP!*Fw4d$2eT)MP9Uw@NR1O8*gei#NF;f1@aNCX zDw*k|Q|uaQYu4P&F}Xpff@4bck7b)GNl$T80uvmnJiO5;MP^4Co!Ik7s{jEb<8(5r{4%|!m zk0opNL|S*2=vyvuM;jBX-YtlZO@U^evq;le8(Ijxhp!G%Y`fM&^tF*SI1*jxx{XF( z8W@LE_>B6^rpU}5KameR|46FI`41|Z4tQlxuvf3(T1c~_*so^I>|@(~bpxue)%@5V zC>ipXK{d&ux_fkkFWRHam@DABZZQLpP-}F?tZjjZtOy6s;L(b!AIwxs z*XmcE`!=V@_fwX7XvmG33qHnr=GXBC4)3~Ye7<$h`!iS$+dW*@Iz7U9G7dQ-ibNOL zJ)bZBANL=?>(qCKKZy34#htXpgXl`hQ|Uj97Zo(<<4%oD(CMB+hAD@0bDp$O9=KzLds0t3Cj4J< z$QOR~beKn_%y_L#epm@H{%Sbs$8HM5IJ|`q=18LjT!#e3nv4h3r1I{KzRzojL zjOHOX*=y&;(jJ%1tNbAh5pfWqf@!mxO&130dhb4^>Cp{xJt^FW!n(eWBcNOCp=GH; zbK2K!-gybDzD+yad;a)rP^l_+k+1vO=ubH&bDb z%*ProulLV>V;?MiT1k;hpC#hZd9*E4U7`7xo8!?=syh|V2K`bfabznoLLVOQ&)2Cp zB7@Gsc7DS%gx~XjBk+&!xw4u0oe<%r>}e9`cP&x5{+_;1@2A0CPDvpNzd(-#PKD8@ z(uBZV=gUo`1<)9a659tv;uOEbrbH@ulg+Lt}7*{shdPO83j5_kD) zC;rIxBLTeRIurW!sxXMlfFN8hoC_t!DhvNmh{Nz=7wt5UN(Nb$ZiJ9M8*CglwXlLp zCv>%$gamRCi6FL}J}$4X4l_Gxe0ZiYvDSuhGr8wv@Fk6G9No~&I~KlgsEiD%?Ct8{(l?^kM!*WQJy4mGS~t6rAz%uLYhO9ik= z?hj6emC1q>bJ9CJX99>pxF1MlL8@c)#?)E{<&Pen^EEKED^1+Lx67aX&A^k>=_yM( zGtd>V&}80S`QsT11w2L@)yaEjwPFIx`-B#lFOy99+6Eg$yRni0nmS?|BZZ*!)cy zEu&M~Rp2Lh>qZF1GWwoB+LlMutD%W_FUQCAT%q+xk0kW->&_P6ju=4oxBsv3?#qY^ z2C=hWqqg(NWnor1O;U!u)-`P#3=mW7;4LOc;VXN4I?l`I0;gNfFj=Mn%OG5m9UX}N zqUbuoJ7-fm$ogc?Q{|zb%HS&(3x77Uc*wMM3YI4&&PS@u{<^P8m$UeO`S}Pxc3>2I zS&xntOqyMe{GLJAq2R1(euGhB?|@j(>TIWj@^1ouo$0PgD)=AF9-B6#EofRjOg`7S z^gz#Zb&7H4^2A>SU7A>3yiOi~Cs45o;KD~FM7VzJ>F}_32Mj2PYW6^>$f_QzLCbF6 z2eVnwi2Zr0be5n<(i7fn0w&<$j7v#OObg#>Y}^wRbJu_4ND~rxod9RxWl4n%DsbVg zH}dJ>a&Wwj!nHNs8AWYnbT#Xsr>?>r$oA(e5=rMLfi=oJUNSrcD!>iHv|~uBZ0fdr zC({G)6#?rf-oIVV5ecg`w*wP(gr%bOc=CuLU z{1o?5gZNLy9LlM}Q`|+?g{RYcK8&yQ`~`ue3}X(>g%aM4e5Du1J-=vKhP(w z4#0y)y1p0O8fzruESDC3GERx>;*zgD#yO?uCA^4>ajIEH*^n$&3Fif_6#?DX z8omhcpg{L^Y&PXot;4wwf$MKPp`v&@^#v||e-d{r`=;n&F>|`4>?@2@aZtArv%7PG zJAnF4Lq@p#VnC^LQ`=_(O~TG~BJ_pHAhTIxh?P;(h~I+zH%m;)L;L&*vz#u!9~7wh z9wsceW84Q~RL+O-_YKsSGZoYfKCVNdQW9?aTGPg^RAwGX#7a@c^8eMZ&u7 zccLu#vpc}S=liijPAv&wg?nX(8KF+X^9mrveY$%5`LP}$Sdt4FZh|{hbZom${X{QD?316+!U8E=-lE3MvCA$CQBIa_N9+OV&p z#?_&gX)Kz6nUmb$t3;1NlH8N@w6TsDCwx%R&u2E+E`o$Q*p$Vs3E~LUFU%aTg6#s! z$qHzHRPw}FfadG8>z5hEZ9{ac=}rP?sK4gx8Ad7W!_9jt<+(5b>1&SvV{IoAK%J>1 zvHMT#YvK<12kI;aLdalnfbDhujG*bZC?L|n8|lNC@-zErZt$=9n&8eoL-I5$5o;#L zWoG8?K@$@_hbT1qsopc^^X!F_PfBzda1`YoY7zVhczro-%*}agbX*chsRab18C|Ra z2StBw`~W?P7Nr_M@wI}wywb0pf5g}4EkN;g2Y+k9*Ry9TW#(+dz?XmO@A`km*T9KQ zxB#X2tF&iD%o=i#zhX;0U-(=&H)n51~_72v;ttg>TWD=YO`m@C(g3HsA8~&&MZo6~uhmt~@ z4~v^C-Frh;&Dp1cilK%9hP4y3z@;Gg4Jf{bahi}cho3z*Z;7otY|?j!TxMS3?v_(I}+%crwHKVMk@jK8P5xOt|6)6%2sQhb41{WezI zzTnyhX0;Xi$H;bGS(u(Ra8oQv2_sv?bR3m?F#2$}FD}{D;sKXVhXrx_N;aAPwZW{_ z82GdAZ*>giwBTNg@;RDlxH@Rac|UA8_jG_C#`vNKY-q+;kTI7L{avD%H|XV!|Z zKY9T7{$1B&17WrEK|cuv9=fTR==YVEb1+_SMS}YtSXxH~loY6st&l-2_6*|bLr8Rn zU%AXFWk%DsVVooRHDE&0Q1FR^h#2r{uhr`b6UBK}7Ilm6_3)M$8PzqI^T>|y;4P96 zQu~9{@)*YmJB%Xo%1*vXBy#`UhlDkjK+FGnBK#eaNE|ZdHYS~A*B`x?94B76<_-u0 z0lnAhK<{<8520Lq>%J$Vx+^G@eL8Vw-WQT)qsHw-7^(yUre`|QRjMkO_ZlYZVk?Rz zDh0|a;?6Yam30S@w97p!b?)VP1k>wv+!Av?!Als`r<~U+?t?hb&s3A2Dvq zt#f?AiYHMyDV7kv?6yEDk3PR*;!=SrIov(CrnMyA^xeLOL^ zjEKU$lghv>v^(qY^w8O0tq7erDW^?mdK+6Slh4q~)bV9W`{B!njSWK*OskMdRj(}7 z5j0~&iGyA%Gvul`5Mi((SKS8<&zkz*n!Ce`onn&Irx}#mskWZ*pFH zBzFM*HzrHRa`qV6yY$}XYMUCf!-+P!EBApsp0o1?cEXAR81tGq(r437S+ z)yvBrGb?qWk^X3`;%Q1+oHW_J~+84t3W>(&2eDd4ctGL~Ioeq83nPzctSZ$s0KrKx&pRh#?v~M&+UR-7n z!pv$Ec*#pf!s!FG3g4w$1jM)4rVO^i9_T6I;vR|q2#CPp9&*c$nGj+rr-z|otjhT+ zRvMg}3B><&j9(o8a|c&*+!MnJFh&9{S;qYg*md{90R{)4qyCsK|oSa+kMLYBF8A` z8_!lrVo{?#5pVoLEu@Rt>?6dwiaGCO6`T@nMo;dEq#QG;C&+P~0W@E86wiKqHRER0 zAe|QH=!ql~sD4%P3Zg1>LZ@cD4Ydy43YyIw5hv$JoM#{}A029!omVR7x^o|32Z3uu zCU!#awJnr=V{P*R&i{&d5YxXXM~jaP>hQ!^=O<^oR=L9vm|K~f&AkEVe0Q(-n^0eE zviHqz4x?rqz|^mn!XPRh-(XqPI$gBK3nox)q$QJ>vlYx-&8Ib(`@c{^kmHQFl9pg< zLs(Q!qBH(sw>VLcD@c{402QF-024FLpDlR)%gU&^BR~ zqYAmLqMK_D?E9I%zxnr_7`_%l;rdanh;BT)S$Gm(E1nvUMKag+QKqIWh|-YFD2NSG zlj^<*S=qyykZX2b;wOA-gcz-*u9o#e2vm>w_fQRTE9xBCB@uQPHFVG|ZlWZkIpB9LK_A88NaE+XRV zZE%C|JM|Cd$2k}RBt%Z-8pC0$1q`HOwR^crU;Z^;_aiEfx}@JJ1VWzQSR2vSc7In2 z+yyI}m0R^h&?#%W)6;3pHc|6V8B{6;GqGUBi0p9Qm6S$v>w!xE*5A=$fc1AyvHD4l zbt&oBj=bV*Ayr~s>A&J@mhpTlv$!+w%kd?cECjk?DlE&CAGY`uA{I<|9ZeZobI3E} zMkr;C2cQ6+brRFPf#41%RYHB0 zqWh?jk&f^kQlva&X$xxgW%pMJ3E)d4Rq)Ou$3DA~3gt*jZPy#QsEg$r9#D-AQB`h; z|GuGy`V!TqUHAuP3dg|wj~wGpcO`PI0q5icDb7jo1;@LRHyR)KS1p;Q94e9tlbiXp zn8JIuDz2+|%skg=;$HZ{<~8vat9}Bri6$mrfevHG2iQ;T!1%!Xq&r_!tLb2D@y&0O zGu8%7c>Qou6$@fh2v4>zfYV^;^LLOu8os|*1;08GKxs>gZD7k`F`lfG>sjP zRfnnEi=8Eko+{2uMv%vt$HRBjsqq94$=n`;*&R6xpX}{8x9&-5t%uhfzKb%MA>_n6 zLG}i=YNuI!0}+HA(&N)eC_MA(%R_bj#3p3(oVJ$mTkBq!&$Yx$HP}x(!9Sg*HY9mDnJtMiXMQ32qnFno``=Ek^K4iw2B4vz?9Hu zjnH4`3NI?CCX7%hZhYiQ##Y<{2KM`W-+`;;an@qy9rrM>mcsk41qH(eO@ffOJfs9t zlzX;dzd1N*>qoAh6~|@;xo)9>VV*BIuVJFMGy2D+G(;$)l@{-`+A8n#2P9d#F=_z` z9og}p;6`}wn*+9vS4hQC?O|=a)|w==u(FmZ$^MA%`TmQEx%eJNI9{y%XlkwJ$U5!> zRp6h?8;D?LZGs-d28iSu=-ZFTk7w48k>bM=t#tWs3k$4lH%RaKNi#N8{w8nwu_M6S z&hOisvJ8ORpOc$^76vUOB_n=6^kW+8L%$voW%Yji<-xjuM^Q@>bcX!JaHttACp z`@K6Ja_@4p9}2YddKod0@p=(6kkGIga-Ym0>|xO3-ZNo0fFwbmVcZvHbA7 zV{h0MPT7(50SW3e540fi+vQ`w&)mdu1YeYl3k(j6QS#hsAKM{)eKYSA5M+8Hcu-fk z40jFr;q>*wgSR3H3trh=LPI!(j0GPbpJfw>{ZW z7r|;PZllCCVgk%Qzp<9c)rM;HZO>CDq4m#7xTku~pYgR6ct3ehKA4?yWhI5(T8PM0i=AFf z0+lYYuT#YjS2tft>(H0c(l9>zO^nZ?bN6lZk%)BqAi1`Tc#JP5Uk9#+tQ_UCX;>h7 z;T?QRbQV)4?AIZPzZ0Xt%3KAEh9BZF=?S9-_Bu*bN!bEP3*iL}PjB<$H7zZvMDMPh znPyyh=`&6J*hQQLO(WvbqM4p0uo~n0p$GbQ&1mMt&BiZQ&p)~^I|nZ9asg|kF<@@D z+ww)DnB>il(T@{JJ@f>^3B+i;A(yQjzo`a0(d!qo-V$N@S#|rrTo72oWL1uFhSO2B zT`;mHL?R0|5D}0NWrPN|twQ&LNeEB4`y=iTX1?i>0}-mW7cC`-BfT5_Fj3HKDOjE~ zkwfd0MOG{POQn{;DVMB4nb@4y%`{pgDM_=Yo5J@iOjD=`9>NHmn@u*@19mxef5T{C z-?qdfSeeF7y%ZxMhG%(uYXib#+2FWfsIkM@+eF6clMK`y17?VOLztT;98Ny6Q+*C4)xRS#d zJkxpnb^jJ04Jo^^g~ zJecpWv#5vV%XuelV7Q-4U)hz=I~x;~T#k@#pV_~VOr;)T-G4zc0eGGNuaJz8TC=TF z&JV6Qq6MUjABS&-BxeYAI(7G!9-@gtFlwD^QwKwTBk*wFE5uL5VVOc^ff4wOpzekY z>fU**oVf;b^aspC56)q8$oOPt7g`v_wD&Ox{w{cuD1Aa6@I$E!|Fi_@aAGnFxl?RR z9(_iG;V%?dca|&+_*)UmNh7ujVXb;Zvp!-_InYnKQEN3SwD~UvviG4WQRZk2_3Ahz zOxi+1;(%J)!1@?{L*b}q(bC1Hh9@mnyj!$d6*z2CCRmC zu|zt|QlbMJ)&RAV+Xz*opU~rvV~4cy?VjW5uyfw4Se6+O_Oz61!MJypdYMKn{fpo; z04YB@gSy)^$f0mogJK}#u`*7_OqzY%Gi9||U%nI|q3ykpX2NI3Cu@;DJHzXb^Eyzt zou^vf2~2d|128epv=n>Wj3FQGYUOe6!7a=C5oPh(Ec8o>Si+i$ex!rl3K8k*KGH;^ zBDO!xpOl-cuB4l?Rkx=h@_r(c-mQGvp5w64u(Fc%IP{HADu_FYmEMpyu!DLQHZ`VlGd%ZsJv|-?o_qFzyr;xY zD1n6$ZRMzC%Y2ar96T<1%^XT*I&A{<3@je^WR;S}e?cvEETAFE2?~Hs{q#!|nhxu) zTzQ7o;ia_qYl(=@gkJvFsBf!Uzku42So1Ypx4jHp9k4#C$DdH^4EkMka?MqU-dKwb z6MZ!zRXTRtKdsbhR!+ACvWB?NWsH#qLn-2~gM1K{_`H?+XT=9nf(r5Gohc5Y=kvVo zg#uq+DK1FlSx}$wdWyz^xku3SRF9oHzI&U6j`njs*nQS{@#%?_# zOD(&;ATGgdd1;l4(dFEdj(x7tYvaU-*3y|ZocxkraIKNN>7K0_!Y6@ zPm!w4GKqJLL-v+>OL9&ncy#JTo>?#N(v)i8d}hY2vm#~)`W{RHlRj$)XFQnfuaizq zeVOInn2waCJ#7?*oHJULjkpBCbyHMi-N ze)m3p3Vd|`hiqB3+Fc(GuI8Qc8mm@zDKT_$DOhsF;ncwU9x2{@)U@bQQ9YN9GF+sM zCGJc{z`J7Wga=OtyCfwEDQWkmENF!>*w(r1*1-gBb8lyBC)W#6gD#Nilg8$BMg7!4 z$<)2@zDGt5?ppY)csMGb;khT4h(L1&pg@Rlb?(12_Wz_>fED;Lpa?iC$-O)kBlWSS zkQ^VSu(yiS1QNkYSdx&$W6RSlm5Ah|D^M}x=c5*{kw7B3%-|WNhNp zH5(30emVnP7P#2BS0D;wArKa&pYVp9H-+G+(!xOj!`#5BHm1y5=Omw-Qv}D;l9A^Y z-+q}CDa=kEwuEU2E*?@%oG8r2q_`T!ZL?q-k$RsQ7au1hf!5%&`k4`}98Y?F@#P3& z4cLJ^IC>$%b_;P*Bf`Pw6lIhY5Sep+e~Skr8G0sRkn+5gA?(5MGRJ8i;2+EF{S zv0z%SI~An#_K~uY-9D?7uD@n6F_O%Sqq4YUPwAs`c|T*lPiN()Vk8OX)@I^}JFj&{ zMESpA7!nCh{nOKU?WE{YA;ZJ}GC;JB*uv^fTmErDWilwG(KG9dt99OH4r_#1ChPDR zu(G+QIWCa=o<51&RcTPxw7dto`C$4x_Aa^ITt4cQJBcxDjZTrwkQo#AQEK90*@H;B zIIy+VLz^=6=qzC`)?qt~my|5agYIlBbkW=ymxU%gW5#<+-i_AeSVpiT;ON)zr{7x59 z41|3l7Rr+ksg^HmB)nDq4^NszIO-fPcyWe3vOrsZ)wAu|2kF}r=`si=(%OE}K5v!+ zuhE#it2oIpT9mD4$7?1&SD_z5qr`W!wEHg>Msy><^H=wy&OUuPWj{jZy(fMlLWzR; zP2za$kL@CgMV?)2)46ANn>hVc6n2T^e9mpK8{xf`BZO~%4t+`Q4kya$wS}oDx(^XZ zKMJC`%l()#GYnA*1-#Yit z*JHrr%rg-GTKl!j+uEh_-g{vHk}UXaiG2iiHB-Cenwn4F4f?Z*syU?8ZAtaffs&+t z94&Rlr#4N+o*?ya2;RU|4}?z5EqYzfO-0;)TAN8jkP+`(MKC-#M|PtFsZG99EsC%w zrePbQX$`xJgYzfwMxy7{U*OR$ToUzDxoQ;+Hxt|6Yu!S8_LvWfpQvr;Pid|&i~3uK zO|#`omHX6YuCabrlk9DbrIMerI;{AJ&F^oiuQv`+iILp@@9po0VYbcZ~>)|o4Hd8CFGk{nk%UL zR3#gVdDkzTbE3Xf(I3Hmc`~I8YY$)P>pJ~Wzl5$>7>w!P8ZkvXgNp+@@GfV&!w4z5 z+JZZlHrp?D0yDFQDHThOb*?on@=}$gn)3f2As9lrLQ`W???3`}-Li4jI6XKfH?j>k zYbA(yiG#k<#^c(ZPQdS>iQ|9tY45UKBzk{4@I5&I7c17JRFK;ys0B;?&+oVQOtmvP zz^{`tCPArzV|SB^KfC}3O)5a1$Bll>qCYh06t>Z5+)~O57U%mA%y z6a6RglB%x|C@wSGh18#ze$+!e>?EMs6bae zUXC6ff!mYJo*bk2L`&Voxq>T_*m4$DQZX?xDa{UI`kKE)C+SR~2OoJkx0e(vE;s7z z?YHF|bP7G}zQ-SxR;$(epZuvvzN_^4tyX{GcC5x*F?}yHBu3(9Bi=G$T>bwu2Q$cl zrNNKe>Hmq4vylxbu+<~{*uVvNd$_u)0cjZA?G=BZ8m95y&;@CL-t^lPQ-_-u*6*HQ zvKMw1;ZOW8f_^u(`blOHf{WRT^wl_gcmLw$6v=mq8hN^Y1@k{HY^rbK zIha;)!(?|y@S>q#HD}w->P}U!lCtQKI@S)oc|Q~*iC>%&Nmok%2t?}F{(L+>Y@L54 z6Kz``NW8%K1FjTx83%npSJ*q6Nq`q1I|cJms1~=yr+I z6Pi<3DGI~%Z#h_5t_24_hU4Rr)aILImp}S73C7i!5EWMBQ}lw%4pis8Cv(QVy`VJH z-0Uug1#Sp_o5*|CVsLLe zT1%G7{ubaXPf+MW&PGs}($s03L;t)dFTtY)w&&zmkkd;$SDY@eubP=1?ti`<%}Bfw zg-hzx%sYA}h~fK__>$VZlKHWExQ z&pf@vWBhjF+OE$6Lf`~lH3F?9xlmCXC@3_q`d+_K^zvwzT^r2k10K%SP;HI-SB`bn z|1)JSZPYfjVWEq(i#qSBRnD(V^s@)fKm)Bl0{2EOhGWH$Mf~o~_b$LUnybV% zy28`!04%_Nwgn%5zZ2X~UkB$%1q3Gw>wo3~`RC2Cd-ekW&+#j73FtEQ%s*_i4e! zAG2m-0^lspqwqi+_9870-g3kwS8Ds0Yv&0vT5d;amS+O0c^akoMkS=`%RBco} zc+{D{`FG~(xjw4WgX)1Q7ew&tW51HjXEl^r`@{E;tS(9vDz>BYJMR=={vCQ=4xTiL z{Lm+aHzFEJm6^`0%c&;H>>O^j`}l*DUR?x@O&~DXhZzi+W0r zvX>IuN%&&HLj~({LQW?N%s{4Wz`EBD*j(R7#v zNxa;NHazs=gi9h0^eNS**)78&i4?n0;{N4odavTCQu_V>$SGvCQ3Tmf{ejQnF@Vft zeRMXa!5L+hH_zNoenrs&Ib3JPTC60gwRE$7w%re6d9W0W&f2}MS8d#?5mLIt0iImY|u zWhqTB5Q9w*c9Cksh&Dw(I!($Y#W8oG9H4twA;U0}aS1^P>TnqBR2uVBM!D9Mw@4kg zaa8E*Ra6-ZNltQNPgCPDhEkLZ=_l}iA3zmWp6iRIbip0@f;t#YDPlAmd;upA2}i$A zz-rbDpbn5`AuZH9JY|<9W=3LVb))uq;Y%ZR8o9@>I01v-WD({N#uBj|Fz=#5Q6T3v zxYGPl2V9l@qi?7rvmjHUm?ftr?7@sw^J@p>>jo9h9)1xlumPr8Qv6dZ#l!mnT#8d1 zKEURu7G*T*n_UWJ(1#Ek{r(}TcOGC2vTNRhy_I}K1R)+`JP#! zXra@&<1jdi&HZ2Z+EFn2=kPi(0_AY^3rbhGE%C-`pr`K1F^)qayD+~JOPweoj9L4p zN$4!mbyW)$?IyTa5({Iq-G2YMqjl^TKz^oHhMaEjFyd^bwnUxhJ7|^98p{c^aMGk( zqEf2lN)ZdeQTaL44MQ=eMIu(RIah&OgC8aY@u@KX<11MPHIywrKlF@hnW%eETg zd{WNPESEyMPt4zZ9Iy3(b5i1H}_}?}2`d~QyrwJ98q1l0@2N}1eCQa(9LzS|K zpXE*Zqvc;?MfmL_z=Go0Gb5A2VaMm=Qf+kTh**Na=aKs~%K&5!YxEILXrpa;Y;2QZ z@Std{&XMOl1gU|hD%!{@dRqKO;bMVV;2fa8?7w7OxFn;5SkPN8=sl9#v(>|CYN0p? z8o5T352d<4)w!9*>k#m?i^+E*Z|3a-x7LGvqw}wv@AS|pLv^zRB4eFvO*|Qs3b|o| zvM5M`(F{GuA=e+%^f*E^^n-54;Y9Y?(gTE{dGV+<-b6Ow(v*`w|ugKtP05;sHR zKAjty(01jVX?aA%41E_|?$Y)AUP-MqM5px{Q^lljm=b;#ICbXpP z2n+Dj+x%>|+0oG9Y-RG_{%)05iDgCE`e8`==sXrbKqyM=Q!DB1eOZF*fXo++jgqpZ zEY_7$iH~wiEy`DMI=KE}1A=Z&BD6;xuqKiEWa0dBaC0E-hR=dkfE^VR@yiu|NraB~ zRO=d$?$-krxoGbGSA4=?gUou4n)OfB@(RK`7@enna6QMl6We`azYHX z2~YO;$z)cwaR`DfwRai&xSD81BFXF~|LdzCjHG3fE| zhPt{0^hk{vG&Lh@>FAIujt_hl`yDZnqjGc=y$F5eBxm%x6{gD`opL?mJ5e~Eo9w-M z`kz(3r$bse8OCduD;i0z3dbSR1%2PtS2yz zpBMT_(5r393LiQsnIR~3gs;9S0|K73tdXyVXmjxN#L*Qx8A7mU~kLt_0o`)my zI`67?TPAtp8gFRH4XXg#h6UR!KCBarmYOoI54&U2OiYWwc;kF3Y*^J&9MNaXcIK99 z=uKmoIVY!>E({j_HFDOofuHu|hI|n#@;q43~#Z zZ~~3Id@l7&K?scmnfOe79Ag!Y%w&ZPg4?5Gv^d#L2jpkSe7ZRbnvQR09ARUClfr%I zWDc}6eJDXr+%d5K4)$gF_fo%_f$wD=SRbTPetO+KZj+MLVOx_q??~AbeMG(t26Kt|>m@!ChFEH?TygIY9;O+{QFXRB@bm>3Kc5 z5`_y-BU+JhmL6e^OD1I6faALrJ9i;TPVxY$k_8@%Y>`G2LgNQ!2=ZknM4;AM1Lux( zy-Yp0aFxFt4t^`Pv9zbK#uF%8IE^O4w>XVg_2dN9x5!+%aWmlk<<0uD2nN)XQatNP zNc!vIRy>>?HIrLg%$gCAXLML8Hf(8uDcNOnIr$xyOQXasN(O7^x34CNL6?2n zsW{6l^qSqD{3l8esBd*imi&q$DI>eZ%mk9hR$5M*WDG*5S#u!@O17E$u9=jZ%FS|b z^T&;JP?EX3xfs|uT#7V1RPjfA;mU&u!|zo`j}0VMb!BI2?2waro?I23X}B?roKQHh z#m|fqG-V9LW98Dg{mj(%`BNsVhf8N%rNLW+N$=u8Vfs4VZ|;An#2hi8VSLqXlBDoqZVP;gh&V8PQ0AIJU{ zPCTwMKS;ggW%%%1wn~m9qMfS8$lvD{f6a%-2ttsl=4J3nQaroez3mk-YitvP*izM2 z9A99Iryxde!$@_3+=EkX!}=-9-k^&*l4qrJSvEU@L_c_F#?tqTOh3q1!lln#L>XZ| z1dBVg5wjK8eZT};T2K`s)>vj^gSQeg<5Rg=HUU9aKWl6egWhG4NGlI(<1WVu+tRgE0>W3YyC`w z0f`K9-=(gC8#a|AI{XAa2h`|jUl{!ymEk=-{l4iUGp`S981(C?kV!6UKk6zy&SpQ- zaUL?vEb~!y+-2QkkhJnKv_S3Br6>z~@686K-=Y8Y$KS7)6_*NC|!4TZmf_$$7cD|>! z3o0OnCRzI;JB@!Fjd+5j<_>%zjbkKEwjy8@rU zmwYL9(#9>%5QwHcCmv;G6Xo^XAP<6gL~=AIjzGsuyyR7r=eJVR6QZm?!P^1vq3|CA z`G1aU?iZ3`VGfp9{gUZv{4np3rx_$^g~qm~r2v6-2!2;a99+*~^Ws|rn*=;C6~nJf zixg}VIHh4<2kz43gIw3<5=- zFraq}(O&%7K1CL;hmWonKSV(8i%P6kG9#vR4!6!xI0}{dP`&j~l&=)A_W}Ew8aF{u zD{1<=l)QaVd+v%9Lcbw9jHF`~aWWZYv=*2r&@w_U3;`y?9^;KP6Y1FHP#lo*H!H#Q zRH_Y2l2t4l5ui+D=}AU*Q@Avh=`W*#H_-k^)wT>T4%FYyAL{i!kz&I}mfj4Z!z7HDv9R4EOq={&y%uVxB>Vj4`I+3(UWB5*vyV+_lj zBx`&z>RmTj`!u1YV^L{&OwJQ>Sf4;s|{l-0QA{{$Acfy_?>03OiP@uap zC;fEPtS0~~YHZ%RhYT+)lk@9i@xy=TPGGEd)`LVU=$`avv4^`BhNS3T@f19JTf6`L?laR~t3g@>AKfZMz0#bQuPSnJvH_~rzfilGn_ zyFOiA$QPy)es>A()W%zei7fAC4!-3TztP6A1`GHPCnKu3veNx@6X`QaAUkTQRG(G8 z3B{xxU_w7gK12ydb`!iyVI;92;B~#n^<8ih=88#FQFmW?g;e>OxK|r_R@y5?*$*2_ z|8ODcVa_xDU@k;T;*e_%1en}nEPBQl+WVNotib?m3v z4uTsNL9)#VWu%{1r$&|nI`URco*60?Rlmms0x~ULfamiAtn2+8l84_9;X>--8+%>) zuEvq`e$nN=?$JK%CzbrSt$3T|P#8|nmT%;E7|K%Z{hw^=_4fR0MI*fTBQNAf5C;1{ z6`-?03`Lo?^^!E1daH$xb~F*!<)5IeJ*niXZ5YhlmRrwa|F+VZYtF=7}w(qE1GeUx`Bh8(edLt@#Ms0LzCB6(rU#b{{JsM|a2 ztFd9xng?*qapH9B@l>>{))D5xG)}}mbK+nxuDZ6a4JB_BEHkj^u9G-TU?L0g5=I$} zg|epUB~s%_#-vb@@52KY%DiP8&(rp17eoUy#W}m|kKr@ayLy=rDXtH(NcRt;x(azI zxu!<|Y^0)K^deGV77N{uRBmE-moOttQqiSYb6Vp)=Nq$^BF75j&_EG$q} z{(^{>EWwy6wE9Cx@#wfhP5WK0aU)oEVyggpnB0ldTt+5t9DM_UH>PKQRvwa6k>1vj z!{@<|J2ee9(VTNli42H`{n*bo`rG$5#{9IOv`iT7bz`XxF9O#JC8=O~up-(*<9w** z8gd+d?{hr?PJI5Krz1qZ$hHQe3pn1N?-P``awtp?c#nz;$1^1NuFw&k8U23*qa|vc zXnz85Ze>1cF8s3woEY^U#ly_Wb6P0;T2Nln?t|UwtJ&tAMH8|HPKbY74ni_O`-S)K zOg2#m^z03mDW^?J-U~25kHrX_Gljda+oIWFQy}j(w&&xu!O!nCeDe@vOxF@i<(_hS zmrngN!Md{5ECld#xHDR?Yc+td91& znL0_>KT2>h2#yvLO!e7_Oqin}dpZ1;f%cSqV#)nAFo!&(cI<$mloECu420x>Vf*P8 zA;y?5d~EL$kg2q1mlX2Q9Q#E1$~V_CS}%jUy0hc5$Sp1L$;%&H0`^#0E3Qn1HFp%s zILf#GIA@{ge9~x)ED6fPjfv<>>L|-56Yki-Ex@bp)Q8eesoH3cxpm_*%syv+EMsNG z3iBkw>9K4|Viw|5_RHINTrKDnHzTyG}lH16sPkf$WiJz_DSXRWTcRl|AN{hR*daxm7A2K<4DO=13XeWPq_;%!J$;!!XUE z-e(V^e0yiWC*|lXq@I4^+C$3)1Y$$70(KOrHAgLUvM)ZIJNN}cBzZJ!boKi08D%-! z0StrrWZ{XdbL#`k<<-iuS)*R%`RZCgv(uVt`FS66E?p zTMIcI(_xe4T$mkAVevWYS6}aH=M;itzXR_f2cgXTtx}hKe3xJ)QuNX8!FTHYMUjo< z8&07wB9;}vSAQzP5oze?q9HnKmNKYHq6n%13N_9qQKF?9z5`W(Qcb|jfEOFMm-)@P zL6w#`fc4?^9EUIW*Wi?!^=(ZEFQpe!v(-oYd*AQujO89&zW)B`aA*9lY()KrpQY@- zWd!gyzrB5V;pZRoMf#W<|9iKd|9|gr?f>&IPbFJUi``eL>Rsd`^yNz#^-MX*`pzj_ z`y{uVs}wmAVRK4;ruHWAh*f^bhAluxEs}{6H`r#smS71qU)mR0WUIL@1|-O12(aaO z8NNkboJLID>?{M8i$vj-w2EzYh2G3>ZK4U2*09ZNSCwjS`B7qm-*Nq4j!B%7jQM#w z-xXxe`ae8q`2X!6?Ch`C|3f?-73T~HP~JdmPoQea0a%SQXZ7~b)l`PF?^xHpQv)gu ziTO7dGeMof(1nOh?l_-5b+;V#R)zfoS8W`I+P^#pXM8SJgAby(bg=F4A_EfBTlw0& zKOm#H#T17O1^N^qgJjEd2W7-;(b?*9FiixvG@TEHk%FB@|#9N$mqB*hs)?vFjcx9{z^wZuC{!bty@ zD3^aiJDj)qRa0B(@(Ez6ygoAv*=C(jF}jH+ec@#h#e_@p&nA392V>GlfOQ~*mmgCZ z$gX1BmSG5Gxe_Za+o0G2y|OUSkT0B~A+52ZXkJo#^nB(=;v_(ad+=v8xd8hray{2R zDCb1QZc%;k4S0(9T#6t*I89+7zVkZ~twZ^o_rV9nx90#uECf2K1X6+F!`%sS^8xIt zifki>Hstl97jG_jS-AX=l3>$!PKOo0q%v5ZaB&r;{xSd=TpxqGf*g1W?V^}21r7_p zQ6DveRhiZT3(bI{L&#kPqv=0ee3A_~HDW|7z8TS}gthW^X#h(;>V3!DT5F=;Wyo&k z8gq{I1*xWEns8E9Rg6^R{-=xFOZN^uEKsx0u47y75_~nZ;Ch^2) zV<9D0vuDm8w=Xh)rnT)rUi$S0Q<0fc&7=ysXsx6}vy}NZn}t%GbS6uR%djbhgoAHnysir0##ChHDeo*T+{V^-|o}4#CT2Lv^Q=0RJE8dz2iKY zqScySb^iH#v#t5W#DUr>Y8|=qk49o+Ya^{P*DoYTMy4!9v5TTF4F`UCo2G&~&T$t` z0_JX_jhcKaZXWe!OQ`byn+Ap?Gd3G*HtsLg+7)MJCaRE>vXJYvmBmzGtp;{ekA$!k{u_9glG&E{MH<>Xj;GgO&hA~}xm zSt?XacRZjs#JK5D<9az2gA*aY9-gzo)T~)rjot$++FBXi3sU(xN^l%cw>3Ga+c?z? zPBTKBMp6=I>M-&+JaNqqSI7<*VuD|d1y&b)n81Bgz>RmevD~Gae<$*Jpq}8Gr*TYs zY*GACp1Yp^5*$Nf;#~4;T5^3rS2yIH$(Ba9;eE-QAk~cdvJJxIX_q$Wvq0 z5JS^YRhiq+tc>t#Ln^bv|0IZ_vM+t>`@%Bp*px=Wq+MGc4qrNw&EqtZ_qJPc>>$)Y zv4%(9g|2nL3P^kd(NaX+>aE(PhGSED`Xx^8C@omD+<$U?*}vMeO8&2$v%HMp&`=D{ z<^M-JHU7WbJ6Okmd5~w#|Gz%}udRz!a{hZlvfd*8-JezP|2Dp_o_YS?M@PLH|KB;- zKV0+whj_jk|6enH3*#5jFg(rnr9K8i8&vKadq(D9_z5v#kch?ribD8N-bQs0x*^#> zRvy9CPY?X<;g4IK2V)xpz3ci{9WN3LaGw45=%9Z8<6x(^yI%hf@yL~#(7bW0LBs`V zuz_;1t3V|y(LJx#&W=K%SX{8O`slNAj-gp?hP3lz<*fi7Nrx55aq3*YQePIG8R(^q zE#Ysu+h!Pvv)^90+j=e_u0%>h{(l(>2FZ4Rj^dnTOox_f#D{6bSFnalm=`Re7Am1V zPy(t1n?Yu%8DqE`{?=wS^2rlj9+y#RYyoE=ddlpmGYw3Bd4DkZu(_$bXvU;@$f=Sq zJruDwG_T;r-?C%no0-$~T$*$FP0r!Mv$yQT*-nz|^4TjLSZwFPIp@FK zgT03Sr?=+hOb zrf=Dw70$)Z_IEwbR}S=FRf8zW)3=71FdWZxNWOb%R90nqY2ZJPRT@(9Ilw?*Z%PL{BW?EER~I64wvK?_ z1n^8IZFn6e(WIC_;OP=N^;Nunq#z#mcviFjTipT1DHhVeJp2E_;a=VTf7n~^{}1wT zitTPWkk&XFqDjGW2-p*g+dU9KZpLK81Ur}^nvY3#8!;K$PSSt~2awW8+_Kb(?fECD z?re4I7XWRhsEwuMlg+j3!m2D-;}k<;kBr9gqk1lg;Du!X{oQ&oQ( z1`kKtCp7i)C$98(=gInBk%X1VbX4Ap$rJyr?92ZNGjfLWF>-6JU+yLhg^lsED-XaW*m*qKCR=T3YqW^MsPpOKFxrGhi>%%+@=pZpw?PM9nrV$vJLt*3F8f zEAOtuC!MLIUay$~q_w)p2hxfk-XcH|M%lA&K1sXsUV8c2SY|~Qa|GpSxd9m|U5hMM z^kl3Dqd+Q9X;M`4>!(k!0$R#&zE}fa9gH)`6zoMZRj@ge)?C8%c5W(jFH;dWa0MJ$ z~+vS>knKLhEv^WMVYyH&Qgt zS;W1aW%r(HsztMCGYR7=Iyb!tbpF^%uMJNS>(rK&?=yOxuhcA6CNVWCnghP;K3_{R zY?ig;AHFZPZ_F$_GnbQ@ucz9mbwO2%&s$SXHd)WrqZWT-(Fkr}ysqWzu{SBQQdL_r z->qiXF?+P_7!44_Yoi7jl+^PD0X9iug$*Mocdgw@k$X(YZdTxYW!JD zSrxL=OnW~4S#7q~vUg%hX)Mh^OvS{mywt#PG#|n1?;gyV4E#y!M)jp3R z%&75pf-pEE-nJ)K#%?!;Wxb5yUte7;khR3ou9# z(#S->y@MRpNZ_$Gn(Jqo=Z^Y+<>{^@*rmU$W*`D|CtJ3aIKKX>c)|GmAPy>C7L+ua1K)r@J0 zXO90*i(Ihwuly?hm0o={KBB{A1C)gwE&(i>uhdS-LYFL1`Jr~2|s zCCyp0qjeam)n+ePU}`5(a(siMxIEpPZ>pxeg{rFoSutsC0)Kp$;2I2-ck`jJG*N}) zLEKKiO%upn`9E7I@}GF-@c+HN!(M~`@2}%OKgeUqf2NYHnvfJF(4`n7=zK9rZlE8_adho8XT# z%Tz`?WI~4lb-BpfN9RRy%#M>OmuLB(a|^s{T3_R|LFV^3J3aClCS|GZ z*n(idPsQ?mTDy_!Hbg;JnKFISJ{C~~(zsz_0u=scA9AF6ziSbvLL zmfU~;-y;6M2le=0M{EC&2YTxIpZn$iTT6G9MQ$vNcHh@K%6b(ck9^EY!e>e8VMzv^ zlO&G6)>wAN|~!|F>p)YCiM$ z|G{oO|L^|((fa=XgFG!9zHUFYg(p!ofGv@K7vBMlZNt`}bMBQi*wbCT*VTr?s_O;cwIZ`=0zCi<94+ZD1b%?;X|kKZiR9>-g^v z^4M2`Jkhslg6bQE(j7dui{#hQN0eSr&CO3CYSnw7t=+aAYDSg$Th#;XiIzXt$BsYC zvlRdL$0VLaql9K;nLg&({|X2YEW^4Cgt?5{7b$#N49WF-g#%h+@d$ zmg3+Vj|lUf4uU*!tVlr`L~Kmr7>!~&5SQwrWVDSk65~9&Aqe7Pm+x^BIvtdd5wHxj zk!EBVeN;K-{<7(#-;#KWXaY4kNGK&4ilc=1j{oA~@*<}haXRRPP9`)#e>=HAVU#h) zA4Pc={uSUm{@|Zk7yeZ*#-lF(NBv|sNw;ihfP?EIMZ+j2%-QnUZR%|K1AOgl`S~Pu zw*JfMpugcPq6I^zFJ3aoPc!-t66B5_g#>p+xs3kfTV!cVt^f0v$1h&L^e5qReav3} zz5SzJE&ktbZ-1Tt`C*<8I@9xvj%6MK=QLqCj$?-MF+uN}B|dDx0j}Tej-q^A4E%sj zx}^@&W6{V)K$~btGidWfB$9qDtl;SQ?9^B9q-7l8jG%&%5Jd_4t8N+%rs(~zI>HP( zqaixh5oTBkSXHAM(tvey%&u8CD-sW4RkCk0N4G zPMG5uQ_=H#`~L}Lejz#fmBbTtuCjDQgrU=9NFC?u>MEcKqcL$fpeQ3LMK}zRK_3B~ zObUo(t%hg&=*uxj6o%}HAsek0X^3;edvu)33qsKQEB*1RL>+(Uv#}rwKpy|+l9%cV z66IauBt#(%AmM?qz+S>w<=PPAg>)y)F-Ee?`}3EtULIe(ynJ)~`sD}fh%L*+9Dukt zSctn1A2vFb-9CI5sWsWw`f?z z@f2wS8^i?VyAS6N8B+U_K zDyp_%G7{Jw4(yq%8c(-%U!gH(Vs_O{Dh>>XG^e1BMOWuVl1CHrLX7;X zob^31>n{qz3uC^}lM&%VG^S7}hqeJGwZvdk$~aY4R-We#X%-MZ7DO|?-R2Z!#FMJ5 zDs6}z?!=s9$ALvDgpufY%Fug3aiE?3 zu;F)&pPSo=W+*FQni;8)xnwJ>-(+xXOm7t>FSVU@Iu%hQ<0#>GmQrSId#@T}jfYh2HA_4Hx>CpThF+@J56r{a`lhLnl z-=1ASYh%hddQXm#xD%zbrmG5^>ML9dC;?rW2@T{B8fjqE(vXriIxqOK)Hx2rh|dj< zIbSSy#DGqSqzMcwjt+Ig0`M(Gw9ssGKtG~E0YoxI6Fdcx6*wlPXu=tWl2vI2jAVjS zUU;Gq7*Ao_0m(NZBm~)&WI)B+oR2BZfM&IL^)})-m_rIPUqgxzb%Z+M<2OiI8w%bv!Im4ofH!0#gN@L@~ zcM^KlEZ`h+CPif;#%Tk4yAF~ij( zsN=lkTbE$>5tG~&-A0@ZV?JfIdC4hX@i3?m;T&CcdE;GcR%Ufs`N~HxSYSr zwyR2ZZP@MetRUO4UOD;(3WDgLqPkpIz*s6vOK@$-{3gB2k*umhl2A(xlW{>%N}j84d7{^4b|*ecv`N@lG1>U>NB;ugeg6E@k3afg^?eOZX6*U%FPj__yb$=! z{xp65e6!qcUaAxL!gJg>v$xE0OH4*%o=?P-k#f45^QBs`&Y1f6=bAGG)mNM~1=r%U zrj-|SX03Iy&g%_8<2a7#ElO#KPR`!JZvC1}Xf~~NqjXdcrj^1B-4YI;I0IktHiJAy zIZdyW=(ZRED#Td{P<@o44$?wVhZbw(gnurR3l)>*%%Tl5I~&f)hHl_8NQ=vY;SsrH zB%n#iE_0gW_>(U2muIfKZK0s|dy@~Fy4_79r<)b1MWEu90iY@<6B?4gJaeNkChoTS zL2q%C>!1A06fXg$B{3ttAz2oM)m)7nBJ;N~C-AM7#Y&jB+sdl~#`lzlZN*a>)@K{W z)-Qj{rO#KyDkuZrj4nAAF8MLhAQ92Ss0!y*a9=_|^oobj$+E$^dahFB4dQg=kN9S zpHHn-DeEIhuxWgiB*{O^+YKb_Bl*@|&hYLMZ5#Bg?%7!@N@cJoe9Y8hNDDSjGy1Xa z^oG;4{JG=z{C!VW*>ys~f-C~rw9i=+Lq&?Psl=2g__KnI&ocUv^O=uRkSj5Ifysm> z7eardN`MP$M3S8O4#)A|3=}I2KXoMNw$&;xml$%Tqo--d-}QUW((B*QAudc~bL{^I zz5V+Auic})wg2aXJRKxe@=B@8chUMf(4IdLLm2)2`1LDKXv=aEg5L};@PY&}&WJW% zsPj>ZnK&WkR4B)^7qRa+W%rPZR_*gVJ-0reQH2(bH%NJ26&6N1g1e;rbO zXyesi(BPV695|Iqh!vQ%uJUra*9sck`Z}*2Ec6Wkch%v@{JNWy9 z;P7ZjzVCI*zB;dZ-mBh$<20-aBBHY7v3V@TsFdkswDP3ZM?)MlAw&tzB{6g^g5XW% zsk{=rwpQ2apku9By&#htl5xz+OwNeqe4R725t3n)h{K@;L$5(xnM6s*x(piGgjq0H zKq*!FPRHq>pJ|2&4#w!3fF;i0TED51+u|56)Ysw^KPJI7I9m{cB5C0(GSCm27Gfdf zJY_0Em(o)>9WZuD{>tj()t(n2m)x$&lp!i)rwL94+`lEfZy`)VqubI@WGoF>88A2# z0mF#Ip+ZAQq+Vh`q%VSDgGh)poQN(@he+vE)i}U>K@w2nkc@~T+j1PDzUmtozR*g52~70(hUQ%Q?;wwlH!G4v3>fS>YEf7(}_ps#Xj z`^e=I#%|AbIta;$>BHYzEhR3T^mcX+cI5}Tsj6~F62Jc0SWo2#GsK*VFIO*9FjDo4 zKgmc)l03pO(<@XtQ%_$@DiH}Gggqw~o-AAteTek&w zatyW#^r944y!C_#L(PC77GyXjp;N)!?$O>4CO;#}cK2F9BU+K5G2OOi$wp(^*ygZG z6QJ)@D3{^-Octi9Nr*L>hhb@Ks{tkawB$NhMn=R^53949tzFJB$GrA7?W5r zZ)K^tGPVg5*c)Ydz~ofRJ6X&z;!z5+QP-?A1s1Gk&ohwzlxY+*L?OV0uUj%0J;^ef z34>&UGLnQO1CkWHhyQ6_N06eg(qKly(F4ac8L=oNVjBVeyA7SOC<%x>J(G_q2{`)& z@S&tK_)2CY2tpE%mdQ^n6Lew^x1qd@EZZdVC)-ltER zTYUNAL1F*XC(f8sh6D2Di_dNXbX@KMYr%$3B2Mq3cbkK!n-V++aVNLJh5}ABk z6AVn8hB!4Bxtv3-yg;{OBCY)b@QkpWZ40WujTk{O!8b6$NfE~m((6b{M1UXVbON?B zQ(ok_8q(|*XTmimBYzaUgOX8=`8=GS@#?+ZBY(%=@q40KSsi>MrBy&eDBF<8H(4EL zHGmBVc7#_PeY}TKzN?9dFl{&p2m@yUMF%$#7On@5-lzB`(?Kt&+?C}Br9X@)1im!T zmI>H@w+V~0Thl`XgNj%&yaz-x(UWKsd~N{D3*ckb#^=q8^v%I~jmmq4(n%mBe-!+C z%B4{TVW+i&5k|upk7PfR-OHv8Q+vJLoL-Zp6blDaPE|PtpD?7t?vPM%NrePw(d}J; zW0~GcOdjabm}W6^#}f!7MzXgYw4bBcP#RmP z&r<#j=i@$dyEi!NWpZnXrZm&K44NvLC{zlbtz2JawFnt)_j z&2VD~@r_YZM_qJ*1?|gbgeU^XAC}StgeeXc6J`viyM*xX@eG(Xjly5 zh>h#gnkTpPz;pnpPdOKitST7r8{38w{j%4V~yNu(0~)C_Ai7!9fNU664tW1 zYzy0aQ*vUtU)ENH7LBgI2OF8}3hwD?v3!WEGY2zP{W5Fk1R+t1&d2~5+#liO?+E&l zmb*BbM6e+n$%Vu;caX6Tgt!jQv=A!Ou%yXR8=7~{n|Y}VQ^rRqr!mQ-iG%}!5FQSr z1VY1$Ey-n+HRE+8VH>6G&Ub~?M~NyJZF zB=q8Fkl}3V4GK0LfMr2~0OpyVqYo_VI1wc|pBdqJSl0(@$v$ER)fgGY%1c~9Mj0#d zua1RHT9u9Pe!&CFel>(<<%BT~n%U_Y4LLKChrzs%chkjIfKd?BA}nuV4Wo?Qa=_Az z+(6@JH1yt5?^nKmNS&}0)CNTk23c9+jD}($ztUTBL$Ymk;sH~!K=eIe>c6Aj*9N%l3fjoVeWTH7X zG_B9>-|u|C;~=yU^OD6Z>~R>x%1!nhy@TY_b&(X?!_|@Ba5;Bmeio&e7Wc<3XMdI+u)xYJ9vpPbRnGn`Cz(Rj(V## zjNl9whNaRJ?4f)U0w)w9qij382;8iduMPgmnQ(ap%RdSnyH#wp zfT1+ol{psH#uGY8xqR;a*zGPvgzVVVX z3)yG=7h$EW4hg4eJpHT1epn9di8foK{xcLtL9Rc6U9~L0zbi7e*(BR>rJL{#%lngd zyqQP?7%zF>6OxU>R4uc#jY8^>pMmWnfUqwrejqO>y}(46KeHF~<%?-vHKG8PCR!t0 zxjI{z@q8D7NEtHR+GK5HN?qc`a8z6!4K-l0!*3&A6RXGPZT~WYv8=QNfi%i5V@YFN`-6tm*+%x7`G=*{_Si66_*3 zBWVmm!y>hWT2C$Q4FJr&yu@Lc5ymbr-OYuX5$;*gjEh|P{W1y{142SP0TE+iWHd4R z4B<58vt#+z?6!S2Ir+ED=iJ#+(zt!lX3NXWSf$>5=B(0?eb&T&uviOA%v=RAcbhx2 z>*nmKtjuJR=2J^E7OeQlZ4`6ShNZ5xyZW}`**XyZlCC%zFt@&Ws;2a&X1v@OWA;}v z1$G49m$sSR7)geuO1PJ*OGz4{)xv)i*wRSM-g^v@&G+*-NX#ri9bWzi-dl3#^sIG<%_mt1DU1r(}_qbX+K)^ihezl zg$#!?)8v`5)Euap%Nf0Yn$kE5rcXaq-97mGP`SE7k%%xTB$7CTGwu7Q1tXQ3VEPdE zeU{LA)@1=R+qYfAa|d+^j69)<@^X21{)&&&Re>&CV{q@?5y{;TZN>vZ9#tPerAMn} zBAHvm#)Li3y9YVi5*$A?Ro*RUu(_s|hPlif%&D&>ckmrB(VCmbYOt|Q-y-RVmIiBC zpzb@Rd{8aRVX?(GD_5&6$*9$svoy|TXgMDRjg9tBtS3`C*M55#;3&xg&5a zzJ9PAx92o0IoV^-8$MsWCB5>L-5 zg{v&W<*S_c8xbN|H0GhF;)c`8=2Omvhn|8?^F@?BRWE=9aX#iR;HD=Df0eJ)HEW!Y z&meUX%wB1Yd_zd6?_5$jhVnj=3IlO*@k(4o5_f0hCB{5{%a&tQ(swwT=_0GSdLib_ ztGf>MP&BCn6011EdZPGqXGDIlw_AxUVVrAOAG8~v$M~~m6AHa>h}#10xo)(w?$fDt@tc23a;r!Jri29;2dCWf+c^#gb5j1 zpL&0eGM4`y<>OyP6la@U*0D|U%vEPi{*G!!du=T(l1J%lX`uX^~k6#6=J^W zjcYbUs>N|OGA^@w$n%DOgvqlf8^8W`@%HlcY_swW996s`i=RE&I6MFCfBpUP-TAA{ zO8XXSSJB2RY`APxrz8l&jJNyS`PRsUuXl1gTZ@|y2}DB;#} z=@;j4brLH)*`ehR@iyHqHv_fc24$k%G7X?9uB+72j&MJj)|kIEk7K*0P70cmZ|I8=CNAePuJ&A3>Lc zdOJHibv&);Ob^2{vMDuZUh!RC(F>!@0CB>rh&i=T;~kcpw)dE2%{#yC_6%LK6s>wY zGv|8Mh~)C^NvjwU&~yqoWGj^*s(}sWixRc+^JEGSv9Za30^`SLgAzzErpUb@ID_QX zMzj~$QPU~tb5sO&aj9F7o!e-IBdL#`d^&k~{`T_c(^n!V?I^4rC4@6JzNK6|n;jFM22Ho)9P9(d6V5d^?SZWBuM88OafJbr8#6C5Hh zK|M2@frfXPZfb*^--|icrN-01QCnX|XE|Xx^8Q(wG|}l7^dJ8r=iz6OfG?*Az|cB9 z7m9q0a@4anS;JR*i`QhjP=ig>%zzSG;cO7u2~8o=gDx#%!91z=#2YqVzCAww<>gzG zW`opHsZj%c{@fg4FrLs5?eFi;EZ+*cLZ5`ZhIRm4zq7+#uTn2R59;>f?fAwGj5lUi z;QV+l)|(YAwdP`eks4MJtGe#2Vn2%woboS5+tmD3bNNS&`Y|Cu*rk%uY_ zXmQ|5qPiz+GOm{mv^vXL9J-U^Rbo#^8nXgfqlzYwOPwNWlXAB&oZitGm25AYO&Xs|5{z0*6-% zRIBzDoHLz&Cg;h=$Hv00ljh62_)co4J$W{n??br~x583YuoAbzmger`R@e}T|4n4$ zRmp#|G6K$%|BjC8`Tq_N*7D!OJdYy(DfhpIN8jpL_6j`p=J0IP;;1LWJZYA`wAWmF zw<~a(Yj3>jGl!fxDy|u}Uc04t#V&A?0OmTtS;{@M`7*c?EY@LaP>q2H?G zTGbkVAFH!K%N(yYv(WNBYpIU(bw7_aD@`r$Ur?{NfOo;X!^bzpw6&m3Gj*<$)Rh`^ zB|^8iyVx1AHY*->GQyvwnA$Hgn$v*Bef0K3xC)6_L1(JNwyMm^ITs#wyUKS3@}J~G zvIUDEI8XlT9o6oC?CtCw?yu#)hj{LH|Dzg)u6Yfl5vI;64wkg%2Q(uTJbb%#KBEFk zF2HW)gk$^)Y^Nn+WGnwEb9bUopXRq`1bMW`q5q%*5Gng*rW9at;oCA0nPhfm0k0{( zny9&G3a(lKX;neCAa3Q?MwII4iOx~le}-Ha)#8hcPAZuq7x2$tSj;d1 zXv!HXOTvsGry4(-2DvOOe5{{Gf9@{-0guEv@noWRFUUf>7I2RI*V{R)-~T<@-(AOl zdXT3hH>pvMHe!;XjhNEwB1P^j^Pr1d!>;D??Zb69H_;Brh#fdRK7l-%mHZmw0|Y%4 zAKf;|uUnP*T%p1@8j*+WN;Zw-vs1adFr#ENL^4{T>~tfi7dhk|m%*<)=)cwAsubay zsL`H^(PsN4LnL2rqbKBJ#6-wu*=7Zz4f2OB0z<1nQtlQyUFOO83y~rbk~bzJADux= zbR-uaIxCaX{-%=BLZty2(oC1DW?A!L>gJB^kwVk0p9gqW!2eU0sy*ON{D0?gr^){h z*7lzVdABk^6Q`q~xnpfQ6JW=xmH{ zL~i&-GwAs2RBKT+qg5%Y&o0hn#$%nUQnJ`IqX8iy=gw}GanAVrvFCr;_3w{KJc&jL z&B(%i%v=8ld-eFAyGJ`a>-GN-PY0dhJSUmRY$qmGC=~`p6hl_l6bILMM40c$%)Lw` z*<*-}NgN}-zJ*>VN=Dl#BQegS8v;(!#(SKEP6s7qBvR^bq!2`jgu*xaFPpx|D@_xq z$w5LXfy8{ocl;L@mlrwBh|@tQkTwPV?c@T5QN|p96y;s`SAg&MgMVgS_*cCckGlLH z^^@Hs-Ljzp4migHhiqrdXSb=dK?cNFF0VgScCl|xt++7!938Oa$c z(rR96$VaVHD+^Z=oP=hQYF?(1e3>Q)S5kzF?cH~rQz3~m^u9i@nZQb-&oe^0qp_EA zhSyb`uUl_w6V!DX!8vKd2BQsdqHlwYNtA(u-cyDWnG&22QrM*-FKIq*;K4`7=)L7w zXB~z(J$F-G+*3(FH*5A2baIX&hH%WN(tDQ4$+ZU33=G1BB&B?WJBb7bV?HZozHz3L zW$O&szT>>(Csl+KarjIEA}VRQ3AU0b$0`vxuREl1Om9Vsbdqp(%leLo{!R;&5No^= z012y<-JH7Sc>^LR0g{c=B4<%Z6zFgCmuT-$;t2x8vD~1-iap6kboMmVu@`*3gx}&RR@RzGi$9ZcIE?9AyZ zSTnsLU$&BT-Wv-|rhw0<-I-i*W{piGWeMMLIvw<`%!p4-{NMEa-Gl#hoQoW1IsY6b zjnCWY{TU6BI?|vsdA*@>ZUaG)DKS{yf($5X+TJVGBTkz zBorjVQDVSxe0J(P&M`xnAESym-$q+o@{m-_8rs$tBq0T~4>Q8XTD=w3_`Se{UXk0V zU?dY7GV(DcK`y8!CE1W>6TyQ#$f+bDsPiIV!B?$E)hkdBatZ}?-4#EwM*I-T1Uk}; z+;GV-p6Y^mR*);-`G5cC|2ogn*47)Elm6D0^BkR44v#YTESHa`x)~8(Ml7;h%n)Xb z1`$xE*ejALKZusTK9-q@5;Vj&9G!fRE}AL{nO5>`iHr=|i=R zO4X?tz~S=^2WP<>uJ|ERHFbqH$PGz2MtKJloD?{YrznUCPPo9J2f4}VD9S||@0h3q z{I9$*iAF&vgN$C21OoFAGg60wmleTSAa__~kVO~T5L+hjCg_5*ZP=_gB*2Lz$E)qy z5VjG?1Ao)!n9{hC;uAw7k|UNE!L`;K0A$-J6)*=hVUUe{Of*SHTC!>W<)rjii`gfos%D|uRHf&(_;Vmsf!ka*OPv6e4h%#?xwniyS8*&O#=(i4)l~(5%?iO7RKA zDkf4Q!VIaxd|!l=WF+S-Q|@SW30IDT10v*CHJKvYH~?TIVDIo>%39%kMrW!6^clS% zEXX3baE?Bsmml#YjS2efd`2FBko><_{RIWCw8U|RJ|oz+b*@(>vknLZI1th;3DIXM zK7l01+c1pWsuhtYJ1NCLA+rnLQ&y39IZwL9!Wb`w5FyH3$&}PT%2;C0`#%~zJuf90 z4$ELk?HdE5%jwynIi?a!qUDLwayP7$T@8o=QaewEky;(vyF^VsXWSs;(X;?(A7iLb8G5M^L*W*uFW5)9*O4sa!chT? zAR|dgz~U&P4DOD&!cwb#pbMhHoeflDC)&S02vlT^3MMnYBGC#`)IE8J24$CuWNm#~0 zlJ^kh$jtRqrCl>XA%~`9y0>GJ5o0Baxk!sa9In@?85^>~rLp1;bGiM-i6u@i?mG)0W&ZpxV5jAYH}XtCAO8l~7AO z3LfLSwb7X|Tlokt2xpLJ7~>J2T0tUWw#rE1oL(~`v+(wOiy&@ZX0>NzCezKJ-`Qz4|7V!Oa7YqjW|jRNYHG5L2A6s!^DYu4VGYE`q*-H3wp3eG~zQNQiJ%xF(^ndff#&OOn|u zd7hGbHnyU9ix?;2F%2aX&qyGYUQ2CHQu* zK{AB!rF2-+s7FHSB5Pq$2vKl~>16vEJDe`Nim_bkK zK&tPxD^(*9dKm*J|^s721CG<5tndHl80Dk*w{v-ZozbwSxEHtpp>qO0X27=1E;idH z+>*9T>(YzKfPm=}^s^iy1LiSi+bGhM2R@2ogE6Z$v|iN~7FbkGteOmFr+qjoy_gz~ zPB#V-38nTf7Fay@TpeJ#Zw7#R9`Q` zsYED$3=NvJvzne?Ro!W+&#PFgl>bn999yt;VW*(CrK=EZP-@+2=A(2{-|u|CQvu~} zb149{Ph~SGPuoP64GET8Q#sdKTwp3UqG2gUZkEU~a7ybP<=Xr}Lb1F|BDl7MOQ_mt zF35xnssT+{o?)=H!0j@IHpG?>vud%tyar>+BuYZoWzfhbAY0mCr+5Dlj&z64nG(2Z~*R0;tKX{6o&!MLYP4u%kxyelCU)bNBe4Zy+&sxZPPP;6(U-OGbKR2Kr zuB{!<-vNn`!f_Lp9tgs~b561nQ#Q24J4}sh{LaLgk!?_&Tica`1pi9{EV9PpzX&iF ze<=6qerLtz43^=5$aZ+|XHiy8AF-<9XVI(pF||nTL6qF`<9Ymn(_5eBse07(dJaD$ zp|BnVqvLv!(LsTic2G#I8^rjg4A5xw@B(fW*Dmcm-|Xc6^|;W@rA3yO!`t^h90PsB zPu#7W9ZTEi_gnnO=gT_-4{r&d|Jw8YX@q)+J1drng~#Dm4$430%Mf-Z;LQTL+DkoYr+3%o}s#O03x zzCs~3$xi#Z@pA3?=LrnHSx`dy!y9;|B?^^p@TYU6c(drbB$ORUGB6@ z8G(6i=0r&+Q(i9m_RsK-A5c*yc06?9WyJo}3`lCH%nq7P+)zJ{!;hsRkuvL93SE-8 z75IrfEI%hrlf~}jU%IMNGhh|R#@6Q2epGg}_%zZ8OvtIFi6p;1&5wCg3$WVR&oZ~9 zHhPD#_XZkiffnffn$+5Lo(geXS}gmRSP8gn7xXf1R9ou6*4E&tmCj52Z_ty?Yl<)p z3hSz>bq>>5YS~T|cSm$&B+4;!x~oN)d_6j$hUEtcn&u}L?uhlaUeQQFd326#Mt`v5 z3!H`L@IM)d8*^`42#H0JB10}@QT-@ls$j0{o7UC;6J&bak&lcYTK0VeUQO%Vx=2rV z>#~E*;G90d;Qn}*?^pryGQg8?LBajR;ct*eHE6MuI@S0u+NjJ;6=;a?86hALLm{G4 zvQ$Kzbfa&x_FR!6JMrV)sjgw|p!_`-oi{HJ(EDe;36CDMu2l29SLoxJnIS-How zAeDwG8|f(p7X0u-n>uQ&G&Lc5qLne|p=t?YDJxbPI*{&Q*M>S%9&E@;o`aEFQ|uHf z61OV8Dd&Zlsj6@;7tHi%AfxU5{`EK?x9mnv?l#_$zCMX3GP|I~S!YtMS{^xd?EV54 zYIw;4Fkd^Y)y}KXS3ujTW&*5L*xHznQuwl5YO^o0ZWuee;GF5_b)2MGN_=muFbM1WsAHH0#rZ6 zGi(1`zY4KdT-XtvyRXiZ~QDuT%-ChYXgg&YUmk`w%9 zT0Fh8&%vkTZ!ppkYm6_BDNHKsF{AnuhqQPBEpbihFbn7iX?-lCZW&F6YK8OevVaVK zWeCl^K*;2h$XA*VV(308KHPiE@x@c2wt(F|)TokT{$pLn=4w+UQX18~h6RdACag6L zciCh;Keb!Zrfapn&i(&@GF6=fs!P_<|7o6uwRHTY8g$GVxWEBO*jes!InlU9)ra2DT+x?SweE{k0xcrnvijS6ab`(v1eQ!_c%5k}ch@=Vg|m`8>(jkU~F$!R84 z^X%bd&BT71Ji-663c!CvHWR1QOi?PyOaclSOyKiP-#3Ie?0*JsuInBI_$HQ=ob2%? zmWUC7uCX6!r3$Nq!fV-O^yJDq-{ReNez2E63pxM6UVMH;Q0Yam$l2+LiT%2qJXiv7?*sL_c^?fCkpKc_l01~~|&%&`ixZ&v^J?2FwbDc=M!1HxK)4xGS6 z(8&P9L`j$TfIpI zqf*C(u8jA2)~rq2#bv}&aA`sLU-qttdKN&I`3yx0`-2GbLOzM^+f z)Uu>G!`U?EQm=K~p3%{*sznSxb(9x!WGeAUo!qK;7W7e_v{ji2a3a)?6qb>QwWjJP zc_v)WTAY^3fvXGEx2|YW89Zyx&S94$R8mZ7!sy@8SDP)s_8`t@)%za*w#QVk1NM0< zyq29=oowpbqF$N=0|#@|zF0)fw8ELth}y24nN?KV|77*{%5!41oPoMS+Qb`1e%Qs4U8YeOAf0)7PrOaqF#AZg&W4%VhB6ilA(0bL{ zVN7KMvRAKTZ|7uAXOgO(z7f9T6}u_{@jI;eF&8K#vV8ne-`z36`DcLbiW@ zx~uB{h_>M6|2MQHiK2Z;(O7)?9QJS)tj2qA*^UF)_W9Dd5v*q-&B=Nv%o@5aUwac5 zDAUI)YJ`y_R!_e>2bL`*1KL;bXzDgYsS^uH+*HrQ>HiJaa-{pe&9x*#6Lax^+{$dt z*vFEND3fi=tdx<$FW#9SFB_YA6@#||gYB)(YO%aJ3elvPV(C4 zmeof4Cvg&~?pw#nJ|X7t1LhUKq!+c8H5txMmRu*Ttj1Jzig`xjq*Ce)C+j?28zvw5 zkyb&fi8Z(+>Z+Aic6|v{#UgONsmllFzOFw%W#)&Bss6qdFlYFyr22=F_Q-5}TlZ7W zge1ziFoe%_efv4ODt3J1Kd!zE5S{@4WD=sV+ff6;+cnCc3!J<_St*Y7F0>}i@-8W% zKPDs@FJ&d;MRE`OLlQf0dK;$@iHrJzsIMcQG--_ZOGu?(FT_83`N4EA=Gnbgl{dYi zDi#^A>6BL`ZVRH1r2aeZQqC-aiuR*g349vLdj4DrPLh^m^o#u9m$Br#YzC4a=EBM6 zsgyB1f|g6onC^Md@Xf7{Zn?k-@wttcyz#eWAy@}=QSoK(=f$iwG8xT3+z7=Bi2bN) zN+impaZ+5EkN9Y`bC_Tti%zn&am>{RN9m>fiifD#0?X{JWQj~g9V$yojD851pWqn| z=Sy~>Sy1b+1KZNw;s0j&6|mJmc(qkeuRj!XL31{~dKNg;BPwkRsA&-zz#sL~-| z`ai)lHI=OCNo!NUcROWf8Q8zACEB0h88h<^1H6Ogwe{p)UVfe`Y+hbJrNPfKITzP^P8*E?MWfl(mOOCt{ANjP{%chdKkgs z0!&Qfhx|qwSN>wWyYDvS+CQoCgyrwPkWPpoSNbi%^G-bv7I0PaUsvgnMZ~B+Gtt@8 zPzV7sf6lqe9Mi-)ebXN3jX9^scpp`_9kn-nr@Hb!Grh!s?a&2BPv=8VR>f$OAkTd(C~ zGXAO@xHDyq!*a{UPX>KJA&L4HM<&uOxu$+c(i+5l)AV8756>ZH(@kK?;^_0-O?{Oz z;PHlMUX1>3YefXn;$oF0|9-Z7f4j z<)7rknFwk??M-Ou8gpjn_z5+n{l%Z!X$IBpV_FBb@3i#MbeH11$y?2bxCS@DWWGY7 zQJGu$pT6+SXutWU#s{LQn+>0U9?akVzl314mTUjl3=Ffyj`=YzXSJsohJa7A)6b9j z{r++M^Z(CDd!}2~m+?1rpT|ur;KK{Yc6jM{=*VdiJaH2QJiVs>t)=p={+cBsO?&)x z%(e%tG+cN5o57TG)_HB!5WXf|DW^#OVjnZsv+14Frk3T22*Y=d)`7!0$FT~9peJ7W z7Gi1}yVuJar^Q6E%k!V;-4#L-{SBP)HE&T-)M$l;UNWTKm~r9@_!DlxI~ABp-D(f{ z64y$vjd26@efLZazF`7pP3r>l%_dFqcBYJ;qfplz{1FBE!SlLIh314Xi5Jcdi_}iT??Jq0SkFmUF&m;QrtvmP#U>K=re>xTW5N;Cor<$sZIbKoc${e9UfgTPFpw_$@@4DrZFNH$@w zD$iBT@`Lx4|0s75@pLT0NGKEXy}91s|GPe(uV~QM@8|OWyq9Sq`e~lspEodk^S5N~ zXDs}2Dv>L3eY^hC;c-gA@Iy0UiAzyR;Tqlm5PL1dp+F2tI-tqkW3f(|2?W?!p`I!byEdNoK{e+k{IelnLKiv zRF_UpI*B-HiHGf``aMm|gae}`dwCIVnBz;Y?R@$r8wOxO5zKkSGXf!LSIO#pmC|4P#py6460!A9k% zl0vB0Z(2XAhOTFOjeXX&UC$K@1TGgHB)&0^}96VJwaAyX3{1ts&Ugp9s?M&TNBKe2J^Ao6%vlI9ZCQat%j!Ze{)tMkprS_wCjRHr3ZT=@tOi_{PCJ{ zRhh-;_X`FZGmStNQdv~fL$QVA+5FkzowYPyH3bsBXvw~Tb^gP`*#EFF!{DufE=L2q zb4G_>w+9E@UrRp+oLy3J&aKEPV0OP!^)acOEwUi>j&R)BG8_o&d%B|B1;wuh&+nj* z7Hf~zkO+y+s$S39rTllQXHH?z`xhg5O*NXW@Cx##cEYg9N{EveVazIUx%wuEZ#&`q zcZ9y{AT-LL_C({JRb$z=7_b3v&bWNUge2s`?9=QtNOp+=uE<#0;K|M%3+K!Va|V%U zQHi}w*l%ZW#>FIED2rNIsm~l~sTkDo6N%>Cx@^-w%`*-U+XS?1`%m6M7mlxjX4Pxp zmmd#U1P}aIz_QL(z>h-==U)JJkN-GtVB5XRx5iAOPTPkEE=B;dKZZ8mA4>6?EvBU8 z26A5>rheqE1KiodADpL7Y$aB`tdhTnGX*=rtzLT4cqtYlG&Gl|M{SJj@OC`^ z8+>&@fc%aWhm4iPOR{+!2{(mP+}97bPd+uaT}j+HSAqj9w>0jB6FLt;dP1^vc}J2s5(PTU z;n9HIFG(YtTrnjiR2d}dE%Nr2fu5ehIUBhmM4reF=6!1t9Hu0e>x*)#=Wheh-W}xk zO!wagZzck_H-QU^9;t3K5l}qGcu0hex%7E^6^m)YeH)AEd5V7y*KKxV7V=%Q3TysFqfcK|9naEcoum;-u&JE?%_$W}k19e0rbt{## z0ZTHAYUxVva$E{SLsjqh@W{F zl*Fr^WI!nDR6Sfpal|wHaH_=Bzo@HC{oLO;B z7|Efb(G%Mj{g-Zz?ii}c%)o%eEJWXl2MvI?yB zkx4v*1rI2cZPFlB@kgTb^P^=!C9{ll;N*TIn8p^CO~fj+>=&@#GRivnykWQH zh(R~&Vl^RR|9sd`U?f} z7gQ*Loq#G)6rVm#0b9&M*IXf3MG7}655jT16Cr)L8p)0v83`Pe1zwT*ed`<;r0wt| z7b#c#pQ&Zb8au92Z|BwxEsA)C1`vuPj137Y(S6pqiWLw{E_va9-z_iC4o=&WC1+o} z0b#pCw>Zvc*n^pz5A`p!L+b|+CJ6hcFO!q_G4@O)Wfd!17eo1mH+%ksYEsr!1LhEn zPVVQt3Nx;kC928NYGv;efT;0b`8U|BwE5QHA7Hj=zfvO^MamnwF^S2rr_m>uDYUaj zRt^;diN3ALQCs3bf#gA+(VwQb)m`~hOqPJ(Wc8V_&GGQ&CM}x3qvU^QCe7!~lPV%E zenWgQQ2rfq#v`1t02yzeeX{mkb%6jzNbE5~eyiP}*gsOmog?e80$(zv0LP8w<=M=a zPB>I!JtrBR_M8nRqInRiDE*3wCwpESGRV#U(spEqDXz2F^M3)3P@VL$4{RnBL@hz= zHxBV9C+yxq#96s@T`f{q9r&%%j#YhW*Tyf%WBpLm=Z$zT`OS-Q`U$OeRQxR+s^Uni zjPznP<&$DL%0XY7VxT>W9)ZZQRI{O`pE)ZALj^QR!L zao|{>?{D*6QYrJ|gF8-3rz2Xyj0bm8nl2a{ap6jR5MLy9eT*CQ*ar%O%L%y(#A}V^ z=0&=$W~@r`O5bqGFIIL)mXDOZNEVw7!qL`A~Ilnn4x0~=iHxVS5NNbu)=&qv zly7P1F7l!ZOXD=dhAJn3%)PIEO=nG^6E%SU9YlnY(81CIUm6NSW?gC5X}F@S%dNdI zGW1GaW_&-QZDo+pt&H&oPb9C9Pf&^a=-wv&RZaKTxRqbr;um14ZxvKZ1lli7n5R2^ z60-;Dy}X$liQujIGbTB9pUD#|;p z@-znKDhf>#o7#==U;SomCL>29I!v6$4QGjLoS8cDzN_b)f~hqF%lh&eAz)1`C)NCI zLX283lzZ5^%lZ?0lOTNKt4w(X#5xZS=yBTK+?E+PEN(P`7+liXMa(OCNmMbU!3(t| zbWYSR7i;d&AGqcy1k+BNZ3=yfN_h@VgK)lejMh~7w{3aJsSuJ3SpK(iSP}^tt5a9}bg!dnsG`<%fsz?_a*+9GY|hEPgC$D3yxD_QRhSm|C@m)nwpQpF z%ARmgCAw|8bTlVYWeK>REn216s1ZeZfL=4lB#=3!Z3_Ok!gf98JhZy9=iPZhVrvJs zrt(#k-uycs)d@Ts)S1O)WDv-=9Npu=Qz(Aw13XSz@$JxZ;gPS#=I!`W9@a14L&xkY z0>;U9X!R~%*q2r|Y(J3sV-AI|B*nYw&Ugg$*xe-xXjQ?CjZ$Gr6a$`klCCOOhK`;w zt>#~`A;}ud5ho}5alEt7=OAEP{|3-m3!SmQx0u@;WvnWaG{IZqxCXLomR$QM39&`n z<>4NKUWJ@D%G5xZ#3sI{*J8tl~CTr9UK%l%wpebP#C{SD4C-s^WWiw@De|H*GL>snHzsn_Pk`cSS5ZMnf3W3*UM#MqJ&2>OMHXCPE;zXj493f8~n= zqkn*}_K!*lyRQ)X`ZlO(uDFTdGN0>9Ku0HM%k1MjFYj97<42-Sv>X}lzD68s&NWa- zoCrhz&V;B1*a*BTXT@tVtm#nFpzKNo%@dd|-v$bHyeL=BG`}mnNo9exoB1OZ7&C8k z;Ya-ty9mv-`o$i@R`Gp}gz`Ig%-B@+pz|!@U`z= zSG-Z^OPJkfo_(c2!KMa=iwKE!;jQ{5>%BITS9c5{c16vPi=rI5{o3|7kjVCm#2#_L$xOzdQRrKK0dWH(l!(Jg+ zK_Eyziz#f}%q})8cuXFBsMxGVL_{_2Ls@G((UtzU??)~`*du z>-E88SQg-XsYcmWD3CTrQ-nvQ{#@9lmpG3dQO!!sI!y_cyGBl5Tjf|1DlMhXU@#$g z(vn$MhkCHq6wHC;5^)9urwT)3nBhY?eq@_W4T62J1)@d+ALOXf>)3X(WPX@I_X=n( z%Wkk)cW745_;+tpk&>qpsTr=3!fyUB1>?n#C5)oWRV}g|b-keRO;Up_gMIW3q%-M) zgK@VE(8cmw*~S54O|v!ND6>X1To3AmD$Q&p8j;1*XQ~)cK3|a8OPX>W=*3gX>d&y{ zkn?DXnuvY@4;Rcw{Lb$=GqJ|xqk@K_5k#doOOQWv02C22nWki~ z1e#9xgI1iqiGOllXw}1McZei}(+1kRXX7N{`YJv>rLYSmCAqk7Ku>x~vt% ztj1Fi$BJuhts>{;a&iRmVvok0=OHiL-@W?)KEJVbz0T>d1w0xF4$7QG!_J66LK(rK zmm3kFK2ya}DJ*A_(6>-Q;hip7{<_bX)p{NB-l?p{?x8FV#W^j=os!v@FPGYH%=EL# z#4KI(8c2ygOS``CIfF6O2`Uv<=RR{^$$?gIZ=g&-w@1TJ2ZFyvz38{aOLH+OhFD7R z5_e(GrsJT#{Qan8n$c8<+&jgBJWiYPS)B3}eO{>2%e&IBj7Rd}2Oxr2_fbr)4r>78 zAwPB^p9^ZG)ZAsnKOsq`ZrVTvhdnqGzOymU3A zZ=SYyxN8$FfmYZDjN*yRIWv174L*~6CHk^&*EK0=#m_Uc z0cT+BBVBRKN!f~;H~wlO~K8W=3H9DHDj_aa+qV!d)0R<9-31$+d}(sqlTbYjt*6k zWD4R!06~+SNiGpbnx`0PnB$)PIaofMe!RR0ArLlN9$$XiPIouWYw1Y|=%OReLl6vo z*|Jbp>itg)so|lMmKNdP&PhTQs8I<~fAI8`xcD(yV~&P|D-<|qjSIC=zE~~BI{Gl@ z4=@aqSV}fIP{wKGFg2Z@xKU9_%evV(0w(29&2k40FP?LIE~EW41|D#p2?pVGgn{iR zaW@3sp_AJ%TbFC62sdO;>vR{h5*-~f#bA;Z@1BxpAtaB(Gi)Jf_BBOKJCH_}d1EHL zj54CBBWob1J87uU7n>$rvx!EOVnb7T-F8PDR#HtaMf>Wl>OlUfXK(qD9{SiTrja_2jfg$?nF33;idx*}HT z8R4i@Zp5@&5y-9Vh9o|b-$yx8$mi${motDX?=zncL-ekhbL*1wHGf+Pc3V^TScAJg zFkTDaXl|if)B17P)G2XwF04MZUif{%@XoUx!sx@g^)s!{Dd*pXzt#SY*N&xThOPC4 zMub`60}cU*RoI%pZ6ng|gYPx7%Xewg%ws7yr|T_h){(80fjLNS<61Y3@`psMnp#jc zXK`bPkb5yRPoqaTDti_5L`e(kL`kqNq?t%I0fEOHz9r8NYc5P9EXcLMFN~-8`a{PLC$N!XMRw|Ok7b6dcHm*=gW)WLJEgFi^+4~O zX0<}?+zF$V18q%hi+k%g4z^EkqIYaaMPd+LQ1?u*vIVTizzV^|L4rb?28(~GPNdKg z)z*F5G1n+2vY33BUIm|SPM-6N87S^+6!cn&P|x4tX#`Vuoo>TPH?4tEwJxq zSP$YY`*H${TeRbKdNI`CF)WK1gWDSm(fc^kL2F6N!{#rp#%~Zi@vyILNxDQD2~Z|R z8pyhy_Y}P*+i;BR{71E4TUpZXOmR%kHO2AKIA@AV5E|TnL%2G<#UtuoH9e#*fdfh~ zZ@J(Zu*ti0oE{LYz3C4JJA*Q0_P=tZG0+4p*@V@J9Z6hWQnpo2+!{k$7 zPdljeLE1t>>lc8)Hv3>BZe~pq$JHgaHO_TcTX`ja%Ynw zGm&{)XXGg(F$zcZ1-Je%3dlFh()Js<=x^bfv2$MHH3(2WScfsdCIVHTHGNQ8l4oysB#}QVz2oFWm0k>PvSt<|6T9X zqBmj9PFTUj^ZSI+UDKhDT|5R$YIfOkCoZ_|XP#Rh9e7-AoOcSE9UK%Fx;MM+!jj7{ z*3iTF5e!42T@Jco&=xL9BWG()-wyxFr67aQz-Vxx!`Ji_U!>#u5#tX$ zZD;}Gq7QP^8Wb;YRqYnQHW>lS#{Q@l6&eY~5crd8rEl={rGt%K>Ol+Xj&{(!iF>KLMi^wgte{y|c(^g~Jde(D%6hLi@iPa3f2pmu67-~Qq+*!b<=ft z%OvisRLM}B&EIBf;C;8QsTOF3KuuAAX3)^8hGStORVG?bIy-3;)!Z-{*Y*H9-RZ`w z7W0Y16ysp_eQWE417(;}#63}n6o7}$6^R%R)Trcc;Fd3`g008o4eCB4aGtwQh%I__p9(OEh>oEK5M)K~m)W}t zICCqt6n`w+%UWmuSucY}nh{R&v=xa#{OtA!ZV;)&3c2C368&LgKM58J7`T`Eg?|b!E!TX znU7$F$@FyLEaKEM(!{@>8-A|B*`1#MTr`9PDYC%snZ2h{x6S)cS4RxL>+0;xdwSLR z$mH-n1uSZ#qU>)fT_EfH9tQ{=bq;|UwM}76|Iuo{Wl?R_dD-Rd+RwxAt4xg{e8butxPW(>kLZb!d(#*uoAIMswBcsWY)-7Dh0d-VR`D!G2zIU=vfaXvPa5QPY*f zqIs)(++!o!a^y2KA)}F$faV29Pv(6o`cyOksAcCsc@eui@Q!Wv<56c#r`utertTMC z+EE%5gI0OEhDF(gYlG){J)Xy1Lccwj;-7VhOi73hDEbPxbl2sv&4Yw0yaz>7s{%3& ztcVT>|eOjy$~$4&Vt=IggE;DjOs5Rr}gi8Oy+gcb?;;g6FtRP z;)~9Dv2v{W41WzV7j!lk<2VCAGtmRVw#I7rR9IeFJPTq%ufBYxp96UDE4y>}aEIuK zbLAz;8TmpP z2LV^g7=?<45}Kxfum)i+YBwS^IS^F`Aey)`O^(IlsAN_71do?zIiT-mcD}!k`WC?& zm7{8?j<6A^65SXPD%gzF4~8wKw-Y5_d#AjhTzt0(l9+X6kp2seL16(4AU+;?B9FBa zmibFa_BM=CS^dOAQCU$`6I?J)Xv`c#a{cr;zd6m5S~le$!(QoYMOTL0=XWvCh)WOh z-B%2^+m69v)#SL4*N`A`{8N7@^=gpLs2a1{sfP2UDt@3->o-J&UHltF4~2Mp6LD+Y zH?QxHgQ1$wl9rH`0J!Slv=+OJJNIW6|j!m@#-@^kY^NECPsA z8!I^O7RV*hmeM6AkEiOKo3yT%o7LYEWQxtQQ8;pokussp5fmI5H%xtj!xPU-B!gd~BAb0*jBMg~_WGqP3Pu{fZ@vW}fk%0Szk&fU;+W#I>jLh31Yd0rf5pMbhdbpVgJRRPG%x&u1S6QU zIdua8*MSZ)r(leQX9=v7Xifk}KBwJuDue6TLKgybQ z2^MYED1C_j_-30Y2^i7ZJrk`f#1=UHg{ z?-nCCX?Rc@Fk=;+Hs};h5sO~4@;1+~aP^uaRt+Gorw#CL)`jmsJWFvVX+JlOXwUU3OPsK&Q(few6^4VsCX&T4=|Z;%(bxY`+`C~kWGxFaA_su!5u?QH5~WtkxF zymp!=XMYr1+J%k%!AoV*sN_AoTX5{NV8{3@deAL1NH*S0;bWwozt_^PH`cl(w2k${ zv!Jz-;}gxl;|#+095>~Hv$isw)*SiCcx3t(89&VX+{%(ZTkR@KI2l+?(!{033&?tu zm*70(IS5|CDQL6KKYMgC_l&DqW@}$DYtNrQMH0A!w4o zyhIp_d#;b3S9Ioir##f4rD5BHC0nCvlIz;^TErCY@yfq@u(~nA$K_o`X=?f?$w~%U0;Wm@rt} zR>$^mxNxM*+Ytoc3Z%p|U`HlLr#|=2^${ zo6UBx9e=$M9A1&OYnZO`!OlwRF43UT5o0X&AJMO}t0ceWf7Fvxp)`CI&bDj?fF&(N zlnE@O8(8ipE2zz;WJ79G1StYY_jpZBLA%zbvA;iy7d{O`etfE`afq%*x^f*C0Aj+~9 z6y8IA1jk8g4fjaF3R|K4OF5eKwc_1C)Fm3y)IfqV>2r3sWmkOsQpeX)fX+4Rnfyg$Zoy(C=cadm>t_n6wPmA=O83Oj* zec*m~%n0XEug3gTp;WP)w|YPi7fTo_F!#b}7?xB=g*edrxL9*sp(971ui;E)_|Uq3k?92w+~WqQ_;8MoR# zd|^tZVk|6@MC^v5Zl|eAFg)>crYF8E}XUS(8t~$|0dPnzMpYeOiWT%6VBAErWXgbh=_NVSbD0y$kIOx zHYx2q)cXP?XI&y1x^fQk6|?eCI;=HCu*`by%v8{HEc}8&&RLKO^CVMgzNBW1c7atz z!Xm1wp<)xIeAja{CHn7OJft*@@!(1tGv*h#*i=;&%h0(URyokD zrN7MxrRS!?E!bo2we?q*Hunebo8c#HNd(@g~#A(-}V|dyc!S zY$C0aFrJ5-F+}Y>yZ}k!i1@|IU}46S)77Hh%7yQkAmGL)jib*e4U&FtK;V}#Z~+PQ zx^tP^UsZ)c6^m|!?JCxp&83Pjz+$+_-?Lni2DD9lifb5>o4KaAA6q*ooohKfWwdQz z=<|jss(R88{M2Gk%O^0G$VQtbT*_jN7w`~<+O=3Mvht|k@t+ZNE*h5G$5iH=b}}7E z#@aa9VmaeSHyNhGu=>F(B$xb1Xjt2(U_6?xF$z-!dG`Q7z4Y|@(bt8?ik4MVdB{VwaI8r;M4hN-%c#hCEx8-G8CgWC#N zvU(CxRCz}>>+JYBE0V8m8TMox`&)K)QHf7^wk}j7RHhcy(}b|@c*{>afPiV<7595u zyh={q-dKBGyOKvC?JwT(dAJaDxXTCRSlnaei}b5RsJ+e(a>9va9tg3-oS)>rb1Lvz z!52M}_ATF_|45Af+NJjHX@SJ3cI!b!Qx_w907pd(JZY~yvHiT~t3hz_;;#u3MFIAg zl^0xfvlA7Z6XU-@H&qA%GC#$!0hMtTYzo2=d(?S+VeF(L?x%J8I}z`*Rh@w2B5WfP z$FUmBI@@Vn;G3}$e00HMpPvXVS9dK=nM{*z1&NpM2tokyw(X)}!~NOl5cX4Am_l8k z2}!9#_*LSl%%{GuLo5E<7nOvfl#^IJM7yeL!1b!BI^jEAlWBiR_E2k11RmY=7)w9W zHWHeylnkW_%P0w2_c9ESgoUrd!7D5K$|fM(EiO3BxUK3Q2EssmcEI+dFqzg*RBKC? zbGAAZvs0E&+Y70=#OdmCLQ(eFo3KEmZiI>b>^Tu>`}g z)WLQ{OK6eQBJ<(DM@cxd7;pPHRt7aQmvO7m7rWOgtaxq ziCA(A+?du}!tfC~5dkxL;c=bBFbs%h=KlM1&PsS`iS&t%m+3ToD203m573JDc{MBK zq2*a3qNz}1O7%^JOJhCsPcD*N>Hcw4!vj4&#V%SBH78Z^&W?|ibG|oW-IX!`8Ryh$ zX@oA>Hf0wk;a}kDkNCZ-{qR5XZmomk{;6QI#MX*T+8FR+?8Q;?2UY^bF(oLh|1ekY z?8ET-H~IP-)XdMs79f9;rQgT3}3ucQ^9SF z%udF~xr>_DhSzR~bv9J4f~;&h*Hz+-^-d5C?hlN4-G$up+ykd5SZAQP4MjZguejvj=X;Hdh^&6{jVVM+>5nf> z2~^a`TgpBE`V3QLGF*AAT#IhGa)))sL`t9AML06`PEr5f)f4>9LB?R-3kAMJ)Q$gm zar!&`()e9j3A2kJjDdnBDQ=k%D{*uSufk#8nq9)GuYsX;{g`y1z^xwVqVgO5#zCn1 zWE1nqx1*~5y++49>mu7=(>p_gwsdTV8;z;d`@t3=t}V-HTNIo|AFKwQ#;q1tv0aS& z&5GYV1u$tnrh1)gz*Lu>II(|u^~OrURKnbmbKMn3R`%0vJCWC)N5&B^_JE%~3q7Vy zVGI85_DO(|U`plmCG(w3iIKOnutJaE!Wf-0P6|)PV?$ zgP5bE9&wr{iRPhK+3-W0`R&*qljR@NNuf>!&;Zwy}$(edh+nwz}cE0#1 zeONI#_9R9CJ;c$<*URl*ZAwVV+U@G(d=G`hrRn$4(at3I1`ih}&-xbM751J0Begns zQG;eIf|>^C15D~v^M=n>n_zXvua3Fw1r(<>2jo|!wnyo}$LuCn)P3at1I0i(zei(* zr5Vw`SwyBuIHt*nMa2SEY?#iIOERXdW_!9xK1xK!cCnpf(~J{3cSTbbV~JlOKL9n@ zrA%mcsfjqsf~ul5R~kY=-6ryP;$rc}F-3dP?oT?bZCAHw*T8M2+KjY3x9a7=ML}UP zk)fU-WDg{xSvb}UP63NU2(%5m_W9$|57n$1ey51#M_vgz=VhWbeQ`)Tn_!SCowre< zk1hb2Kb5FALJd*o*|Yd`Ov?fp0Q)ep2kfBj+FG~nEg7hR^!Z{`sP@$gwt5QemyOh? zL3aZ#Dgl^Eh#DSj>(vh)Oo6< zshD3f@@!|rj-0G+j@MvwTri50u27SRB+y2{FHTzqqME|bZlOI80c|1RkxVEYSj+fj z-7-OH)ZhZA!(Rx{Wfq%fCZ=ReuNa|38}Ab_&FP52DFyeEYOYhN>gjEt2y$|I)_;9) zv~GfO>c~T5@evK;pi7g(m-1X1r`U{{)M|e&dhn;97VkTfg=AIo@W0kt@Ui2gY(I(M zh8nk5k<}O73c-vIsk&Zc>dpI;Lmg*x(C-7%6vGdpe8A2Q?!)|RFk@}h#}*9D+dY>e zp5{g7e>rA~Btb4ZRL*K6OIUK+G>VMi3{ef~U>{h)+4AbZK7kVcQD)*lCDs?cCt>fme7Lu8WS4Hk=1kun9D#4xGjb4Ry)T>=Qd7BPfza`&N8qU^F zc}6`OXaYeVbE!$!Z#rMIv7+=FG!cb#%ZRrk&Eugws94*)PL5 zIKx?hdvHx%mf;{uz5H%$1d3Nihme}A{@Mj?EOgczE$A^w585! zlTsc@mbqE(=abaEKP@!61%Diy>zH+W?wa@sL{86jfoT$Hi{TIW<^x6#nC32gIH(L_ z9OR7R6jfmLSPIK%NHnotg3WRmtbXBwX@V;+FoP}eea7W&vaD}}+fAPp~Kl0nH{I<0S@Kfr* zQ5vh^n2S*z1j%QciVNn*z3TKlWvo?^KA9Eu2NjhruH7b)G#R$XHYKtV2hSl#sa?c~ z(#4<<+MkwrgW@!DnLP^$+F;eS=@^xnRw3|FX;_}l4-C#sY1iTmb>U$SDWC||-3Ac8 zTW6@7nx?#Z?rw%VItUL7_wi~~SjywGH;|!1LHYi`L|Rkz-t+h7BEPOxqmtGETp;6( zW3gswEjACzZ<7JMKv)eqZ)DD$w(n1-s5ZKoT;X^j-G^Xdn`;s*YQnF4wHit9ZebD0 z!aAZYD_Z#d`~g&y1;~$j%6}u?`H^pWHyrzEm(Xk!!8)h%X?rIjKBQXrGT8v5*V--Um`3MEZvGI;@$I!wGX#6b01~}fEcV`4_V}CGCz{0%=gQPaZMC1&u^-YxW%mK}Wd+k7p6J0#b`aan| zdbNLYaP(%ld-CRVK!P9x>Wc2g_HCXY&x8b z-R(zMacWgvqHAEWVW&lg=R6Wsk?Do@RFQ+H3)BT4!1EbMAVub=ByvV;n;wuC8!tAU z)9u)Q32XrJHZ)bMC>NoKb#8twyVS$d^K(IJXGP%4xy(5v7{kVFGVUJP|D+k(u9B*{ zd6?oE;PkC`obKhVNVpEtXi`@qgGb4!SAy)Fyi$bL_zZ&qqZ9DQ#(Oj^)dUAnq*KU# zP1Y-2sfp##;h!cf&vL#>z>jo>kI?$#DXCP$lk9`4j|qk!38*<05{H7T;+cpVRQtW^^t> z{AmuaSmk(m@LMpX2_z+|>os6&wikEJs#HyRli_T8{loFo#U4=MjIT6-m9;x%O zrA-#t#0nmz1~i|pkOjbAGM09VC|VT>gcvbjbCf={rR2!{3gR3{T|7A#(7 z+v!w@%*qP+8@M3&^v6b#(j);kHy4SbbUtyg8>fdEM)dx&B!WwF!DA3RwVw53uTxHY zZ9{=HKm%OWz6rDTN+eHnlgh+sGx26byXk0+rOrKlHfBUa3&FZ=YW5!a8T8{AVL;~q z2ZE-ER}o)GpKsWj*B?l{pr!$Vsw#9w|6)jebXPT{cE@%*Q`cg=*#?TnF>+n<^!$r41z$KY* z38`{KW){o!jb15wIUI{fD?uo@qahdT4)UpUnbon0GoZ=@^9qCWPnhwXWt`%@nMjOo z1r`~EZ4yEUTv-QCqKr{}>oqFkz^ts0@25FL7*!&#TqhqHrWtVx8YT)>thxQd@o3~B zNCisQab%E@QSFtF%yVYKjw_R-XQu2U2N4ljahchOE#rp(H+7J^+fF(#fOe%rN@yKL z5q>q#s#Vnh?hlIq;X2x>fNT>P#Au4Vvimk@yHg<`*Pv^`9KSY}J~>F%^+nWx{9>B> z^~MswX85%DztZ+Uh8x?6aP?cuj|vEzTmM*xutRmb*l6-h%Z9h3CeE6(f;6r2LUKYt zOLMIbE6GWusL!=tqb)MR@t@==6laEsi34{@7j}oxplL0dKQpZ{%V@o@BO`J{tm2yr2J-k``Hph%!Ov`na>E=n{^(#hn1FNMR{ay zkvRk8>Gx`C2IMb$2d_?Q2bT-GOSB#rCf20}W-H%ulk3IvYs>EKSV>FCrypq+K^T0R zb5KRj6qDubmyh7#tn(XTHfplj+k8eM!KC7Z?9ot7GO1&8>-3ZAB>i4@x##w17V=aw zNYHRGjZax#1pnSM4%*Qyp%WgG2~GHgl6J@yr)Kb&(Ci(g?sfB3xiQSUj@BNJ8pQx# zZJHo+mpq9E#4rx0Cej^HUzSrpk4sxtGPObmPK@F3k|hzQ1{#H_&X4&Pob@s{g!-(b=cvrnmzMXW%3>EH&wvlj(Da*(_Fp7WWx)$esyI!Zi*5OGD^ z-&}YNDehBDb*I`=V1fY=9eWYQ7TCKY_9>+atbxwrF<2$br>VIObOa&uX2mog*yaKL zT`iOGD;Cez71|)p8kca-k8XT^mgwAY7Kxc_MgbclKtH3naZ4If7-ZnU**C5L7vSXdBbuv;d$x= zaR#FEUb2kb+&qO~k2b!eT7=pFZqQL@oT9#A9+MHv%?1d9g3)$Uq}V9b<|`&iA?2QX ze#}Yo*~a=2$Vwpa>}C;?yC9Q3hfW9B05OIbfI@9(aFqWAwsB)M3SH zf-)H;PM(i>Voa@o*G1&lVb{S3xw+{rFabSz74{4d4OUPcj0WBm2>b@FIkZjLIZq;& zdw;R9PSTiB$ehNp7aZb0H*lQe>O$r z*cQ&FNl*XO19`=A%43N}4GJ}lhg4hWS5+xc3k6uucUhs{(IV_eI+G;Ks2ulT)UQF# zp5`Wuj9D`4j-4cVe{#4g!DvzA_xd^@8BIp4A(T7LIWf^|)nc~qR9%ed-hn?c(7XDk z$V=|*k@qKH0ei(I&mB^tC3!txe!_gPp&N8`H~hj}*Gtkp5)nHVS>CN|EC?82z*A>% zmXo7*ul9$>?@rEMK3R3C*0SWPL}Cq`zJpg?A1){ZTg?zxi|Aqug!$V4kpbGS98zPo2P5u4u4Vr zJZL()0-VPBRj`L-=isIu)v@M!wH6TP zB!rs169z|{$z$4W++aqwgtqOLG6P$7JP@?xLS!J?KQWjh&oFyqg#hE9oo79AC`LRf z?j0-}2YiLhSxPpw))*WaGL$+Sc)?CblVBwtngU9!Da&Yu! zK#q&9!IT3%V{y=wMrtA?E}tM7Y%@q|~B)+_NebXzt9i2~WVXkhDB*i}yKd zXFl(p|6fM_8zNiM%hPzNKHBBK&1V}s8zuQ~Yh&lxqx|;}&-Hcxn+|!S?O(MX0unN* zz0x7y^ff_2O;SOgu;eS6(=MC=btJ4-1%c!Y+I)gR{0A>d53C#Q`%H)&UhAd0nmY;0fujJ00j<<0gM1pF0Jg`-H`8vX1}!L}S$@Pa^io^i&LmC%lWN zl|%nlFL@FjNYb6qv};>g^U9ebI@wE6%#^|xio(C__~)ppDs#0jTWhLcc^Vf*o)kb7 zym+NbxkZbyrt`!>-6fl?lP746yae8Slwo!F{)>X=lFi7?&95C-7^#^gx8A=Go*NJG zW~`eX;fCwbmCYI*Q~4vCnL1rm&~(?Zn87$^hE&zm=vH&_4<(=LI-g0RW;e;n)|%d% z)m5=Ltc9!49U*N3@=nm%eES7(Wxy2Gs({sTKw8z5t*2yPtkqL5$XYF!dKs>YtQiOO zA6^3_YtT^t8&Xa20}24Gle10^N(|~E{YIJ$+L*2-^=2XeKFgz;U*)a~z|3Q%2T9cD zNPJ9`OjHDAmOVMvxl;cNG(hJ(>9gdD$T3(==Okd$i-PvFgYK(VmK!!HDQ5IyO8qjE z%ZO(rNP%hBiF12k+yhv}9*Pf0)ou@Y1~T4t_liMG99R(S9j6 z@hFxHejwdlcnkT*W9>`YS5jnG_f|er+&XHOYTGVVF6P_9rRzaHKBr-w-%mTB;MbxT zQ+na%opi5_oqP{voL>A&i}yt(51V^|uTCer^uo)=J#S%BWAeG;X|YL^dQl5SUPV%*&nsV}R@ znbmFUZbO>z?pm3`Sgf#rsdhm*QEYX$jDhJTl&d2~?d(s1d9_n@m2QDye2XY4X`0dN zrL0}2DR<*SJ97o}g52EnjRC#->Ox@NE7TVC>|6Cx0}$NYY?`42H#dEArZ{lc57G(d z;v>4kls1+BOeBlQ(|!<;-Zvx&`u{~_zgv{+4tn3L`!8I4dJT5h`=h6ItE~!F%tO`! zRn4JQSEvE}C)d~A>n>)MO;F8t1NRMv?=Qey_-L!pjSBHD?)jqXwEa~f4eRR_t`~*= zRvi)_kM>%uyn{Q0}~KhWTS(~(^9ro#z59}XY?Yq8-ptsE!u$9 z3j(BLWEbP9oI>bbx?ppjF5L@a4d`F66+dXUZLS=?dcoHGYDIH+BYdEJ>P|6!%EGPI z)^=|2pKTO9>BDDWX-!L%?4{8%^J9#x6zCJkIU#bdl_h`HU>8|UqFFup=oo$wcKo+;nUT;epXx zc@^{l@T3{^>bm;9_0^Bh{XR?G|2P(;%k^`FqrIm>CumO{t@BB$dDOr6$3b+Zl{?<+nTfI{|y$4qg-FDGh4 zhhsJ{;!OXW&i<#=7&y4@`9SD;okp<-O+td=D=eDc9#jK1%wB2GiH#pdK9f9~CL!5* z27mI&>Ga})e&IPO&umer)dgs_c}i_Y#Ykwss~UMk#iQ( zsbt`q=ejnc{U~R#&KC&o=Vq01NmjK-khV32MDZyL*Lt0U3zDIo3pQeHdN67RyyLkS z025-(0q`2yM#L6oN;UMve6;w~MH;^^f^U1H5s)VU`{3o`6I#G%v4D+~O@R zumuen>He=S=?=TM-9qIJZsYdo;8TsDS7TDuE=|8!Pp88x(T6z(>J4or&I)oDgtbi9 ze^hKKYa6J)cbh-zGbZypj2xzX9s;^Su6k+3go5MF*zg)@*aqlSh;BIrwbMa@z`|4@ z6^K{@i~>YUa4h$Va8No1+~gb_u8@gtKU=hQUk$n|K`F~bu(5!PEsuI2xbvujm`p^X zGfT&33zvYN-lGy|`Qj1k?9L z!TAjJ2WDQhQzfahaRRqT#E8b5zV2?(4B#YMFZp@XSjKx$3%oT4Z84dxsb)-O{ShBk zsnk*{&x?zC-|NtC{Bc#zUt1W$)Szt9X}iC5SB~X+=Vf>(1bW0m2I!{{JlW{gj2#Vn zEh%=J=hB)}L(bEn^_1wm*}~EBtQ{89Ae0BdaAX!e3vK z&EDpB?Ro_L?^+3AwOx>sA>D1@^s(L2TWMOVbILyDEJ166`<=ue)FM#=XiSbpM0RzU zZkDx7>o}!NQ#uDpP89I|)s`9pA?`36ziX7Vl~!2sssJ*`Z4r}-=oOdQ6t4TepGG6r zN?>bBHZ!f@%CC|6PoK7qsP-&hntZ-a*(7` z2#~TG@lYN9LVz^VEf?*Q-nETtRQ`CgHiymj8nQg3^{8>}v(Y?jH55rmbNp1glqt7J zDsNowQO!#}OWC^%!2E=!zo7ZruMk4{0$d$$VBZF$kZ3hNkRe@#YihJFQ~<55X%%3Z zk{JUcbF)3q^xT~(Z5g$P0dPK!8HA+6)KN|jEPa!%+21iW5nXiTvKEUe3@(Ri$vvm7 z3RnLcLf;NvV$>G_`8n?LFRBB}tQZg|ONjYUAB9oDv_q%*V{Z|n?;#1RE>Ae2aRv5^p?7zTxb z%7%JDRtrMp_r(gvbfJPQVHppHLj78xytvHi0>$WW(~Qw*5dd@^!ca^WFP*S_PGQx* zDx$W}kVh=Zc|IF9SC}SntvWO`L)9JAVa6gZ>t{~q9DwOnD`BVrtS@S!RSw~t9IDMr zN318GwXDR3Iv?42M6ViGT|+HE$M8HBuq5=&m+tZd`3zqFu>AthCfX9xTKZ7h%X(@K z5r}BCv&8F9@w!p8se!6i^&OEjHe#RN zLd28q|NXyDR{!^4I9&bT1D0HUzTh$Yd_|{mzP9$iuYdV}Z+`t>H(l!$Qh~9qC+zEj z@>t*TU6Agndrpw2|1F=Ih`I`(b7n&iI)QbI@J|ruyc))j@I>Z}Mm-zF#gBpKBtHZ% zUyMwBKEtG!CcB_bY+AdRdtI`Mp5nc(kMSlXo5Q>E-f(QyDlT5l4yoX}zR^l&^SUZ6 zKbMtkZ@T)hRtUW&W_bg@qnTlxX;2-nsKR36xodL7(c<(|c;I34heb))++g0!Im-NKjIUzi}yb?-qDkqxI?hRd6xT;GXS8q^IHRr)= z>S-<}>fNbV0{k_7)Bcu%wb%8fh=mAkC5Yj5;kDkI6S2T+-O0kHzPm};mTT2@R!^kP zUvYEOu|GU4ctzUaixQpxO1GtxVclWGmBeub+{`N3Nr>7mcAuIRgFKoXMAzs zCe4+odMrEXfD9okO_o0Mgy*~P=URO;jzXugEw_wn@Ky17A7;U(rp1&Eh4 zSPp7Y>~`CRV`ytEJ|+rk`&H{%j3!SN3PM#XK!-%b;TZdKls5EcZ?sYGxz-W2{i7*G z<{Fnf40T%_Ii^N8gh^`AmDVrcg2s70JkMhX+ix{Z<`=vOoszT&;Ytk_8-{ zIxSuq_^u~^_(b2KAQJQH5l3E;Ce>b5a+dnmNB`?v55IqyQHf}^L@xx{$W1Pa9GJ2G7PYl`#k0ie2WPg ze(9x-^m0cH2cloB@%XL-30%|FCZ44FkfQranGu1D?3$f1)~lbt7$9zPHAid;RY4)&5CkKD_e|!q-SH5}u6u z_S(DD(8>PkyZ0x1Er_j$)i8}dthF3i?w;P~toT{d5KCGtqLE*IR#j1KF{4sgXXPJU|Y)!YiT%OBU&@6TX3FtRL9?n z5p~*o7fGpChGUk``sf%Y(LYI~RW$E!O1+?ica z);U}7_H1icD~(o{X}(}{*KWyf-;KAOgR=i{_RE>M+woxD-dc<=y0t;UtvDjv)+0ak z$PYEF!iVIBoVEQG@j{9fx-BPEjA&s#s8r?a;es5%9)<^M>+t?KAlu8G`JV#buax_l zv)=B5`|)@hbAR?n;cFh1XZPFF@5#N{Wqmu|&7lgq-^LJPB`a0{uWXIbGn-+q@vuXY z3tK(w8?L`J`HtV*bjt6HiEw=^F3*+S^O{F+>pbV1psP>4))-M83Ez*?bM`$AFF(>O zf*=7ohmZiU>9L455I^L}rJu#1`nl9=ShB5M>>7NvE^9@No$)dp_v1bIBd7g%58ljU zH(zc;lKHnRAB&pC9&gG^Fg@Os|6%y<$D8s8xhc0BZEgx4%y;wtbpK>dVw`!odMd!5 z?EfEVS52iqgt8SY+HU^HW+n4Y$AELy%`#J$2}m)UpBD|5ZDGX+@*;O* zsF851hTfkX*0)v9G#tOPin%3OA7Cn!;J~sootCyT^2S5P*grbE?fjH+`luoM&()CqkeKgye*)Td%$^XSIsRjZT@iP-RC_j>HY(hbasif=;;;`X=+!~ zydCr9)OL$Hh1MElwFYWjTHE*N75b>}d-MwRTU$a|R)Bh6+OkLIQS#_K`o~do-LL!T z@_Mcs0&)*pt~o8<$;)(ICzx1eIdekr7g8`Vg`|iaG%qRX2rJE{oF1Jul z)$=}g*K)K%Huo5suji;O`(+hWbC7+P^W?TFst20Ag_KkeID5-mkj+K*qZ!$MvWDsl zgvn^O@2W<~SW@Bh81&*%-SZgq;ufl&0?_+W^gPDKc#MtFwlVHgwX3rumsZ5ymVv#U%H@mC(gNz12brbiRV}TEeN?pk=PFwC!nmE9 z#WL`PlEqV<^!COy%X>#oX6buUku0VfxrIt(j>4f(G&nHfD0kEJCzeSJpHQP7xev9- zJwdm$ilm6F?(0TM_zuNnLW>sl&w(xK=Or(dRx3vbkoEimYj#oTrvm}aMs~ldznl7Y z%iX?XZ#Ub}?x(-kc3w0UzKyq^@S6Ph&e!_GI*w@SYfD`q)YthSKpVN3#5DleZ*9y& zO^W+H82*H&Em`h?C0DfNfYR71Z~O`u$S< zexdj60aR72cN>b-;`|mb8m0-i&Rd$+PT1-tn@ONfQe^pVoQ~;Sy!CF0uemCnc|$5? zwduybN+5uc%l~4XfiAa`6ICZ23*b(q%^n!@_H&*f3++Fd2flzwf`w)%Bt ztFT0g7!SF+OxM5n;<_6g_jT|dYv$k%$L=j@Z_}B3PvdWmyxb%@e1QniMkzs>b&gHp znaM+jy=jN*bnm#rh1P;*n9bWS2e&!Xn(WPJt)ylO!1orcFEk6)NIUgoy-(k+r9iAT z6e3?jowgKM4en>;OBqx(16L!H<3(4CK#szAqZsap;7iERB_dWQna`nLHADHy(Y)N> zP?iAn7P^Cmq3lo6eD;cG_dA$U@2SYDDk_IUrXo5PS$ zZ0pqCOQ1lR*bb_pcVnH3G)JX^^-g;PeJX1czbqyKU;XA zp6e$Rs?_Yq(oQUceQ&GxuAnz8#ak>wxWYSONyIXiHM5FuUJp<9k6!Jc92~td)*v+& zJ>xu22mSv1)fPa%L1sE`GN=dZN32r0(X>z-4j{#|zmKi9jbLy0@bLTHy&vaTCHdWG zxQ_LCtUy|1sLPJ6Jeqm%oR)8E8n24pydEC!p8aq-e6@SFTQls{5wpbKNxz-&?piOU z`55)+E$tq@dsDH6U8vnyjOytiwp&XD7i_wX%Ieln-u-lNdhqV(;ONcmx^P6IMXsux z^YthCb>+jE3vd-qV{i<>K524Z0LZ|w$^op^#>h8>XUUZsu%?l#{ukU5=G?EVA=k?9 zt9M)a!=Ro}`T#gsKyqKA_5RIrZ4(Owc4!*n!oAOo)yc(+4eo8 zO<&9}SU3w~_Ja_YvKW?r>viK$)}FE7xYh9P(R{Gdx-ob`WrFU*fZy|uf-O=+J6C@u zB4Vd3W??QepLWDNnSKh?_dx0bT@SRaT3Fl$9YCrhD??JfbGx|U3C}^JW&R#?7Mhp2 zZRa8tu^7!xQ?Na0Y z;-6-q`0HrVL$lnRx8;{HLYr@Z);uRr!G11VV--t==WODfS@3h<9Sb!J=S9OgQTgHU zC}s1j%7taMT=fQXj?vJqmZ&-hgW3jfF}5-vP;b5l6Y22H&Gi?wE^A*0Ps$5t;el0c zHY@g;{#L22-=M!JZL5XqRIl@w+nh~|U1{A1s8&&+1dd55a8ngy(>esY^vE@xwAnz| zziPkdvy_o;dKGp{E8aEZv=wr^T-u8%w_-`CejK^ezgi23MljS}SP{DBZ4EUKi;Q(l zW%FR4$lT;QxxVgyLlTjXm7J^%Nc;(}!$w(a>U6-9w`?M^Sy^wz^hK7z*@Q)WIvEfw zU7?X}Eld6zJLD7Yu50s3+d{PR)XrMAP?75zUa@VPwpmKFT1k`Ct*y^?K<; zW1g|aJVT3x$LA(?{6b?~&Q@c-oyM*CF~gt~y)Sbl%(c9jZ*{T6tT^%6f2XC_8t0|n zbYtQ1S>M>-X>4;s5I5O>EId9Z0NX>1a3MnygNlN0s4&}}5kspeP-#Uy`Y3%XB3;ON z@h~j7KA3xHCjQC7ye2x4H$+>kPQ_5PQPDMceT9;@_Yz88*S@M(-|{l^jEqoKZt@^7 zQ}6RW1__HZRrL*c*cc1xnYozX1qLhvY|$oNP6flu;j-L{l?j_<2d_n2dS1JRC^Yd{ zZmYIzTcfw*iWW3mYFZ+t-YU(vXw$qUxTxx>qOQ|_{^=`xGMa>AmIV`$@LXg(8TG?4 z&2njN|8h1-W12JB596uKS(b^IftBZ?eQa!OZ0u}r!@nCF8|8nuHa2(uw7LE5_SVMc z_OqR5f7;l3wz;$YC$jNnhmOxw<}~}$#+~~r5AF~0)QGm(ISsYG4P6sl^C0+WFS4UG z{{wCwTqY+XW+&5_$+emmrjqSjdTWb=$au+5Khl z0v6Up39AvK=FiDOoc}zodA_8YslEoCy1U&4!2hzasq=!S{7ojNsT`1By4_#N3h8zU zPa+;Ff=BW(BcaF`**!iWBUP|;dqaNdhDOONyX(g1Ms|ONf>$g%he9KktFI7oL;VqR znM0Xt$8@@SZPYw{b8i6LrrpyQXN3InYbh8e{`GrPqsQl=pIe^)=W3HAqs7>OIp_c8 z_OoZD^MCvK*5>2+{}9iEod0#F`uDhkZg;rf<_OhG_A6!$F-*kNgV~J88|4j2+{|jzXEXaH(=+73EV4Q&(bo7OvsTjJIN%N{p8|?OPVw z*DkKz`?D1NcP`tP5di1Oe>+>x%ktmW_SPf)e~9Nx8FAd+gaaVD;IWR)@tSLX7v6)q z1!6I%PTaHOYO`Y~CYWeckBjcIW**(n(p1wwT)wX31+j5G*WlNWcAzJ+95TlZE>INI zw#%+0QUP*R&u;GxGiJfPqBXKcK9lpEZI*=YV2Z0`Dv7uNp94AQ_s1-r^yIi7iBKZp z8X9^v98Gz|`ZNrglsp*?q1Qgi$6S(+JdTM-;u-lUGKhyT6H{`b{)X?+6s%qvMFdx4 zE(p)v8|;dLPv(k%zFH8^rT8?hh(;tgp_qKG(N21(b#dtgqN*Rpe9*mkDSsru-L-%e3mV|%eA<6y8+iveu^RXI|OI%pb9hZiO!x7z??co@q&qm4C^KesMDMWJxmfq)HVCsq)3>(pxs| z*I0kDc>FQdN=s>z@M^G7LHOJf;@W`cPhO!vC_cOv((E$ zBzZPVaZb3|GDS?_SW;wvi^T#K^{Ye zD{-c#lAY!>XBX4>l;vyHo8TI+ptVI@ppbyOoB&Nz*|+S#E4#}65VA#$bb+O7D12k} zW^>g0yk1UOIPksMya}b#T*#2dJQ+C?TKKT1F%PM6>oPz6J6-*Rh;=K!omepr22Ju?(1ME`aFLWM3~Br>6o( zz>=%Nf!A!m`ev3}f0xs97UpqJ{hK8$XJ9m?U6cYFXtXI2R}cG~pLa=5k>nV4nVev!efh_U*Gr`_Bh?uCMF7tKOj@VxbPG z_&sU$;N#}}gFnmc|1qOc%%luVw8{kuz?}WRx%Irf|F<`uKg$0P@_vYzZ~;E$DhY64sEdPbAq)IL>iR|t|RW+B9TTa8U@D~5TLo8@wge% zzteT4VZ^kV^uh6f9EpTg#O$SoP~DZ96l*IE5#?>{|D_`Er$tQsM<(zKK1ZWH?&bIaR8?O=v<#EDFwbQ1}xTGb&ln9JP%Vc_MR~ z=t0|RM)EgD0buqF5~&VwxI^E@lQ4=(F-fdlPHzh;c41G^{xCi5teuc2?`7 zQQ1~&r0YP_zuG!yIo(zid*<*xVZ z>fX;~zu)o$Uj3IkZZ!n(%PwLTP+#LVMgrxQ?$>0XoM32<0m1wk{EF=f79VT7b%Ec` z?`Uh#dfC6+<$n7Bzucv0OTRK4vuGN#%)hqvUZaO|uQ|ca{9Y>YBb=@(-|nNoaGLRa zwkMLDee$D-yB}^HICB9Q#zN-1F{hF_)9JkYl6|Kc7ck72O6jA!@@Y=o>A8KuuU24ZN+be*v1ZHTgM2F|D|@=0AyIi|#`1Fvzj6rkSRG#+aPfK*7ulFwSx z;9=pMj>yf;-;kosD~tekOQR~wgbxUJ1r*=7#aW=oBIe<&g(WCfITj_0&t51d6I$+B zzw|RYUROerF3^%%-u@158E|>~ka7;?>V|0}=#W;hZ17_S*pyFst028G4#uC7Xz_=uulGDA9+8VogEL4-<_P5dYhLSvIZkHS^MTJI^D=y zl)kH}IDJs}kH7-%?f%)x!QN>-ycmOJ-i~Va(rFR}>#`R5(6!_)bRaBeFf8Dio5ygc zc9YzBV0Rl2mbr{+Zzltx4gLdzyfXj`xq14f&WWb{!wI?K?<6-7FKYc z|NrxE%l`j6&o`bu@_!HU+>!s&YXFt4y_ipUevphZCav#~%QEQFYAY3S-H~@VavQo2 zW-s6f$8DIt%H?AyCW@^YkpG{(H-B&2))qz2&-yFy$5ZX z+DUu%m)mPWBqX7x2yOtBUjCxk}MI$>|k)*`k;AH8rzup8}U{wxZX}>*qI63TScBm*{?F&x^rfSv9#_6oAH+ zuWPo#8@fr4mO~KWTxhn5ofwh3Vspsx61V6d3vr6>7n7ZS8HkKL6dogaG$z@)uyd&lVd?prL&%AWEnmw17DS2Z|E`uNdNH!F$}k8I_i?pcZd z(~A3*V*s`Af6w-w9~SdJ9qc{E|9Oz7$V8bXwyd6w_4CAhIEv|+xRmN+ZHidi1~mn% zQp4KzE|VBE0p?C9bA)SaT+~!trn7S_XK!!gm3gsQ@JfK?UXTUrS&Ro?kp^!l%&N48pxb*H@|u9N_>sa%hp zzUMIhb}09%$+Oe5lGUZruj7l+3ZL=#qPeRRN)gZzd6H=h8V!`h(S1Pp*R6kws*s^2LOv*UB z!%}B5ccqn%**%;kf0nBIr@eBql6y}t6GC$6rEqg)W~3H-7K&TMQfdHp=E;2{@b{ zmhH0QaF>DUwQaoK;}UGX1o_wc95{whB zjl|b8HEg}A^g{|(wuM+6ZL9Pl4yjM!P7LsyCS@W3^8XP zZq1e`+f|sI^Lbde6f6Y@q){B&T-^$(5`GnP<`W@^(8vWRlKl<76V4gG3fVM=jrsvu z^o0yqBGC=zkuDq40F|Hq0`tC){Tb1xLjT{3Y4Ep>?qWeADbPoY@eD{liz4D{c5tw_ zf2jNTr0dM2jMEx+B20fo`lhEjo$EmR;%}yC$fj2l#^+dSWKARV^!*b_=j1A6J`NG2 zc)&6dzoI9c%$X!tI0!iEr9Trvxcc$0gb0bYpD0kSIGGSm_|8)^Cg5m3+}c50Nn;tI zpTXYWa-|enFBDOwzAh9KB&sO(EMI7xOeRDjr`#g2$s}J47UT-9M4HwL{zf5LkWhR( z81AEMax1QWXA${!ehdE&2YY*SL?i4=x**>U0yM`Tt_1y^d^^~irqeOUG160Zv~ zf}UgxoW4uDTG+G|tUm^tar%t2vFXLK02o3=OUe-+VTYuVBv5e&G{rM*Ulo53(PJY762Rf+?t} z=>LWnSooaAGWYsJ&-nmVeIfEC=Ar)J6a;&Q{%V*ZMwX#utc=C^gj4X6MG zCPJwD>hkOqy&v?R|N6hZt}|zLlgg_Z!2#juCnh9%GGjtIrPMJ4`Jj(u>ZF!+7S1*m zW1;5?G_n0YoKUzhIM~~Bmhw|w>w;kJ&(>(!sUVW^)r5wUaI<*1#S-pJ|%m95nC-e=C)6Ci|T|R_-BdtZ|&4zpAs?SkPate zagcos+9`LbG1>x$K!x`A_RzuJJ{kmVB@gxn=wPq47>;hyvw2;q@vYucdQS8^pGA_h z(5-@lz)oTM{CDphID=yt=945Ak`p{Pjc8)j)&|t=9qSHJH%`VO^}BAZ6^}r_?hItx zx^Q)-BXy+N%0f2Aq32^M!_4I&KwapjTL=}KwU3^5aU5SJ<99sl4!hl_g%@yM7K47u z9f|%j+*993$f6IUt*nw4OotWXwegMU!vuDp{;eq1q z?xu8d7e~`O>a#mP;dj&D@5ExN{`>uI9v|MZ>GY0G7k4*gJl}b;lfyFzkq@Z7 z_}_+{UdaZa&HsP!yy*XTc=)LQeVE5F{M>f3dFL7QSj43`0DO#n_2D#25Os`WJfgp}iUko6KVgH;3Hn_RI|LT^qc;{$bd3*HoWv39Y?@o$on~hG>uqU~~Lo zh@L-tcJQo&&_qD558EjxICv9oCrz;3DEc!{iqkQTRndn2$tjeyg+PE zC>Pbo$$@3Z-3fWOqXy_vmT0WW1+0)v$7rNemOx0xA^Dr5BzHR%^!GxxBZEh_L&mO? z728I?rF$NCD!x1Cd&kU+p54@w;~_fs)TZ^aj#oWBW>yQ{ZpF7NXms4R zbBdLvR5Z=u6CSPa{whHTcoHFr^J?B5I(l$blF6ZLOVLV4?^$TH~z0RZWmw1(pOF5`)>DF zw1v7|M5BQEihzJ-$!DCPqw_PZT`JD&`)-=5U37O*S8D)8H#e#8yHg_7*Biyss6Rp~ zWVdpTa;LXqXJ#cU4()#K&{|H&coys3jNbp+`7^UhtJ43mTFq!B^Z(ZS|ASKe=YzrE zG5*hkJXLoYhW4*V*T2^r+Gu)i%laH{we%?DSABwC9oUvjh_V)i*41MPF~Q?0-5m`^7%1;D1ewdn8({CIz#Osl%gbNh zT%258{c!g3bfqdSg-^~dkG_9-dUf{t?CsgnOH=vy;`He4*_+op1!YJW;JEqVyD zPl5ld$maz2;b0GDV~^f@hNmyUVB>S7ZCWu=lN-3y?QF+Pf1L@1y?R$P+%dmN zy}Z@=N%aLzzQ5e3HFJ-xkZo+^{9-F*6id;`Cb`5==Gm+n{@N4xngtN-v};)M8mtkS zVmmuKSr7JQ1!^wV3jCDb!i&X^FgQfB8&mYw&+q>u;B516$?) zQvR=l=Lh?b{NF=7)%>4P(bwDr6?s0Z`^Sl7bM;sL{abcTqAE;adr|dXslU)&8X~8m zXk8ZyOqcuoJRG8hRTe>m7L%Krz?Up(Uf z2YIUT{}++*v3w1_Zc@iDDvje>RHq?TV=Eb`R2sCUY+O1?<7Y|B_=hB7TrKtDYRMKq zcY4J+k(~M$#T<#v=ciPuikn%s>M|)HU0CWvKpDYY+XUKWdw&HXD~S+SSHI?jYLnH` zv~WdjEMZRK5c}YWn_IY+j#mfTJsxX=t+2{EfEUkZeKhy>6q{|k_A_MVTp1PA?%A3X zup&IoSwh!O3;IPQ0)0&V`S=u{mH2;&C7C3lSkVZih5vuHe^`qDJ$U5*ALfDEJs~0W zV@QQWT_HsmbuZ0YUlQ3>@6wgeGGE#|CyQPZL7Z9zQ;dzA%4b;AdzCBbcXoii*L47Rp*2Csym4f^pXz25|jdaHI>U4JDIgx^bB=^5{OzA_+b z#}$pW!cy==`$BBsHZx<}%!ICbrJ(8M*Tk_DZJYf<7Mkm1One!7>R%oaNx;Lz+zP0@ zAu)|i7D(?{aVlOhe`D3Dt%%LF=Fzq2vlhF=EU3hJ1#zr`AVRT47O=q1tkKaFah)b! zCVYSc+sDT!8WmAA@}WlR0$Wr>)2NHkuylY2gi7%?P({oQREWD#Gj9-jW0Qq~2I_Cx z+k#2}n58UHH5sC6&X=2|xNM*DC{RA&vx@vD)pmvd7pQ$pAFcQQrTAZmF9v&$^4~)| z1*)VL_Xm#WU!rL~tl) z-86)M!ozR7CXGe=TteD}nQp=c=J#1ZzJ0QNGbIxBdODL(j}sBIND%cVRxC2=^=$T~ zZ}!v=PKi($`pL(Hho8{?cmHF6?hwAYMo;_y2ce(dzCG`MG|b=rr>8rnxA&;~MD^H3 z-+qg_`+Iv`q!r$esQgQQ*2MoSg@SIw|A)_u`oHJT4j=LVgFLPPym>5@W-_v2te1z2 z4s9vZpO4U_VHIoRL7V&Usvy#9hD@#qIBeq~SBFb>zcvO-b_!O;Wp=(7Vo#@`r_FCC zbhLS6Ls6TbX&`FbfS~q5S3CdxMTyv&_1h{m+9ue_s6mIpfUz z8;K@qP`0WNXc~?`KKvQ1$FF}?9Q;Q<8{q$?LxHvO|NBM!|7`!!|KnkvzdZimBp%tn z6B*bjT-j<9ngtP4RHr74*lHdtZ;6oI%mDb&Z)MqMP5i%-D_{%$-+O*ojQ{xT#k0r! z9}n_;EXR?pM&9@0A$VWzd26GFKM5v(IE1v``M=lre+P$qrTAYj4j<3|gFMUL|1B8> z=9~o^aax-og+GevS`nvM6>S+)t%cD_1|0b%wDS2eizN9Vt@&l{OW*BPX zyxBa?37$-7M5PI7Vt>^*eA~ztEA}B-NVsbU=e$^X1gSM%MKG0u=Zrny>VcKe4mMem zew{Gz${6}xaWbVs@>_k-i$MYMTU@WKt${v$>O1CH)kR|_6t#m;o-KtXo*K`e6n3@0 zY>1g|bM>rf)`rjKbF7f%`);4bvOmUB^v5)UKf1f9+k@cthTLkCfv$BP)At+@Ey1z0 z>LCbta>9!iggD z(yrQTvLO85S?o)ncK**(M3PP;#>tv}w8?+Z4vP2xdwVY)p_p(K(ujB+@8t68QZi0DTj-e0=PW`$ zA77$?a?$aoRQBQj8vKqo{+;*X|Mtaf+E@RxKgA;IXAO<9f1Si=LPH`tUwPst?tJBq z@pb1bPtN1cSO2%Og?`4IvP7V>lT*?07Q$x%>3HVSxZ}});Jz-;*>9aUzq~%ZxcpNf zmG<+0ae8#}>eQPDYxmK3{s+$o&z=>||KP>`!DIgS2YI%vu{ipPgmVNQ6`jsk=qwTv zhoR2db3?``b*@n;L33;9y{Y@_wlP$+M40huf2Y&g+Co1Pi7rUY&{;HLoz7@9@>wKU zNIHtFAjOIy90bVJM5-a-v^1r1mW(~h`l@_S?@gy}5(iik)o!PAbx8<%pMKL&sm83B z7cgJI5BT4eD%9TrV|v4rp@8`bSfhZl`wE6&royJiHdFMLA*w&L&XA|a6;C;aorNQ0 z{h5%FArxW4C?Yp#j(=kuEwt95(>XFPbO+|O`V~zH+79V8L5qR6?;W7tcWANi?Rk4U zxolKeDo{8c4+6cv1$rO@sbb|%rM4gkMBU!$}g)Q%)vxyo#x>2hYF^OVM7K$Q=w@45P z+oh(jT@*31TpcPhw;C4NSU8lTLvvY?=)>C=ZKK|Fqd`!4H6%jag zwUr+YS{tpKFEol-a6$!7V0pezf+>;SoJL0rOhY^l$tdgRoCPRI%c41r=scMtY>SrQ zifFhFWc6_25APyefUFD`{s3?#Q6*%b0j^B7JaYRfz?GG%ZhbSjGJ1I_IhJI4Tho*} zU$!U`rfgH|X!H?vtd$nFq*L|rApDIi_%l3mnQ!e4iY4)*r;&?nTT5tS5&CnUtTm&9jLAi6{J zd~X+Zjb};r_CURzU>YWzyq$3(W-JW4L+}V3A#<=5C$Pi_@6d%Uy3`2Q9pI-Cz*SJ1 z*8`T{5C#2v4P4#SzPUR@hcEVbk$wY)80o8t`&(@leS$b;(1mq7tpn{2fi{*+(7CCC zbQE*NX$V255sFzLT);LtK}DM1Fkxq^C`whhZLOnG!2D~%J<67aE(eiS|JGK{AWAjG zZ4eb$_BG&=c-mI$4!v0r&S_wnXe+xx14!a2x|uOx_BF+)!sWo@*y79W5%?63iQ@8r z?;P!@L99pj+(C$0>(K4wiWx@R3~)IPcbdxI16*;ch;LnVRn&Nww2WhPHhInDIVXZd z4G5tPE^{GOe{3DNOlBRl?c+$TpS?i|c3imL|GL(U89bLu0n&u$v}s-{yPKNz2DiDDP_dyER*L$0< zqmAI%u{3sNNT-`{mk#Ewru;~85}WROn>L|g9V_;InrxvGu1&~$MRuFIbu?CBu7ySe0#p1@ zBOkmRTv`{>$aWkjLbAElnzXilKXCbNxaLj=%~ku$lucLj?7SMT`&h?(a-53h?o{_C znyJGwN(T99(Ad;P+jL^Dh}5dC0bVU!Zt@#PkcPl@Y@*;hK0CP(+WnCPHm-~Uaa;k{ zs$?=xAF~`cb^G2fn=R^0V1*juzUX#UE}T)WAEVM#2-I?l)yZ|%T{|1E?Xrzd+@WJ_ zK#GyA)5u2Lp%J*9u0BlwS22xD11>!Z0;*Zl6d--+Ibj?L8W10I#{y>U#nbb}gggo< z7NkX=b*J`^0QXm6X<*EPrvmkf^!utEkuSf|o}$rIZPw*^L4nS6mA2M5TSp7m612y{ zlcPNgFm6GZ7wON2INrkbrO_S>SB&MXg-1>2%lAnvNuaOs8{q11pe%LadfUX;XK(={ z&nC@aDY(wTzF=kGa&W~FOd?>a$%61(7=tmpGItandMC&v3AJsA{Y0ZFdhhFrArY!>3LC+Ji-SH1W~^c30eS)Kv{|LOYu#lH8vKTT*r`ex*P z=WO@43>-Tiy0G_I#r2d---@w_AjZ+0aiWJzA_#xu-51`JjYuH3ByV3{=4ZU7{Jp?s z{;k&-W^ic@p(^B|S4aO;EPHx|rbwY9bC`QXfxi0|4ZOkE3fgYB1zMx?HAaN(x})Rs zcl}pn&iHL=dYFdN>xXnsa|Cchyy+VFw&2azEJ*KPAv_-o42KMQ;8OXG;Bu^X+xFMn zwCFO>ZMeUz(ee_Pa!1zMIaS)Xz_rPI0lqtns60*?wt9YnN{6qrxA2D7vZ^ix=Y7G| z29xgRvcz&Wfws)aTNF)A>sX zc9e&!w2p1_wUJRlbx7ZGdw1yE*;h^&;`%$}_ZI{Qzi=wao5&|iBeF7FO|q*4m;5=4 z2%VE+3vg|Mzg)N`G{PbMy_y`=!If1imLI)G-Fz_bc;WYA&Ljawy^E@+uwvgU@n+#y zqW?Q|3L=d`_lh#cad@ju2hV>#_DbWpxA_vu_`FuEzC${(DMlh0qk!`Cnx%9MHawIi zxM-x=7b;QJ2B}6g>diGHE*YZpn6m|pYr^e-dOhnSA+e<9a2SmzAD=TU{e0f*b?MzG1jrTSOT&N zJB(!@b8Ry2&}f{9+c8^N(^Csqr5#2)ToO-n~e5rD;VwTzNB#4F!WL zxGGI6+TqHZS!`-dk%Oy2olu5L$x9_o0{S1w=n~eZ6srmbM3&V!l)Z5gzjK#B9$|EeO|o*OeS z?X?izA|0;6UAizLEE;} zjI(4qL+@oA);VRd8_A64;gzJZxWZ9zB{}u4MSm+1G@AD0jPzt2_9jULlEcoY;RTpXb zjdZyH*UwJ4fpy?Avdw3Lt62${gX_mCxJokgM!N4rG8kt*=G9wvO`-zx{mSS^T?;l7 zN)1U>-WuRJI$aOF!cl+~-@A+G#41+xCAJOJQ`m$6mqsziHet&W_MAG_jqxzFAfy>z z-1*wXp3~}>b*l|qtZ9Fxp&<%8t>%WmWr-ad5fdG}Y3Db=WuuE1z*=*QX<;}_rnK$8 zN5gfliyC4PGOVZt4p}tSx3l&z*zz0gd&NXAb$fT0dc9@4%Bfr;7cvfa*5fDjc?=0x zyLgGas4`r(@;w5T-6={sp^dja^>C$?E6Z=hdZ!NF=O%e#J$sTca1VLBuHpbj; zUeQ)z8#sfPCr9T9Cvrw2N&Q+gl?`tPE3%r_QQ)^T6JtRMz|O6g;fEdh3B1vvWy$8@ z2*SPWA~~Z1fmEhl?q>vDlUrS|2L-*HKno4%L$dtgf>~>^?b%Q-*J=ov_2W1v*4-xM z;o9WTw;DoL!nK4^5L znNyK9ok`(Z(PJ_PS4bzszx6}%6JxakJu<2pQxwfqS!-@|DQ$F;n3GG%V)Xu0TV^(D z1SJ;NqHhm1>BTI_vJ>~Ql-Qs3W;hB$!h0!{J+dH?>~95RLL*g4NVxdv;V9_EoCwKc zTTATN3EIdqMT?0|mqB)dtc6k41yEJ_(Z3RW+k7EZ2|}&+Z5f1S@7p^xx~4aD%4yIC zLaJ-822X?6Z4sXQ4NoOJ18@Jix4NrW1zZh|v_-hu9BDVCEjhT(8~kXCaJ4zoZg}To z;c7J=&EU%7S2nhOKS!=LU_b`f5>W!~4=&B|y^H9-67mBLA@)vg?(Wc~VsU{PsyJWh z-Rrae_3jk%+P^+}b$VH;5rv>Sf=DHa89foGyG$E5Gz^XQ8zle=aAewWUi-xc2}F32 z#`x@-+zQR$p)rv+ghZ}fg`=R)II_`&Z>XHPI(5d;JP9R@LjnW}^y^w_+sditCJz(? zH4ZVE6AU=r8{fSf$l6UX_h&|WQ7r()!9gJUI)a(#JHfH$h(;;xyvsva1r4wPWm7QV zBvLtZLZ@2j02PpdLd;ka25G#h^lBj^%U4ixDGhrSRB)R&DnkYOxwTC^k4{xMyj-`CY;M0;h{H5F1(>a%Ngkxwb(Z^ z>d$gLQGs|8f#?FAjtQi9_t2SqDi96@LopNjjz%&A?=3od`4WC=5NQO$a+KfkOI#dR z&E5@`{_GWzGv@63JEY&JP7@ugEg+KkGCQ2c-B5Vw7qz;-9KAjMsXVCxL)d(?m+-rK zJ6ygzJv~QM>T=-gsRorDepsRiiI9*OR^|Dg!l*0?^mw7wQRSE{K`1EbFZI@Qfee9Z zq!COmAer4vCoU9%88bP9{?s4qOfV-Q!@;|Ymw9CTgCe5|4;53L=}QX&rLGIt+awBN zMkBea1ClM+H9>fyP!V=Es&xWYr^!Tb1Uso26LNU4x3>o|ad0rFQ6F@Af-UGi*ili_`yg_3q*&phxxhtJAkXy*a75h0L&b z6$mt<@(BLSTZ(C#1ZGySTb0wsU^Ip1K?5t;Q&bh7cuEbA>0a2 z$ZqNrjiX!Js%H5E*VaqYIq_Ns0k;vwTZr7vLjmgCg~O7LOu3FTO=OL3XgMXIdEMbTOIYN5-yvp0n}(_UaVf)Ex%bjk}_Pj56Jm2%KFi~R;sAVNya6xP}X+U zC^v$O@hMH0^A4Q_797T>IHE$QxKP_SBS_9d8j9>v;-&1{TH%_oFdOC_dda5L$05?M zGx}&JZ3PM4VQ*0xHywI)VKT>bfH*M$q-&J8W<<~kl6jEZyiWbfNmvy zXe(S0-{2k9QR4&vrrzjixDd13V(dI1U>&6(es{KxlC~ADoG-^6vir9w`i|2&8YvOr zbDT%Lk;D(2&6UI>E+tQVnQ#|aJ%jdYR5D~DOiJenAYm)&sf3u}5DU3zU|v1EHLaoR z##hOa{b^Y;MM&R<;%CC?r1EB{M7vcDBt!5#r4xH0p}MChy#v}0DYCa(VgS$r!3;m{voM52 z?JlI-Nd%!os`@jobh>%1z@`ETB6kco2ayDev^6ktG zdsIGSO00k#OCwjDkPI;s>CaFYA!%gyH%t`Rlv2Y6Z{Q=i+?1t*u>oTY6pE+WO)-IQ zgSm)gYzib15P%tJ+zg<9Po%r1dLwX>##CN%n014y6#h4p{vBxlavFg8d4$;0<@0+q z7A#Dp$@`%ofb1Z;IE0q?Kz#K*FN-QsbjqX=V9m+f1infsx+F#y7VhE2=nRA!LE4@2 zDq(tQY5MZ?;`Pz1(_Q#=esp>H%bSan-FuO=bJJd-NqKlxiYvWViOzZ1*v#!m|_`V~ejnd26eWmjrcn7AAx3{uRkC(0@5vQag`-DV4H2 zurEWeoQLSopj>t!t|g1kG5iCS%q|=>qVl9@2Hm7-c4oRk+YATJ5G(D;?r)-qc2Uf# z+UV5oxXxa1IjWYKFR!f=u7j&uG;g7589YnSG+TJ3BUk{iESE1s(Ut;pReF{|RxKHA zM$a;kU23+?tgDjP`Rh@(Y(}dUbAMXOmJO_$%YS&v=0a2@?iDDTE&hkmH4T7oViZn$mb*=k)${j-qIIS z8C;1V{2!z_gr0`$n%r7-X$luWPz{`FIKpN+y6V>wfMsySScn_OgC^M4g3C6n;j+J4 zVAEFb>Yy#NkgKjEH0&;9rN76LYwLiz#M;(Rp$2N1wS|(swiR~ZT4HV2gDbzb8m_h# z)>|&;MC&y>*WRHc`@Pwx7K1@*U@Z)(C>5mw*oP9vD1cMzJCG(jFh$#32ksiSLi1Av zm)5^m-<_F2>%{@rwSF(jS}a`kZXl)kYH?|}pCk+xuBg_Z!AV+S00Zoy8Mj>>1sR-t zQ?Aq4y4r{i>4uuru9CinLL}6z;K*F9sQo;u-ggRyw?tIi64%44V`32@9{_M*ngMn` z#(u+&bYZti0HOfEz4-p03(5XMhG=fa9_Fl@E(AbMH zlS?SAqJOKO-|LDZ4N+!gT*%xT`2-cw4uF~na%ym`YNdw28I}wq&a@e|UHcSIs}HCL z>{OioPJR7*Xq+gvNsAYnG{WNJ6p;l`#SY&@A4$7YJ3?VquVNR(B0gzyD+aeTz*brGHw*R3<~ z+Hy$q25?#`fKAirmd3u*Yz30R1y>hpu`iOa&aWYNk&z~ZZ~_9u4s^@E|Fw4V8=nGP z&IYdnuN5x$fNTzz$$-2B8-`9mJV4M){S*xB$ZrbzGZLOHDkr6%lJJy zaAC)$lTU?0z!p@?mFV&tMgCG`;0KQ9h02yL!ZpTH^wt0F39g#Bg*9+VJgu@>yF;VJ zz&l(wZeayn)gEKDa5+w6UcUS$zG8W}8a%Tq;A(Wv+BAp+yNN=EgA>>2x+*kZUA#Mu z#IkT)U}$4o?YYKF8B%M8xdE<URi0U166^TxGsN;3?)t~nQR|(lw!PVrjdxu6ZycY-SI_uWK zRdU;{f~(14cSC30I=D(fTC3n{a@gGv0jFo&CNQqoGrh?hKBiG2vWuVsTx!0yM;9a{ zSP(C}$7;waj0y^YS1eNdw}Bthk@ZdSxh}4az+=wlicv}gVlm-Zeb~g@y9%zTcsb|5 zl@3_n3KT%3LW`GSsIl(-D4Z-@kg0NkD=%JuVmD|4*He)UZ_#-+P39I;_#CHx0e5IL zXb$ULpY8*2EI@My>tFsiYdr;6>xSAA)FSFDxj?&le`-~-WHr|`; zd}`h6GO$*!quM)f>71}c7B~s>)?6PtH`aQ;QK4qcCV57X9eCBPRSTDr&Bi38QAzkz z91eWb>zvP#ARyq0Kw~mtoTU5Bvk)#FWvU7$y^Aj@AcOnx?AsEM(P&D9&PTU8T%70& z*`KITyh<^NfIC;OllhoHvIZ6g=t6q}>8J<{aiZA3T=~&pgG}8fBw7>@PMSziW@V!j z9H?qNZtA&|@_)3YVlOn>N~tQqYzioISA^|ssgN?QU^=XA14LEvWojX6jxf^-kzO`UhrXZ*RGt)<` z3ZVt_D7?Y9P08A-4t0IJ{X3J6lSGT&0&`8#3Xrb@W*cd%2D3K&er?44GPHk@ApH5c z$=`+LBcYXcnuCnX^_E}4H4a2m)S+fDmkB{s4dxmKIcp_pH(?{9|eK7YUqzSXHf=X&;B-82i^?BHIk+8YjIeUg%ve5K7fPt&9}fGF59Gp(cs& z4n?UDSOC&QhCspygx7?`OM}@aDOP|P5qZP-wdPtqNK9qYbB{8-iP3Af7?{XHLOdFc zSrP?Eay*$(UvpE>L0WlyD&U83L#H1=o-0zvJ9J_aTY}XS@lif8t&)7jeELMC8&y53+esrbK6 z2fWBE)qL(9XvfPjTaZp?G#cIFdDw9>h73E%q%cx{Ar!DVrqK|YTNN5jJ)%4Fd^U$7 zoH2Qo@NkH_|G6C--grk0`~Cmi*41}})sJ?e26S+x&d!xc##ai^A^7!l%m7s1X3Lom z2q{TiXrOwiTMdtDUboiGdK`^L`B`EURLX4z)8b|p>vXatH;!SR%^Ri-QuC?ER1^7h zPo+{O8im*CSYXs{txC0~8uJ7=`S$v zhjeVGsqdCE8Lfxr)=yLx;sgh8qVRU8^BX%|W+pSOy%vh9H8v#^!TVx{ISJ|t7XfhR zp>_&#$@7Ql&z{xaLoWLIu`CTBY&ru=Y8fsW8rj5PBhOlFXIZY@jOwG^&dByQ+C?K@ z&=I7F8Ko)DMzDF9HqpF|6?^al;|O=^T1=f^T?Z(geIFz;wOu=-+!U#;y0wMQbSfoh zfStF1-LU!(O~?(JgJ-LocqzzbV=|(thrS2Irz#3IC&(OMdjjJSM@A=zq27$$q?YX1 zpP7D~?Voj{7TwIXMii=p%z{h5L+Cfbq9MBLSgiiBlVSL->nKFKN{N69il<{`V;)+ zCqlA$d8}KhKP65T0$4+vxnH|06;KYW&#n6%@^WUMOxvmz_k{F|KzM3upm1g)(-RU~ zJg&gIAd?}oN`Fg;d^UIEBkh~g(}>8kV2JE*E4^oLvJeMD=h`1WhU}VVp!NQhQ2)B8 z-M@#0_6PfWgKw7U!@RM`;^gw`{Nl~4)3-mJzPn5p2<#Y3;N)t(eDja9V)>Wsni{}; z9F|uWD6XA2j&Hmvm9u1=2*Q09sk0B$sDAgeYtH^03rV;;yM^U4k_-o; z@$*A7QzrNV`B#ENkvcYO`wiO^i6pzqxr zeU@&B>e&%QB5kCVbq?eVzE*W&Mh9(Y9YP_Qbp{NNp09KgIvw-k==@B_95S|oCXAq( zMrfKTJ{mlL{27_!A^N-BXSXcjQ)oo5bszUX#+>~|eA!dG`BQ()apcd4=zr9I^=R;^ z5AF9qs(;|OnfC7ti=1ylZH`V_79~O(4lC%M_MaNVValz2Fr*%aIk`oz-d(;`o05|_ z#J*M@|L6V>X7$}(j7Nn&8oMZkRlV0o1R6F_KwGln5cNVWNf^3;9tuG}asgt@Bul~tVu|+NWsza9JJ_fZaOT=i zqlUn&9{4Eh-RxA5GbV@&kzzh>IF*t_9Ze8*tV)oLe4hxB^9SB* z-D!T4TntS$SDk!IYRwYPwN<79!mm49E0m&6^C- zV$iv!Q83J6j&$ZkVnsF;?b1X~n{}mZcXp7F(Ha^uR|<iQM`fiq4Ur?VIl(vg4E8@eeHH^P%LXJ;zocn zkr(Wl@;V(8Y(|lDCA_F*)ge_><8GKM8l_iaAUsGqC(a3mqX0)c*acGD5@pPYnj;gv z>bFGHRB?jRC<)Jmiut=(_&|NW^ZuSH5x?`q!uQO2dcf>09~@1IXP3^hI9mnc2fLU` z>%}zP1DB~g$U;?P_u5* zGQh;^Kl|qTp%0PZZ+kwre@&7wbQ+oC4{u@;9kVDBL$rssQUtMh<{kdO`I2WrNKDE7 zhLS#uqF4CCOFAJ6!3@#h@NloGX*=)PBJV-@Z}twp{l+`kMc?cle*2AgsQ!5N?Kj>t z^~dvXzww^$LeXyrdroVrPvPT`1mjE@r=KnK_c3GPcSbd?mJm)A38>7SdnYkOrshzk z)a2S$ft=PIQTJp9IF?FuQ;ohilb}uehJCQ*vt|q)<5P|4sF3>BFpu)KjC#m9mpv1X zOHq!UoF2$MBJQb!v(>3OR~5%<3;kW6&s}?^6?~U7Sjg&l=)#ofxCin_j^i1Xq^Ut? zw$vFdv}6zNbcvQU#YO65c#4zzaH|ALEor0eOD&Q!j=tyYMr85z-<|E0I@L`Lbt_q) zR7q-#PZ13~6MimJXfi%M3(!ExS0K;Y=zyj=e9qKW=?K14IH;(}$VF58dhhGKaxIG? zQZ%!xdQL>zVTyhl>Z)TqV{jNpK9OF`7<K4JsY=C(tGI=><~aUN+gJ&$Q?dbTd=@PT1w?~2=O_%t(9Fe^QGR(f_;Tuv%oE%z3+ zs#Z_q5Y4QJ5c}8gJE6faU2^dPUQFoGrw%*G8mKqURev)o6x_D|@S# zVOv3>;EIzujRGa8r0ieA}99cxN+RcRtheww+}viiw(czBna##bQGZqXgeYGj>B3iY2#s z6os)X2&t5zL9jlXn{<2u+7^Up1F9Xb(}5LDH-aWyiShkT@t--G5RYtEufH!5_^pfG z{D}o)HA;%Ft&fT~RNG;bA%$nhS&)#$ zgp0naa0U8JGoGvb0Cv32KUo6H2qqsuHT3N<??5d_Ufn-P+~Q<+(Dd7 zlMwUp7NF3UBNpF`Z2u#BijARbA-JkV@8&BQuzDKv13sH_JkE9(*tHpUT2V6yCP5*T zqn$7;(Kw-Dps$3i%`iwX3axssftxxrj(MACoc8?o&n}?u-|)?LsrHm4cs|gZu?Gv6 zDLsvyR!Ui*Xg@1tYVR*!d%s-!VENhy`Pw;V((b4F7L3VdeXU}X=|)_w&3v^Evgbe_ z1E;O>aW^H;p=85>*NEy;aqU((O}}yC6S^>*dqx}pM;P9(XUjMx%CeY!io%C|w$LwR zJY(z{y}Nj6g6&PH2^Li-tBt(76)mJ#g$N(V1O+SsmzG4G`Z3qO66TD_I7E{q0

~ zv?a+0B@KQ2h(;ejrZ@z3&5%7KyLFL=K7AUYPoJI^8=z7xZG8*c2-B&riRVn8a-AuL zhE!4mu|Di&G@0j-FjvgRSLiGXv)K1wqGc?093Ij5#5hRR3K)RZEMD+bbHG7E2fDRH z=pPV*l`c%KO;Gu$52=Pu8*FmM9IMSd1bIqa2LhOpX((_6f-Qld4FHTg18VRfcIfyD z25V~}gcjrwlJUsFAHgG64!|=5&@*QQd|_l5HMhWeDpU=;D2&Y3Q9s~@$STGX$oIdi zZL=Yf5sOI9H$YP@wjf-IV44;}X;}718!fU4qGm(wRvszPuVJn_NSFS{EQYZ|hs?SF zHRRg3kgAOtC>~KbvW?vvdLHb6^sx@?fHTKf$s{(KMU&|_rRc+vHF5+mN)mYJRM9b* zdKf;50+hu1B0o1dR^ZP0KjaRakyX>s>qg-Ju@F$*T_Ggs?<7^&SR-{REO<)Ry|D^Q zi=fcRgM{(kk3^0@Dw*LZ2q6Q2Vr)e+nNW+}=1fRfIiaoQf6^2j?EX^FjIJ0CZp_Jq zen8uHfA4fUXAxpN&_S20E1EjQn|iz2%y8K6J5DgRvF~�#g=4CvxUNL?G&4G4<=$Z2O{b?f%t%saI6JdUAE8#{02Pq~AAP_EpWRTs!@=&Zn#a z1O3X4=n|i-5k)Tj5IFXv`8Prax$NWUuYy?zm8?hO4ly9YhcIjnS>E7q43et%Y%5 z1YdzA(ca;qHZpLX)SA~*V~5oD40c!QVptkS%CTk1&n_k7cuLS&Bnh8jpP=pI%d?&I zNLL+xjE*nQP{0+tkH8m#6UixAWP8YV_5SGhgcI}+a*KWj74f-g*TEFvDFTQB3fs~4 z;0nFx3)<6Nr3j&iRk@v|mn`2PFfpVw&pe52&sCEOqLiRkV9^j2n)ZA_J%!dOOD|PQ zjUdBZ97)OPIFZB=f^=(kijEEOn%|})x6)3;0&<>=Ln>x^&=+JfEQoUWdeQSavF?As zI;)ko0SeH0m`reQPD zTRe$+H0tSSiq4|uqA8SlfOMA^MwJUz(CK`Co7o3d#dise$_BihanVIwvsDoH1MCq* z>eQSIUs4ql#4>#v$e-wRhK?4PhS+##E1H%>V4al8a~-Q3yTOnr4Dlp#xJZQ2OtpjE zcI-Wnt2os&)zLa69E1vB8xd%nhMYhn`_cGz>zN&?o`5SGIc(XeV-!(G0x7vit8G=M zHv6j}kl|3FT!$#uw5CUMgKs+^HP{THbF6K8Eh}AuUNXeSU<-`H5Q?0g3sbsd&6d$W zSz@C)8|ND|QUW-HyZiKMG|~i3I$8JtL{PY8;i$Y?GOyh)&GKEjs;@GuiqH&S#e{G% zL>UY{JG9|9d*2+|Vj=j+Ug01B#uq*Lc>d<(>g@bet`rUyszX6)3n~t|8mP=N=0Co` zN4p>KUZw``16~ppUd`1Gp6z|??d|P7KRkqg_xARR|L*PW?f=K%@WtW&-r(@y`N4ne z?H>+a9R3H|dx!~0gv9(m_BQUTIJiH_^YPOk4Z-8{_&h$3&*Sq)JpX?H0096089XxY H0F*}nOwMm) literal 0 HcmV?d00001 diff --git a/charts/rancher-monitoring-crd/103.2.2+up57.0.3/Chart.yaml b/charts/rancher-monitoring-crd/103.2.2+up57.0.3/Chart.yaml new file mode 100644 index 00000000000..5b302cc1df4 --- /dev/null +++ b/charts/rancher-monitoring-crd/103.2.2+up57.0.3/Chart.yaml @@ -0,0 +1,10 @@ +annotations: + catalog.cattle.io/certified: rancher + catalog.cattle.io/hidden: "true" + catalog.cattle.io/namespace: cattle-monitoring-system + catalog.cattle.io/release-name: rancher-monitoring-crd +apiVersion: v2 +description: Installs the CRDs for rancher-monitoring. +name: rancher-monitoring-crd +type: application +version: 103.2.2+up57.0.3 diff --git a/charts/rancher-monitoring-crd/103.2.2+up57.0.3/README.md b/charts/rancher-monitoring-crd/103.2.2+up57.0.3/README.md new file mode 100644 index 00000000000..e0b63e0268d --- /dev/null +++ b/charts/rancher-monitoring-crd/103.2.2+up57.0.3/README.md @@ -0,0 +1,24 @@ +# rancher-monitoring-crd +A Rancher chart that installs the CRDs used by rancher-monitoring. + +## How does this chart work? + +This chart marshalls all of the CRD files placed in the `crd-manifest` directory into a ConfigMap that is installed onto a cluster alongside relevant RBAC (ServiceAccount, ClusterRoleBinding, ClusterRole, and PodSecurityPolicy). + +Once the relevant dependent resourcees are installed / upgraded / rolled back, this chart executes a post-install / post-upgrade / post-rollback Job that: +- Patches any existing versions of the CRDs contained within the `crd-manifest` on the cluster to set `spec.preserveUnknownFields=false`; this step is required since, based on [Kubernetes docs](https://kubernetes.io/docs/tasks/extend-kubernetes/custom-resources/custom-resource-definitions/#field-pruning) and a [known workaround](https://github.com/kubernetes-sigs/controller-tools/issues/476#issuecomment-691519936), such CRDs cannot be upgraded normally from `apiextensions.k8s.io/v1beta1` to `apiextensions.k8s.io/v1`. +- Runs a `kubectl apply` on the CRDs that are contained within the crd-manifest ConfigMap to upgrade CRDs in the cluster + +On an uninstall, this chart executes a separate post-delete Job that: +- Patches any existing versions of the CRDs contained within `crd-manifest` on the cluster to set `metadata.finalizers=[]` +- Runs a `kubectl delete` on the CRDs that are contained within the crd-manifest ConfigMap to clean up the CRDs from the cluster + +Note: If the relevant CRDs already existed in the cluster at the time of install, this chart will absorb ownership of the lifecycle of those CRDs; therefore, on a `helm uninstall`, those CRDs will also be removed from the cluster alongside this chart. + +## Why can't we just place the CRDs in the templates/ directory of the main chart? + +In Helm today, you cannot declare a CRD and declare a resource of that CRD's kind in templates/ without encountering a failure on render. + +## [Helm 3] Why can't we just place the CRDs in the crds/ directory of the main chart? + +The Helm 3 `crds/` directory only supports the installation of CRDs, but does not support the upgrade and removal of CRDs, unlike what this chart facilitiates. \ No newline at end of file diff --git a/charts/rancher-monitoring-crd/103.2.2+up57.0.3/files/crd-manifest.tgz b/charts/rancher-monitoring-crd/103.2.2+up57.0.3/files/crd-manifest.tgz new file mode 100644 index 0000000000000000000000000000000000000000..4228ad81118c9e4b703dc16057382e6c3dff43ec GIT binary patch literal 308613 zcmb@tWl&t*)-8;?OK^wa?oQ+G?ry;)5ZoPtySoQ>x8Uvs2=4BFJ2}t$)j3sPz4!dM z{MdBY?jC#g?p||_ImTQ>kuV^C|2|GTzqsu*H8TC0dct)7tTo13k$jq_>v8P7+Hvka z`#hrCVMFtfA3wU-J_LQPg5SD>H~Buc-3&UoeBK8S47}enb`A{i1%0pZ zZud5*tc9pCKXr2p$TQQrJFUHBs=D(K^z1-wzkf*P*1kUrl4YXiIgMXNg$319XN_E; z9jthF87;nzG4d{bJVi3*2!32g-amJ{zwc=nyh3@$O1%y*eWF6G zubVtRg_Oj?R2oMuv0lL`-ys51;0c0c99u*4L;Kj5v@n#`JsFib9hIYtp3+o#mX4y2 z*eypN|9g(FfB?sj7LsU;N6%tnfo$c&n^TL4BO zd>&vRl0rr`bLPp5D1@oUV6Hw)jK`pkx#jqTU?Is+MB0kK#08(1BDX+mo#zOL+pUN9 z%b&2@8?MOXMMI?AJic%tkwieuTAl)}5JeQt99a~ec^26yA$!=FBQ6iXbBvu|>4>nqE#OR}LsGZ+Z@=eKzFI*4foAOX&VqH^Fz{e9v@!%Q7jhB&z(?D|fK@#B1rfq@j@V%+3 zz6ujISgOf%k`$qx-Eovy8baQZ2&8|I$RKTq(&T zet^(9z&)aGEOp-n;!(UOsbI5=#@71u-2ZhE*Vh5R(V~-MrB@9QZw)%}tt6Zq?|vBN zjRP};pWR4!-yP8Epuj$}_&WF-r$CVYY=tfV$YJM-eY`3K)nWy0D}R`Z`Ji%}PlaK6 zJ@d13;8_SagGV3>AIZuyPGZ<=#XuTw>D`XryawOw*}NmX24@A!W=+&Z-8jcbQxk-C z$04J}k3%E4w^eMtsqi<{^%*)HM4xbs&7SaIrn{OH&TixQ>(EZ@<5xrNs?NXVSe9V; z4Wi5Tt5d7yqUZ5xgX*I;tB9=NBMLfH2jVW zwnb!n;e0Ii${f6PvJa(=Ut$ERBwwZGmH;XrEZR)%MK1~`U&@*@8dPFal=HatQTV04 z_G{@&rfK1}Gx2N2#Ua2&;wXL9AOE~`!M?9!Z|Eh1k|y66b=X z@)Sfcir0A+P&~p{#{G2s+esV+i|QbM1IPS4s$7V3_Ym%N&Ui9T1rpUrrMf6ftMFYa zO~Z6~NA(ERsAMF-VzGU{KtN5qZ1$(<2d<+zHFs=s#z+_AGiM!S;Jr5EQICiohI&G2 z2{t3s#3eb|VN7}JUaCD0&Lhw374mAL`<1bmuI-f$sS=5#uyP+8vObDG!$?$zgn|nF zYhy@`z>Q&%$^aK$T!-S=`_qh&W*RcGR}j+s6cG~N_t)NshmSj-9pBJyR0jrj>~>-= z62#DQ&ZSG$y~NJXHpTmthffTWm>z0mmi&)K@==DC*^vEGW!m{ADw$H(2#%XE=mVPi zWY_u?@k3nh<)TvA(KA||z48cTE7n9r`5DiW>ui;|%JCRVpD}xnxJ;`kGc6!~WRS=) zfsuPORxQKo+~gL`AJ;E}y-?TQmrLy1P*r4nza7J;#cPh`7U$w!VIi^9HdjeEGp|zj z>eg`ssQad&6!aB}N~Lh6+?CO_C3|fTd9mglu@PHzN0v&N+XkaLwmK+&iX9xD z(i#Fj)^0{;Qp`GE(NIYjs#wS;^>HZD;W z4U57IDG!Mha(v`+-sXJH52({%ZbYhUU!&jw^*PQRuTMD?WP%^{?MPU_Nk_2j{% zPeEZh-|}1D1$W?IA`{8)L%aEj_d4K38`0n%ZU?C{zebu}(bKvWHf4Xb`vw*Q;)93zX&bQ-n zd-rj)*qqu!lT0r)0sCT(j)q$K&lc|E6~uxgiuuVncyOf@$bzb^>fn?*PK zs^A_6YMc*#Kl1f`7X;oa?pi_b-y`oue4+v5K@5^hEN&hl#|O=K$GYk;)_1u$ruJ>t zmFvcD7g@|8Ir9o-vMrLfb+W1b1JT#xU90l9M~I>WT+);2qU`nfYx?DogXc3OSMJ>i ziPix=((2SDdML$uTLC_9Pdwt33m)K75K7Q!>+wa1iWHgDvT16xOhsi#*&j%3$?LkDVaEc0 z1f4WBn;tFgawH2)c~PWQd{e=fia!Dcmb{-Zoz&3Edg@FhFa?m4md8x%XbKK+gGUDM z%Tq%r$mw>PfMQXUVxD?R`!QJ5D5^RhL0Ah|>O{V6c+yXazX;YH9c5N*TqG8}Oq>1r zSn!SyRc^ZdzP0(d-Je5b6AdZ83RB2+6UMaUwxsQw zKR=_7t50(v*rbB2w+bz0=F+xfwFs%E1rlx_&=ie$H?s8ofCw2%5EkYh4s!N#fyw~8 zkf?}eA9Z9dH?7^~mV?@uUz@GVfXRMNIXQ`>8F8oYca`9SO0k}Vd}{{awsOm7T;y~2 z9(fQ}bHIquHSr{9-*{THO?IeKOx975USyG4lHU3D#eaSt19h*RKx?M1a_!k%Zdz4l zrd|iv>eVw8F7Cc#?Zrlh5)LP(Hi|$R@L!!4C=6-+LeaVXww$FzOA*MFFCKDT{M!x% zyQR2vRrk4y+;EI1EQ1S!<}<-S*qC8qDYa%ZW(HEeuE3|hfzm)OM_guDBHc?K7_1&t z$>>rfi~*!3&31?0Q(|2qiKr&Dy_tYjKugb8l+P^$kC5&e@gRV6LmmH(XG6XvAE(6@ z-o^byC^M>`)3TM)PbpNr@ljL>K~m+10Z}vG5#AmLU?zxiAdCx#+{Lg*>Zn>2&y!OD zU&V`C#^=L_q-%(`e^MH`7fgx|kjiHYqm*^dM~G(q+AIlxK(LpSva%94aS;$huioFE zlya>R=9-D;$k`EnZtME|wrn`0EJv25NK zADL9We0ng>?B%pK(J=nM0fw2~ork#U}O|q@a(2%Eo_2@7D zH>6Gj7xH&9uDA85{w&IJj%*Y(uo2+uEHm8Cr&Z^E%;Mr#2t@~L-)4S|wb%S=5mHa@ z>K$yuSxR3B44kg__G#iIV{|OEE2DHnVjqNa!(uuPRE~c{+$Q#!r&{a~^Trjc5(!F( zY+|)kYyh+YZvEs@f-&BdM+D@SBON1}*c4~(zu?-J=A#hg3#7&s7?^#swXM&uI#tK8 zZO^BA!L&8$&nI}qU>_E0pYPR<7-^HKF|P5L`pGp%NEWEaxkB1}Q|#Yt)i!t?iv5EO z%p7^#2h!8{USsXl<^PXW)!n> zN~fU6KHZV#h5D+G^Y*bNe=k9&6TxpGU*$et zzheWdfANIr-_^Vf-p!@IUkq^ch6UoiT<@LSbrz-w{(5HV+8Q#&O~d6#=rsc;5S=iW zeeL584kSXH?~}NLJ7eKfhI#in>ST@Jczp%JUu1|7N~&o_hsuWNwyzSOJl`$sn-!a9 zUse}DyGPDlEAMbNM}@bK79}UAq}KVslX9e&A^5XRFe=sw;~f@&Hv2wO_MRbU=gL32 zx(D=}ZOUM_yb+-WLN@zyfz*FUrt!DDwU>EZ+WDfXTD;FTEP=X-D7=i?vKoSRXY#EIZGjVy8fd%fMGHWt%^?|0oNsONM{yr=|&cnk` zl!{Lb&|RP1_C!xw?tA4bm`!x3&|c)2KcQ!xh39#ppIE_*X2 z!!4LQG1&{$$r)Mb1@t&sBKL&yVHNma*&SNqYi{Euv+2Fo>efs0c|3Jfja?$v0*+;Z zC_7Y(6RL(Z?`9D5=6z!@)K<>Trww0(CTMJH?>C^oPQ#|{rkbxMZuk}%{1VxE)1VCq z89UE_zF2d{PO%HyFmygX%uXHX51B<5Qa?p^GVitZ+i4?|X+r zG+Mn%+MguChs`VVDuVif=tuglIN(c=8`f=PLt^Kr`RV2Q`gY=QVkIS;yXss3pn`!6 zA_(aN_j((5Id>5Nmu-iC)JAyZvtT_zkpGQeCR~V14l)i1+mw|)cF z^SA)K-M78xi7z`8F7Wo;vNx|6K@HYtKMV}S&@A^4pYM|TWAU;M={3{>3k01LoqjO% zw>Wt-DUQI%4AdzOxWt>1*Xn;Bj3aIpx9K{n9ylMSeG0=bW%((5a--oJyItm`_AvhHcsj}?}Zz7&dzWB;WiL+0|IT_ zgX%z49Ta=LL1?&Yv<&M-Kxo`SD3FyPH}{*?1nk-dE7b-Nn@iVh_cyeAu1{Ne(HqWA zpe@%9KrPprVTq*Hx>7SNYn!M0zUf^y!DV7$W<(qBEUL+8nEBD^wc8dvA{npnwp1j_ zh&eDcw^rTmA8U8F`53@v(KZK~(Q350Vf(&<^AL7FT6=Xww##Fxku+y*Op7{ETaGam)8d zauhQ5HjF-orIf;ZaHtiWv~3m@N?h9BfH0z*psjTUF_@sw@+4vN`qpB`ohgGMaA-^& z>=Z>sIMFjt7;-J?8HK-&4TBx>F@xbY(VR!fXN5p8(`XKFAS?BqCLN1t+y@V zTOheE1%I~mz=Ft?_v2xP$(4ryW~_y9mP5GGKz998{S9!&>Wfq`MksxR4lWQsEI~Vc ze3Kw-mBaK+v?jG*T)A`+e>j4*2j40U)&7R2D_*A)+0R9&8Wa^X$gNm(LMDY1q>fIF~>W4z+Ng|wR^0j&)N(` zZ`u##nugxjZa3H%O@m6Z@%|QGu^?E6t$boSS9bk>tGlLsk<-*Vc-`K2#nqL+VzurS z^87)iW|O{|hEzPT#pUHQl+rW=9;zYJp6Liy9ev2#pht1F)G``$>2Jz-JoQ&G?%>(U z7RQwtvEpc{@;GsrY-3GQnZA^Rg@&+MH5DC!SSiLrL0Tz_03U!RdQBq{ZY71xd|C(rUm}he zJ1^mi_NB`EC6^+IAq;!zv(U(4w{(C5X-guE z#T#3Oji%0)PQhsDDWVQUp{K_KALDKmlXMex{YQ1I@8U)E(#l0IstWC99ujr(MK-~9 zqejqtW_2m;*?i{YAzb1_OoL0}aVdLlfxSOBRz+FuI6m(7e~k4L+F^99<1* z*$PH+o3L`DO{z)o`C(6WU9Om_p4`UDj`=C?42%Py`wn(%d0WZOk)MpWlZ$hkqx@F_ zG-YyaQ8w7^%^{0IE=Uy!MlK-;X{h&+CoY4WICrhr_8Q5M`YFY0r$<-{wSaS8m2v{|+$RhnGGM(hE9#z%8MieiA3XyY8 za-WT-jKp3n43XKiSm~#yKgBpi)h!&pprqOKo8{Psph!6#i~YnMHj!tp8xXH~6>yvT zm?5i#M90cGQ&4k*&l>~NC(NHq(lr%FP_FQ za~uw1O6Yu_Ev+E&AeE(Jij*dfIBh1`ZQwc=QJql;kIpi14}`P{r8YS13`J;w!w=0e z5Yo1W17y|kjvvvs#u~&GY##PcLu9E0W?Pes`03L4_h3(?$i`#RrLkpTBba5N8hAtR zF*Fbe6!;u=8ma;h__BuH01iz(kUgthK15Zz<)BctT9DDXTV=>SD?-%3;A^%mknwji zvwOwt;*9c8{cNaJ!$weHc@(FBSoEMw?L;BGO3Z(S=Zy=D1?(R@+nNp#J70&FKuSw= z!emO|BJ5Q4U;6+OQHPgMaHf~XvqG7;+R8i*p`ibw|6Ndj51`nk}2v{fSFS6r!>F8Y$HgQ%oakLVaO-;o>4FR#-8C7!@!sb_X{)9ALW7P<-U(&)}|> z-jg_V!>mpR)8_6I#-NX%A323QGl&&hJkUFs!i$qiT=$!fg0%Q+>zDpIkJjgz7#X5O z_4-6TEF${{%T^avHtV>h#3u!GiPWmm!UDJP;Z3~s4g7pO9n^66hVt*bgIqtWhr|XO z-L#;xpqLLAZL)HzFe0qgzv>F{>F(iJ>NYUW3P;*Bsm+)Hw0Zf0==w(ac4f#4VI~8K z5R^K_i-*;x14Tb`u^zA-I;M?FX`7Q6+vgPBm{@02$++lv*-YWlcZ79`AAhs_d3SZO zayVOegFJVke638*%<`eY)jC1d?)tRhQJHSuzADGukin7K(ygWi?dttj&xRY6)c5T2 zM9_31vJ(AO?Gv^70F^VdiS?~uBX%Z~)ouJxK{h~5M%7i$u%e0CC8AWA3mu|FMs!1^ zjQn{{IZ{Q}eAI%SBUXA~SXd`FWU}EYNrbm-cT(x1V6g#=3N9dc-rXVpu&>n3t0}Uo zJ21u{G8;86*}7FurW5|{TB|JC^RP@~2g$OWlPQTk{ffW+cmcgJqsc=m#B3b2je~)Y zZZEEU9-$L+#8_h@tkRPyL`KUuf5y*kjDeoo@vQKf`+2hUTi;DUealE2$3jtXS&~Y( z0c-tHB6<4UaFOaj$xfQGw`zX)?`Bg@vgd_{pK!+!d(Zmcq9qO^AoXfVVLfm4YJsL& zc2r|yjEj`Iv>B$)4T~8of4@5AX7nU%XQcPia1CvcUMNUkHE5y`BkiGC zBk1&4`GwV?nHqR0`X9mpdMp}=c4!9KS?Pxb4&)g4*F!&outXRuFohT^aC1}C1cC2E z20oVa(^p>y>8#96!Mv?I1}kL-XE)D1?fJ}CSqH9YQk+6l%6t=sO*fwk3vGB$L|q?> z%et=0e~ZfsyddH~%aC|?J|DtEs|w_Aco%cPQ&|7t9h&Vy0+-<$5g#DqvW5@v`~IQ` z?$`VGu-9P{&;|)127Lj{UkH4F?0y} zXVLb~$SsI)a{WaQV55=@{Cj}w0e%8fB0Amh#~b{quo9%M;6~h*Y7y`S7~L)Fgb1@X z_J&af&kzmF61$WfI}SWtFTt+oTd|+N^%7vMpvgrL%AvxuElkL1CIb>f8pTbD1==>+ zV$!VmM42iN15!Vk0y1Wg;LXv#ZWvoK46}LIxb{uCrTq9n-z#p?lOyfZW|p>V%*JBt ziK<)6`Ysbe?xCqcS|&4`5hoj_e%I%jrRu^_-f;0%lefA~&pC4`aeX6nD>WJVd%|XA zNS86Enw!wqlclCr`eh%ZWap|Qjs!bj{<0V|rHB9-G>;^a?tBURj5~_d1y{?O$`5zL zk|U$bf^HZV*YSIx=5UVEE|0%@TRM3eDDXWXC8BPnSzwI5=~A(Cg}*;fHcl&vri_`N z9Ve`4UiNY!fQe$}XpmdG!<|F?hh>qb$|ec(eaMkWshV(lEU?WhRtp@y!?lST3}h_~ zP*U(FMc(f^7F%2R3@3O6*ncKGLAABYmcUPDa^{;v__!RpBP@pt#+gH9C9vcBsfG&2 z#$gh@=Rn3i&UKGB*z4t~L-xq*N?)0F%}C}_eqj%jSv>eG@yvj0=kUoh zVR~50ZA`g-hd{AD(!97R^F*anjdAXH-MfLsHjaIHq`~K_rSQedaKm$--E(TOC@pGS z#_-+Z(89@GjP`}rsWUjj3nVx)=r;pJRMw#GOWs6B4iOpsh)VKT% zT;K|pMCTkt=iCfU)jfGWot{j6tz~dd`sqMf(%-z;@0wPO_P6P!uZciVxfr_kSQZXO zzhyXuJk*2fgeMS6M=V?sN}oUxN|ytnm~Kb$z_J4Yp7zA`z*+=3hWOVap8>8qEa3~` zKb6)WE41T^c586D-2rmCy$0$~Zy9;IeHtA296i^~h*Ri7_%C!jh;;-&j+u(J9u~P` z9AO^gqzKMBf*;67*$4lTX!43RIn|Imi?MGRY`}vuiFYALC}Fr8sr)yBY5g{Wfv7KR z|6i!>ARbsApnpg%w8HdfJ~{=ImN(Bbu;vfovi*IJ{I8?|-RuiQ+&Dw-gcm7=2n0t! ziUr_Z)|jGT;{)q*?-ek?VB`N&=?xuRRPLq`k^V+Yf(SR8OZ?{m#w2vnHw<0z+=SX_ zw)r*j=eiXAm9QeA*4k^n#~#Xvxzra8unbh!Ao%C9{tkE=NF~yc0pNE!P!h2m0qq*+ z(5_Te^*{%ElXb4v8nl0_VP#iDZrp+94rd18c>?O?Qz=GjQD%`L5F$PGz3dA-qZUH+ zgwsLh!m&Sq46o=@XAwiW^pUzZTWh=r8C-h22Qq;q{TGKc>d4JN<}~MYg!+M=I~7$k zog3ntYk98N5xX^8Rsw*zDh+AJHXfFkdh}t>*DtK|2u@9Xv_^y`J9tZA!{#mcyvC4ZI&vh)ORW^ODn*>~D4%gPwuLZ5al!~R z$R5Rbkja0{TgRHho^}xA62sD$c1t)F38go&YI{K*5`qek&7n8xJY4=J zIE2+9YOtm`Q>u73Ldt;Vw4_85;srf3B@p5TAnK!yln_{)X(%G{)%HaD)Ak%qGgML`roRS^4OMlrKP$VcaHmkC#W?DXr3I1d* zn`VcTO(5mK^8L^P%LVbPbE`b!w~4(zNo9^P$w(i;g1P>=@?1PU=(j4fK}dp(U|dSR z<}q#RpJbI7^84DXNja-iAEsupDR5O{kY0w$G(}6~g#{Yt5Ou?W`KQbz7OW?v$l9NwS9TQziC)XvTVF8acS4v!O6&FH+AzIvok`7&>Pkp&*RnnW^vhs zbcT$T6%<)KI8OwfDeLhx?rbs%sG)rRnp}Jhf4zXw|Q$ zv^s>O_?f5Hyc;wzR5E9fAl7>!MWawVLBc^-P8`UsvPD|hNn#->)Z3`HAf-!RHQK`h1J`HUxLkz>sH0XWuyk&g>xX5gCzc=NPRQwP&>(~VlXf(HkvI@ zi&1f|%ELK)*WP2Y(e9eq5HOuD=WCO8cYgS36}F=yju`n!ySa5!3MG zX;mb8^uq0zw<{^^jkjL}B9_}20g`7$>g0O{Dp}0V^3fQudfOI@M-fUx&4WtS$y(ZU8<4V_#P~M}H+?TfETe8Z@=0qK% zDeZ}$8*YM1lG$58^LUfhX@N0|TQuZa_F~W_-lwhyG)*}GuEuS2Tumjw4zW-LJclCz zy{*htKGEAja=O{F|KnCMuNMhJ$RsMZeiusnuVpLbi9~oE_G-w<$YIpx=lmv8MKiUx zQ5PZjaPT(xo7yBuvI2KC#`-Ec&WCFTI!=Z*nN~N~48#GiKL!jA_WRA^hvIL%w0w8J9*N+i z{i#T09bmB_jml{8?B3iq^}3LDU*g6-qw7Ox99e$f?D9~tbzDjO|52?D<+B1Gg>eB( zX2&J$Z=1nMJjyc>G*v#rUWRehkfyafWQJuS%pz1S=C(d1OpuB)-A_n#`_V+Jf6 z^EjimAyNN7GK=bMI?Y0`8L*552dekzwd}IU7qV7V^=x1btmkB{Xh)#_Q~K&*qrwaG zXv@?MUZOT;$FotHYB2G8Hn;y~Vo*_v=(Ww(4nmd5t;pbe z3)3A4W;6S23XFn#S!^j2>0IHIv$!zUK;rKE=iDf7s=h3_{PdJ3Q;jvQTv2y~;-$p1 zVt*mct1{%phxn1IHvKNWv0uk3p84APU7c`N?%$sKlYCsh(=(=3;_BbiI{OnaHXp?n z9^Bt<|GEb!N^s;y8`*71Bnv3Q?Zn=}|HxqVqWlRvKknmS3lh>0_zB{W)z!X`oglSqH*2$58L3`XRy1o=k&g40-Rq2Vli81X35!A` zVLj6ijDHD}{}NFqNf6@uYr|vt&-0rQfwaqAO55d*jmdNE$GdQc?J0 zj;I=mye*$KM2#AT#iO7i?lIWHl{^batpbIQWOf(h*r2tNXX=oYhAM*wb2H>n=InUn zoWC}p2S#_{o1lz2=gVP9}hHLE3NZ$|;-R9-|+`phu0#wM`+MbWrG<@HV zUQa)Znxudxb)hCsU4kV}`Ca3Hg6y$E<)P@}^~CAk_jT>LS8nu`Z8c!>^lyU|Yh9Vo z6#9WxqP{4^YI$9~$=oQbeAQ%YTD1vTIWx-5{<5jV9w=!Wys17IZ5l%jj{(h0c7Vf; zIU~|+&wt3?NJ^nk+v0Rl^#yXFv6-I=O5=yVepj2_A#^UjpA*O^09=TB*$=u1AHqjG zc|ROR=XSKqpJg?+E79eQWF>DjGuolc;(^(g47O{+u2V0xF5E>*w*iLy@rXA7$U3tll67Id!7;?8x~yL0XD?OpzZY_08faqPd2Ci&aD)!ppk%YEEB zH&;yB+dZmzJvX)jVhG|)7|gGt&og0nrZ;Ek)~pWNU^R-D?FU+N<^!{_mrm$sn$nNY zUH5Tntyr9P!X87+cg^t?^Oph}wR$+q`F=p#tNFH6urUHzhsM06_??5@E1>Cq`VNC< z!i?`%)aPHguyvyMmccew{-TymM#&PH3s+lb;rz>JMqfpNdj1-+W0Q#CBgdj0Ofyd3 zDP(sQiVdaB7c#adF@&kbsS`3f7=%SEOyY+y3`|T_bPY-CAi&cP zYz;|O|DKo{l7gG+{v$C}oxt*(YcVCuvzrMQ&Ar*x z&_8Fcmb6L){ndiJn1s`_vn=zREVraP8b@5g-@KgS}KG{=s1ai8_$=`rca&J`lr_i z%pM*x$)Qc#m#l@Oq&La!(T?^IJRFPRE;iI*gcv!Zgl4>{t>J{hSqICGqKy(rtO6qA zlR!U2VJVC?-)jjAMn#o=s+5kU<903kWCc`RCEhICM9Chz;66_EVn#}w@67^?(z8OH zH2r{WH5^~kk_rq_%-S!6lon)@^RcV<0kP}i6UDS51a5>42ph(cCoXQp{J)DdxDi4k zsQ)3-D4Ie}D;7doQq3YyD-M89E1II`=1Fn_U-lQg4TRyjU(C1Vr4<(W?1{fAFG$85=8!#5gfADZ4-2ZEl22o7-zZGfz zqW{%r{zs${i35r>_jpSi#xA!0*6++D)dM~Fj8s@v#w~|OV`DcbHz_4QyrJJh=f;{3 z>_5E1CO3U<&R6865B4*a2Jb|cGJhtY>UTzA9He1NNYlbEn35cugo#6(8593eHnnsU zLRI?tf7veKV@0be;TsjgY#@Gnu(30)i17_0q#UmPopE^2?b>?~RgXh@5sAQ>u{c-# zdb)zrUIE`3Xh-j6D!gGoSGEWJwoTa9@LEG!`a=X>TQN#sISrwelg|6$WZL+%g~hs7 zhm|efub(kn>AySdaQqHaqfSV0z#IX4z#Tg?+fPb^NnU(M!k3V8u=y&9vt?=ww)9^* zqg8ZrSSw`!{z;?(S-HO0`CS6sJh&hC^?FQ+o%4JWzI#PV#eA&fWBJ3CzjCmfTx3qW z=DJ(8;yCQo1iPa2K!2+Gv3vD}5bn_<2J@-KT zBk-d_{gwG$m;P`qDE(jXJ(1v4{%t3Ob$RDJ^Lj*3rS0dCrR~5e1WwT7>Ln1@r6u71 zes9SZWNukD-8fi(gP)it+tamzak(imkZj5M>ef^a8e6pF!7porp*XLn!oPFBm zk}i)I-sWKqP!}|)b<^V~L^K>Mf=~hIRYk?C z0&FFEd-+RIEwrPMmNhRIl?3J;WtY1 z=v*0rfl|q=LnZZN`^p~0vjJVe}J&+7fKP>^ZT#}4k6|b zh5ilmkD?n`0PKSemo2q< z{`q|v6)7@v9$m=jib2%Uy20|yt#b3!h$9X9K+N1T)sZY0WyyU_+d0WC=<7N(Y~G84 z%ev22EztqqPGR4EHFE}JXRbGh_ptUQ{>YH}z0wc49@^48cZBYObr51AF`|epPitgi zRtjz9l0+AqM+<_p3QjEIYsNC~pmGr$6Wy;XB{+fDnZga`CekN{x^ON%muDc{Q1AZ8 zU>Oi1&&*8xX*uJfQp=y|s!Ngeo zug{s>fse?<#e%n%=Nd^vsq5c~h+?)CXqmg@vQfAy1IJoOYmPCf=#N*eWHm{n zid7}q|4arXKTbsRRkCf1&Oo*nb8}lngo)W7ZF5Ymc)=^lDX2I0xvP{f=6E(mrcq~X z7w)U7-*j8+o0gpjWCn)EZ@lq!C^beXXsz`3tlm-5YX~J4eZK59Z|w3JZ&cl-)fZT00@v9Xai|Ga0cgu}xXEkc?5J`oU540~ET(b6oIfvZ*;$p23UxHv(FY<`ax};U)=RynKAu7qsoiQoG6B)IvDY5!*t{wR8GJ+G09jkYiAp#f__G8$WQyKnKc|6yj=}>=TXA? z#P(J;f;J|$;UPQwl(mTIjkxc2M&2$m!Nf0v=Z(==dSb< zeHilcM}42@aBHsVn5PZRE*AX&IJa8_avfnXPl}T>&gULP)UXiYUcwN(<_IFRDu^E< z_xrvXAm4uw^brW5lEFC-^VOYfTiG(0x z+wp2UuJy`Ysh(&~4INf)h*FC0Sf9@i+3|^T-sNGRx3|p(PuPNt7Lsxibbz9pSEVao z#(nv6G8bCu&@=nwrP~Qb0>>;Cs#fGR!3P^JVc$whxhOB?Zgs@sXC0-lpG^WBONmh$ z4L5~`9=!tOX)UyGw+oRU zkL!^Z-gtY^0l>YhPbKq*7Rq-`;uuEO5!Y*p) zdw#SKnac|prYd_&QTF6QYpY|}UNnE9NV4rkZq-{WJp7oklQ|+e@eH~vH?5D37pbi6-Y3O{My)uo}FL#?B zxXUhGc>E5Qh2K6Tz;YJT<9pmpD(I8q!-81FQduJA%Ac->xrumuo4E)-&IcbK!!-5R z?#8K5kCY~H)O71t9Qz=*lzsO~CbAwzEZLq-G4BNdExEg$q_XK>7n(neSkOOB3APr| z@dFrT%G`al#5gC_+ic}A7Nrx+O;eeHqZ$#QC;o8RW};E%LDB3NS&3{i;Y`EpC0IXq z4syPKpP+4=Yh6!j^I4Sdr%Y0cSOq`9f2BkYc6kI{Md@<63XQ(T19LZR$UHaIN1}oWA-uE(k2L@>XY8zgws5y)0gE z1o{5t)Jjtbo6eC9pO07iB7q}!e0+0~qxVw(c5e2Pa^LSU9`oAE2p=%FH|>V0UWC77 zS2|{g=Fo=!`2M*oh_zkl+TK9$?PF+Y`>CI3x|c$VOU6=sHqgTN2ciB;qGzsee!rmG zamd0=H&rNfC?ttZ^!Ka5DgGQfDLO$CemotOU zLt7o6&L*VGB*~zb&a7=6u9L4KQSG>-ACEv=H$%`{Lm5I#G?(YNw6B>XwFAPa1)lRw^ecyQ zj?x~vX5USRsVe>VD5m@bWk-_l%5}smafS7$pGGoi;!Bld$#kdVNV3wvStjQc*6k9Y z3AEor04BS=J#dzJhVH_gOv#X@Zx_ycaHras~(DH%Wj_#r4Zv(xw-ij z>5fSvwTNsCbnRxDsKd$T0N_e`ZK6rlDIy(?siX@~j2P@C|$GSCnL0|LZuj)k-FFU9(nfc?L*|e;) zNit2;FP+Q}ili^%`pq0P;4cDRai`FE!xb{`3j{^a5>r>|1fKJve8?JFQA9ueGLKJn z)2|YL&S7sWxz_5;1gBz0cRh&;M)YfNllOI)(#fr%$4zxz{lUcP7C=ghoC<2n8Q%$G z^i#sw)?Z2OOvxoGJ6%qC!JjSEW*~B-Cm~p6U^Lwx0VM?cN4Kp2OPE@Pd)h95EokF_ zy4+B2dm`U{s&n1wzAt9~LUcD2MrH)vx~>iEYeewD%7(5y5r+Rm)j0sy z5;g5QHcspl+qP{dC)SQ_Cnq*fY}>YN+qTV{@B9B-b#K+GneMLMYftSxGgY&?-)CSe z;1>s{*Mx2A(tt7~?t!9u)1T)8L*p|8&gTi1tnvvWOKQ6+O#+srxEjDaDh1KJ)@!GN znJB<(4W(4o`&i1xAYl8RbE_*?MatVg^E$gsjwvuHEe4*RJ@_YOACvYwD<%cnXCUY3 zlKDg(v6ZC+Zxhq5XfCd0?bhQc8Y@D3+@=s_wW=PzbvuidsF6)GMw+!#eWPcy6IYXw zDD=ctXjcsxFIt+l!<9`YA$a{{arB~R-OXOTR7T(5fRswDZRj>cu&bkbvFj7zHRvqe z6FZBn6OWdO6qGb7ON%O7%Vd!QVBxv#NL>CE^S>RBw0l-_L)B$7mXfcuEPKv(x92E9 zAkO~KChp&)LhSX?{LNa!jXBCi6Ynj>0@aR7^JlqsnKEKg3&l3!(V<2LLc!GlVkikH zjKn23##WVzsLDau)>=9QRoIVTRie>*asmgmV=0Q*q9-`%Q8z$NIcG@>CjvR3eh6S( z3gmx(w!cmUpqvQ4_X+bO1vfh$Bf|HlHsWop~%K$%>|4G%= zF_3yJ!D;d7m>BPN=P}Xf#8H;l57C(kHblK{Xbw>E?20e{zkB+3xuM&WpAx4QQZg&2U@b zN<0D`S@57OLif&T52VDVIb>N%Q7T8%TGoZ7ZvrpKkNz|UmUfD&Y}5Og4AUhm#cha; zw=jG&Cvp!sa0H8^2bLoLXn5klWmR}%$iDrkaNaNYB@N16-^?E+f?2^AZKmw2ql#F> zwhfj&I^Ab33S5(g6|qsNdX_3{nZ_S3)^jGw;sj^00Y~HDvQ`Oojj+}-GecQ&+gYcn ztq4%U+V}pZUQJe20iEO`_kDvWHI>;pWif7XT{>ESLM=If(tR*{itvGIPJ4^=@^P#@ z{$pJK)oYhUw1=PZvjD;~0}_CQ!TtI5(f;>e5SHhI!#fEWU=8@@C-)d zCqP{DFV@oWr*z}8gB;>2M&TQC^Y^4a$6Rg^QxUZ!nW{Yg{Pmg6>|l0wojVawBTY~-G18MO$5x|;P}%}m_J4gp+h&`z$zhK~}{AjD_kuK9U$10LJi@NdMT zwyGA3nAoY**Yl~KRfEbTJ!XH;=N#0a*Ai}`lhcP{u2zoBCe$?;iP)eAAMrf|OQ$cC z6>eG zDo#w2QmL7&=6+gEh&k*;r==DB4dsAE4tcFIq|w&dG%e7Bv0f%*aXAn}s;g8Ng508E!l1`%{xdH-S$T$L)DVq}k#_#k ztVTBh{FZHZ8eJN@7GC4CPlj-OVeUnTSy>b+Rq#9yVprYor|!p>AN}igR_O ztvNZYIjNFI{!>you*;+eH}Z0r%36utgFq~E=9qst3VU7ucN4H(2r^vSq~&fJfZwfG z-@mx@l0>tgzhSFc%?-bHfndE5yiU;k_ShzGdH7Pp7fY3qdqZmw$}}iTN|dH%&DN)u zYXN7eUR|NdVv@0)6tqt3yvsO-oE4IF;gOmAgEDwKY${uHJU6~$}96a0dn^Bfs*|Djg;i*DmvvFLuppq1gCsUqL`9b3n z35&SCIdb=>n4ClN7#a-Tumo*it`ss_!M?WEM){+7gH2axA4(JFIxKQ1S8tk5hzL8i zFTxm$#XZ|dv18Ta*Gy|{d>X*aG>H$n-C)(0)FG13R#kTD4&sWGD@_(cRt1Nq2Irza zYT1RY?LkYItZ(-{^uGk$DRQCTVK&S0ib@|~r1yftc8S=`>n%|yAN}-LGly=Zp z>kb25cwAqk*)JZg2vU@5z#p#&{((dX=XjFicp3t3$DiL4fb?p|`OUOlmA-SyWM|l! z$JV~Ei>Ot`^*y5ie{Co+&cFJo|8W)}pu{e(Zr{Q4)lB3X;x-yM%B`%|8zHFE0<|1v z1KL*mo2CsB>SNc{8ix%LgjO4ZwOWT8J2z_D`>pn!`3Kz~hwQBlZT0b+(($7_B~%-A zy5Q^Omz46#sj%~HQG0$7lJ6?|wLW5g|B^zV`aU;jcXPIdGsUFUW&@P(NVOsz=;X>? zG*uRzJPFk)BHx%ZnWTELX5THiR8QGi`EDbaLB^F2+owkKQR>B3;iN)kpYm2$ng+Lb zrKE)bx)>xl&`5=pL-wIAeDrDm%krgC$3kbN_r~hlw0EWT zm4){VQ)oL(m}p>spEvM&GSnHOtzc9Rtx<3%a9NIK4t0yV-C2J!3^tSm?@pSa3Ar_f z=GzSTUl&xlHE#+^f=Cr_>xf^a#i$AoJ@5VE&H@x5#{wr%RQrKEuMAZCB9Ih z;)S_5u)`5dXG03HxNu#x3n@DEbFm%wQyC8QM_n2Bye_bVDFL$tql@?!o* zxNHvH?p?H1?ze~Si?)qges(NGH2hmexK*Mbiut=Nwmq~uEt^rI#i5!o6v6TUF(su2 zIfLc$UgTX*W_FH1hGA_2>28m()z&&4g$;$p@i*2Fk^ry+356vv0we_Si|GUi3abJ* z5N_$m>rRUme9`%V*!`werkSu>u~!9?7Y(dx#}On`UfzuM^Tg53*epuEo(jgebaCbK z;S3QPz>EIe*bePzM@o7O<6`Cyxw2j%83$T6I?0SO$t(0l_IhSrQl=`L|(ZJ=r{6Q4BWu)Lke?vgUhy4IU#lHta#V_Ak*71Vl+U2QQf|-Fn&Oa>&%CyVL z?}N!fnKA+_|AdV8R%W0fU!J%G$Z<~sK{M_Cdz*p%73r_Pa05m7pL-6Jm^-k);;%@H zUr_&Xf_j7z#0390I|l{9sn74fN4Aq1Q-dORUL|kKoslcZ24*b6odFz4hrVwaGB5oG zbQSfRABooP~>p{OQ7v631tz*)XqewIb$V@15{#t*(lFr4^F?b{ZK$2_aqFI z*PaK6*FL95=Q<~+2M@I+0K~Nu2>i+)7-k~~7`5e|2Lb$w4G3mK(68O+Zx#n)b1tZp zqnpzvZVoxZ!vGf0Nm#&-WvBSxsdC7tVE!Ao2OarAOQS$20v%Zu{{)) z70`bIGl(-NkRE$1z#e=5J&bz);h)3YiQym?I!xKcLZatPpG&<}M7B3}UWpcNo)z9( z2+3ayL7sfo8J@=jt!q#V88x{$Uv3yRoE~XlnNnDf8Jr=Kt$QyxObmE&6`TC z%-AJA!Sm#%Xj!V3Fw0l9cba#*HqkLMa7l5SaR)#mU5cFQ))1Cjz4-R z*hE`IPLqR@~sCZ^6ChPZIR;THha_hL12k4_<5c`do(vKttU$9UuP_ADAAh_ zvq_Z7y!eMsz0AF$GN$2aMzN>CN*v5 zbY6I^@N!+eT!@~zh6c9?@XAfwu^$KD&*dM(haC~r;_h{9jOm<~BVq03Y&dZvH$EHI_;`Q)!2*4{OQm8)i{+!#|So83_kifS0LWJN>mV22v^41zLphvp|!^ zFHy_Uyi{zYvYx6Jujg`m{MPGe!7|L4OJMPSIzETbuZZ!SH|B-0T77ni6^ z2BKLchyYp=S~_$T>GBSI8oEuAN|g~wWV1zN>|E8q$ePq<3|aALW7Z!4CQUR98CSMR zvyYMi-dNV0<)n-`&=CkjV<(qj-(b_`@A)KedDL`C&97co1B?ELB8G~Z9gzq0P zYM^2+@PFw_uG3fNq$drpzyI4n-c(-p@F|#4g1tbc!$Yr=nfd&q)N3ngYyXCpPCKcF z0KCkFv+sZxVPyfpB!hxl3-sv#4RGOuw=aVc+X%2vH*g~&C&YOa2$DhOlO?QGRL0W_ z+JaBNtP99JZ}S>el$7PHL{p;l6D+E}Gr@FS!lXgq<$yLr!&C#B+uE?c&UC|dI~ae| z{XOAPNB$5?a{)yN>$K1`XSk`B1?r{|g!mnU?s#ELI-8T`ZvC={MU;q>CiA$WS-5zW zfSHuL`~>nlid`DQpm^~#OL~e{?za8Cq`FlJ{0c=hMjod1^Z%f&dSZ0DxjkL|<>nh; zR1)K5#w#oU{9^yXjpm~H3p+_`V5gyL%srgTa?XIqSpxPmz4r|9){5t$7D(ID|51^N zZ`>|~cMUse`fY8=Wmbz=4M_Yj`$!02d3X*PXf_$BTyOY{9pCduWz8xl{uKdi9a#hD zSYRu=>J#-P#S$d@NK~B!hfc(UeY_08W@6;e8bW~*>S$o-P)oH5AZAk|!oOb*R?7l#s+;@azGZ>;`U#NjEt_~l=Y^Lp&z&xe@F_aaz9v3}6XhK$YU#|06hM!dx zc=#1|>=W(qG^V~sdEOQ!E;WbqL@)-Kn;;UH zS-_fgs=L8B?PTU>3Nj6rbqcwYYN@x)Jq@S26}4(NX$xI1{VR7sZ`tcmpb8|PRy~7?L8b+*^B%b}cw=(txOh3H zz0j^-Jj?5C2`4&eDJzM)zeu2-D8QOffsi7NageT!-teRv(&ZJjQ<5mOAkXFrA}BFIViB>^N!Gjv6%25B8FtwBvJzpu*YR zzA=IQyk2d>>Oq!IpG1a%Kinr+=g2Z7YrDkSA`w!@>g@?B^kYM%KNF$0GDx#n!kg|L z_`{uUdg+0%2Xt?!=MYks5QqpQT>R^;2JtO1wz;4#$XSL#k9$c>4QT=kb6hX6q!Jlm zRmNAk<(h#wayWrkV{Ss-3BW;lB(`8m#QDTClu;idscl>Cu)Xq!&VC6`w#hiD8a1IS1J(u;B!J} zL1H%JhsC_6sJaPlXcvkL+9&$yM0xdV(zQF4*_*9kK_JtC{6Ogic7T2?JA5+fKNe{w zy83>?XYkfq{I_h%AA{3uVC=5G)Tq>Xrj17e2svw=>WA7P>DpA_Zf>TX28iP8WUFx<7EI z*UiIo%xpVS_KWr-eF%lUZiytx?ZYs5rJrtnXFYj4jmY*phW(MvFQxs=IWX!yI-`=p z$JF$J0+`v}7m8{~buh zPQSj#Bm_w121x369*kTni6rI7A}wtf`_{Qc&_|?{a>FpwLXx4)i)@Y>UcFi~pjCBF z(zQt<5zMwc`w`v9Qm8=)-kh`_$ec7*4H+m_jR}e^=Y{PneA{&~6CbCOWt{m~Y4Q0w z-)+;Td>f2s0QS|ikAjo-ME|)0-sbeJySw~X&U}R9!{x6C*!_LouTJu=PVjC7Z5n=G zpAmI2BTj1|MEHzmhAb>!-~~IuZyTuHID?MaX_OFmgBgwO;Ad8ZZ~ZH!m0kP^cfWpo z77~sRw4PmCZ)3!e?jB~MIVIX=?l+99$6ZJKnhpOSMAQiDBs4u_#T+_ia%HkM2AFsT zt=&nP^G?kkmQ$MrQK*uf)vfRkiBksJV^GPx3;`*Ns=oOF!`$Y49zKh>Ou-*ZN;2L+ z5IehP4xV>$y9j2o_VXw(mQc2xMyVo{n$=oQh=txRPv%^19(*?GTW9f|L&Pmqij~|W z@9IXW3f}#1vC;p6F=Kpv-X4A!liS~)&C^NKT~)qamLC~xK3VKOQ#kzNN7p|B)NF5; z>?qRzDDY_4s~*+~R^EWX)?Mx){3{<5d7;=+%l*9kqH%%$IqIQ}b^oxKW?S59Bhv1V zi+DoA4S<4Sl>as*Buvf>p3mPmx374oWkuw`R0lkC_&guqHx~^g$f+$|tlg?YFflAuksY{9Jq|A)T0?%#zB| zT9zY{lEA2R@cQ@vl;~>ajePwa<@Hb>BDMpU1D<*Eh8=u4aaK?pp8b5V+U=b*gfllT zVy!#23MHW0(&xAKAe>ZXoy9Pb!~&~!wayVkrxs%|tO}`ume;C^j7sL6H$k3%WXZcj zfwHZ?e~YoR-ldlb*B@#Bbe*U}FSSAb;H?=&rDG}TyV5VWyw6w5!w-T^t?+8f{!;?2 z5;vVkksYRd{_<}3&^mn1{Ij{zt1)yPxe}jPXXUV1`+$*Sos{|0z!2G~&qZVHqSYav zewC9hsrVdc7#}1=s`j_hv%6iS_kv-7%6oLX-usRsk@@C-xgziw`9w&Wj z+Z+xvVXLa`5Kc#=uFWiE=u1vH@|^1{U5dV-B!MY@7F_Z#Fjy)n0lZ3!C4Z-V<)q03 znd-g~AjnKagm zdlnT#Da#P=q$!KDAy<%h` zJd6GnY;t5Bn8=pjw%~VlQRWGddli4sOfSjuW5Huj{7%Hy@I#}*`c#*!J3K=a+$=wn zy;0ZYIx%G%Ih50s&A#wg75o&)M?vt2IBsGE$B%Cucj;q5f!IJ5hVVi+2QOE z`35RJ%l3)VDm82k_0+;S8ji~4d!x9tkzSf;yXVLFP`#W9w>quZ_L5Y!^4q0dyHm45 zcj>xfWmN=iv2`-~uqY(6i?-7SRGZRao`{%pi+R=fNw?;A8Jpzru@AuH_~AV<{q5>M zg3=W_4GY==gh*PhA?XZL_92d?CH=wK$L?}eYhF_A8LY+bB=0HN&U6&rnMFUP})=Gi!<`CEZqD53)g)`C=s{wt9yKmtq{j zQuqV9(P%|EtVw&e2AiT|KGOjkjQ?q4eMpcgCQ)<};Cf zbLBShS#0x}t!Wf1RIN1p#J#U|F`8fv zZCH?(2tOKNSWA#g#+L8&DgV?!*%ZB8(B+MFbmh!5p<4`=Q?Iv`6D(6#DVO>)&8Y@h zGEY1fnFvKc8r=vFEd;2% zg7A(DmG^b1#1|FB{3GjN_;{8(pZ0j+EMonQqUccT&5QG|uyQ(>gkMBj*CqX|KtTznzbxf3FG-2Y^~x)f1q4lo_cV`zkINu(_2D&6V)W@RXJHu+~3y zyOV9FEatbY%q?7i_si5?hEF_cN|1gh`YNS^W%GW1J=_&H7u)Lco{QShCW0_mimpbI zE|OgD2wXRVqh+k3#^0CTPOeoyhSV_BCr6!GN`%N>n=3Vw6je~D4(?23U5Ikd`ZJ~) zvCZ%E?ic>PG|f&$f{ITHL(^NiOq|Z}*LT;Y*R4eUmAcax)yZfrL6_QhE&mJeAEU>e z#=g9Ka49X=GKof&k@)c)bQVi(51T#?~?-eyh zxohrEgIc1<5LVvMiE*dRqi78Sdvo8N+HZ|kCaK!K{2>kAy010GzI)tyr@-=g#_cAsv|yg z(U+YVyQe_)mag8{ypZoLP>gQ}k!P=sdjLP<`ueUDUkhczc*{;lft- zB<1YqJaYg!hx9i6yG!~S7;}vV^)r@M;xdx>V}8U@TM zI2rAzK`cc+C+m{yna85k!@`$C2CDH0V|CV)elfMnhn6ugUL1KPN1tT(; z{Q3O=%B83$tu)?NS@QcfDbh2;H8UvMfa-ewR3Fmq-3D2=8qXJ_wS=kQZRJ62Z9TvvRCTGvLHU3ig6C@~(q z7{Ly>TSdj3-Rq>Yh<#t=JQ%G4@E7uHYUbj|0hnQ6c@N!OS*huy%Vd^$zk44n%@HD{(yGr3X* z4T;AIi%U$FPnFF|ym*Zm0Fi{6%ToxG2a+|#lB_3u8*l6Of^XSnJfG|n0%*C#$U>XZ~Sv;zd7`BFL z1=h9Ape?35xZ(p&ggWdhx_M&^@%<-KMXkcsp`hCzh#j3ozY1*-2#Q&= zsJE>`h!ToQKn6}_l9V2fC^h%$%xI$LpjCjs8X7h9w$&eV)N%_;*u@#M2UP|;(eU71 z^qvenfUKxSZ7>aGb;a9~G90TVoA%7^%UV_LHgtnTeh7My6@jjlbPr%9yA1_YT@*r{ zumt3>y|DyfpD#hpQf*};=`IY@T1vgrUDLUbVxL`k{q1=4JUZc}!TYK6jPgO~LVq02 zcR@@|M*8Ib8HbjooRcOAJF!c?%Tpjzu3Rw^W*0dt*ER60tzSnfOhvatvGyy9rW^-@ z?x1QNyTz|^09~Z*2)zsO9q_gyQ961$I{&Yp7gRn8I$NL<0Fb5In2`UwmyXSZR<~CW z$Z_1OF#T!5p)?P}4$2mNm_Z@wBmX3~PivBqNPoDfZ@N_S7L()Fafa~}G zl#JsZe|;Yt3L>}-nT~VXv&)UVx2QOG?16V3Z-(F^48&b6i#u)uDyB=yw`L1@f;Dqs>{$L zq!LpK!5GMb*79e`7dO89;81_-vQi}lVP)&A?<7z2DPwfiDkq7wwWH68)tMN>NzvL{ zyWB91hcVkTe&}Ew*j&dKo^mO1`Y53jk*QhsCh9a2%dxx48snXi!KL!v%h6<-W5c`O zf+sEM$`5gIOLuVgy~Wv>B;QUyHJV^TA!x?V50puZVE&nH5=WZsm3%bzlT>EYwozQ+ zF3-O9-5PioPwB>A?E}Ap#F{oJ$hqfGNny}r2S!rD>OEgsFd<|WN^@+k5Yq0)4f;%XUOOI4(|ch0YxuQ~b9z&E-fkkA92G01S+V zmezb(Xf^7xSATULGzSR=Qqg{(d|DF2;~NEu^uHiMC@iHbe1H?pyP+{t=`m`Cm?n^U z2;e^jC7r1ZEMnh#Z(F0>$934MpG=FP(U%ZqdI%GjdNOG@jjnxr^y21*#h=K}R!c_C z{Z)+<)g}%OLG+Ov_!qKVz3($R%>`De(o(L@rU}Z4OI^bJ!V%OwHqkW0wnsY5C3F=|rM;e9t)a!US zPm6}_9#n;4#VnTWu)3WIFl!TB{b^RhVGolAA4V(jIV4){C}a0RcD(8R{zhF=09^Mj zc{1cN*|q6wvi@WbUxPtLECkH|D#VgQ;s%~zHmV`1y`Ul*+@Kj!Qt{|!we z@h4vMlLdNtlF#*4^53ko?XSUtu%8U}i+m}*JzQROvE|8#l{OGiZu$y&2|7qwM$XHJpzPNiW}A#J>8Q;?zBuMk5Apl=WY`GHkJcif2&4i2U5yh?6wXM2tFqjEy za$s+?dFV~N;G5VOwc6Uq5Nl_Ul$o_61H@xv##buQo2}!vaQ6DAyHz0w@BTKQ{FH@M zh6bxMAL8fP=i>O^5T)He6d28Q{#ZFLv8|Day$X&49Y5p~C76a6RLz#T#-2~tkj&^VqgUkfGrinB?G`ipv7c9X%&t673%z|FC%_-^ zR^h8(d>k(0ka^+}L+%Fc=ndM@D=_2{(L~-kIyBjMFx{j#9nhJ60Z9TaL9t5fkKVW@ z`-^9^pT-5TZt3v#v*_((COaATNyT@&TFz(R-%_qzy&`Y@ykXvwJaHq-6ZUbAkIPMV zzd5fSL$f)HnQILeO`CLOd}Az>`1dP$u|bXN*xOtC((R`7U~-(8NE(VRvbt=2sDS zgs|7YHCpQ}udfNDH{(fg{T+of1~p9S3j-5OX~3$Qw=!Is>D?v6bbCVLbS}Tov$7&G zPE{@Qt701ebsvD!;@vR8b)txd3tx8#Ekqd0lZ-kPOHR6)x~O_z5@Foe@RqEf@s*xp z>%+@m;Frh!&sFQN7lm*!uOsG@4?BeX2g#Aj3knqsgm%4bE$?d;R<}O|&W*%} zzzbIqF&y=w3PtBkY!4o((9F^c(pg$SMN!5~$TLnRym}%5Gdimd2RtRJhFHLzGTooP zj8^Q3UfUIzO|M>VI&74JV4^3CH)*o2;1M~e)ecuc%m6)hj@w@p|d8HN3@^mYE ztjnXJyK|Pt5@{PScYq($@;P^MH!}hv^xK|NRT%-K@8*b>ZroIZC;LsL35Z~5D9XQOy!P8I#XV`bGant z^+0Olmz^ku27@eUPMk4QID;UDYPdaU(@y`6VH|0Gn>D6};v|_gEywe%bx&fo0p|GoAzbNBEn-zpT>1twR#`6* z`Iq0%WB2!Edc?QK=j2P0r)-3C3clJe7#?u9!L%(VAxY|&?VlxQ?nf5ltKtjXcF9rL z@EJL(4-nfw6@DskOv~4je;l-ndMG#Yco6Gw|KjU)dV0g>Otkr>m=2KiT;G5LJw+oo z9^M=s2wB~d@uVNz2ivwKFPpW>S3ccpRkO?us zOGGKQ*#gW>f#`^9Quih+orGDnZvCzP)EgEQwf*@AzZoyYCaWbb!ph84cADWuiBo5P zyn#zgY@?0XF5C|K`-@z?%=ql|-qiQwgP+LZCLh)BHiyYB8`8}F>T51lky-u5H3KzNi4|3ME!(tGLmm|hwcqXoSoz9CHH}^ujQPFN3;&^;OJ=4AdCk1td@~>M#q-~hg=gO2pxYh_v~9` zX!H@$V_O5E<6C;fI=(ciK9_z+`&_+WY2oxsBBOemPD8;;x#H->DS`9>*P2*n*$w{o zaV-4I>P&3cj-GytuFGNxPq>OkP_kX9%h-Rk;Zk$48vTgR;c$4@imh)J?2Be5FOX0h z)mrJJg$Dt~(1I+Lq)|Q^EyXD{CPXQeyI6c9GA};oexb3(bdA!Qrdw#Pu9|{pEZMXB zWV)1w7TqPs*VQGZDlm=WD;Cka_QC8fz}F=;9(Di};FQcc^f%>W{Ly_j@rkSFkIdYh zF=!X)HycF^<#EMilxHVOyMVTUUmwv@%S$+-U{qX+$2lb$D2!2x+Fy)mc_60y43*~J zoBRJwJ)El3P-*<@U8H26fx_#?|7OMf&#w9MzaJRR=G>bz67j9Kiy)@+?#8n}71w^# z<=&ekzHBDn!nHpY{-3=6e_}sVqGrv0)Njc4pU0ji4T;p}CX(d3>Uiv}M-L(7R}XrH ztM{CALXKrLsQSz>$u^YsRBD{|-yN9{$UM;YUp}asZ?w~@x{u6t^#~1BKTE+G+U+F$ zExU`bN(+~qt2I$pP0)l~>mZR!)qa9_M#$8A=C<+iFOw_?{^gw*|G8|oyx<|_`o z)~nZoEve6~@CbesA^Hu1+u^oWT+^Bu(1ZGGv)0`~%=8}R?2sZ)N_OXRjTns`nOw|X z)hS>vj;+dA!%=m$Xzy^eLlYmG`W)=-~mJ!#>4bIMe zWZL#SMXD;vWHq}5 zZE{frdyc=t=4RwW(%qO2gww0>9eo<{bES@GXGt@{C0$kW5D;D(;Khnrx96PeLcMo{^b&Ah4E+l z8_u6fpd*8+n_wR>*)SUbHH-i5u8@2Noa-v6${|K;mE=`#-AejC-)Gc?a)i;wdZ}7* z_qCk!1}+QaH+)s-hMlz&Ik6NNhU7*nRyacDe^LWK!F?2y#@sNS{WWUYQd;>_kGYKI}I^&yM>j0_%6;>azK zG@X>s_;`| z>D9mBE+GtYQOtQEO0Ig|ztu%T$jW0D;l~2md4qftN^r~}BN)mN#oRAdqQ2He9wO`}IWz zsLy>lv^V~NbwmjX^@w2tU^p4FFbTVJCja3we&@|uu2xkTSerg3Ky3SAwK!yzeG%t7 zB^tEk>6@9x!%a`)9eQlfg)cO#3Gt$eI0|Eiqk*Z?z*~ zU)A~f8kPatK1e8o4y{n@91CaXAIqu=j9jqh@@}^Pm73szO1&0*eko1K^3HKKaP9bc z{w2-Zbq$5AVML=j0oke!ZP!acG?hW{iv)GPg``JSkag?zg~IySU)XkkTgyk9n#@Vy z6b6@s=7RQ`!2k4#Zq+!h9uNWeRdEE z{7<5o+L*S5m{~aN*%ThEOCj1Jh;s5mbd6AUb=s0RV-yawUuUHq>Wue9R z>{32H5aM5lgNxf$-|ug>7}ZdtCH*ypLmwKwIQzda0n-EmkvanQ_o64j`yW9O$#p5^s7)aAPK1C7`2 zZ14K;Rqm3xDB{TwWa6LextNf=f7?2u$?TV`$5kGum zmlzFPP;+3>3!ssWqJh%PMRq*yqx9&|ApAX6>s7T8sM}giMkk?L0dLGYamU@B@aoWS zmikb*s8X#6xvt>cFZB_ideedPE5U1d5V3Usn8%0Ui9i?kBUG$9YD1EdTv^N z+%<2lD_P0+^xhnBoza~q(zYKA&KL%AiM8S~ksB_rmps-lPLNWH&2PiM4ptPcH6IT9 zs7kzk0^dNc@9q3Lc7a0d=PyL!R39|K=}F0?+@*?|(9KE?IF@nZ;JgdAfv!xLh?A^L zcU4ELf6IP#Cw%16x+PxvYLvg{5ioMu!K5RLTiQ{s#8In>4U#0Dqv(~iVa@uJCl11<+5)Ks-jqUGqTHhCYmQOD#5__yhNfmJ zsn5JUgGQ|Kj#ox%Ak-M8Q!+CIXn(mpET|Sg?mC@Fj_{wWNvTUnv0V+K@02` z0t7@JXFy4+XjSxs+)Xi<^_MB>muEUAyEo;LxK{F_OsSo%T*q&5YlZA4ZBs4D7$$B1 zPSFS^%3R&|z6wZp?q{Jbjm|~|)O5{!2Oi3ml-75X>wVLkzpR!HLFjF4`}SrqZ*jd^cLe#Nwuz?7a#>{ z&`Umb42L>2Z7Bk|@enHv+Gr469@`Yz-3OlrX;}~mQnl$P4vG+Qa9I3Z`&b~5sMqeI73C0CV6%+?W zT^0q3tmK=(Pm5E2!S2Ll{xN>8PtdbuP(u0Gsbvu&@BFDu@OJ*2;{)Eg$BxV@mx#Aj zCUw&~=veIgxUt@2{%WzVJe?^3S3y_3EC2f$M+o0>!4i~&6v`C3j9!J2cemgdxZ%_QD4&9#FZHhY11k|1b(5bZ10#dF8G8W>{q%dif?%quXq`cJd? zs`W{mIb4P=X>*{uK4c97^@CA%n=2H`0$EUO+ihoF8`gFcaUg6LfvP-PSs|VgHiQRkUa?x}XwX@n4DkW7j zf&6?8Ylx@RHtKIE&Fc+DO0gdu-oGEpR4o(W1&hREePs3qIzs5m}&E8 zZPCT#V?mySw}Y z#m`_)utrSQMg_0*FEt+(?Y`O~CKZmP0L?(!>@PXofSfv$l_M~YA@xSamY^#eo4^Iv zx7AFB_L|f=M9S_#j%yO@gAt1_2=h8`kQTcpnAL0>Bg6TceQl!HtRw?bwl( zum@t_dk=3V(mekf$q%S;|LslXCs2VhN2My*C#vU)jp?`VXLk=D&&Y!DNc$ZiHA(`DR8wJhp4IYHh7P^Q7Pv<2xCnkTWn2u2dl0s1 z56B6Z^wKu+9|6k_ZlPe+XmIrsg%vUkR&uDLk6L{6G)RX2YApAf)kKtd1go1~DS`32 zz$E0PS!*VxY$MVbmKb%^wd>y|)L|;_=W}>Ywbh>?IdxJyHnpPs6Z;wO(|F|~>@cAX`G+y=^A@6h~Tq(S=Al`50k}oy>DHUSZ>=9_04G zaW9LKs`40v%8Aq#2{9CV*~8n_gB$seq-lg;lZHpA4ni%ap`Y8U-KqnP?1Q;VD86G9 z;)t;)Buya5hi`V|9mk$| zD4)|3?ORbzh?y&?WQbneuLw^RLT*2=#suNj)(NANO6)bNk)-+LoCEo!C1`70N@Svx z;oS!Wx!xm4^G9>{M!QBg{c0sFch@q_3x4K3LZIWK34oQrQdOQ$$1Z=`T z4Ub#8!Y`-wf(U0-jlek8;skll4ZFp$N4tqHY#`-DDu~Wc=IvTZCgJXA0J(7uKTx7h zB1+XlyxA_out!79*MKg_Qp1#SQzG{*zrT9%qW z3=M_4UaZH0z}Jc1nZ*T2%6EW#=k=I9qU+?Q0njWT;GC}Nz=~h;>2wX~1=V8VSML}{ z6Ir@LxbK9&sBsF^r#){HD!)xOG3}3R48I&ch07{liR7B&-WWDK)y60s%+s>W-8I;+ z7&)!in=8|t**gmO5UX0?zHKuH%wKzo!mH~w?~x@@b)Tz|sxMid=Zp!Xgfk95A&1@V ze&wu+ZMn9Jk?fQn^=-_QF=?vJ zjg*nne9R1WJ$Eg+YjJ25eBA7IM{RISp*0#V-r*9HbQHOOB+rB$+Wd;I%g(skdLFN= zScCT*v~jn&ZT->OjValKYs8`_(%49t`!3A>=oM}oK-C-JJXZpItju&AYlS+zgs*Gb z6+aQoQa%xC$05rt!UB!dUwc~Gk~bQbZ?>T(vJwKqose*SBHe~lIQ>yP{@z7#<|J)F zi55g+ld@W(ss|_fLQ=y8%_8?-$haWl%gM2h*+dZ`iuU3{QV51Y0x1O_Wu8)NtR?Tv z7^W=(6Vf7vWKa4RZkkayFI2Incc(r_qrkhH($Up1UZh#<;N-cJc9<+APIaSCw567X%=G*yA<9Lg_M>-@cX~tp=YY2WSmmnO zsssN7q)`xGS;m0j{-5YbCDtOTn<}F#5}&+RslbaK-g4nHQEmX9=2Vr7YQzFolfio& zqaa=VUJyfm*m$gA@miLfOhojLC=O)Sim#_zJP@E!1q)oQzSQ8o?!>{Mes?+VMl#(I zZLnjiDRL!k+Pp9`UE5#RA@M1u$J+hA>e(FNDA){ZLpS_T>M=DRu~!71vpgJcEyAZk z`_pup@N8Pg4LU-7Sa_wx0Pjxe9Uu#$rh=%!;!}~OLP#)(o2(=uJG7Oxx|1WU2)o=6 z_bDt;2^LdnqtWZ21{T1SfOUpmvs$7Xg1Io8@z@0SsI5%ke-u;SpiC|!7_xuSWSf}= zOQ1o>o~mlA81d%@9#?Et*B4v9E%N3I^2tw+LZIu+*D5uKsU)RWU^+{tW%4iiR;MsL zaQ{G%OVM;=O)^kDJ-XlaGcWC~TTtpU167!6Fv)td4ya^UfLpCjk0*a6>|zcgH30Mf z0s~-b4&wh228O8h-nCn-)L3jL7?-O)$ZRu2wLPx<3ho48C}$jD^V%vSDZdM+9vLW*K6I%@@2qTeU_rHp}-(>#EHJWO#3)MLbS%2d?jW|n0dCO_(`8mpyy9*UwZcCoQ&>&WAeKbb%Vr&gbM|34A=xP1-#->z3o1T&^0oeBo$E429_`)6>y-o%7 zVc><4%{(p7Y!4_!8HBUo^}JN?!eW@MMkvU$TqW}WqTQ+WN5RO7 zoz&CYRKWO~n#`x;6p$ud+v7>}?PfHKy7zL~G-Fiq;iY$mB-+FgEZjbT7t@&cI1k3e zvpp$B5$3y2R*!hLd_4!C>kVe0v!h0_JKsgN1g7h9gx1-PZV~aHEH3>&EaW!`HCJsm z_8eixRyz$=(k9j7_bh`9(2=4*H#y4A!k+lN#aaSN!#p%?B`FEA;&7NoFo&X-+6lI+ zX73i*O@-P`78t#RD#A<-+6Il!zeYkyHFVW+;vAjx|W}#BF)VLOdv*w3 zl;AOWm^t~^oW!~B)|6z;1~tEj5qJ~S@A`TC?5f|;?fOYL1uHH>(j^YVj=C>W zq)RvlD;62+5QSsM+(bLlC62(3dUFqv-ggdC{PU$_jKvnwrS>Z zi^3CA{knZ#U(dtG>FM>m_#n6mi9wbA%`*F&k^3$>y>=Oi+p4Wkny+E(W+ZFFaNY1L zNTe13bTj(|3n(%c2cSmsvO#Bt^@!Zh-g;z^O_YPqds31L(`fv`(WHV-MF>{)=^%>k zZ;y&xk<$pua69R-F=&drGpQhG81_Z@8#Cl=<6Nzkz+tu{?R0Wm5>eS&mf-jlz8Oa_&Q$^-eCW)xAL{{c~S_ zAa@F#?~{LsQ{+{5^K(Am=<9zze)4iYUzKO^Zg#-w;-8Q35T|blOeSEw#|t6pH8{_U zD1JNCJywU8C)SmZ$EiQ&v{O^nG+l!aRGBW8RKhnLe|Y1L+;f-kV%MzO25AWNvtSWG z1)j0hb#rP00{DqDAD1xrJALyMZ##V^_svVhw)d3kao4j|XJo>-BLysO zr3=A$W>r0eNPktUKKfiB;q#+HQ(|_u7UqY)^cWgw)OuW2>@=FikiT9s!RF^+6j&q? zX&oW2G#!ZtLwss5m@gTb?`5sfQD$otW-8cEfc!lC%>}HN^bqe<*Syd;L)Wwv&Mlz* zhHU?7Ki#_<%40N*^xg4!lLsxEw-@AETT>yNJrB$NH0bd;uuj$m>6BX5%60 zHlaO{o38X)4lWK~J^XpfmXQXSO()dNaj4yF`VB`GmjmsG@UQWgGIvCt?0~LXWg(-IX1j;`7a`5RMHNkP3Lu&7?$_k<#>|4B02w{UZG zUcOM2AZsbT=6Ho{=uQq+7X?!%)CJK|r=5VN3@X{as8 zX329;@|1Q(UVQ(H;F1=dbhZ_8aP_Vdc~Bv_Fz zQk92sXSU@#XG%52P-hO|sEn*xcnPk){|@WcTr=N=D*wj^UjO9^%QEw#QNO zQvJ6WGYouVB`stb*&7s_c~1?)V*YT1R_0<0^)R_>>*f-^;h=7> zOsa_rFJnbdHpc5vllYKMZQrLVNbI0R%RjcDcKsjx2l{Y-@E#Z(LY)sp=?Ag%f&{U) z7ANFJzB4=g7UDSk1atvOiW^k)F6bKjWf`#uQy!7i2IQsF6lI4<2qDV$#;;L4i?@Cp zvaG`rpRMgSh5P&te{5vg=$GOBd{%tE@qgjJUk7;A@;~5j1AF^_zJ&fS?dK<|R$3fS zW%FwS1-Sn={`+Nt=k5NCmIuE)5cZ=YDV{@oE_K26fvBx+U|<&d0y$|)?+<)Zw)^^m zIQV?2wu2+x%EQuI78r5Z!9K+c-Kok{6Z1Xzif;>03}YBhSUS@M{%lG9lpnL~TfE_D zzQxd7$~)7{OQ9v->5gpXwfpqFr6l2|4SvQ^vWm0Jv#+y{m)O$DyRNcNTm}mB@@e)y zb8RTHT`v!_7@Bq)-97w8SY>NXk$H`K-;To|`_-t8I%w*(U2lIcwIj?l1s$ZcHqefZ zgv73d=ZF;cdPiBi0%R>M)M!Cp+7iFA$dB!{_nC7ye|)V4c_iBy)Ay>}jEX zZ@(AjtpXJe^znh38Urv2K2!P~A%<04M!WJ$_X0Os7V0CGGZFVbMdw6#NVx)o6vDVJ z36msZOK=7Yp?saKDr`9f_+lA#M`9t=D(aP0!(fabtjzIo)^E6f?a#RiC~L5~PW|Rc zPeOLNS@Qmmwu;l8i*Tx0Y$6uTTo!5=6ngbr8IQC+0Y5|x)HReQXyB>AZUf+V+lGNx zt0%p2GUg6X>LhbA^vQU=h)SKfxbJ=n53|Ixqy=-+?z??3wYAN~7}{{2V){-b~Y(ZB!b z-+%P)Kl=9{{rivp{YU@)qksR=zyIjpfAsG^`u89G`;Y$pKhnQSs1!g6$=07vzmv&p zN&T;k#ONqiENN;sG#497BuWWV)82xSnRzYB&U^;1K~c%GD5t`%`PV7xd~g(8L5FTw zY{2KP$E#ybg|G;UN|buSnpV^v49J$aZOv32-$=gBKROest~-Ir)vLttan75NQchP3 z#znXiJN(4*&&GLXW%^Sy-%WGlKUIJT3ML<_PERuqa!b8;h6juJrXHl)Eg#^fVAmUg z{JRTg{`1T&EwY(0u;@3Kv}1vN!F!U8r-OP@r~PFj4c%7KP%&6|ij~1pwkHn^!A;8< ztoD#keOx=rEGo5t(v*YE>=6!U#f-kW_k9izF*z$uFc%`%y$k|bR7E8rB|I3YaK3!l z)pPJdlBYc6bEo<6o^1>2Pd?vo{hOJ;8?OGfm1a|IU1smH{*7Q3&+};B{bRmYjPTtF zq}^tyTgo5aB<%vuk8)?WE$Rp$mZ!(HE(E#MnnbsfsMcz2s2#jxYnyw#iptK!WNn=@ z*v7gId!~hKR=cD50trg3tkxkih8FS0;0?oJ{PhIjJWTNU2|qYc0ITC7(qN*LS7T*& zS6XTIHhj5%Zu4|p!rHd+`J;C{x8==!^_tG0J?cehL>%`H9}PbL0UmcHmhJ-?wOT*b+sHRG(W=&9 z)^6=Fkq;KdaZ@9IZx1Yw_lIotFT<6zzMbvkRwgWrQ@y63e&|n*g8H-81nh-h05fFbb#W;( zjh56D9eU8heMW<1%K!~EL@lhoHOKPvcVTTBc9o}KwH3>5ob;$ByN@uoo-MB%7%_n% zi4uZdlPoQ3aMWqhBa?7V8*NGBUWSqPt_8!JjI+dN=TVJfe=uvLZDb-bIx@6rarhGL zM)w(Lr*U+LMkuEKbjr}%W1$H_b5v4I3=HrcARoBQACq#@w|{Z8eptvr;A0x>IzmuG zQBkMIK+p6 zdc^DXK&udo3krQ%eHh{ZxxTEZ&%)4eY~kss)L{)m-1x%=pNpN>?`F!DEv!5KaAVcD zFKm^v4Q-XuRJwMTb)&7_eKNJ}R^K8*09PwI_j$fzuGd{=XOTRG#`+c$W@8$9- zK2v0Nw(s>zBK#VaXCvLSf6%)acs!%Ixe$M_OB!*NI6Z;3eR=STxeX=b-PQSL`w{-zYLqx!yM)o-Hrc?Xh)yL;vx>Dg_Qq8)~mV3>VS|Gd&!^eD{QR`e!7 zHr`XQ^P1G5*&JwZ9%cGdmv5P&Q6Eabs5DXJwXAL}43b4;I!6X% zYG|$X-kM7%5g)rOwvG+t{eJTA!(9GVHr6`_NyW@zP}2w98f)k{wLncEqBmu>h6}d( z(I1~Xkl_qBk%X|?jVLy=R&1CFBL4M+--V^6WiT9nJxrKiduqRsmJ5 zu~-4KAmpFxM;L9Q6B0j^A6M$8?+LI7H5XefJw!xc$HS-3v1&=FKr0ueCeu!f+G< zKXiu@dSZ0xFnAGyHfD0Qcn(!@gCZmV9{V1>ZQ~Tr^*2kp8wei0P;lqT!SCc~N^qCw zLt>^A9PTew3vpt8O)9oHw`{R`jm6qL?8<7z9VN`M8;19f_=4%e!wC&jMxf=KMm+cy zpv3K-t3f5)J9Zz70D6kjeHNPaCiR)6rdwIqz!YXiTm#G`MGvagNkYCuoSA>ykM;Mv zzyFW}GCO#H5ft~=l5$yed12vxW$t;oFPHEo2>Bz4&j(2vYRtJK=1CPHpxzU;Yzez~ zeb!$swNfI4M%Y_0@mPgg?7jeFwo5RIBeNxV=2p?}%gkNL@bf_l;8spDEsC`BYOyxg zxX1&b8!Iiu<&|r=y79@L-D&`=dIq$Rv2Y4 zgF|j%j~Qzf?aF77K~5V#IAw(x$UJ_Cdz_cf9^m2?_4In4w!xl{V^$gLHy=Ce;B!>^ zi6HTydX%Ljl?HB+TW=I`e7Apn``lLiEMS(O?Aq!{tZhu`|A6x*-EZV=3N)PAt*_0R z*QZZZ!))M;o3eLkukA}H9&xXx8#;-*;H&R$IjGv)eAZbry;=W`!YbG0oHeA9=cW*q z>a8u^A+c%DR5a0N!1tKhx=)(dfOT&1Fw`~2$hI<1tnnH#(}}Q^;>i{Y#=WMa*y?)A+<=n-9#8Te>B?OPP@+p@kM9Xk z5+4j&)h=+FCwrIC#_>G-mmRW;ligu?V`+n9C!b+Wm_zIuCsw@?D-)SrDq_%Q>tMG| zQ#Wd5*uTC(lMjd|T#bGv={XceGIMn}mc2?U<*^wrG{l(oxt|Vt;M#|^Az-Yp9C*_ zuoGU+FLlLPP-k~fZ%m#`?cj&9gh{RX&h)y<_S0o4{XxhVXS`BRJve^P66n9GlzF6Y zzvv62wy>wYv`&9mI-~KRcsS*mnmqh^#Ie$#-t68Fk1ShZvYgzag5p?%=gDqv`kkTp z1^mLfM+Dz3vJAI**}*5+`ildc0m$Jv9j+D#pl4$_-S~Sp=Mu)#PEk|TXVSO5{m&=K zU-zA!Pv~b}z1)8PxI!BLHFt`?KT}V=xV?+d?{;?me!Kfz@1^f~4!~4uC{D`yhKAoV zXx94otVgz{0oyzzD?MCRa)8YXhSW{`DeU7xYHfd?UVScSCa$do6V(|Ghj(kzshf@r zq=U_ENkETA&x-;TP`?5=8TL7&r(@<>ayp|&p#E7|R+DwboTGq^h+x92)m7@{Cx@wK z{8`RowpZBVS2R3J-z0DAMl6Ce`TGQB*F>7C!+KjU_UKAQR|}E$U)u-ohhLCmD5(^)PxzwD5wJq`*6*I$T&V0 z5%_&TJ$gmOZ)z5BO!{?g&lSUdNK~=4^a4xV<9x0POe`>qQ?h&spok+!-d446%t-Bh zN5BWECm^tSjXO+Qe5!bjcnWF}7Q{3Nyc~698c%_9J2eBRALjwp5Y@^2q{@Oi%C!vw zRq+O{IRn`FpD$U;R)bhWRrQP=>VxCM>Y;%RBQ#dh&IVHqBO?zvp?H&o9zQg?*&4qG zKORVCY!Ua+sCf1wA`YaVe0??Ten@-vZf zH;-sdVmomuTyq52QS~XwU*Y9u>WwwQI|53oy(AxW#i90{GUTE~G)Jtcwck90!(KwA zoeOeJeKWy;^LM6cK%t|zW6#*;#lSeatIeC-76fXBZbnFE7Qc0Q6Hn)irT40wEEMPj z6(zXGMb(7`U&vsXk;Q%|RuB*e~ySXnFTu?C;IjWv4!|~MI3`Jqb9?aGf#tXrn)mXoD zuNOhyP4wEei#G*{`L!%mpolsHc}X^#WMcWkmyhPFhagg2T3TbHA%acJ*({r1i^6ww}g?Sj1j%alKv`UDb!* zCJ-zD^wTng2#lrPk{^D?^@Dmo2Ip$QFAzJI8V8dm&TKR`MRbOn@hi?sr8fYTlM6VX zh@>a6&lm)d%_&N%&$#b|=-QS|RrKjm$HxDD{$dA4C|KjNTAx>Sf<930^Uw~*NRu8% zvYvQ6S`9?hJa%D_ojPLJNTP1mN+^BgO1o8R-L= zLp!9FFGU8<+e$JAaXo?E2AJ1pEsHcQ=4fI!>L*xGVdZs^)T-=6O?sv#d1IHN&pQs$ zs4;`C1g^);Zde3uU|qFFF^l?Owmn?L-ysr(>P3*)VI<@pt4WZ?hGQBmZP`8vUU0zj zLaKO=nb8qbKI~>I*Et~sG1P;YKBLdL&FZ8zyL`>?4jq$XpRPrs6`MJ+?S*YOXu;D< zEi+}TIqi5S5xECyP6u3}28s6=BNyc4HGeq-~8~K}`;64V!b+uec*%*y&Ai}9p=z<5y=bOC4I2_s7X9UFBqmj9b zv0tpe>GZ3uF#jpD-&%bwa(zesrPl`>Bvc3dMIZPM@f=Km1E$c-N|duW6(vT;_k}6W zIk2jUpx*rs0quyuuvVANeb_zRO)UqrVJ0Xrn;zyXa2_ll^{_krG-Pgl>kAq$Hrt;8 zy%h5a>H=+$>|PK5%&+3+Sn=!MoWgac-P%v?*E*P#K|wC7-*?M{de0h>NFjXRz_2{s z7^GbOJ0$G0MOTI~U^2>lCP%mX@Hcu@dw1x48Z2=wbVRh1nh7B*Qr!HX1{Y@ea)9VR z8Vk-vr6G$`<_X#-LI}@^)Ew;`<%uAtD+jSpC$@jP{!Q9}V8)%o3!U?ev=lZJUyBwG z?MPiS%?qmu0gN^*ia*;-gi#Z>j?vJp#$mqwQ?Z~3K9K6&h}}l7>+{Ck{ZL~SS_KK& zz!)(+8CsbJaRNoJfm?I7iGSzhe%bVrMf?e!nDWtN<}u=1m(#IuD`tb4{{cqRnwijE zw;?b^Z2+Kd8 zXZkv;W5MY;XX|U8qMWC?3nWn$V&hwO?-!WL+S$<`y*wY}l)QIeE5!7%d*)Hg$I&hV z(p+V#-VcF~FF!la+83aJlGDw;o|z@ZlWOr8P}*gWbsRSM?GZhSpiq}VX;rr(@&x=Q z44VEhv_~`azY*6c{M@NP;w*hPkR#^8<;617T;)`~iwJpcNsPm;RqAjZ5#*L+Qbe0i zYQDN!K!cN`nr#5OLSxjZgJkJ~ z$=bC_IDdg{&b+i%w<$bm92_*by6iNQ4$*o0ZxrVVF8T9clIoQ5E;`4DPGNh~H*BC` zz%$o6q$xKn8paq+YMKih6{tyuubRQ<(UTm*~KkMldxvXT7AWELwUDxD9 z>kiuh>irr02iu>ZNmsf1PJT+ibsVDA0Jput@PK>iE7jz1HN}VIKa=gGraaJRYh5-w zoM)c&*g4yjm+$g{AaK5G9BMd}Gr0av;#23T(Rex7>FOJ^PzsIT)fQi{b!Yx_E|yLg zf3g9i#(CIiQQG21o*cy2mY3=#Y@LmoV%51X+BWczC)z{(eTSGi#vt#$=qOnmUT+i7Ws+0^ptKAmudK(`oZM1FSJBEqvyEsVASRFX?@r1!RLGhfNGn$ZeYn@9e zzd?ryO9pmtB*QqmeUgO&Hd*#hzZnqbWu>EW$__j1mOt5ttJIV+LQ%Q^Je-;Ct9_3g z47}U?%^|8f3HuA+N(gRj!3W@vsX!K;hB3FXis=>eeD+3w6D9Mg+?CDV&^9b{{dI)X zEt_B#AvIK<>CRY>xt9&k8TpcQx~M4+xP{!=Rfa2Ok-&W3c5;`v)l?7t@P}p*T zs&}R0Il|v5aP+P8YonlbmE0l4g6O@PU-V`FFyB_t&Yxa_URvTTHZe8#I!_ULRsOO! zfd==UBEkyE2*#BrV^9ie8x(*l?zy|WaHC^@4vkD|4(@i>`~8NjGl;I$@Qrtayk7s^ z6g->Y9X%70_M5iTQVwq=;THADSzToL6MsOS!u5 zgzsy!C7XQeWm|!ebL0+gKhs-2MW(cF*!pJ6YU~US=LxHrSAj{^4->XVCa5JPgXms_^xfWo#o&mjFoutJxThp&Xru<+g;Ava?<>6;jp5LG4&re&13>b z2imeqL1gF|4vuHluf*vHT5CDP&L@;jtIi8TEvU3tA>#`^1CW0L=C=5az|)eBQXyQm zQK}j}@5BlZf6$^+@O`R1e@Xl0M(p@?qxZSD%hq|Mc*Dd06@4F}BDiXk1bbHrE!lSC zf&RfUC#)t~5|$g*2rbjeB6iIQqh>pnI$d|oy+?ooVS8%flFu~xST68WrZF8Ft}A;K zkzD9Ci{2KPl&s*0GYnGm7)L}ZUDt|AuL;aG`%<;+^v!Ajl&e_nYf~W`zJ&Em7wMmc zmk~Ufy-!A;%z{w7l+0(5Mq|4=xT@CCCC3&K2U{E&A z#;XkC7f?oM#;qWaT5~n2WVgXaahz)%lif5}vs;sxiQiwz)?1RcRvh$X^LN;{K#le!H3^PfnZqjugeH!rR&Wdx)F~CP6ZEAN_mT>V3meLH@!S8zDlGQrXKv-nqxj@#Xn7Q#fQ|cWo1Z zd0&F~3R++0eg~OLiXy^!8nK+=jzVV7U6>>|!nf`c$2{Tcv=*@B&l4#nwjcE1qV7Pr zjm_B^%Uo!DJKVX`HL0N;R+&{MbPEnnUn<5*!bLr`yLP)Qt$4J~%ZJR!&Oboo*jQGl zLRXCKOm{c~Oag(qP*1(br3npm@pNwDu{RoVw*fgYFX;HGYqGH2yc~i$;#(k0S0WjS ztafeh;gA4BnGT@!?)OMNX4U0|0!bONri9uC?+%%hes@~FWV0>0v<-icmZY1V7B@o; zjlOA7DQ!>w+sxmDN@NUIhf9^C;b{Vl#H%FV{*Umv+_CafJ zdWemq?weSeay_^0Xuezz`cw7iiTz@w;JJw}()x@ID%gx;oFr_MD2SUK$&Pp!gd{3_ zKcX8#Ru_7%x2X+RZUd=2U;hYGpvUc-IT=Yp@4uQ!c3PIXgH|!$Ihm(EkAasqT)n2A zrtGa#275>(H%HRvOaXC^{1O!=S1U18vngcmE*D8liuyt5F+FtPuMiV_m;KN;0yPE2 z9mWA=sVLMVQ;bD?S7CD^P0gv-y8V$WTV_Agvg96)uK(^s(*~K=E){D?sx+EmMR7?d z5qkntx^%KtUJDAWS1^*`9F-*;r?6o6l2Z46B6b$mfb*e1V9v0%`VmrLi}1t%u@`nv zjx9yeY|W`~Iu6xzxFz%m*|XIgdVQgq!aMa$iu2Nb?|#Um>d#mx*L0eNc8cdMX3ZG{ z0KxP<$YutsP3(aGAyoD>@gMEAb@tLl?3Bk9EE^xV+c1?|v}Y_4vUq6u(+=I^YijDL2Ncr!ANtde zHH2pSDO@~sl2ySQA?svZ!PGYFwpg?u2U}t5c@y8X@>>^$e1i!pb~_Z8pzgwyJx}PfB83(4o>@MA@jUt7r<8 z6=u}y8UA@yLMv{xf{Ppv(n$rMVRj3bm!c@486M+IDp%`urCA!2Y1v9gr5N06$O)JAk+=$XjcgHFN<_QJUah zK;Kva?-;ohKAEJ)Z{!mAL>yi+MlX@c2MBHtovMd%mQzN`KH?3m^bQ^-%Qphs*yztF ziJ};Zf|xk{bv8Fs5S0}|gC;njL{>CB>a1JAf<`!iXjce{nmY##+JKZU#$0$oeODnkAt^D8XDi^pR3vgB1odDfB?&#p zf!$aD%}{r#fECpo1l0g{vKiGJ2-OJgjt+670KWwT3?&{=&pEhQASOs53|j$YQN?Z8J_{O`1&FmGkMSbMVTl15cC zizseE_}CEK4In2AoX=T|ib9QyLPZ2&ZGpN!kG3c^^KDsk73-MLq$PquL|KTC#4V5r zC`%&*Aqwa?6tgLp!SLsxU{M5{@VG6>9If`2ULOWkjsyb5fB{;!e2y#Q`WT<&P7k(_ z@6Dil@*0Jhu+PY1PuP%;$om4EFynLIfPSOFAA1F0<8gq`m?2k?J;nHc$pO}pw^D2c z@*z6%07l?41Bwj2fJnMEg%%Ka2LVL^@)>V=26r9;J4ywA!1ers><~C%pLV1k=T!gHx(KT=vgV`Q=;)1s#4{E!G zt*>jRk=yLO^2`;PL~;%1xfk_0mx%z6{P4H3NM)(8&ah}sC&#$&vo8e^@JV;6^F39X zD^n2ymTdx967<8*OtV?pTeI~KbC8}6&}t67@+Kb^J|);&P@;y&b6{-pF7lP0QBb?k zsLn9jI(m|v3{>K~eu9twj3)k_QBhJHrNYHM8^WNm%rlePf~w5e+~>oAo#hbn9PcoQ z?bG0%@L!~SMcA3R(X4Cxo8mL#!ef7ifr{`$N}W~SA~T`xHit<*%vFB|ZoC}J#6uO^ zE4leamXAW3{Qw`4Pn4e=sR%L-BF~OHjCAckEK+vlek4juy$^y|7}}g>!<}AaOB$1% zTDgPY2(Pk`O=jfUrzMZ%QTz1T%MUf*OJQM&6k~RlaMF#8*YoI;w5mpKI8aTaqA}S34j`U6by&_V%q!W}Q69sc-y`?-?Un#G07)uM$ zP{d)gqUDs6r(Umh-C;ypcr#k1M&~ZaEW@%O>s74;2D_X!@idna>Y_L|OhDLk_83V$ z5m($h3cPsb#1*n;h3`sgfW(}p74jbmdL`Bcz(`s4tk%CeuvS`>{4rL9r8~E|jl=b< zOU2>Jinj9278zq9*D^+dhq97RkP7q;UCZW9Ii2%1w52(gO{gnUKJrUpLB)eIf`rNk zlhPwbY+VjV(xs5FcYb5JXZ4#M`s-xMJg=p|r8{q92;Oh+J=-729emh_q!)4geOHca zaL?t@6~0?-I3B0%*7xrU`d!pTNex#q4+fSt)7>Q>m|PT{qXN8BDajx_Oi4BxvX4cAwZXKQ@H8;R3^%P%n+p_@2W5YE&tu0oe!nQde zHi!|CGf}+fG7g*&nGGx87(fr(0cL7wQoHFUgO*l@IG`y_Ev0SEa&GnAK8W@Hf@Uxt zSetRJfqwA77(EYZSLpai+jJxKh{G)-O6k4r?D#R&o!0$SzF7F ztrV7e=6}ic>@M$t&q@^qD-MAa#6k1@ngYwYL36=UnvRPuRhh|Y zD;eG1Hc?P{nPRG!{R0iVnwODunUBCY^(Y0_N2q8ttk#2MIYMl_x6A>FT2!|PjYVT! z#)Cw_rL?M2A3fh&C@MoF8*WCWwPG7G2{@BbbjG)g|3GA&Ipc$h>=zOB+ z3|8!eUzWj1zD`us)J-)Swex`S6D_Z{o>vagluysaEjbg-oW?lJT6IBMyr>RtQE>W- zA@+w@ichV_8cIUMHM{C#>O7{#qUorUPkf%IAWT7ABxtQQ0>SGEAb7;HlkPFlS0H&y2;^T&Gm@i&u&HZ=vndN zwyzPhvUp713ro`b55ga_GnoS;J_Uo=D5*#}Z-^0cOUX)h3TG1w{RkCt4(P`v*?=>6 zaW+I#gCd9%eB?)-Wljy@c@co;L^@ZJy2F0qWI~_7;V&+|kU>Lb;7feK;!V+-;61OM z7)d>$TvAlf@Wg;~8@PFSZdd%i1 zvM#gYA6Xg0D=dH3g3?ubA%rTv138gZSaIavHD#&{xmT^E>4S+ZFnbsn(T`Zf%j^g2nI(rG%Q8zPX#(FL39 zZKDrloF|#v1ytI0>^Tpk(xBUF1q8dc$OUAbOEVy&)5u$VkRCY@z$uv!J%^h0i?#&* z3}R6-p-EGXje9%zzvQr|v41*BQ2bzIV(wt`@T<8razb1o--QWz9Ey%kLJ!9z&D&5?m`Ve@}%Q9Ay5|53| z7abZ8+?cbB^jL4pbTO3I!yD#G0qL3b2HvjQ@AO_9;^wVaiU9+d*jXSD}CYR2>|IrVye0Bz6-A<*vi&RA#@KU=cGCtzB8T!G@ zp3_wN!fZ_;d-At%-7NX|VaWg9YWEQ`!tZmn;BSW>8>*#!3;1_Uh{{SZe?sYbEcz^7 z4S|+gqPapu)H;7HV?&a~O^fMCUAO%AFD5s!dII%|5Af-R_T4$K^tqLS7F-i5e4PUJ zpUy$Vq3>3QkStOwgrG3bjQK-O20M3~?W0kvN1_^aMu|l_JYay}dD@?RxoG5S%(+I# zjR4ZDb~q$f19(rU*kChT?r}ni;uQ&hTu4I{Da^lnj=lz>aQE}o_|{pn+CG^5SS8i? zc7*N~V|X5EWG#OhAax^jpm3v#JLU*PN1|abEY*VQzA7{r0&I|vIrCK>Xt{k4Yipx!+ zUm;uH#&)|Dn?GYaw=FKOFa&Z|6aqVyq>8Agx~{iwGAf|ug$TS6^$|bqp_-#>Pu0n3G|Y2i&y))0phNCj!ipK$i%3zkM8$p zR{Et%>JU&1N>(CHf)Fy#E9+9pJ+{5ya^Lxf7SZ^3ZdmrO-W|+YK8rYech~eP>19U)(zc4tPqY*X@ zdS;OBIBd;$vv3ri=3$UD$6#oVutCM+W*!PLa|BkV#(In(x;0AXh@p!`#ykWjW;_~Z zH1J#$4rYUI89^}u0keZ_8M6-tu`jp9w2Vfx9Pjdv82sp>CIQsSNW98O3ujF0Bhe|x z$GAq0^C7S(4}(HE9)B`~g)|y@GA>$iADEL7D=gxc@d%U6^nAtEaXR|Zo7OW%c&W4q z*__y5;lRV%tEhwA3Hb1+0;5>3xRWiKu5&_|wk<2-UTdC_urnd<>cY$^1Py|HIZwn& z%iP#rQ0;ukgLiB_LB$X6Z>AN)@>^;_({ZopjTQ97+P8Ku_w9CL&*9OgnVhmM#;@fGyo{m?&!5pgT?IFC(uSD9` zh==EW5XFw1m=7~+mpUCx*$5o%5eH59AZ&3eHNY{f>cjONdx+vu$8YfFPJxt^ElLJs zgWC(Xn=cJ;#z>vBd#0v6b|yoFQA%$qCRY9nzu~}W-FkFyHViHIk(8fjAo{0%<$QCvvIF&x&E`U zAha6b+oN!>^996nphZD#!=G3wOg3CYv~#0}t|S=)M?1~Q8=k+!`&$4eXYh9N+2kjY zGm>PrtXMfbO=cM(a;jQ5Pf=2U5w<|Nz*k$0V34A=Sy4%zoj*H&`dn!s=sTg3o%PI& zdiT>Q`FW6{>d8~`GyMBZ|N9yFnfy%16}ckrzv>_IGnuGC)?M?B=F_t_BTZfJIci^PSy1D zj9iJ-+P_KQxCj4@B=3*f{Nnu)cyz2nqXD~{tZH9vYqKp1R;o!vBM_ATy4V8Cy~)I}at++k+_Oy8S>s}5 zfS#kbsccWv6TLy;#*m8>+WB3VKU#BEOKt)w+Lz|-JIEy%tHxxj%Rpk1r*7v8W_kK$=L&( zie7-hfkA5`o$X2G|>UyJ~g~r zDy2Zph=d{nK)xe9v=uu?fg3&~HxxTNCp}7COL_Bc&ux{u{hZ0UNZG}F&U0R^uSCX^ zxQHeIYHL5|oY711Kpq$lJ1o7y{$GL5*=@wR`opd*IH&AUr=?bNv~s*D~b-rgsiz2{G?rop`=2b!*$mGTq+rOWzNliQvny z2d#VDz!}5QfZ7gDQoq8mc?4>;0;Zk)G|AXK%j`xlwIZBF?%Tv_g5*1pB#EO%Tvio1)t(r-Hh#GZvd z5Y<;K#T&hbNHnx;*yOPAOB0uK)>@~-)`<27F#{2?^oL2l&$nIX!`lX7V76K3!=@;V zy85W=+^nUqA|qFCn){8B=#Ihp3$ulW*eUcu`m28eK$xi@t7R}QZGZw#SzhskX0DXl zz=j15P*c@j2`4rbhl))>VSWTYGO1;IW*iomW7nu}iirpB(o5naS%LJc?wHi=X z^r6UDlV@c;UwYm<2fq)=t;<-agiROVQtvFSqjo)qrI1xlSL|q5VT4gtD-U;~u{!D! z)XwPS=@}$JG%xgN3`h3Kc8ioad31(v{YfL8IdAm{qpj9lR(+P6ZTItreJakFzNxmg zPcDcxqk5kXdp#82$FfYRU@|{Gw$>}1rr3!fRN_!eFC3(NV#Hrku`JVBk47yA^|KO< z{>_1M+cGyiA3X$JV%mXea!UbS4$d z20&cC1ynq0P2>@l)fnF7twWp7fxjVy+MKX^R<21QB<5uTi>FfZMZWU+_U9%p4KzsT zol&-qw6=8wRvgh{&-=6oI7WQG-72K`lh4kdoucP#VICQna^2b-})QTL3=mTmqr z*ARFQqD*zHwVuR5vI$fIG@jj+Id5V6IXm`20J{N~0E!zsf5m~IWh8mKtrCfGPTd@aAf+o&aLQ~ z0Dd>;i@G%SZYP|ob$`zo=^lu>Qt}%VYM9sAHLFls(jY&rmp1wJN`LF2iS-^?o}x(# z7kP_>mPl!D#%G^QV^Mb(EQd^y)Ryd1%7mDks;<`sOZ1d)9g*)e)p4j(vQppZS{zj0 z1$=WrYNSn{wu!ul*Im(S86~x-8+BuheOxEdszcRkr5YnC09FdP3=mj9CFD&lD-bF) zNpk#r5JfL2l3}MxQP2gf)~LR9A8JDVQ6O9C-B1Dy9o*%M7cyZ{N{Uts)EO!rETH)^ zT@zZ0I@bXZ9x-0$G`;CG^sR&WtMy`=P^ENZU(CSM>C-1$4|DE;2I!>kat*|7@+=s6 zt8@F!8sEAVN^1Bz&*GHnH#(C;Jv1>gS5?;R48M9PdBq~q)=uU$lUNA2DeHRL&O?M- z{N0)fk+A_uQP#7q_od!|un3!NX_w&fu;A0$x~(nhcJ1hhoZy|ul9WPJr&~{-?&hlx zMO@+!K}(?rDkgrmdDK z$-ka_d-3jvw-;|-ohpvv5yEwEn>=*4<<4um6EwVZI}>Nxt!4mtlj(=zbgNnkquH9U z4`3d<-*%q{gfw#s_5P~(%pN5#tU?(j0YwQvnqa@69xeBDMIKkc>7#2z9`p8JKzM1V z+^AVfohofqBQ`*GUB4|rbrEWTqANn_`VJ&VZtMWt$TjpqzRIYqc=AOik~`J+NW)Cu zldYcSh zJ5$|-dY40rSWJ4_T;2pACfmb}_Z!Rhys4XPN-{@6UcpGOMNJ+k`VxSrOvw!|o|9L( ztT8^M>b%BJQ@2dkg|2a-9R@IK8V)UAn5=E`3QZ<~q>>}{q5Ayjj2wNKOZ87)&E?UQ zT&{|YC%jTJN$%82XZiszO+I%8k%KG>@B44Qe);ut5R+4ZK+Wvlsf1?$vYOdd$F!GX zB34ByR@}i~flcDWy^2cL0VBUpD)HWr6@!ayUGwe~sB zMVT-Ia}2k5M@O=%LIA{;8RMfecoLik` zoha@O!9-|D3!P zIrFD*zhg8dYUiHe$L~sBF)ufYI)c;^58*1dI%FO&oq6j3Bc^Jm)D8M9@*ftNn3-Z& z`{>0t?=DAm;+uDurVr|)m~S*6n&@Yg`)U42LA8-nzwF4U%~Ada18=>pryDYEq;%<} z`s%~L$`bS++|)!Xgh-JVAqGZXcuflxK9tGhVHgd~iB7 z(jle+!!Yd3jYf++R^r*Pc9IkI+c{WKPp7~$gK`fbe0KyU^WdORw4?kIK8p><_1^H! zyUU%+_J(VIuid1%p& z@uxN>k+;*h3bc8TTxebdwACE@f~_K11aO*h{WDLP?jb{juZd@4MK&qCftvvyzQwe0 z+fAIXVmXh?q*J$kdEK^{zF@_Ye0gmpY{y^dvm-#WCE8byCCT`ld2kz}(8EN|D^bRy zh8>$H)a<@9b3xuX-%^r@EMrN~Dj?=K5SC<*>ZW6n}KzY+@+Br_Z`MXyaFW_)82xP2q9mksfnXwNyih@#QpoPQH<7&Idb~abXHk z`L&mlf``BsFP+y}hGSiojGeibrt^<4MKp$!R#yWuMNnM>iu*nW$j4cfV@?~N?65i3 z%`@%MzHWUV(jH-9Rstf%0qJTkO0_o2_F7_90)uc~OIA+t)t}1Xh7LMt>^tTyYuY!o z@7TO=WcAn9^Xq3gD1p(sT4E$VSYEX_&^y!!SoX`0Z7gn~FJh#BbX`k_9>r?3bx+(Y z;g~&~KZx><=Znb$ua*<-13ib*?&rUasf=$N9T#b;-LJJ|^E$(T8#yJk;5ugS43yn7 zQuFjLp;{fIPMRQM+IYY!b*a4?zv$K7P~_V^qDmv{l{6I)k5s~p`%funIn#8-^G}8w zgk?-4vTQ9s@;n_L^^Qps?rjT@`aA7}u|8&BGKi5LL~gv$Z69>+-s%qjYAt`vCiwPA zkv8N1={SztMmfKMh#kx*XFPGEOmm^eBC!~k@6Pi^K7#c3N$UMk=jIN>87JdyFNZXI zxe$=esj+Q?qL?n{r|k}q>p)AoVihYJB%MU$xvntsb;0uMiY9knugIf(@T->(c5wKb zWrGQ(Iy0>a6vGYlOYX*B*l&x$g>PsPXGt=5cVq$kH9}ZnFl!+S0Ja<)0!-2Ng;&Mw zUCqPCIcc4T+AM3OzM1jLG914D>Vp-|bakY9go>Z72h^$it;1!5X+0pf`v=|5-J|h{wV`MJ>T8g@8kccrH)=N{v z(qSW-wJ2Hj5#x}8_Q}>>g1b8sq0W5QeGR3?*f&5T^Hv zmUl`=)Es#V`6bfgieXuWd-N`?=-SQO3byrlIz2`fB1$4FTIxNozTanUkTETyem){UI$5m5fsOq$l)+&y!L!9md#PO#GI@Nu=8nCaI-+LU zs)?~6oPlSxI^c|%BNFsThB|&m>^1>$E znE;~8ykg~?ChXL;FtB3DR;={Eotj`slk>Kud8$DM6IKEvVIL}1s<~*2{Y!H4g)d26 zgRTvF{-ewj@TT}cWw_n2X154^(`@V|b}^))S_{imtv5)=1Xj2fb*X(`ro0Fg2u!M> zl-K5>l9X(rRFKkvphSp}ox^mu3yp4EF2m8t-aeuDqh|D;r$)8`7k2|9SWv00iQp=i zQY739#Go!`+EB#;>8Mr8(YhgHQv(Im!Sg!bqsBTHDZ8p?8JElJ(MJEihcWods_Y?H zMPj`TJ+^+q1QKyTNodAtSz%+!$r@LVU z9oX1J4{+javj_{d-ME{Ku@MSD9>6a>2rbZHk<>~NLw>C0tch0$m+p*Mx)BKy(9Uhe z+EhH`5421#t_JhR+9FwVzbllQuZjAe&Hz{$a~M1huTAgRFX;vAGq6=cv9b+MmUl_? zdbxb&J0b(8*cMJTw3K_89zX`w<0F$LTT%G0qlOXATu5Y+o8=v)LORnfIw58pm7(ttwXL&&hwCy#M{rlhePP zoP7Ue^6!8D{mJ_&eE8kzUrv8EfBgRR^yK9GuipIi=IYho`RUK!=k@9i|N8mld-m$@ zkwi{U|MIU<&1*?~Jg+9AoZ!$u2dugXB4&WL*)Po@Wto_j(hsoI$B5WL@YZ$<4lZHG z#;<(5=#(HKaGAa2ktvMk?snor#a4wVX}Kn;HjS-E;9ocZ7(0XhLHkl(?vlm0Vkxgz zBY-sEes@G30Hjuf-ifoO_`KnRgC zHQt60wy9O&+h@FH#(@Ybo zQVGyAHJr>G>SkpTaCxLH#lYfuW%jCZ1ga&Z-iXp4m_;+`H+a*c$an(K_n@i5G}~o! z1DL)appJYk)gl+UzQ;>d?-<*_2ZP!V;3&E!m2>;NG|5rnHFY8=EobS^RD^0M>RA>Rbn+eU$CqH~B@F=u+YzD3vNCcVK+T4xGPiAb60V~71 zM$rQOTBpOLEMpa9urLeTkI^$LajzEV`<#3u(a}2Wo|!)<^t6>7Ds`1;#!}3WajFBB zbuQ1y?9_w^4iko&ndQo<)c{tclBod(9Gg7SzX=|tWRLfG-i(&bV+NYC@N;A5>bhVZ&NxIHyICU4}>LeH1`vdJDlw4Wp zW%M__$S7Zt6Ui8PWtKGla_twF(HdE{W5t9m6qO?un8TDqK6eekuSTG^k*9W9_5M7( z97>48GPMO*o(dFCG?I4%REKw&goCEic0RPmLs3k+2HZRy57`Pa2`JWjMfR-# z=;np3n5cwFFp8_GmA;~_n|*}xk_}L}{Th!%tURprfFZp-Mt;9<*3S)F6&bDKV>yn# zbFAi$_f_p_nx_+*sbzQd?Tb*PfZeIBP^_D#Dype(qy)85dz9b;V5=ERQ>W4lm{J~c ztfs8~xcc_R8R1hlHGg?Y8Zzb>sKf$e>}|>iEO_XGV{i7-RO4jm-u{Q`xO(!>lmBq4 z9sK!Loj9esA)-Wxe!`9Gwq@FGuS9xXuw<&M@!Xg{h$! zY;FY<^-8r84DgMAp$ff(ewT~tcXsKUAVMsiPw2hirh=7X0XZ1#14E!^dA24iE-QM+ zrsP@)9B$|W&SXviqo1Xu&MWSZK2lHG_rox_LdxZw$U0GnVo|b*5r;yHzmK76(4a!- zD0cOxCB?|DA`NyPi$3RiOLk+qq3SV{Ua(3w!I^lTDaNR!7j3zL!w#k)NR?`1MZaQzlPHBgm(=MS>iifN z9oB<;Bhom3UZ-?h^Kg@Vfyq4*%v*Rvl-pqiRH|+@Te770+gq9&>2+c&%|MeT@Vd#< z7?(&yM4`DxdfH<0Rn4V%)OEC5sDu1tpVh2h$8FfiposER=3-p2py%XwEdfUh!&alm zgq#zj;!pL280@I_?J*8((s?0BX8l^KP8^DJCK}0KN#Z4#)J@8yTF|A)GFD=NcGUjk zmO%@}1S+ne>+8&#f2^vK_tr|^yo`KCC89~1xCAhJnW#kb&N#f-W zwR-86-Y^!IPqdfuF&lv1!9D24?v23|dbhEW!5zB=P?I*V+(m9utX8%Ip<;VLS|!qj zn*?WNU9dl>x1A9_Y@u~8N^+4;bh5s6-e|eoL=dA*f#z|Nofg`lwJ!OKt4qzKPjZnX z+rxX~1UIy#^(r?=#!4S1pFH-2bzqr{c*OHmod<9%f+0iJNx~wxk~SN_h{B1P14Cxqv+qFL6|>edTDfW)bNNv(WnuP?Gxiy}mOH+9k6lIMj_@#)8Xq?gH?tzx61b_otaje*O zhEkg=VCBchaa**RW}bKDOt;e~&2Mqrv@4^YTG>}JTLY{aggBygC04ZJ30MI6-0`lh zY?TTVOA)t4K@v(2b_Hw!+o};VuJoGMgzK$*G-b*A&2< zs8)d4yphudz)+{;8(jk4o-Z~pd%!*rCj_Al0qq0mxzQ$qDLJ{wily`B$b-f5CY!q2 z9u>FUQ50X4Ldv&>?2sA6bRpKjV7mFJi9c%MecgN1#QUy)22I@IVOw|AN@s23qSNWE zH!yiCDjlc*k~JmcY@lRAOQXO$ALx3Tvr6Mp8K1yA^WMLfGyp>GURo`4HlE=M1~F@k zR#yU#)Fxg7otEWPPJzoUtc zI19%Nt8*Uy(_6P}BHVb(RFhxo1V6h0`SylrhgF~3^>R+$u|(Xn5*NU$4+YI77x|Zz zXR#h3vs4V)-@U{fYUv{sN&I)2v@gMicKkxo{XJ8&WI|NVW_{p_-g_}F11(IvtH%2BGD_&L1 zggM)t0r4qmB)*<|X`TQM6pU+TA8f5nM*#p$`+V#K!O>B9}bX zxhvXR7o!P`6;r@X_v(X^jREayEB(4lqzz68LlM2?>@d0#D+Y_UWY}1@nz}CDzKqwS zMuahv4NIY(_r6X~p;u$hS1(V^)#dLsWJz^E2{RRO+8skU=tXU8?J)VDhb~;Q5a- zN!a}zrbzquIQZ2t%L|Y;Mz*7Ai;bnUlg}l&%!w%V=x(tEbR)5O#=X#Z!$(r=X`h~O zQMP7qZ%ws6@Ib21=7n^?ehLvJHZ25-`O%6kA_A@2*NT-3K!;1B%3n3t$TJGBE!Xw9 z@OyKOF)!j3E`Zg(+bo)R33;rNI=<&12)%|xr;nrAZZW5uk{dP{rH&OX6xaE`O1OXx z^#4ZavAmR1azP}|7a4PD7_*-9x@fYhMM_WlvF7(QW0=oKb6fLG%s1rEcy573UX@5- zbNi0&*!t+K%a@`l9$n@~4gP0~2U!^oVj7O%w@CP6qBFI+BJeoT_K+e$NDU=m8PS*~d##yE_XE%({Q6zUXR0HHjToTqOhR`i3vcupHl5#S3qZ3#l z+-=Epwp53xlgJ#oE%lsj7GZYlZ#BtAQ8Y!V6Q(!9lcD}sQ>7(bkFU9!FpZMQLSTq) z?;wnjV3ttQ+Jx;@q^Qe6NL}}-yJzA!C%1^kCOs6hmhA?C6jGhG97XY>jnSqZOaMHk zeaoVMAfl+7P*POeY^@t?Dkhd6A7k8&wyDRiRHaH)ZVXr@+8u#(_;6)wb4i9pxSe|G z_AD6(CgWa!$2AjmuGhUm`PzL%RqqX^fNh@M4NYQTTqEadoHSy9(F`2_<(!oz&fRR? z&$*&rlM0424R1Qbx$BSN9FrZVBXlXbVz_N(MJslO^$_QoDVq{~2(3iQ=Q@J$+Hr=S zA(({5gK$BZqsW@!-JcT`|7{pv5`Y5#0tA+lQ^7(QN53pyknH?3Cxb?hKP3sZ&mb!}R`Je%UKxb2&gwQoW&FHlac`+QS%PqP6S8dPWDaDsdNi*~qJF1cJ;pC!+o zbD8M}ymWa>x|!-?nM?6NQb8UV1R^Op6*3ECOvN0h_mGsAh*eRF6*p1xW=-r2z?;?H zl1S|t&gxuEZ&PXJ%NL}g<$_g;4$14)jCny#_giCZbm`7xTf4K>i|q?<67tSagbXB* zA?Kn@Oo*o`6^e!9)XaaY&&E2hr1#K{E9!~E|J`?Q_YjOwcffv8a+JY{*MHh1Zl(@uceDS5>ak`HJ@ zd6w0T=MBDNGXWtlklI(q&@WkqpkV;|AgU#id{t)^&6%iWw)QTkk2wk9!!=_ca3jPW zviA1&VGC(nJTVg%R2xiyN|SM$p4*exg66RgEYV;qMOH8PAT-uTt%u)~_O2y>eEX-; zsf54l5wTr~`d_CQaMN3!o}{c`c>vE#R5hPxD5 zDhc~gmDKz4M66aykhS#Ygyw$M6{R}tOBhIh#lirVxDR9~cRUQ{k@hIGzK+u{O+q`C z*>Bh7hn>lC$)R>wY+2PxKqYSr#R<$<0FAHNJPfZvZpFJ_Qu~>-BWLwfXF=`lT+7T8E>7#$K+Fuwz7$PR^D^=aIQ1H zCJ($?PIR*JIS7#FzlDE-)&^9nF49!zY$9H&MrW8hC0@@Uwdah~JpD_^DmF^w2^uf9 zaTgxC{QDdLd*jLm>3cgC4gIASvK&jrur*`Af}!`UK1~gjoq2&au6wv!dCyHECn@VZ zAD*-*(TrA9vMOYbYs+WNGm3k-dqn>P{wJ0a8ekb3dzMW8ti`;%MP)8ZJ20l)ke~mW z@LG7c8sm~B0rcaj)vsIh%{P}X#}1Qv#t4G&xjj@79zSsN@E>be1p|nGJ2;xwZoLfd zqD8_Qlj%z?@8ZgEZ9LZAT7P}bp#~sZl--}pls-L!Cd6dxw);sXIeL0DCD(k#Gg@W{2xd{y%zb{7 zxF~HyAgJxh5jhbhv?!rD$(R8a(;Q}vT0uZ(`&ev`?A`obq@zTaW4zxvQeE@X)yu|N z6;Mn5`r_4;e4|wveJmqaQ~g(K8t(X*0~@tl%^?h4OSaw8gZu}9&tDSx`4h;7>$GGRjVrj8<19-uk7%z0(2T>E9eBony~A5m1b zwR*uJ+=u1*CF#|@i*A||J;l!EG-}|x*Sog^cjJgw=n`ML4T)H-uo}(M zLY~C|Tw2btTOY!lH8}NQ?uu1_=7U?d!(Ywq9Szlfty9wdp$nka0KXblIa-(4k_Hb- zA8F^dQRnYoUA%nrYPw2yVNonk3&C?gL_?>Jwt2*?9rJ}#$xGue`DG?Mj+H1xCKmlQ zm`4o6Z!s8oLMLj3cUNnwqvu=2b)MG4j#*T+iXX{QVpCgM6k_8hwmm0G+%hpXHKy4kN3#<%NYEMZ~ z@_U}Kh0hSUi5G+tfXkpD*g)!0<4MTe{N4-4y| z_jo7ydJ7J1tnF4RbXMdvrc{#Q^OVT-D&zTGQ^aU4G)N*N>IbuaOGf?q1>MtOrxma> z_pe4-nZcl}_SxSIo6x}?t}=gg1icGL?7wi&BUJkxcrKTTEku#IFIh3er|{U848E!-G0O8TYjv7<(*br__6#nb<0Nk zvHbq4j}VH#<4FTM`oD|sis!uIv=Zegzd`FMYeK1IC04k@z>LX*xD{`~n0GnfB-rGI zf(`^@ewiz+Bm~^tj(H{g z5<+uA_2}VdE_GR0kle16(#eY-XquMK@eH18NvaGcjGIynG`C*tlWJY)T1Pi8u7HXp z|MtfxAjy69?2k{plmBt^PDWdjKB{+dXK`kHY;h}w7_cdnly!~awThSOO%97Mz!1{r z)QFeU9%EyF<*~6Qy$u;t^ulu=f^_dqCWo0G_}Ra;M)5a?hk2>)hrjJb(dR!7HFcs6 zyTRYwLEhF-;M;iZ4d;fc=(HyA3uBVd%eU7*yuSG2)oU0xjBfE*B&(&C%D0JojNi3G z`zttYd92ee@3`x7^7hN?mZxsRBR6b8>3ZsRRh{zr;0PSZ-s#{^x0F;hC{P@(oF39P z=jkx{h1^l9=QtW6&(U~CR9LXsIk+xrM8kqhIVJ=181FYSi`qZPd`iXs72{`8#$c~? zSKTkVvUW1~PIN!EW%ETJ6j2<9>wQ#12pJTmhW~lCmCj%@fJS}TNG6VkuAFLG!|Qv0 zctc&7DS%(uiFpEranYgnk#6$wz;J~WlFgwcRGL#)hp)L=%7@_0JC8Lv7~i&R`@3M4 zsCQ6$^}hY+d|{qp*UZ+OVJRw+Z;@Hk_h8#;e+75E2m`h?D+1$}tt$!Y4C^ydi22V-*u}N>4-!&b+Z}s%a>ZhG?{2pFESFb%E;zQJNSv7UcTIVV~VnR z5ak$M-ZkvKPQAUMJ?NK+t%gb0 z{+RJfV?S$zXh+xfyGruLDXG|qVj21^j$JG#BB^L*N?Wq(fib@{`Q*tHv^_rg)1UrC zpg>s4629sZA$Zpxo<5zD|9$bs7t&93xm<~eRoEh!5SOq#Bmd+2>$eR~s{YkWSR>J^ zHDZTzN?tnQcdo6ER?A?8C4ftw&mCx6E)8cefFpddGzG}j8q0V9Ngm88Xcsn^HyCCK z@3AKdbm5F-{Em^igwo^wToQw!Tr#OdevMv@FKSkoWL^-W{W>EfYTC9vM0zuuVn({U zmI)+BUJ%#}5l+6=JFhHNoCXab(=Z;IiF^UG7VwVOF0JX_I=(%&$eD|KkX=*nW|c`P0i_w zohfc>0FK*efs!wBR-!#7u4okP)NF$x{0e1gb;hV&F9}^Npt{I?%q-=KQDpq52E%a4 z$knSi5CaY(u@c29mSb{b945G0fQ=YDCFGS96(IU4%6fvN1qO#;lF2L+i62e?uSBeL zO_4k$@DN!tnwsj(9m&k^<`7m$&01h+<#hQbAPg{TKZgxNhS*P3<|-SGtux2?Z9rUGX) zN#ZGA*;9ja1l`f4inge6c8E%xk+nESQM_ZCQTeKIj-?0$$JXQAwj$>iH8~Gdm2*#Z zId>?Iw_R<{gH`7|P<_rrROsBLM&}L_woRSR!&T}$RIScKRO>uoz0QMF>^w-#&K;_D zZdtc;hqb<4?apngcaE*!xm5+vt!j8~GX`7K@!YkN=a#d4uxg$MtLJ&(ik^q4=^0=f#DJBQCs}v%-?6v>{DLVgV(9?tLsb;c1 z6>FShzMk&Kcy^}GY}o* zZ)h>nrDUIT>o^cP6Ln>>WAXQh;Q|EoN*M3ko{iHZRFifpD})}jV_(h)qsdZ zBG;4>IE`faQgzn)%DU&yh*2ZXVs!#0wBN+8#R8qt2aoj2Ta;s|6|il4)>dYV3MWlKPS-u^fwI zVO~wqtOPsRXR)9T%_Uh?7uZ~Y7*M@sQer`7vZCc3ORZ!TJ4;t;@T_^FG>0BU;n8&q zU^Cv3dpw;ULSTs%01jI%;~jE%(TF-)?zN5md8F-m z$nLEBsdv*DOU_n``YN(EaBs7ZAx5QDbjU`nj4}uf#7|)YoURBk&Zy-Npt3V6kSimS4=T*bH#>$ZFDOtB&>VI>tT4cqeC6s(XyqPjq(UPQv%v(;V{oB zd7}@!k&A~a8CZ&}t}F!V)+Z-k&@p2`L?mlNvN=shcUvEc@rST}bo>D^*5W?K+8(J6 z;kBhDG1j`ir5P{?A`#iUj|??|*I2-bNU|}o6nK3+Q=@s*ls&O9b1g|evV)=|ibsR; zW5jKQuZuhU$C8L;_7mwHNkmuz@FgT@vF5cT%tw631A{Gn{ItOM*NA^ z=(ks%Hk$UOIx3quj_6~9+T8t}IsLR% z6|FJxt;S0jc$xG$IO7P-b5UV1%o&>EM>!IS4I_5*vP&oXY?SP{ZLC#uKI{~ zlF5>-=x8}hU0{>f2Mxel&=j-3$GQ7R!?IoUhaJk%g7V04j@^CHc9NEZ>&mEE@>yce zjMy_)x$;?JjCExk%{`>HZP1KLbsI%8WQl>f8bvW|iE*tfqey!!G12*Qn>3^H=SEwk z{FWG!M7fol>z~UK^wXLq^lYb*DPef{Rfabg$*+?a%H#Pv*q6KWy+Iq(C80_De4sLa{nvCo(cNUl% zG1;R2N1$wVFI$pC%gas1c*a*;XM-tr%H&8t!wC|YCPjUwiA`5*CCc@gC8&NJd9XFD z(bOQ?h$MAcvb@UHA6YWD;};)7(p>EcwUrIgLS|*K(emTmR-%<>jX6er%kq12PfIU; zQ{T0P-C& zyF1UUzBHPo(yyL|^BUV%L92?D`E&ALC+~m%^W^j|Cnw)Onf&|Te}D3R3Lk!V`j^w6 z%^$x%Jv}-3{;M~Cy}5ezcYgZw_j$d#!@qt$`JTP{dnA$5)4%-d$cDD`0-jeBQBH75 zpF;%XC~{`FciY#^(chG38K)o+P!bfo|418mJM}MEm4B#xzlioX-re%4xKKtAN7$E< zHfDTiz`U$~OlcL9^7zSFmVb#@HH2`Fr1}`B_A3rl! z)AhNUK(+r)acWX6u!LPCiKz4HM%=N80F;g`VS@scnCjXVVr&Uueb^*-*bp{ar(CJ= zTdXnjs9W4IPg!bzh+JZtn^dZ|Q*uE{mhggWm;{R7h zUczF(W{JqtZ46f%G#Zf!++oosmb5Nys`@?6c*?7_jR?c-rF4%VGJA8Gg(;gkC$uO^ zL6aquPRq|m%(jNGx8Sx}z2G@21wdIziP@TzqM}vw3fk;Lb*fH!X=QHKjfO?TONV6TREj83V<}DnYXNxB+!K&D~bDUh(X6vJ`bGr!U9( zlqx}QDSd7ZgY9<8 z3hMHKP4&Qmhhk)vMJZ}8C_$?)aJSKrjYk0xt<{&NAQG6jy)y-Lf>U&`0vCO?Da?$- z6#W-lRPggs+*8@$k}bHb$}uB?540DjhDs78S=Cv^i_DIWw-fISF?~w9-Vy4U-YHg= z!IG{8<$mUSV%xy-3hE3%1U)NgN$v0_Vzr|36rC`HCV^8(*G5z4Vo9@R6r9+XT^DD? za+-U6r|l}ba61tXIhJZ;WWn!Q-dU9-!esOXP3~Bpl5eCLoOGSjl`ec+l;WPN#mUkR z?Led~nx+w#*+8eGShed&)pA6^NOg0F|420bm;63qF|w$lxoaoxy``Hrx?!f%*%cbV zSNnYt88lOD#jlT<$|RkIqJi!YUeMTVoxvNt>@;}Jmb17@Hpk^GZS$~qG!>G30Z?Tx z83^s}Svxo`-4ec=rFwe{jIs}ORb)&~@BR&JZF*0gm$UTz-K&e2Z(dDT=`KYkM;6gK z_E)fL6?gWZb!sTqq9zX%aTayuyBl6SC$Dl@Lx~+)bzT#c(}0!4)xjgL^x*m>%8yVD zZ`abY<;<^VqX~5>9!M(41B4vhE8Pve`qEfaoXMP*!avlBSQVvMacPbfUCwRi8k~GZ z7bBJ<+-{8yus885jZ_wA;U2Zye!CP8lS)i#$tK(z`xe#VJGO>E!00aP2U@ycCBVoF z-z#C`-4`#vks-U0BUYcYDmhPBu{_t*w`WFs9-fuRc_qsDazw}G2|NW8k!8A8jF^)b ztXNu-qpM!;x&IL%Cy@$rj1#F4!t0Xglx!9vYi}#~26vUSaLQQ#QSTF3Mtd&F-Yq!j zXZCr2fnTTo`bQh4aGmv}WK|#qq+4oTKLYx><7lLHwPbn46SVBgaKNF3iFX*|q-Csm~lxLxyUE ze`OaB$wfNsD0r+7-qwhmRgYzhcRjQhv|;|*o8g1r(KIcYl!k-qRpRa9#nmPG zE0Ws1PI)OR6hB{z73JPQg>DVq)`O-~?g1OqnJY_&XhL&QIuO6L(p$AY`3shY-Vlj? zJ*AMUO04YBm`bgsP@?%h?}UA1kutF~?1cDriZZlBt=-LBfUZQHy5cYNPI*<&Ob z$;?r5aNn6TxqefO<3x=fLL$9u?3Cu-Q*#|=jaRUH;v|6KWuN;{@l&e0MlX=OXsU^B z2N&#y*8IzgSu&pTqFXI~Hk#_=l3VC+HXtAu-Y2u07^70w<_!)UdlI8f?9w+4g zHZI}9k$@*pUZsd@_q4Z5{EY+Q+XC@)gUfp@tp0jSHt>|L#Tn!BA={Da*Mgc~%B+V7 zV4mQ5$G`dPXwA#S$@q)AAv1A0eH9_teKvZ;RDs1pQAB&HCX*FJOFxN`TagD!OTPfCJU`Qr<`Pci74E@F?!-8W?~^m?ksFF!EiG0NQ5C_DyCFutRkIb| zf*#siuRH6N@5<16o2j(O1-vO_Ecu{2Sb#?38#kwV{4op|^edb_Qr;@40X?x{es#o^gtIV7aC8?acWa z6^}+QrC@0nvkpfu6UMFJK}4;m+R+a)lPvupH4gXF(P%{QU5=JrbK7?)@$qo{mJ#G7 zlf%0(8eM)nk-ikRwvU~q1crqvQ(Mpb_DH!3wH8mO`@$nv#{x$0#`U>zXbLfk)0;h; zyO-gD&}lwXFKvmgUbIf^|sxrMo0kHZ#-o39ul*k!6H($T_~QUb9d0Y?S!^eg~{V zHw>pjj|9ZSczFNLzvc|>e|Ngcl2LEx71&xzLS3{5icI!rtNB<$QkVmo!bzlwVEV`w za+&J|MiREiCArF9+I804BX=oeJ6-7fk6-)GKovlM;&vGb9$&W5G9hx4lN$%f&8J37tF3 zXl;ieI|NX{%GdG}Y1q8*qnefE+Q{;G>N^=ajUw3v4bgk9Z+E;hf^zJHvmc0@GEtDl z>F=yAa7h%Yu_sM8_QDxkX~UL|11ckBruuxhpF2zj2lDrh6xdA<$d)`|JD}N}xd~9t zlwMR+0#53|z#tO}aWZqsYRNjo#e~JVw+Q*klHde@`5#Wu`zHnA;C^-72@F3-5)+>O z9ujjR2I*zOo}fC1!i#JZ%Sk!4J@RZ5XR5J5j&bsB?SjF5E;cST)amb#_j50MyyvqNg4*Q!`3V6d7~9^$2Z0yW}D`TX8z$spI>8I1k_0&Ke{4%V-)E z;NEyWww$<@Jxv#DdZqUwPBPbYWrA9cjM)iX`fuxe6ABZKq3hBvGRbbpqUlSvmZ}(B zA4R=-?gN2NRVA>`t?`BLn)7oq2=j`zZ@uMZ6!PfObT5F(`t&DV;*#y`RX?ghgvzA% zZ_3{Wbf6c}_hv)D45W4!HHys&6!~2KX}i z5meoV`FN0dwlY(2TTplPN4o8f+i`mgGiN?7g6s!RxzIw%qqA=TfbopwWu4?c;<_NP zyP#w|ih5OMXWYJG+%pZ{P;0D=1PsBBXjP;muiFrhT?qD!(CI&q7qlS_xBI~gtkjxx zY1_E&E2X^|WH^Xc4VSD~4Ddc*^RBPBpz^%%(mn&?!+|5u%I?-;4=hsHyF0|~4RHEi z-0C0Q0af|8S}kB1v1Llj@#(#f*Jgw{@QLPP;uRkcrLM8keZ4>3nAE>$jOO%L4g+QCO;3_`jfI(M0U#A? zY<@Vu!Ty_CEiYX%JE-~hw!J=(tcVXy+S6LX4N_!rzp6~~7;z|@a4NV$Zy{d6y6#2D z1fb_{p}G2fEYU_s<)CYF_t7%AX_yeEt;$lvEm9h&K|-u7qo-|`;E=u-TA=;)GpNtnJy)RnUcAY~Xu}z(jBM9@ zBSu(X-^C)KR@A@oC$4S+tD`(1c{i35&|Lx7uf#BOl5P|(E-D@DH+x7RgMXh^ljyqY(V|Z!- zB-jS>Cy+Zi|wdok^--Xu-FB6ens62cp_TV0Ndt zsQw-oF0oP|9Jl~ML>b$S;8)_+(c|O%wBVHSHIv4zmodcO?qhiF^)isZEO`$%hMgie z!6Oxl#<&m z)a#E19i4)%Z}HlNfWcWFbi2V4iFrP0Xmui@O=gi!&{@U?&Lee>6e6?f)Gj}vG15T-DP9>t5F-8DU4XRZfv|HkZQI^6Ch~(E zy1Jw;>Tn)erYYqSZf6x}0!$GiDohV4=1}AWW_J9W3>7`jIGZ>>TU?jwd=~hx18(7+ zH82}L*AG2u9O9_BQiOQmp9UCudg`@7jKo~d&ZCFhEB^ina%C(;N?<&vZ=3;R=FO1@ z{%2Sb%4HocgpLbe9DkY@UPq1(ny+bvDCvP>%^GY*=FQBERrjszN?HSo;X+@&uIDG~ zZWF$q8)VU2N%FzPiem7CfJ-Ax)Uv?1#hV9Vq?d)p*VWcJU=Ng;sEF12xLOxQcDCQ& zF1W@ZG@Uj%yKRh`eOwOfJe%ZTN|g1P-Ips9c z1^nH)>E7WrY*>MaF{f<%B%4mvIZ3e?$c)e$251M0yKM9s-bJn)xpryLUkeQnBaY+n zV39S|7S2v}V$FqQ%Src1M^B$v%7oq9POb+~aAJMb_Py-9x>1HrXUv&b8 zR-=*=!NyA@+cOrqr8J-hCM&LUq;q`Oyn$oWs--ionyv-~UZS5Uky^C6aKEDs_sdx$ z6&D7bega1c%vHr`T0jeYUp$07LU27hE85{d5Z_?>hUA`>Et50O>Tki>TCd-Zo;;smIn1TUr0Sa2FL83XkBkT)bqxF*SA=C3K3OwvNyH zs>}I#jPzWiUPnM7vzPfjh=|UF0Ix-1m=AG6!y5OK#v%J^rhi=6!VUOgl4 z!z$(gj(w9tGx3RWTyZTVhO!g`c%;abZ+#=}l&C0m^Dc#wpw_Bvp%&c&T|+iHWpc><}_^SU?gb?t1zRf!kRzZ*0ozrwOu7`ogBg9 zotY7yWidY-MlZJo#e-s{AKCc4;7zBa7aMMVeRkDfQf%>LN$F7-eI7(2|NQMF?V|%Q zow^G!T%hjFW6;w06cpB42c;dy$*ai>Ubx9V@FN(WZn7zjiK-x9QpKuj2cz;|U;%~N zWP91kI;1|$uF@m(>Y>0p1zo3?Goo6tw(Bk1Ydo?_EZH*?@G-O?vjxZM(!b>`oP8JA zIq%{-r$ej(3LZl;QCV`A9TD_?i$?T#z^0(^`+z#C@xV{k^mL)e4CU<|(b(rJQD;A{UqvZ+A zn`ORXn1XnQ7o$~#8_NpIVv)kM^!9#JIIzR()bRkMUbH!(CNu_-_;$;oD)&&%4kmR= zQaSrJFG|ULlM1{Dud9bZ{a^Ttx9e?)Ex(gYO0&AGFaU<}81 zxfZAU#$d}UNX(&!f#OKYwoe49!|CIj*XfoL9d{*lltNj&hJ({$*>_%2$JMJ*D+{*+uerL3e_bCP#TN?sRS6WEK{xi zr5!xcCwD&Ldrwd=S8DS&tHn$k(F#mT8%}FDphX5s9-f6gl8YXOcOTa{U220j)^YSj z$MY|X0Uy@DnRy2@b>47+7uDMk?^s}_OubPC^&zLK=uLk@h87NmREqXLVI4dr zffq)*)g^uLRy!oPlvOj_;I%g}sHhLA%!3V;EhOU`bfshl&)@|KwZ^h40Wk82&||f8 zNI+Lo)bJn}b$N&+Hv)gC>RZ&v9wXNw1?O+`x@!3G{gLgL?);d$u% z|DtNDNh#dr?15m!*lJC0u=xqKO3`oV3DpM38h{YP5xUKSKw?xV~Z&dmyr znnP5b??%=UVsoz#ew~><)xfX?S7B9Wj*v=}8lz-Of!8rVQ|NW`y$L!jKZ}6}#c)ER zs4Fc>J$u{k9p+GLMx|7GT?fl=h^>Nn5(#H}PD0J`+~+>q-?4(cQ)aqfCVKcD>k1Tk zAr`s0HLf03xgRW1rq;WIvOyX8_s6cXJ!HA?c0c`H;lXq5h~C~eV1KCw!F6piHZfmx zeTjBAw)+@D^g3XF^EhCS{V1*x>+$eG`HdsM--YBx^K91V1~zi#qHiz zs-Z6%O<>X4#-1Voi#&-b73~tv#R`uFlluf>`zI4zVvd`X%~K4$%dv7=FGHRb^*b_w zw9)X?si_0p!ZlisI~A?qRCdnV?;bDcJm#oXVZ(3(^Oe$JwaaOx^cG8p-BV86#m=R^ zf3PB?%^Va0cbW&2VC#D_G=3XtA(PxzW|Sa|t2D^J2*Uc#ESD7Fyq9O~e&!7__9D{^ zje2{rZLi*812c ziPvnDb9Cj^ZZT&tL6U7?n7CNlIQ{S6VMSf4C6u-|PC3?JDzx07&eMRnwq4dR|9Ei} zD`>B$T@XsRN8H~BiU;{YC+V%(92K`q?oTQFGJXQ{J~Mw|tbPDE`;=Do z#SS(Cd-?5)Io0SFdvJP;8nY@$T>e=lwlXu$%z?j#u>=s|%;c%0;NB;BQ3cstxZF(X z5Px~{d7U-Bjumxh$nXDlGp5i^ztVbsY68V2QfaLO>4{U75~_E|VBK zAe=hK&Vwk5^`QwunF6&wtl;$$svPlkcTqD@$!B`|xolYiI{KuYY5ix1&EK`Wp780e zh9EdhlWNBpjy|?Wz>PsudOeHG_aHtz7bam18#ARIl51@W^+Yi4uIu+M{=37;3~jZX zHF{!8$<}?m`qmKBJc?kt*G`V_G77a!c?kw9hnH5j`rk{dEt=&~yP1Da({}h|bEHQv z#}!d)d@3!9^={CNp01x@fR)*e9<$5Q8?!{N?S((L@>>BqjET&I*ZEiNMwJ=Pn_Y($ zH?)!iFdKJFUgHU4hmtSR>?n8aRH=3WZhcd$(iGQQ;B!M6J2w2Z`HB2=`6c}Hh9~~I z121N#c0ng1TWT3y=&jcoBk#&;>1&!uQkNyka!KI;hH&*`imAnH78^>s_{wY*u!QR{8A?aw^9QI+l zY=r!p$coByZ+(<}cqN)q){pNPlkQ(}hWl1D8 z6dR&3aIMQ|7HnCRSq-8<<4uMphh1|S`AysSmAG>+4l<%i?!8z&t(1{;^ymyv zn$g4YuAp$?C0S1C$HI{}38RPt;S{meu86r(ChR&=8D5jr@-}$+C6eaa(GkX2dRxb| za>?~Y6)dGglIAH1>7qoWCfVkiT2wWoQ+t&ZUOehmxvHY73R<4}5?{KfMw!Q=+sD)| zt^49qwOpXzRI1DbI9JF;Vr;~ZY{r7dh_SoEhc!#BHqZnzXWeFv_3vk7lD zR&dtyDhIr=nNQIKTrDElqM(o}R10>MhDE*tMyk`mZc3|2J=P?D5}lqw{APUx<#U1$LTUxip+wHbPaxt2bJQk&XQ1rD`tz zlNh39L<<9+dk3c1k8PuNikXa5BT^F^B&{xbr^EM!P#^Y}lcdpjxcJ55*e!tucYmE- zAj-QdG`^?85#@WBcy7b4&CK6TdPH z>^K&ZZF6;DaGH6%DpOeOw1ApyCJS{S$l{OT-NyVbsn(i|@n$mR{up%BStD@j-#sqh z?*?NMZ^#+GDCAi5I7{iQNCW90o-DXg?cj0>0SL#7BM zd4!sl^jb^txc1i}8a_+eCn#4Or=7aDBpQQ?NmlA<@Ki|`MloN=?sE5*(G(pZP(7P5 zj0wlPrtD;ysYZ|ZofK&Z&fTS1kcz44IxWc?>Ns-Jd6VrSAv8hF!~-4*Q9ZnKGkhZT zv%&Dbu<+Zhc!B+0I^M=|o9B#wHoAOFIw|S?WWrS-84F1VtT3sWzJITJ-e0%c&=@rO zPU-m<9B@l)mM3Hurrxq|A5b^uReNQBf8uHbZA5_f zDvs8X3+K$&k=!AV4{oB{BV}w=ioQs-6`PSk+SGr4Q5XDWk8mUY#2MsbiEvvtLcQ*k z!W~uejM~%j_A+jyJVQy6NI(tH-k$w{s2vyi2_@MaT3Gc~!Fb#E1TyJtY;DI+_AU(A zylFPQL1?m-Xk?0`3j_`y^fSKR7`(HS;XnDD$0r~}ukS9#$s%sAJH8LO8SK5aDx0(v zh=<$Y^SmbYj_RmPy=(49F%TJ{(7)+!SD)h{nZxyxB*2Sz*zh&#>+G@m`*hpgN0BM3 zEcScuawh&VF`4L2i7z?1#_(88eAPw~;1ttZoP5{F5_ZZQ4RhVr`>JZ#bo=yrv-5xc zcr&7R%-&KaWXqeH2yLl4&Uno-k?>g75`daG$PBwmtEs=>NJ$_^{WqwYHDH&up3Iyv zZy8~m*eK1lP8QZBpuJ@^(<4R(L0oL|v1@a=YH7zX#-%wbBxZ0U;U=9t-BK?)hIjuM zbB8;=-Nn=Sy4L{fwt05>N3bDVm`}G@*hY!a?sxgIFzgF;ADY}>L}r8^FNU3w*{x>@ z#l*NTS+pg_yM>mk_VWXk=~X;Mf_kK>Pi0aW8?J|oKq;hETn0$cb}%+5m$(&-+V*YC z3{=YGn?~GY!y01lR}=0^zJ@(<)|{LxCxNBlgI!h-MRnWTuFF0_96?I?KOXxYTH}F~ zJk$9~rrC$~)xO_*tJ(Q{`RmIZ2F(rcaPAdT#b)#5n@R5PyX5l?r>j+8LagNK!sl-A zH>oW;`RaNni=a)puDtWC-)RsASN=6A_gnY|)=+|&$zzg&j?|;;CkQ&X%j>(1hyARB zLV6u|M>B+PFN?&Gl-(3U0iMEXL!E0iY36!ehTb_1N0pa4S{8-ZDC4#%g||327(!p6MkYhuz4yjAfUJ{7nY(YkM;VW+@8HRKB z6W#CsEP{iU4p8><)Z3ZTY^e#RNpl-1zjk=-ZI|;)^lY61W47Wa^G1#nwiNY*NR6lOILPnN)NURE*@vZRJ*;SMmnkM; zy@BgSh21(#bVD57Dwt>E=vY%-J#)3sT`c;zgz)#Rf#ZvOtU_L0LAqB}KcnBK7Wodh z?}k_=P-i9++MyIrffwfi6a}EF#z;35m0{|LNmiW4T8r;Ab$4-i9*K_YZmNH~?mfe3pr|1{b&np{h#Ccpau3Mr@vT%~kJka?;~Z zlGcKCL^4E!Pl;_Pz>hyX`+* zTwL$xIp5Y_@96j24bwd=KQYXM@gA?&Wj^lLyBa+opQnRy`meh|{Hg4Z$BXfvuU}c0 zHsIL`;=wpEIpmCn?aZTWm-0>1eM@I-j%|GAWcCt7}|z~rg>ZM#t@U72WBfCg?r_GEDLlg#SXUpZif1f*;}Ap zP3e?rflF{Y_TU3zqQZ0Q4Ets>!Sh`xJ%%?cD6MwGxMB=>7_!20M(c6j%$1;XIAqYu zE3k?_CoGlFerJK?4jm}HDrnKgBUC!*EHupyZkbVlaY+IlnL3G7xp+RbfUJWoIfIKS zld9Tj$-gZz_Z31+>+li3`fQY$6RT)E)~?z$^W$P)CkOAqjgno4wGO)v#N^+)xc*xV zV@EsiMfHwWNv!+jfBc#iD$}Y;6HLAhDe%>r&MEFttqi2Cil%QOjHhc!Vts%-N)CnX ztT&Jr%1qvOX;)X`YqIZ|zD%&wDj0c56(fC4=Cq}$SmdiguP4X9?SBv)GDAb z0x*&2aoN`0qg^||^K#Rto0&;INd8&wB5=IQ2g*F*m?GE7?an0N21fs4d{1UVbu0&YZeBR>e4E4O`kACjv9rEPcdVhAf_C`h=QDOgxsqGF5s%8Nl zoQ*&ieL(^(`5kkPs}Ngp2a33@oEndmAgN_EYMC-4)Tl`BfQiFSOeRSp5tihGHXK(G<0>~KJ8Z(<1b5h|egBrXg5zDG{nk%&m2cY_k{13W8( z^K^vbsxkn){5i)PVy{uY1Zb$Mbz^LYB-UMNhs|gfqoPx9jL~o~F5i9L<5rJ{}mFhXBc} z=_EYhC%xz^0jC4F)KzOHMur@J#~kQ0YR99YXHaqqQ3qfe^Yk8D<2aBHYz2k%xDFv* zXtW{bTSZ52G9S98rS4~4^CMqeRt=s<^JF{}7qghYD{zE|gTo{MvGV-&S9d#(Zai!=AD6ulX0O0z7U2)|QbrGj4l*qeqQ%z~|*#UC@KA zFG9-56?N>Ko5?q>n(IXqGJ;159rf2+$5%AMqw=k*((RMNt*iX)lPr{r4Ai3(l&d7v zlQ@){7}T=}l$$Wrvmlh40Ms)dlp8P9GdGkQ7t}Kcl-nPuXErE62=J%0>xJ1-?|p^Q6p#K@Jk`&SyTEIPvhZg;wKD`REFB^5!JqT-QGax-B6gE z(DulS4K0O{tvcKH}Hguc#pu**Ko+QpGM-#y)QBN;Ojh3=fApm zLmux|x+2A#%fh|}vcj@)W9O=jLV0nUWO2Im3%Hz}pLTAb`Vmehk^U6pRtNyKz^4zv zvpO)yEkzC=22AgCMj5DnJB?n;5Xx>@$d#&uiJ_A{t@pX4V#sbWTxf(Oc3}Ee*fFC= zIM1zVav#*EHRaF9;fCqqgw1OVnUzq}JoebUygE0GHAgRJe;&2Uk57YSm}=#80BE;+ z+>s1WPSo{Rno+}O8@<~LHpZ_EdOXu3%gc=rnwu>RS0@>E5D-Y)y}BD>Ir8t$WLeev zOtR!u zGcD1%)Cnd_;}?XS8tHC^s!gru*=~tL@`4N2Diw8ZgP=kBu?7Ux;Fy){80xMFS#P9t z8de`-prbbgA%ulRIXFb8`+_<+l)~=y`~!YYH^-}47}~YDe{V^4A5FW;MKP)V7DYws zl2F;9Af<5`O(TPrv-0o$Q?_Lj1iO}YP}{F{Z{C2FU1WA7SB#p`LsOStJ*jw+UcVgl z+ZOMniL&px2c6NuAhhLZmz}g=7J{^4snyk3;-Z~gR94W-qyWEUD-V8tU?5=U^y zvs?&<^T+C?eC%#MN}_bD8bxMfeEcbi^}S{4D>0+GA&aFlL#9QG^q(z zC7fZwJ$xEwI`x851Z9%`e6n)T8QtAH@3;z=2WUmnR%DP_Z?cYc+V0en6Rd5O7Ra4z zN$ITiO-n|r*3H@-t~1zmr+eKzgg31_b`H$beih)hy|{yC+4j5YZwK}*Zy}6b0fgN# z^XD(1?5B?IqYnQ-s{FV;6ARGgnu?Cc zf9k4kG!>;IgnastYuLfnb=SL5*fgsoIxRQ|~?@5f`mwO2L6TIkvPU zYFwjekwcnBmNUk;MPyPe!gd0#Er1R~c9s?-=qQPsYihT8z>*g9w#BVTTu>pv;!;z+ zXri~{s;X|YO=J6n+D?#8@(0+bZEdq5v`Lu;=TPEh6Q%T&nyblKfiYFT76fwSR3)GG zh-lv+q(EXl0Ib!kg(&k2Sq;V#ZQl~ArNDosP*bF_RHdgDY_OtG@bnS<=XDVHW3YQ^n{J4zl7NsrO#qW&9hAb~Bltm0kd0j#u_CAr7ZjYxKnu=%u%l#0)&HsHcSBq{&1jCUldo2LaZ2^ zISxuG5cBhgW>W0Ilq96YgE>}ImJNHjpez^SXjM)w#KC4O9BJ*JiOudh(FHX}YGsd~ z5Qq|lIW_A*Or|y*Z9f%*y|sbMjz=6i9KE8QeKbcDM@rOE;^ckH!Q0oQQl>8C=xzAt z4FvdNGi+-_A3hG7>lheJ4JVPB#Y+dNO(;tWJ4pytOy$5iR6GBsIoS*`M+p@aYl_2> zSW<7B6oL{Gd9gkz-zlExM^dNiX!Kl3$`0-v6U%6w+iFe5?mB7-O*d-FbD6M-I;|?V z*O2*-pQ}E1i=HtiLJ=;<*LPfyofF!@j+J(!0gbiV&EOa`^nn2uN4TJWcSkqGR#>4M z*AqKLLZl1BLprvfU_cdNAWS#Zb^xFvuPx@=sD&^9SvvA40sA26Yy`v;a~{3Y*13Wg zt^fnt)0?XsuApzT-$TfQGL8wiI$)0=!Vdnd7^P_@HZ5c)H%O7Y?v38ng;t>d^r&}n zQHKG>*p5(e+Jphdyb?PbEl!CHH(2ubM;+NQ0P7nQ?nyR-TFcP|)F3u{4*BHbY$QJ8 z?gWj*)nn8%+>aVE+8tXe?{W8d)4N_Un~i5?CqrUHW6cqWPia>+ z02K;rVhWHL>zju`kk$Mw5c_3}%Fj70Cg&#lzur$n0$Ko6UjQVSP9i}#@`yX>3$!j8 z^ye;9@}_S!0?(xm^eRQ{4D#|2cU5>9bGukw-;}*R0m@~9h;mt-`9r8k)@br(Xo4>S zusg)Ewt0x#^nC59r8BIqD$@y+3Y{sb;Ku%gCmUYl{I2Z^L@BL({&7lc7ak}T^BF>^ z(a`@2T-bfxs1s`#<{elNDZcbbp#7k><4~LXDlYv0U%xcy+vn@x;^Ll<@3lXX{`>9l z*i5ebV~<|%WB1~oT(0|N9Dimtd+7u>SXuqYL^DWpJgRf;q`3RK0dhPMb=QMG`qEg~ z3%o5S)6pb|Y~a0CHD!f@{%`xW<8k$&CP!{s8>G3~j4WJSPAyYm@R2oxOdFBzQO)Oc zpi_=4ag-d*#;5$0#k~SD&^hRIbK6g+GRG_5;z{WNmddJ&aA^KpTS+i%2xe4Joi+hg{8cv0Qhpi}0pL@#Np-ADXy<7MHqyCqIdutkh_Jm)XCZ6=YP0V5z!Tr!2;DX+cYsw}u{+jUA(E0@I?*2H zbOai8PZKHQ>T=VJ0*pXuao|i3iJ6)@ZBBmK)=imr|JNeU3PSX@jYCMxwQ7-o5>Xk{ zlpg`}>iz0c&ZJvTh8y;FRcMeq9}T1|d-pU|B{T;u&xT#Q>5VAJt(Pyh?!R;n;@5pO zvF~T)o}jPiZ_!Y@u#_Eh(YgNpoXh?yxi*NmXO1tYV_OX#J?d|~M@BJ*;&1|fYTJ_t z@nAJ~l#_fYIv0}$Er`chfIK;kyWqd-oIR!k*uc;fP_tL$jNx1$-_!LY4ufO0R6Z|<%@>yrBf&(&h4P7JdVEQV>{~d{$fVl# z#UiXOyD&|eMM1f#tw)dg#I8w_rgO%%(9wMsSKF1#6u;r)I5POV6;z$T4N6uE>U!n- zP*x=9DXYGd8qy!#XeC9x2`P}{U~M!nPu;EdTfkXOZouMB8_**Bq`5%yzSIBjbF2T( zLW_WL2PCkL%983x*NUeo#|xtcWbS3k>Yq&nbyd#E;hv55!(+Ni-GQa-a{QWPv99-_ z?P*;zFOleJYwa$p)x{U}d^Xs&(%4=^I)N*&hF1$+D%a}Ri3XH!0aHb;N?OS@ufrL) z7nqqaClD>Wv1a8BSe z+prW(`76_A-*Y`l>A$|wqEM}2S9HAiRJm2?HvBm+m8qX13rlJJb5y=9N+6mNQ3GgF zrYd+(V7f)4jI**wVNg*cWl;<9TJ_p7QV`$mBoI#t`Akzm(VH3*d%*E0Q7RJt(rn6~cf#VrgyeFk0Z!C0Y z1=gdh0C7Z{{y72%mnO;Q0@YfB%uq5vQT9JIr_BCx7FaHRh!Ow(1=wgiG1kVBx$6YV0>@HbNy~&22!ql!z%O3O1(|`IUa=?jwJ|) z0)uau$z~hwV4D`>(p;$g)mzII|B})kk@5AzDm3Nk zT)?@#zGrmn+(Sk<0p{>2D8>br8HD%IgnnQRBp2RIV<~Bpmc=WN^j9P^ly(V?z^`R( zyYZlg%IyffO+J1#mUQ*N8Rb!DHWROe+XGu!X2;I;F)z_>DCyeB+{>kyZld(!R|&gz zd#QitK2;!GWZqQpsjg2 z38YctWtpo;nVZtPT$jKfzHcw6! z!xvsF2B}%ww`KQByCXxtu{kxxm;GdqG9{pWNMlrAs;#(;4eAHWSi0jWJqGj8FjY;~ zLVmMDg}m5Lc8{!QN@>++NlBQo6qI+25(1zCuI1zDL1n|O*NBQ!uJ@f~szx>ASV^D1 zSab!L^4qZX#~zRfV1NHoGR09yxlxHhqw(@=X-}gNeoJboqPBK2aZ%|Aib;U-8QVTT z^zi%PFgX$n;A*oFZ@u%Ezlrn5d#94Nw884m=x$mUrx%%56ndJJ#0uO zV@&~CEH++fh9*cZw$g1o#*1THSMt?FA$l<&gsinCiA?_5;K_?DN+1uD9bGsn|Dyrk zJTGk`5(ZpWMe@KJRk3W%=_pEspl&tAL0QI+oyv(Yk?Jj~%*4Yh0Py}TnG>OVYh8dw z;a`O$bz^AIP{_=+%5Q_^c2f*$0xh}Yzd8Uyj(Uc$x?#9Nn;~-%T+!CfAPC=UoqF=7 zN7_p4u8mg$agacla+RsGyhL`yO<-18M!;ABoE&$amK&xnask8?X6)(L_jW=xZ7!HR zsNftD10+$#nPfjo#?AKxahyf0n&jNi%t&j|X4fy)DX+s-m&P&hyP?d(2%e@D8bftS zYcY~jKwUChz7|nLKLS(cfmy42y1ld?df%09v51!~Rhyg@i9rBq9udB1{5j^Y75}T~ z^=sq#uBd=;(>l&>RheK>gsH()Hfy!A8qFN9xZ>IGcVJ}yv`Mr%uj>`e<*vuG_Qy~jBS{Vf(KUuiif zP(B*CTxC!1>Omhv7EFGx8M2&Z{38~9SXxLqA_CBAxf+gM%%K3*k+dfSBQtBQX>IRg8)_SSwZ@(OH`z@;i>dY_FjJU-Z0YRGtl-)v zRqKby=>k_aZEF_relpgK!e_>YdEWCUAIYLecW}G|llsOMXF3)fbQd>yH9sqF^%}qE zRV}38TFbPI(}>I!hQNh5C6pgP(P^hP<=({GZx&G*VzWzs>t!ReEPlvoUk1C|eO&owt~@@Kv2g$A-9wYB7AF-}i}(yy5=yZak|&d9OyZfU%6t zF&hsdBu??z3opK=ORl$8D2tt^QM}VyMFAxoqw{K$HUa`LS(nc}2JKoe>Azr?HETJh zF|9!Tp0R58^=Qo~IeQhWmHZs2f1SG=(=Ucfd=R;CJMfM&`f)U-sv)trHl3EZ7bna1 zMhmB(Kb7aFie+`PUMrN~ySyosTw{FCyS&ZTFplvbVwez2W0>TQfJHOK;r@TSf8cr7 zALqXnsYh(e$=F2J5|#C9{NufvJN@3dgd{DqBpV0AC{vbTa%s`|G!4)_j1~8|B;LuC z@{IC8ZFT_a2t#m~P!lGzIOgs_663>qa1R>GJ(saSyR)v54~`ElcwM_;%(Z2LR6S0NW zgqJpU)UPQ?XK)+;#Yisx8_`WckS2tTMqPg~@ZkbU)V5oVbwRlwhl)*H0MANsv1cfB ziE9JGh*q7sO4*#6hne+M29JM&=)L|TPK=LFvF(35&7Sy-t5qm>pKu30QkfQ^!6gDyB^y?6$m z8bW}gda)V%(Q>lCZ6DsOowh9<3cJiG*h$9aEA}_t4)CWNx7)C?c$El7hXZa1^V0$^ zI-Kfsm}KFtMKtrjq-gI(th#I)t*@w(-_KHX;Wp~|O{P?{eYmSQnT@B4ECYd3uKL;$ zS}dhWySXyYzdov35VxOBko`l$o4GX13h=i#uOhJ2j=AbFuMT}y|87dm_~B}nMK23n zJ)EoX&>FokupypL$-ls_xl>omDbFQA$}ML-)q-_SoT4=YG=^Sv;@PpztuFk5P3qV? zw2K!;tY$q7&tTTSKHn01o%Vt#XL#)N7LqTuf=JuJ(gAtIKt5htHfou%`W79b=lz6$ z4lS63iW=rWddT*ogQ4V3`!5JjbYMAJHIJpDa}I1Ch%tegsOTHT`5bl zr`L&M>#&nLX&XCWTTZFZi<57tH?%K$A8AiWLun6ACK*LzGgqfvRuoBSfshxUa3^xd z95w5cn%G!)tL3Gef;ew*7x6+{jNQyB`c2)DR<8=u9!7C_{kwsUYM*@7tp`w+R64iz zQpqn2O)=}5V~;F_hEu|EW+{|`BU@78OIEBSiaLIt8~_{^3s)2fE~|%>E+|T~g)k|f zbSa*ez&tT#w}9E_F0XD0=WUKS!7t|>3@h`=+86p_xyqTo5a;qO!55k2U=V?Q@Ws9> z-5T_pRN)p2X=o$I*egYYLzk3n&f3*na;@$=EmxM9mgXqM&1EF~N+SMWd%mXE9kh3pOya z6bhIvokdxPPombg_Ys^#RV=6UN!zL&*350@uaKw#16EEh!D}DmHH^a+BQjN-xr}1| zl~(=HEBlIi)u)>d?GEd*jGKXu+T|-+GoK#!Vn%q1&CTO(4sR0FBx}XSD@Rq#iY5+) zCnK0j-u`2YwYX5&Kxz4j$dYJenfEO4qgX2cu=;vTeoLfdC>V(0>~`rP&#KcWS5L$L zLkulep5i1w- zkfp0DOU0@Rd^5iW7s5dKj*hhk*F)?TTca(7KEh5oEtAN7rQ{&f{CHHbzU}>sp{2y7b)!#CmwH*T~iQ*3bhpKOmu51bVjcwbu zZQHi3iH(zFCbn&3V%xTDPn?PKa_{}#`riBJbX9d#?OuEJ>a%uN{i^z4ThYi#215dB zZ#%zkM52+9j1py#gL|M zeJu#oBgG_UD+aNL{8bl2^na2fyY@wV38Z;?%OTY}sd)DT4BW1pEk%(yikw_X}Oy1ZV+koo7)aVl7#ufhEapd~$4iO5>jTf8P!_4GC z%5aYACgJ~j#~~MI9&*OaVOdCEg}oi3)Gfig$g)2!c4%XFWDM3Pz@|uw5_~WZwC~LN zgt9|8xA~L0-6uGcXZWvc+gUoVflEd~r&6=uNT32IKT^QRha=9&>>AhzXrr+)-u(Bd zdY@eft&j(;uu~qL?ewZ>a~Zs2sLd(>rkh!OKSU9p)D`Vl%l`65viIH1+E^- z2!8i|>@##wt}fNTz1!!vta+3%kamT- z(|I{u1bON$S;G83I(aYJkKnGt4}jO@VzC!dM?jk{Q&^t{v3>TV_TeDgcJnc|j&l0f#;oz>@y(o@3)a65_`V$k z59e@^qT~O|f0Il9&1`<}k{?8pseS*uU!K4B4iuk2J6!Ydzrv4yp8hmg{*U8TRLje7 z)#%tt%9xW_uF`}scH|kZ%!Q#RZ=BTgoA>j(_NUf0b;(OkiC2AR*;1$c6f5z?dfql^ zSV(D+BIU#$P=#Ws_|x}ulyvE!X(NDAE!aw)l<^<|b)-6Fou(csY=Kdb$8(ZdsG1m` zxEi*Pzl^zx4pil}G{0r{{3eX0_KL!V8~VKKQS1KMUcT7Ov~odl5BJhvgq$>Fx;(^b zd}AvGR={tOQd>F&@jeA|X)xuxMw4mPam7bCD-oxJ8$kKO+1~QX2LxQ#hTg2l8g5?E;BAmW6=l_D*pWEGFV`Dr8k6q62N0#Ee)U z+fH*619=&y`Ow*F3!k(7&iH$+Neg8s;H>DLVFa>C(k7-RdZP0{bnokNAbF?wa|`v zj?dnNs)#Q%Snvbw6ZJdWV`aurQ2fN&hSv=}VKQ>^yC;;*-76@v&JwqBoE zhNjOizk@bq0m#d(CgKI1q`>^f$mtg!(Vx z`6t8tA8Mi?Pqz!goGW4L=|HbZ@3cRQ{jUg?>5LUH#c9Ee4CR{oE;_Jm5hp+DHt=M< zW)fOGns1XOck4pQ`eU={ zdG-xfQSpE5O7O7E$H&L`R=JXq4SZt>qR{@11Q_K@k1Ti#r7)8=h~9jI!Y`x9Rx<#v zVh+E*u(+{Lxcfv%#+4F~=_n#P)m2^1c_=Uw>cw?yGC%7e8Af~Ch>T8}8eEMl09DWH zT;Fv8F_BoPq$GZm@%I}5{n8~dorK^}%PXOmQ`KgA4XU2SCVUH1r6wK8Ow%d&fw-G; zcIM9Gs_3)Zp8b@2p>Xp1u)dT>(e%EDwidw(mrEWzem;aO6UcHm5eMg{%RqncuEN)A zI{FrGilaoWF5g1x!Y&NGQhNs)BBh9B&C-2aXJsnV)+~BDA{e?^w0wQZJ&bfH*3yl) zmRq%nc@d}}Y(-3qwKY9Xk-bL9Yky@Q?E8kad+-J5^KPyJQED6vDG)8~Csv)yC+vXm$8s|cOIp`xX_p%PB}zt6qnp3=K+Bep{dH?*Da#wF zHFKSFEEcGE19Uxst|5!LZ;pRIovj;5^!s6)HV?ko7+I>+5|!54B8h1UicSe{Ezg_O zu0(HP1B@W+a%F-KCaF;947Z1MT;Lyy8zzZ#4I6wMG52^7DoLyiUwpg>WkOExB*pG5 zwIXVbeZehAasnZ>(9CSD))!D8$j{f(GN;M~G16omWamTEtK z0BSsKzTl}bGyyYGUwg*F8Arb^qZOstdGmTvehlaY;$?QjY`w_7<_~n53H?IPt+PjV zvYq~gf}E1UfMot#Wks*$fW>|`&>A8L;a71?VCBb6tL*!bRVZtQlEG-gwdv1?0DT4o zN8AZISF=734Ik$14|)o}Ff$S-hw%)g@{-dGDkqVhbNKWp(sBC}$=UqjXi-;0YQMHX ze+&v4ZR)H|`A2z_`J;Db3iQ5L;qH>`kBD_>++oAo09y(zUQE$zhc|ECw>BpkEyTCS zM4lP)%v)C%`2cdp(+KxbvS`~!w-0UEAoB6>wAK4PxO4%$1Rx`_v)XLq)mrW5y34G3 zo|G#1mW0#$v($i`oOCgD=bfXZnK~7*-=(Lgf-~FrAD$d9ExBLM2M;F)wY#|iJUp*| zwkAAwx;wl)ZvUL8`MbR@t&HS8UdmB6(N?Dt+o-O)=_LD}_gJd;hBG>1Pdqg-|8h4; z#63dE)JjP17DbY+cU$1;6r%<)l`g7xSWY$c zGf;@PSk$r^zWqJVeRcf$VO#M2o$dtwHu=E2+Ye9(mr<#FrrHL@&Pp+rqw(UA{lKY)dqp2$6k;_`u6bJB<%667yyM+Fs-u6;Z zw%48XpOT8m+nyOq?C50C-U$JxIIEu<+AL>fho>hc%>QZz# zxqnrzVH$wd{9Uc#4+49K_JO>m@$6d}z-A?gOm@IUUJzOLEvg{5&{H6jmj(<~=geKO zBLn)4m%A03l=91n)*x<0#{9b}P;-L&U9Qh5tC;BbgzPe>I+(dDaW9k;I%S<#Z$QLC z^pIZ_OOr_rsHo=BU55yt4E8j_S#*oQZ1}WXYn;H23r{>^ z<)@I9U1{OHd5{KD@2ACX;wYkr3N%L?j-(^SjmHC5-z3sdaFM{_wX2PvVHijAOMh-R zq|Q8+4|M&yFoIEGsmU@*RG355Qu=+Ikjkl+_EuTxKvh|yn}Ri$ZtBr0r|_Gct#FOV zxfiK@YM3$l#+Zwnj1>$d&){9KqqSq!EZ2s~IsI8S*x(@Jl-p}j%-_S}d9HdY1^u#f z>3d9?6lbfPC?f%}Jk3Kek6?Ick=EgDHOOl<~gF zlhW3R{J~x=zLDYXVqFy^X6!|&2#=Xnq_rn~Rg;4g?vik`1k>OOxVlP9vT}bdufAdG zJ5o)OzY$}QPI*k{2T_+cNhIMnN59nDXZR5J8i61D4n`!~Us*=tfj$PYKVIB!9Rllj zwtL;rRIa4fIsg{)FkJ&>29<*S<<5&yjBRFN8)N@vJ5Nhdlo#^?n)%g8a&R61+a)Li zWx`(;i)DjE-&(hSzmrXLC6&W4HF~P_lbx^az5cT3j1~TEt@GpSxDqmr!L<7Tb-=hW z%SLtYllq3y+jMdf@rzr61+O&xKjVmdzUw*eh& zBdrV}wc~3kh%dPaH(oL{$ZJzp;4)j6890I(+K2DRTXSB5;Y2uKjS5{>Lj!Fw;?~*j z#HWMx{sfyrNI?n2qW53Ml3mSY6@zNJBnMh#Ce%FwKHcvNqQu68@ATY(>c-iB0|-1+WDkUhh| z!}N*-M=*_`8xg3;Ez0MP3*Z}-rex$CJ>0iF7*NR+#xXpMW@kGnahEH zsAhhF197P=!_txZi|O>jL?u;g(}{UtQ$ZhCR{;{Qp7fgqwpS9}7|eiR&m5FfC@GJ4 zx-fY=rW*A{8ih$Qc_Qk{)+iLl1!doQiv=8+j0@))elyVi?&T}|vSyH`vU3ftI`P_l zrR+(*U&s+czRdlps;^_?*yH%uaUa{bFwI2_Pcan)Y~HcwS_`4MtGK-*3@$(MT>Q%s z+gN0ptwXNCKWV6_e@Vwu&5G57!qxP4!MUjgkziYWw*wvIoN@hIhwE=e$wFtHRhrAs z`>wJ%k}3CNrz>KQjZ=@|NO8a0E5QJ6t0gs>YRNDQsp^|FsCcq>H%zTyf!j-1nyX0&+I^^oHAc4ou?Ti=V|2uGEDJLon<4;S z`t_THGavC6YWhvLpY>(E@)TE%GA_<+_MTtG%bRqgNC%~E{@gdF#{XnESVGD7a9v`e zPgD>!5h^d9YZvsvpS&OPl$Flo6*#-BmTTP0z)b7WQ}&gA+TPN5X=?4owc(l|=yU^|vIP_cP)$X_TE^7sGt`QJC6OAP)oXZ}n2K>em2ML7!z+UpC7ZdGx>>|= z&S|oW11&E%F&()gl{+wIvwgUl!wZv7==E#RF@TJWoQ`VC#te*NI4(`t!Uouk- z*O}+Ih636wDCK&dSwBlrLGPmwy*Td2L7e<*qBR|32^eU47Z^8A_OdLd-y|qmA#jef z?J4Xn?NP~|QC3i-p@@_SEBfudQrI>J9unAeB76qWAv@i!!cs7yR&%XZOO{heszNV) zr#j>mSs+RNRh{KB)T-LbG^wM26xaM2W>VV(?f5TtE++y2 z4Sl!XE~lXB?^3+f+X7f^0^m{^G#R-b0)VJ#>3Zo`T2;1h;IMPJTC(K9%{)2WMu=*y zfv_!iO2mp%V^;KB{vps3(a*voy$aiEs~0Id)Qy@=TBy6}`lzC?8V$0g2=!)R&zdkH zu8J4}E)!H?IGEL#1KG*>C@8iOwuWK63X_g4+Xyw=YU_I~g}6g0Xg!Lqen;X*1GJw`62gK!z8;={ zFqxP+e#>7l%`OoR-+$3}rnm+~5+?LTuU7v+@XT@ZOv5n8iw=#=@116cbSvz{ZY0M=Se@sQ)E(&n6OJ$+l=t(3J?I~+yH zfpk|MuQ@q2cTko{XmJuES%8{S!niRj{pM3Y31GX|oXh1z>i5p&E(#(zv&QDkh)$^6 zJ2MIb?nGvMe_F1UOxh-k<%52-1v|-g>&L0Q>_96JyR51WggHtQhI2GI!ck6!s`-Yd zxJOK(VQI3F*%ae^cX;R`YHQ2D_>;r9gi({Rl*)arA}jG2D$WBys}&hY*N&42lG@o| za-_pK%oWpr!ffEET)@7%9d?rs{|WQC3dTBZ1z>%pl~#^-u^8Y9Y!gD2b_xX}Yra|< z(@FUy0XF1yF5@PI58UvR3r=+4j5`c$ZT&sZloxRJ;CI^+so4cs19-QFy# zs_AIQ*IbB)G}jb8jeh_4MR-o1;ph7a<0l{cf@>U;XzEGrbU@qZ9yXceP}X^{X(dr7 zcLWbxb?%K$7fpRKZ`B3!QR05a3ewPZBn9(B0#e{E}m%eo49Q z-bIukGEW67mR2FsB-qqw7A(L5ET)Xj@lw@0nQV^^pCLp)^KX{Bp1uw&lZJn2=rB<% z^fH+9Ln>ooJbjGkkZGp=RyUFY`3t+^ABYMb^_e#6b}x5xqSHjr$JY{=V1z%&%Pc3b zmV~bzO*Pi?4>5-oat2v@hkpnn(W*4G=CG%%sE&}*zwxI+Q5*+N`@KJZr@H%@s?@oJ z%{Dz}C(j(GSr`$}r%~gA^^r&$WI$gGpYG`G@cLfqX>?T>7U3wzZ)Kcgq8>l-CQ{G| zvk?C+WSe94hZVqm)M6NI+BdM+bQVm*^v*&|Nkq62P`kt^2H#YlBqATzPW0TOMG7-p z5XBgqU;Ds{ze9V66L)xU%d-q6qVzs81Fz%>kK|a9C7J9LQZtEaVW$~wrD-WT%Dovt z*J6NHb0>BzPk^{VS42dtyqZJvBX$x|QUE*huK^tkqwcY^lE6rk_Mj^%D&`h+ic``~ z5xO%3a7KfGaI}!_Ek(DyIk6aR^&0H*!Dg9f!l3GD6`gNFFn$Ng^9uoeQK|`h(cws0 zst_q~#Ib~HM@6xypU0STDMf$7`Zk1|>V4CE+#i(HO&^q_1uEtw*9H95vcn`{WJLpteyNnObqk?DB}33P2GOKvTrNN%R(4|eLVhIHzo$t3mCXdwmN}) zZ^V2Qy=rrnYmf?OScaJ)Z+{v?RmR%KfEp@YPZY>jYsTb_|~ zJ9N^g4$s!GUpc2Vi3eS5H^sUNY)3FZ=}8F2E-`7l-qZ-R)Hq9Lv|K__ zj~WAz+9u~)0xtv#Ton}wMicG`6UKEm(uGTnaFSnlh36zu+v6KhN90(Y>j72vY9<@o z!oBF>NeD?I=?7GpGiOrxx>dwazVFZ-G%TQ}L_pEAYh|!Yv!RV+EU0_qSr7>_C!*`I z*H@6|1p5t8Jo~h(azNUv+09Q}Sn;po#j7z%_`{FaD1))|Q@(J0j;?m#6{aFQ2}_JI z-+qp??=VUnM%?mC+)n@tjO=|%a9#hVtIMoqR&n-(BRu%?Ib_^Dgm3HeXRP!Y>vOQ{ z4Fxho6yh1_?~$Ny<9M&Kz<^f39+07NRW8%8WUJwq(SycFn5VKvr!hvS@N!3=zYrP} zwD=W;l|kheP#-G)q{&|*Tm7sZ$FiB3=X<~e!TI&u?jh%Ce{91E*+AyS?}bWr8Hb53 z^M`p0*Bg|#wt@ysot7L{9-W^sH7KkTtJ67D^%V~WazCxE$LpZI2)wZZPX30?pc|it zv3L-Sup776b3Ml|7Bu?bkEMe?Y-GUp_7WfYS21C%4oos(jCPJ@lwg_WgtrXvmCI-d z1id7!#%f2{wM*FEcQNf;_la@aWSXd*rNL{vI^?{d|MJ83fuIdTR+v!HgljkuMws~O z+n4`F7<*7{cM1yUrFh?0_d>U=hr_m2DuchNO_qtdN=OlDi0wqMfWx2!9gC;fM}4kI z8L)C=--LaMBzcKND;2F@(#(-#ZaP6RiftTpEVqD(p-ZQUw8oE-orySxmz)xJDAQXM z3Lppaosi@=*afF>(OwLjx?;|sOr4Z|Uo4q+v(fP#*a5>1ADbT9E1m@hgytnPzX5d^@@`_lQ zpMt|QJ<00ya%D2r_Y&!7kyty;`wp}YXu&M~6?JBs!No4Jwc&v|M|GES9pu_a8dqbg z;2x{{zy?imi5)xDbRIRnoE{ug9^{Hw6VJt~5 z7yCxtKJ1`$2cT#S#_8$BGA)IqoA5EKFLDbz#}>0 z9fc%gsclb+rHoWQ2m6;?@d^Qm6(qBKZ&L-jY7{_WB|Zqq!>cq4*YFHRVGxV=5o7OS zsk%Y)8vxg7^Qoy#!Qiav*vel5ul-7t(MIxmJ-l?x{aPc16qG?tV=05%JZZA}arv~A zRog8^U_wl-Yvz`9*-|yg0VpqF&s!4}Kg(V>sfDj~YZgr<_DCBNo2Zq|9(V#wR7wS- zy8isc?-Kt)keM`mSCRF#5~}L9%-}&>%?&u!t=SPt)ihGhmmrcjtR@kG<#gY9ny(QJ zu$qCiQsHAb2?DhX0T&}M1}fqL`VBnZVf)ts4RjmSwLeCBK_S;;6vTx(|)m$WvGKXg#a2;(DXK`uopjUJifR zb*^EymC@|b-mCU%stRT(qC<{8I8TvewLpouVX)u>>)9_4X>0A6MpsaMoj;=`sX;^X zjw%Gs55K0&G+Q?qh@ceC2uNr2qqLw`NHgr9ZNB3R8ATE@E^$z@E(%Sq%CWN7q35xh zga`8@tq;r$eo?ajKPlMQTEfiG74Xwk?-eVxn|=5ntu4;~hM0d1t^+KhWd8zxSPjsB zi?5i6i(`MB>6(7aqQ3h)Vjs+R3}dAV{44#}$$upNjAv^GQNpK8ji*GSyjsyD?nZ*_ zx~4M1b&9tL)16ys-X>?0^>VO_elQa+c5V@`Q$}5VU1ndUu`uA*BiENRdPFLfqPgB* z)OM z0u(Wt5kWA_WL4fD3XVNuMh+8;T^2yOX*cBb@WEJ2MkDb*_EJTVM7btX4Va-YZhQxB zA}Ic}&NM_(?v(_}3y=D(_{u68HPN8un43i~5TjuKzz93YK)$KfF>T`4b?g)h@o7x_ z=+np?wd`jDxJV!w%h)}X>)k%+5Xv2#1y*uwpJ1O{Z@NdR-9ASv(NT*tRXIc7mdVRYtJxvzMex{4A4hUMT5rjkh1-p1(x$Qt(W@w9`FtJTJos(&>7QM`fhq>K{*GQeW27|md5!F!TE))H2cyM8=#BGe~XEJa_3KS#Q#w(%RC=bedCo_6cfF)-XnXedwdA z&8nE=#pH*b}o zYavEG9!2&0c!O3IRJ{}=Q_KCq&0HV3rRL(f)}ATsRW5?uD`nLaoicP%M|E2bRafi& zEX54PkiX4(c2n{DdOjO(2W^lTL&WmSK)rScA}qbofE2A|65)FJ&|h5=qj`&@(kf6D z95Y<1hxWCoJ<^Z0nT+hG-JoHf4$gDF0;AOmfvh0dKVBh%h={{kBtg{B)ee?Ne2jT9 zvu-&)HHBBWNadD@^n(gZ5iaxXBhgYk&?PAovr5rH1>JN(dq^CYg5}J6yKBJo7ozA! zL*<+xP{cYbPdgz&vLbfV7v7r*eCmgvkUcs`ymSHi= zcB1YlOI$(PH@w!=42J`Z4?i(K$it7Z4MKvaslemH;3To_L0_H_D&8vGE~tDI6t-&-oS+4>fmQ zl|S+(k^v35oPy0rX~KMEJ2oqPiW|Rv;BX=IG-OeI{do3usiQqd09A0B!X!sc*+8srw@@Iw!JJ&jo*=4r7yRHd#INQ%N!F5b~AJkEPhog&yHcPcc>jG~0oN$2?R!q4fn%Zb9ot1|x$Bl`onI za?nWe_~CQ`{@;CWEY^#7W$~6=d%vjx|Y(I=i8_Gw?4L31+|8MF{7tzdYe9i&$F2;kY=H{y9 zdi(Zf5TXWp$ONF3DdYr%0Qlcq)HOYWJ*sB`qX+o`IdVzI6`@FGg20&B=SzK zt71%0=D);ZZ|$@&K5=u96Jz%=iiLfZYyptJs-!E!r`Ur~K8UN!p>DRT*{vhawlk%qI zHZgGoIN_H*`vjMy?$#*c{aL38#wZn?Alya&KI_Yc@UWn;LM()` zVzJ~TWLzu`er%_aF@ESF6->}BZccj9FF?2s+5b?|w?Hyql4g>L>lzh=K8X)`@00LL<;eK8GHGxl&fJt^N+H_Umk73!S8O zNM5}vGJlE(7PO%?w?B-EG0o!_-JJrC-3aS3Q!d0e9w7_p$t-B@`6K-4^xQq7lH1E% zsM45sizIJiwM2SYne|n0HX!n;!>eyNz4h(Qlx?^RLt z6v9%ClLxWe!N|9n(MDiD&90Q0r?tEK*drUl`a^1g3coOnEqa-Fz!h$g0d*IYF7Wh6 zbzs>9%JViHT%Y85&_KL!*Q1hY`_*d4{+R}t^EHWG_DFq_kbITtCXIR%AhP*O5(nzC zVKYlw9j>fmg-mgCkpHgsf?jpl@C$ZS65H&Y=1@P`D11ul3_Qa~BgIx@GhNRP zs0Ti9iH8%z^Sh0zn0ut9nK1Ee9j5CTGlFe7Fr@@e5lr=|iFJs4Nd7*G^Yl#A!KZ#8 z?j(5oh)FRwfkMiz-w<4JXkZRv#I7rAJPa*;5^7@gV(c~2dP0=GyL%=)PRHdp0pR6v zUpw=$xvXb(Ut9BcxU5&jHBH6GGEL2{Vg8=U-CH%cysU3vA7`kxb=Z@ro4=fTXn35Ava6#q5OuA7+t*V)2t6lV=}aTog^6PeJ} z-7lI0^r?uK&0x|DqJGU50D1+Dkv$=bGNn><(bpw$UKeDQ{8*@tGeC-}s&97a!cj)< zY~@`a9`*Y3)db5}p;o)c_P1qX2MX(@0VQ_oX9BzhX#%-L6xUHC4HRc4azD~cxLcVq z+4uJ;m~pXF{Us&(S0IpAL!Ww4*+;Y*U^jWqN?0Q;Ly;&8>JRh}ef0DKua_TW&|u}b z1lkUGVg+i(#gxa4-J}f@2UYr-4`4F-wa!h<@jsf;O5$M1vB?SEI@9yZiy?OyD+yUU zOsrEURpJI+$m3hyh1h}bt&>!&n@GI?znupW5w@>*@%#cd!;RG}IF9FDEp3)iD}lQ1 z>K_fO;TXpwo#w-KYs?roP(e%U-O+u7K?<1ams{mg3Wh*7JL8WaJ-R*F30@MU!Z8oW zO|7q04f<9M@FDVKm#u!GD@Q-xC~=r=C8sM=*KWa9pap^c5^;wO&{mw19 zfhjvirh61(FUd^LGmAM!>u(_1JsV6?>>GY3B1dKlsMKh6^9=i`pN9f2`@5n@O3JX% zr7=*5hSn&+>&Y=dxFPzurVFWQwyd>ZyDc2s7}ifh$R^LwLajWqck!yLAGj~=#LtrJPO z;>(_XJKRu)Y!~;lqsz`DX{kG1)+w{r5~Oy1z2xMYu;6Oz*kgPa>9)WWahl8W`N^(s zh;ccsNz;>|F^suvdm)nC!LfLVr%SP4JTN7m*wUyXiB00&EdgE;(2@(K3%0!R`m&Z9 zz`6qVBk0o|;2} z+WVlHR8qz|F;l*%>=qHZc?Z|l^t<^~gNb-fB73yGz>4R9wl)h*KFmDb!5F^G6=8DA zwCoa}{fs{S1*=_%f0a{?YBSUA9mr1RnfD`aVE!$>bL!varFo#Iy1N)Rb;dUPnY|qf zCLbku8Z~pLBwIE-L#l5mdOE5WO`=kfkpr<5eWDDHh)Wq69y(toor}lR zGs@RrGy`A|+h(l;?lkfU8%Xmj`AbB<-`d*2HO_O^+tFX)=!{&SADkHfuF+}S;4uRH zz;$dLdUO=am--3c@h5kJz)vtBq$@#yloI;6jyEORYXtfXz@fGd;QIlXblEtt!qzoj zYZ(u;(^6})%s@?R_NZ`mN%@oul$Jf<)}d>SQWup8jTt@-sa|b5-Jr)$^F$8SYK(3} z0(+V|&}8bBHND-0K~hD<<2rxNerSSUg&qAw^)YAm*}dXi*(7ptd$CpH{B88+4@q3c z__7OxpH7J=U$0JtByiVyb;r1U90V|^HfzJP)!&w4>h6MyV3z+}?7=ilBsV~nf-JBbWLZap_An14MB$iN73bQ8_=-d-nvD0ziI#7taNx2YxwIAifOzE2 zViQ?{ zLD*&7T!ys_=1pTHqcWa8boYFQ4>7LW{Slg9*N?X?H>IeK6Z?$z%HLn8qL?+d9xox> zBp-S*6hv~$!u9a`p)@WI=Kb=dAwH$8)l83UC7@#Q;Lovp`xlN*qr`_4 z9gC}fZP7#RI8=#eEABNW!TLOOM27542QbTY?m)!kNB!Qyg*gY4?d&B$`}oQ6`R%EF!ckZ|@^aJ4D5J=*%+UovJAYogtNULq|%{`rA-+3(@=g z<8$z5{Sfu^*;(oFWK_82-5{*#dhIEQI0v11J+G+Pq1UX)OO;tTA^H+Z}2ulSQ63PL>8;` zY*Z<(sLi-cu#o@jsBsgws%!8NfXX^Z7+m^0vrUQhdGfKxm`{^4zUP6mR(x0~s5r+4w38D8@vg3|ijkNcCSDQ_B8XIv3zTNuL zH4FI!l%dkq11ek|2Yn+^kOei-gdci%A> z$eihYnkT1UHO(oQ!F3UW?GG#4#08M)1B7 zQRvHE@MTwc&vjhO*i!Qij$Jy%noX?`5zj!=hM+e@FEd-hw)LmWq&^dj?#TAqQ*;sp zmn0h>MeV?yibR(>e$sioWv=#&j@cO%Zu%ybldcf!tw^|8HDzbm1?boZnEI$ekNV~A zYnOk@$wUZI7cWlDzH6?k#Vuwn7tZ<=!tn#$+WF!&RoqbhEwO>#z4h$QDgF>gc^jLq*-Y~f;r?oL3*^^6QnG3lj*|r#XWp-4 z<1pAE3>S%|+U2J%+;Z;&7^o_hL2t4}3q!#sYhAZ_?n{W4EGd5!*FvstWtQjv9_4iS zI_C}d!(BUbuIChSy*9(#|GDpu(aq7ejChY$Jy}GhD~HMemAVk}S3HwF63gZ$CUc{> z(%VQ{U3S`vsc&?7W4g^Xn*fAF=b>342Tn(0w(ufS_>9VWp@&>7TWGd}C;^k?wrE5f z(t?~Lh!nTd>?KfX2SrEEWT>)vQiI;wh}u-1X(eehSEuy$FsI^Z5G8!US?8{iktaYr zeQrIcfU=`z+unO|e`obqOG-B?hei$hEUR~JLns;j-IHMIJ==R!t@p-Yz3&0bwa|3hSXXUlmu!Ls@Ar>s*NvYR55XsRZY z^@n*LPxfJz3pa-@!g|Ca;1qXaRs4yj6lKWn^G^;#Fy&2DEp!#FJUhcBmJR`OB}N2N zApvrI&aQ^S(hs>XbqKb=Ex#lo+MRGkq^pYOI#LhorOr7{i~9bT!x%fBr=I(b4WP*r zjnSg&0b2m ztpJ{1Wi*qjR75yPh9DB1pX#iq5K)=#eIQSk9jaUIG`DJx49T$6(MF^l*4HLi^0s$>|R89Qp{$Pk&^13P8G(hLLnuiZVAIxI5**`Uw6}~VrVoXSkS}QWwMp4C)*iCy*^(MqEItO`EQc&d^ zIBYY3cr`*_|K9l75)iy{R{GH%LBLs?+v6WGy|CyWMN5{BsXa#Rq!7Slb_3RKoSO$I zF}%{+B8iipkJD>fv^_E#(qzLX60@z>aNaipIx#=6UQy|@aww5AKDh4Ur~5{zq`|S{ zFF2;)0ori=&PQFD#jO<3)Hql?*2?~TvGgq)47HeK!~a-*<(09Gse4hF0tVLCZnMLb{rktrwq1;FuIYMz9_{eM>$%qRiyoFYIJcA3fpw_+v0Nk^ge zQDZx!;NF#FvXdBbcN4H(>oWqu=uin5;r8vF7{UGsb3xyk=~+oGXd-HwS!&hod3L{xNoL0hV) zd2qW-v?17e#kLY#?|i^K!skRPJw?}4_t{B!O*2UT02oqw31QYcY`@>1KOR+P60x6x zXIzLtxjv7hGJBPNHHLUiS|ln*l5%=Tcj(=dnOJczu1 zC`0!zE2H&?>baq7Tm8+9q}sfv0uN(YeT9Tus~WB_`7p&-C%z+SkiF{2Sm)o+1TP3m zHahqAU3s4nAUBAUs@+YUIQd=*Lr+_$XG5w92{QghMwU6$76M&q!Fd~zPDbil2^{>X zgq)(iR>!4`vwql~5x<*br=eh}Q?b_USwM?G@K6Mnihke$@cr4co7*xbHlbRC!w z0-6ZD4>XY5WU2Nd<*by&=+K+Q&9)LfYew~w99~+(qKC0dIx@1BPPCEkZM6;U-ks+= z)4q>Aj_iPJR)8djqttI&1)sTHrzEIC-g@Syh8*s|y=|5O9>8|I5OVc2whguR41BA_ zGAc<2Kg84WW}U9}Y#1ImOHFbgf!x#-^WqI2^s;4+Lrjq6As;JmR^nq5N11x~$yT|f zelQJm>#0temOGD8-QKIM1>9M&x1*;S2v64ZfH3?e3^Qu&4 z&n^-eTR%_{8ifo&fZ`e0R!R>WbG9G>!RMO?=GS}dz(B5}sZ*ynVOeLLFlS(zJ5y9C z5)KOJLI@ld_c@=mXCGRp&4jaJs$RuJ<=x14!D~%rC)=3`quA#B|1bq(gfu(d%(jeU z<81J+5~P7G20AWE+R00nN`!i{6jL=PkQbB$#T^0=D8+4=mk7E*p%?$a2~y=46sFw@ zqb!8K>(DXS>?5b}hR{qED&BU@QD9oWO$A#gS((Ep$tc5N2x+LMW1FB1nXOGnNRj_P zU;>UubA`=VGQ)&q4R8u`sJ1sDy4^c3(01lA`)6W(NNqte6)1X2b6m+a{wtgc=BRHh z!JUod*dOfy#OKCAKP4o)A7@(eWl_|?;_=l@B70XH>d`@mRK>P5UT8QH&2wy~TC7vY zymN`yXzOM6zLp)#X>q7N*CZ{uol|Yq+vb1tB02fio;2N~>%o=-2tFQ4JrWOUV`A)X zD|1Uqh?Z|PN3J&!19$`kcwTQiQg^oX^xx%TcRofVma<>Yw&r3F_fMpVMn(+nG3U>5 z5N*UAB&b{}xc?7T=M){v(zWZ@#)`FK+g`D4+qP}nwr$(CZ6_<%$=>@P)Te`OYkwo1*cK3Wp7-lewXYBg(4b<;wxNZZmk|lcz zu68DbOby2rTO-P(Z0^!VWjO_!p(I zZjRGGJAvyUBPq&XVy>xQd?(i;|EgeI?<31dIAZWpFnVn26L~EQp^2Tcrf4gbYgAK% zn1=rW269XFG|`k{fyG(czaBxT=_MF{=VApZ?O=xIT^wWM?+o7DjH2GE%?*4*Dn|NjJ@KBzfjBo_yBf&V%0&Pr+jB7x-!h^B$wnU`}pg; zyx3Qs3T(#{4Tft_UyI6Da6=7QAZ5HmKNA?L)6&n1NzE5+Tc|>m3%|&As9U>7T z96wi{5T|L`!M{}&V6s8Gd@G$1n~}4vo}Jq}+q+BlBOMQnIsde<;t^{DjIX}81h(K| zYQo^on!FHDAd;hN0cMFg0uhT&?5%E1f2B_BL<%Cvl$(u#+ri1k&yP27oLS5Bzsk(V zN+8wowj6*S6s-(W#Iwqt3oN$;c7(a>U;JDw+&tXNjc8F^3r!^}^^BedNPv+NP|c65 z9y^QQm86bh%(P?i8*hyB(4c#FjwJl83tt}Nw|pDxy5`OB2W5HjdgD8%%f&8wv-v6T z8#VEInRHCf3!i8gc=-*>97fmkD3JP<(aij z(R`|ZWJ^0;ducMYi9Hh*nW6af^vPk5At*$&W1Q7_r}7XJJKYq{k5cr?w~^*++7v2n z;E$Pbz@z8bs?|XgeKx&{_(W!<71{httTCUof07dTlhw%$pLQF|i zZ-8UhUaVA~@n7CxYmS_D0=Ran5X4)H5PTPX-x(caMn+7tKIR)n#zAK&nfsAtTTHZH zy6LInjvMbt4cozlzf@Y0@7S8HIJvs89CRk+piI>bxy4JET8JPdiTPO&+&l_?0TZWi zDb7RXE$DvNn$n3e&aQqr+UEf$-cgvxMM<_Ud_!wS68`HYg!khM-r=&iKoFFj)63=U zY`-HV78|vp96q|r|Mr;ddbZiS--IVL@&9-Olfx|M;-s}@kL{T{Nh2^ULmm5)arBEI z8YdYK`6;vwnuSIFSGEJ0!02-ruw^n3ZUX4$L}CF=?wQUynKPw2!Enb;1ZTe;_N!G& zXnmj1L>;|c`V5oS<)LfFV7H+g7S47rE4C~YG=?_Zvhz-c4Ql?4)uRcALS5zkt@jBM z7!EcOS^w9l2Kdh{^_55xC?R33`+a|sm6k@yK64>vo!iI|c{s>F@`$A1?|%F zrB`pJ@^xN6w83Fh(oAxqi7xIC$1RI05xF^Io=xP}(qorENXAypn@~9-?8c*FPbyuD z52IN>0})JHPGMN~$x5q9fi|VMbKcReQRk6&FZ>T{@I->M=Ve;05w8qW!Nez?*gzt- z^ABsl7*ihK)bI{jyoh|us0ey~(K}NfpOqTd&}nOg^*euRDmJwjK2>_w^}>#yEZC;R zaDta!vZc^GYKd$W#|5=}+Q{8?RWfGpv?88FmMqc5H?Y}59WrZNga4Q1P>}tEziVi# z_b&ckc`zR)4+x$*z(24-+*DZ0gDr#prfpZRyRz(Z-ZmtU`{Oke%>3VyQ0z5vBl3w; zFhYy#gUjxdxi!)GsMzj2|@RH8K7#zGHcdZ3f9&2WB_+0sJ3VLb)Lk~l1 zxV)1i#+&>poPLit5Xn0qh)4b?lz!v2;5pfa#js_Qb~F(Gr*UnYaSXg{IQ4Sl_~kY- zc7O5sb5>Kwt^XE*ANWsyQRp%3m;wJSDMy^u-nXH6KK&vAqHuXFp417`6JU~fin>kB-ub zSMVyo&qdva0#r0>Du%e?UXdHdZojD++Faumiqy;s!6q7v!}9uv%(F{I_Mkx~MBjrd zI|m(VZB{{iG<@hf*E@q&4TD$Y+-pAt?fcMQnaMhY=|>@Q4i8=k4QA{kwK>+oF^ES7 z-|@V9bRtJl4?H5fX+>VJN8YjYx(4b8KRu)3;s*>~&~I*Zuun8PNm)*^J;P)6WHvi! z?F!iq1L5s-z6L}Dz@qOlhTot?UZ4y<1L=ACKi|L$N;rfEBkK2~8g`@Vx1*kbT;oI3E^!U3A|BqO9$)i#Y1uC8zPe|my#wqYm^_9U+S?#FyP9(koi{Z~ zQ<@6`RvcfH9iRSOEISB`c;xw4fOueNtR)EMi{A_xiau3i8;m}z4!v}A65Ib!_ad6D zk8DqYXT$q+onlwXEIK?M=MbOm$3S$pWKDRKk;G4QjFH6kmG#Z!U{j2D&Z#D_hDVI2 zGF|UwqG3`itLY#7={=YIKcK;ezT(wCpaG2z93|X4!&9U(0%2xyQ#E>bvV!p6kN|B#`#`)C#fG>T8!ZfeTZ*Zq6}7IcvKv6#}Kcv``ihXc*rF#RN;KPjmbV z9VxIj1`mQ0t{f%G*o78hJ4(?YmVRu0)!o$+HK3djeIBHh5d(QQgMvPr`>6HdkmU;w z;JAbuiD#VEXu2`8H5c&;ffsH@P6BlCoM)JdE6ETPm}DKjvr6sYtQ3Q=kW%2g*>y!! zrTEH>n-#v*`Oy6jXizz^3uXxeHm+H7gX4W->l(U(2HsZA1@rY|AkHOpQK9jN%!o}C zrS{1}8^16WvN`_;G$0A{K-%DQNfib3;w1u~$WP*TAE?XERnuz;iw-10`jbY{_ef$W zsB$5mptv1wVDY&ZE?cjg?(Qd?JVw?JqGd=h3|j}9k`*D89C==DXeh*tMcuT!32y^j zeV#s;Bx@-P5DFU=f#T#kZCLbNOsEHXO~Thdi?@wezy1Rn7>oS_8r+A=-zsC(XFVnF zKL3%vDE|jEINus7VJ-3T_$>CB=qest9|=JX1d;*Nt}AAZl$D44ni+o^h&E3LzAiD1 z+{!%!6*3h;_4cE+tn^caiM?*hR6H%?4M4^T*Ta^6>)XPZNSckXTcP%aXX>J;VO0+36BqXk_|roE5iLSAw7aIed~!>4td-R_E;}`RW0t#uLy% zl$Zu1T|f;zz&HtX{4i+C!HXV=qMfyYx#4b`^>J8INLkA)ZbB-uElW=!c}PS!bc^O= zK$XicR+ZJ8HTAXzqwh)^@(ca~(erClc=v+XGwd3dg}O#PiQ^Q2q3n$vF!mJ|(N(+S zK%zbhH)xI4qCvjiq2ZuKkYw`vLzJRLhB!ZY(kF8-#Kwgt9M5Ey$d2AB6SVUUl@ron zbVa*XSB$5km;BqlqlpK;pueSaBZ;RLTkxMLe+McGE!CnU9f6h9tJjL3fg&!pE42?# zgX+i6@R=CFH~%5>rK?$D$nfMv9*0aiUdn_UB7l z|B};qg-iVJC2vb>F*!|FPl$r%8%j#|gQVTr&($pBP?3z?*;CrTVP7uAt$zvfd)90t zelk4&H<)D-E`jIj?Xf?FlCu5yUl>XX&%bg1=GwlS?yXX081m6SVK^8gp^!fgpdKr_ zEGhK0vEU{^e|4|P-GtX>{oK`5!Jl+C7M;ggn^Ij{ecat@#s5L&r+aXNk{^>#D5~Eu z*3`k(Q7}x*rD?E7>8xEJ{fUH}{@5o^*drt47f{pXdcxjhNB8;k-e{&oU8!C^RkgcA3!i;p9lf=f(tKiE9f3S-*fC9eD2$QXEsKdL z_8@=ApRXLEvMrX4+@&}M;NjY)j?*7gQI7WVM?ErrR5jLSZVmzIAjUWv;Aw8&_$Kaz zhx{FNfaf1^UY*5O@?0zETViz0BhXKZ-fdW@hX5=*$;OZ7O$`G1r$L{a46F}8+X`gL zZ9n;)X~GbR25R+Jpk~Kv@)=*YN;(Y#3#kc0m}DWupdB^(i%M19di9m=zofy#TR_?H zs0qm=Sc*R)iDMgmg+0vzbOT*g3vl(iYmP}~kHZT6{NO=LztnlQ$je!)@9jgh*2#6f zLmDOzoDy|Gc;V7Tl=?*4**)UI zVRXz?^!d^1s0C2%4*QMF6=ABPVFBL^;j?ETs&yo+bs%VUu6l!fC;s>Q>A{V6HqQ6s z>1jww1Ml}sM?s9kOAKxFxBExK`^ZgnOiZ-*%kkC95BckJIKumlU;csl{rE5kfB6^_ zCm0>`{k8+v@d#j{_Fn7_TyYfSmdcp9z&V>}N<*1PkB&?8w|C4og*&kPgJOj_jQbzf zV7;U#&ddA&-Nm_l>FlOM0f!ZQD$hEA^0z~E)U}}?ePp!P)aE4bJ1F+HJ@t}kU6bb!$#|__tQZbV;CXe zggO@Ca-W&A1gkmz&LyRqKd5sAU0e}k{=2-3hS#-dQ5Omf~Iy6fQ;A$T~Nt#R7Z2} zhL#Q%kJxyMgaFR`P9T*-IQ28-f>ey`E~P6K_cMwCTcQ5hX(Ej59F6hE>LSFgomMcGXp_+=)7AzwbK=#k?enF~ zOuf_xvG=Ik&E338^63#a{-wlAUgf?qo>AYoK=XjY@(Z3>XKefWganfM4qC=0Dby-NJd+vc4q z%?^EeEq6JnJ$V?&>jjQ*(YQIQ0u#T?%BwT5vZkQYSkPXOS($H1olEcxDFQAoIuBX7 z3@D$}iJ5>Qe{U-}t_uJv7bT`Kq@*LJ2ZK7DM1ywBhuizj5(g1h&YoBtw)iI-gJym^ z$I{sL1SO_PodsT5zZ)rd1$_{$UXd=u|DK4f?(aKZu58jeuCyb6mil20Oa*QK_>Hl` zIeqW;;Lk+s*(eB^zYPeF{jdgyI@#Gi?JqSEG9@+J*K17;4>)f zY~v~#LBLwoD3zb}TdU=TK>Pe5RnAwBjZWw%en;re$&B2c*aAGQs>~cXDJ5R%#Z*>P zGNVS0F@>oq(VZv0L?<`2(_bg4$zfZG?~J0uCYY&N?B?3Ms+ay|#(3F8T02yCerpEmeSJ&{w1P>W&h^G|cQ^M*bS1+!+ zHA$jRGCCfrDg`|itv^A<(qh-eql=mO%qWch%tE22a;8Yn?N31+YoXB!jLf)w@Ll$~+T9Xj zb~*1#ha*PgHvhvKq<5pO9S0=7Rba{xx}SK)Ae|*+{Td{KYBbT^pbs&3eZwS+;Q;QF zRQk4iA0nZ1+#DCrEU%GJgWcxi1X^A(shQ-VM=BV!K?ksCG=xYo<=zx|CYMg|K85~7d8m?#f4l|+{3LzS=CR@mEJS= zQlM5fHjXt(qpY0%fepAyD1}RZ1Q*GHkV;9;d7fAaSNkm{HDF>q#3IEpBegBLxE;x+ zIcv+0$z*_r7EC6HLS8aMjMz#?KjpL_br2>KUvPQ z8k+E|U#s+w$c$gFveSxa=hL{ik9k_v9E#1vA8*C@!8Fj`@ai`Cl^R2);HaPRr|2@S z^ojtPxdx4Wyp+pVv1rOUC8aS;z}j44bK(c55GtvJ^NXj`5iueime(msx%a7V(ZcnN zsd`H!DCE;_9OohWD?#JAZRqn;t5_icVoX5BS7D)hy{!p@GH4}fH|2pLwqvQpNZ>~|&#FJU`@ugYM7W9|z_!tj7?uaO8!&_YI;8X1|iCC^OBHu1u^FvojYH;n52&<1NIlBQs-2YW+A8_?~ab_tmwKeU0y4{g97mgaxf zj{y@TnRdc4)Vd=fdCzbI`eXj_$038g#%Mjix(F1!sTYtx9)-jg(zXpvu|`I8jA2r> zk@OzZsa;@ovUv_)t0iNru&wt6Y#zWch`64wdbAIV^%)St>;WLk<%agN*M))wojJIV zu?I7Sw-<2<3tTF3MYE7)W+o?PSM;JCjZATR%mqN_hWX*dSW`Xg%e&pO?s7(Ht+Je?=4|xV%e;GH2Y-R=#UU_D?TIJ#jMjs|NGN`F zz@d2*-c2<{$5gT=L=Ph)?Y!%>qn+8z3vM-X+(!43HOq7&9@I}W=`iujg1lyIPS-3m zZJ+XS(E2G2gC89|Bcc-S2R2YS9*o7_N zM0u$wO8-#O^7`s3JuR4p$uMaQxz6MDhvrlqOZs$W#_u&R4eEefL)*M?zCL_2h#-LR zE^4|U1`9psD&!c~F6AH8fUP@g{~y%gL44yM)S&S{)L`g8)Zp+3H7NW+4I(hm{cB$S zMGd@5>35tivnCu#f!bUUex1|>Bi#;i5LC$@0n)6$;hEvr?yU3DowjWyfWegZS;&)C z9{}IiR8T97=zIgOir^TYBiv70)K3~0{zZ~1E!|6RawI{+>a_vZjhdE{NY9uMM&ulV zZr(>YU6p9>ribfV^hHhtYa?OTsgheILP1Cgb?`Qhf0_Zx)K!buP(MB%E*Qh?tWk_@ zq{urCFlO4COe|i9ID%srr_+u$SS}5+r_N3mR5dw+0Cl7 zc9&f?4#)MgrrM;ueqm>zYGf1(vni9BYTBc321A%Tz*XGhpiS4le-+G3M~@n5`)t|W!~Etg(9kNja|I(B(m zxH^o=4On{7Ig1q;*&v%*G*2|fhE+Rb0FNjEzg{yJ6NFP&FYt?8bU#Sh5Wze541h&S zzPi%6P#R(>`n9jW!!4xfCWf;w0?H$x=55OLG)&4@+iUeLMp;>$dZ@;2my8>zfky*+tyaAFAKKSb(RQ#YDY z@)0qc7MUtY=?(OamLXJOcSTxq3+jMn-wa%VvblS-8-+qh$vfQkt2U?-D%?b_N<0EU zT`cEZD_uiS1()xC-EDI?tFj5y#tJ3nExk}SH0TXe3^_kIa?C8uMsmz+Y>o`ix6z1W zMOaq_1&2H~{4SaC7HY8MPq>hH6|P?NxND~}TD$fThE>r~*>{Ja=s8m+QTdNFAVG16 zz9MEA4cQ1VcIlK80o5z+DFu4HdScY;@SAGn@p=1k&`Cv5i|6GIa#eKlhky?bxduI#JhQqm&p%&Bdo?NYhMM{3->v(@x(v^QbmId#*)&i(AOw)epM6 zC+-@$hC6rR8@O~C)!Nojk|5%8Omc~=cpa7U0Q`>fTPvU|6;0@qi$T0WXB4sh^DNs< zPf}XM{D)pvI4a&hDV%a(ye9xPxZl)IkA?6 zgrNzqFHb0S366l8297r!G=_a~huOe9$Lt~HP4++oSfcB@00sx$Eq)JXqq@;%H96KB z850(;BWHlRR{nW&UpbYzHgIe&VxTM21ly7vx*cj6brtjBsl;2n3h`^#h0X^YOj8w; z$_fAgARY+E!TPtBQjTsF&>!ow;8}U)-&qAgxhyN4Tjzf{$yK!^==qwo{vi!CV@)1R zWPD(kp}N-et8UDf!%vK9KkZe!3E!T6TrI$tCU90sKgP)XD#@2nQ=bVD!Xg#VjO3xa&PiS1;Io6}fzn}r-c1J}A^ci0Hmg8u2{n@A14MQ4UImf}^KcE4|gNFyX zK2rkl-8v#0zJZUA`NQnnQ}z0;`(w=_v%ye>Jf_!Cd@|)S@ny3@(zF`8f|wYzg+zs2 zX}YKe5l9nPc=sDEG4LioS6ic6qJ^@r0*MJb)w@DGBLxZ~Qbxe54^+U*N=&B9-s9%N~X&@Z2NNll{^X~-R5?ZiZn9(g7pKD82%DJqzg}qbTo_% zWI{arOlCeVCWCHzifoi{t{k7&kXHpsW}_A@`8Qyb-ccT#-c$FTwpx-6V;VY+$&A=& zH(LeA8OXEz7gGh0J$R37LPw32ghhH>8$7S;1dw3}-$7_eSTa;t_pTMENvGPvbEZ(q zaeaPHDFr4e34r##O}ZYckG0o4`<|O<2hr@Q!*j~a;61y}I7&{PTBhTiP(aJCqp0x) zY}!|I<$B2{LyfiH2_0KT`R8N+420kvowmkrJ(Di4=aC-*P9R8dntExPU1!5W+i4;6 zz&i_c3eBa?v|AW~-i{xy$;ll7IeAOolZC7HN57zslVE>JJ;azzEmGD%6jFLbpAt@g zxhgF#xS-0qTgEoR766h!7x7tf^!cxu+g`!AJIb!waF)6+iWgVNGf8K1NZ-CMuofB$ z39}9YXOuB1{HiJ-T!eHcu3vhZ z^DLi##wZ-InL@3halHL+CZN?|Q1X`+lv+1n7|GZ1b9GoedD<48(cQ$9RotB*W&LVw zJY>2HrOGJ+82k9S9hps0=u#muyRJ(R6|)5I&zTb#`fUa{v5=!Je8KgyuECG?S8Ly< ziSf>fye+lF5hKOuiX}0yYrP;qLzMa~p|c6m6K)r+5ynNe z`r)D0DC`W&?y#Q&tFx+Bb|3uYho8h8T6BHFUn>HqF8_XG@|FAKOrK|@SFp$-?AG!T z0RL@Um28~I3tTj76@tIe?+a7`%I(()^P0}KMMV#J5e66OFJ82^+YPS|y!w$jc=&Kc zlQItW#RIJ4W<^JosnHKN+QHWeSvw(F7A+PevCc6?wk8AXEnS)o8iab)W^!~?DF7C$ zw3-NLJ{PcYt{Z3ZM?%JN%U1Kj-^MnJ5nJ?ENwm5W=~s4-fXYwdL@47De`uSC0*d^} z@jLWi0(T%?=4yhRx~7r&X>_1rEi*n!76=?gK3b`g=%oSKVGS``@M_q zl{XClJwj)OTv((NRxc(8-k0|4@ ztOGAB`crH}jbu4Xx6!$-T9Jh8?5hAtz&x~3JVS3Nk@jO42CH*>P0xtzkl2l(Hcpk$ z61!5QwteD8gOO@V=T-dU8i6D_J_vme`szAMR)+^)yFtpb*>Mtm>wwpVyDvEHM?xL$ z#;FnjT~sEzIzn?>s$B7GLs;Tpt;(Tc`CI)CoJRmeR?VC1w7`mkDc+oy-r}+1q+>#N zc5SNXL zwaS1>n~QA+ZGb9h?BEOf2I6fA?gC$2S-ou5E=-3pmPNss8GmB(C#s1hSh3t@IYZrX zMW!{j`b)p9#*{eVF$N*`pYWg`V@)#KF~_krSi(k4TdjF}1p$)*san8jW438tN@eTW zLi~BXQMbwsgS20p&TW#f^~9PsgxAibK}z9C4N5}_PlFnR*KBvb6U-NLNn;1q-x}(^ zRgl}P6HzXZm2Hn&O?qh_=}AZ7P+!OmDZa*(YUw_>WrDw)rFwhqE~V(VbW5djO!cUL zBP3hj7|zC5e#Nm@dRBawg>vZwEJDziLdSpN@?1Ed{|lS zxb1O02?o|SQ<=-@eI7U~S!2|sj!f*Z`~hHs#CPmop}a0gOpS%!&rC>3Lgmd&X4QYX zQbZ+AXh{%Jl9bH8#FeME4DEDru8TrEV>Z?xmDTLOD?o=t4k4qaEjkt0gvOQ!?tQgH zl6O z9(3(ZfqH41x@`l87g$=}I(I<3`6P**3Cl1bp zzz6*=+4sKlvQTnEcI<3C)UYFC>h||XrW6x%hNrMH!#% zi`elapDrZQqnu?^-<{nD6bPTj%lAo!=UI!lPXVOh9lyAlm2+#A+v*FkWG-z7xS&-8 z%S%!z?MXtZwy+ns1&zkub5VLf{hfFT(ANuM7R=XgO{Q(kUB@-HdTDa4kbD45(0n;K z4a(G9wJ`U>NCOHTl&kSujK~r=p^%*w&5{UWCuCRmEq#|J&&}{q%)$FnL*BvB{^T%* zbCvsEF-LuHiTJ&Lqd&o9oPNjq-y>bCO3vMwt$g>tHGA#{)2Iwq_QOt9z6gWrVQ7+g z>A%~bG;pAcH%td%nvqKcfuR;`Q2qKV(5S&y@a{ zYzl*jJ$fqQ7S?USy&m*`;A6L&WRS) zhhq-}c#pp83ude&n)l*wqG^*^2NfRX78KKd(;vSbza)dLZ{slnd)TGvjQq;6kp>GW zW|;jU#RH1>$}C?rcZ+eNu|Lh{(bqcV|uarBACA;g^%}v5Cv{4uM~!59TmVDH&@X1 z6Tb!$979pZO|7%kng6m2`h_M|My7+hHakr*V4ZC;#|LOMv#mbDz`)d!wFSDjVJMZ` zFnyJ;Ka1(};46}MBWb?`OL`OldaS2Fvn-$3wiXRPz2W1l<;p2@8dC{-!MfwUq1RAy z?_7DUYL$rLrT+C3!@JE9!#njN@$!{?>s|eldh0EOzm&ndr9Ls>hF29VUpc$AEvs5M zLm;e=a^iq#MSuH5EXahjLK6U=^I*qWrwOJB=~FEkNRG%0I8-eWi23R7(MtqEBJw(F z3^L(3e-=nNR4x!q56v#Uj-AcUEs2e#CPr?V$TPU-h>hVV#M3wX{50ek?}`kSFHh^m zmz{Q@x&%oB1jX+57FkaL&skDGmNUrm&q8f4x4RFJhvfkPIRV*S&EZ zHBH)fv~DoWSQ8a7)jYVKT&)~nN`}4nJJav$NpE1$i2Jx&_@ppK+wTbc$!^x?W`g1Y z?SEF3*8w&E%BFD%4ev9)(q+mR%iPxBDCTjnBbyPlowoYC*Je(d;P~*2)H9;VURiBk z9?Lh}NK^~FFXgC{av!XK&^u``G7R@EnI8)* zW3imJZ}GnCsU<7BQ%ig;;M+UBcjmn1APm(yS2>h4m_gb+T*~}~J3!nJ~0j%U+vd6KrRO<>X=6*fzhqL4e#<)e<#qurf-qx3z z7hl`oz_=)(IDpyv3%xj5dB+ddN^T9<5D~lAA6uPcg$n;An(P-g80?MYQ3G&;?T9<+ z3YDUg>~`&aaKi@aFDw`5iAQ&1Dg9WBs%%|2bz5(mogVjZ|IA|YeZPzdeKjN3i!$~+ z&~LaYV$#F(h`)h~T%Q-);nWf;@)ncW+P&0`^eT>8zh-bjeTfxn-R?)X{3e}RGljU^ z-87f>2(SbU=l1M=t3SAwd{Sn^T5m5G`WQiHCPfzKMh|#qB)G~SXtRzk2Q)yo?Wkyr zRO4293`o=ou;13)v9tjH(MDYFv$GXKDqoF+9tzw~-=S;u<-z@!(!|4q*DMkjS5(4} zMvuoi=FLVoGo@Frub0rpW)|s_GNNrB^}yElW_EOd-|)Yyi=Ffd?P?oSk~ltX+&*Bm z4LBhFu6^fBSlUJ0X`9yVAC-96+Dv{EE?6S?u-B$$$i{|SF&tLWscy>+p+or?SV8bH zY#i)MC)%2Ke3%p!;Q3HrFn!U&-V}&wb&AUbTsWmA7IqHTYwWRrTxg}(p-`UIOkLcC zQdJMs(0@p!hR(2w9U2;!OEvRhX?<`W_E}p*IVl%9Q;At&taHv7=~k5vUCX#Vd4cYKo`cpk$NQP%Q;+cjp3n0R596Bkayu@#j$_0( zof!R#coov{S*wpjjm_F-i<5RDgvUEiA?D)KF?Cb(oa%egzSmklbopWx)xpx*0Oj*o z?(RM$319$I#rv$04=x8s0-!qsAFt&X39zG$MvS3vH43FbhjYYEtIX4|4R&AH&Z@~i zD{{{n=A2FTPGHa<+ZxSri(eT1w?oe;^S((1+HCkf6dCn##`|SJjc60mUG7s){ z0oPuaA$U4_wWVImW9%?yGBUckVkS9f3_ojHc_%(r;#*C*63fyiV#-THq5KQS$RS(S zj%sH_A98YN6FIAK9hjkskK`&X)77*Bg#! zC=WM;-5{3VJiW0@>Wv7INGAvda4eCp%AUx+o9>(!<9wf#Osw_DQEvc3ybxkaFxz&y zg__E!bzgXfGiVZ?{f02rzzO6K0EL10iPsJ_Wjh^cd^B}?was4H)JZI{I{wEbqk9e( z-fHqKHksh;^lawGb&)IU^L1ng$0qt){rkD3g#sgspKa3(bSoID0FC@ZQ?#yHIp9=A&k+|NHy)7}n{^GZdcWVH zZz69SojufmNt(8?fjwhadWwdc6=tj8nCx9kN7pL4PTRsPY!f*a)}(%l1BbVHAuAeN zF7?LRze~L0Sa^BeBR=tb7xVNRPZadyGEO;EJg>9cFbC=hSASd=i4;>5yTT+U_q5zAJ z)hM@5mucl;O4dV^58CJLr7?`f_Om1b-o|PPHKF)|d_rTQ zkRIp^7Z)E5lLjnP@6Vu-=HT7R@Wxxh zRkT}EnHyXL9ost_T=;vsC?L8MYb8hvJ*C8&kLGAVUB6sjr*HdP`Emf<*By&&#Ed2XPw7VEP-LnqR?bcdBHrK2-x7Bo5Dj0vZYI8GpapKn{2%K-t@uWl{1>jGYnFgJGrg!r;tIe|T7H#f5 z@HL&3TtV>2)k+5h5$E(H5t)?~v!_0^`i?F8U7lNX&wHu5_D;yj<+TcPw=|Vvr(mzA zUBwK2c9CAPqk~NWRI7{d{)|KUIxp<;{PaB`Y8qjETjpIvh`XK--k+cHcRL@=G@h=g z8v{_@C>d*Z#uUL`n@U4UgBu zTMy4#VUPHuAKq|GswBz9nBms@Pq*ESJ7%brXwMFoagM!HbA`raLVk-cQHoW)n+GK} zrVQK~!kP5}Q1gzB1PKtJX{e}~vkKgT*`7KHOz?25r7=_qx|dQ(j>g}N&iW^Jg%vt`|VLI%qB^* zqq$nzcO}~d*dnE9%Mj%PgRY)SK@a{^L7m9dYp{bZWqv@R1J(&FCSW;>z&(Tuo=yJb z3U&SPPZ!R;?N+>c(3dBDY4)C`AM<&_I-IeuA9F_5dE-#NG7b53k5b5bGHQVaN1sWJ z{}y%PV&_Lw8R?SREvP9e12x1EESm^3w5Es&7bU0B@TtO|3bHJwn`Ip+rc0dA+x{*I zbg?T-`qcO-2~_d@G3=_4**h-oWfRA2HpuPY-DkDNOjMDl+2N9x1U(dAZiu$h=_{J( z+OkAh-CJciQS+WpWm}b7h}lw5y>^_p5S_%pTcFlsoDrv%(!r{?@}eJj6G|_+-95JP zT>F~Q>D6V$EguWj-q{|p$s%LX-P5!Of65wgrY%uqLA3s9*SR}dF&bGXkx^;_pB~{W zjQd-jR^W8I%n-rg&5z$sTk&(hUzhehyf#^sQZ*7ahY!ezbFy(s3e7s*`bTK^N8EVb z#(o_v8G*AD9V}~unU{dcpy*Wxt?JS^C43Ri&bK&fXG8`cy(s{Lg*ep0c=h8^UFkt zaifL-af?O71Bf+PfD6rAO*1j zU3`Gh6*)3>5Z+SBsFCQhnkbE>aKJB*5b716 zWGW3J)whK>^Nle?avKvZN`J3V&EDmqKGC+M>S_`Pj>}Z;i)?bRfir#<$lgu4s1k~* zP~3)WX0jRjGHdiqp)!|vsUDl&4ec5WO(n7_7IdK{5>gbWC>;`W8wt%8z;6M}nAn*M zw+~UEy)q3D*X2kO-1U+kC2PyBXuw&O2*93+1#FPtP!=Ac8#-3jvpHS3Qt9l5)v!pBz^=F+}A7) zCn9ECQSq40zLJh1ZU6dmn>lofs30DQU}N&C;d-fi%K%#iKsa-oZnE$H(rc*IdOBUF zZ16ya*>H`Iyzic{tDuAEgk0_Q=2u=kUe#d%4Ql<( zHA4yevAl@`iRwCG0Of`!rp98Rz#Zu>YrHmZ-#O-}{i`9PP&W`#%X(7FIb-x2 z{?;1Zo2e8=cz9`wm9ovYx0l=3 zn~3+v1AP8?&7BS}-e<4&H@eu5?|F?Ekefw&+bpBnv@Hm2D2>PM%R0{*t82|J9$IBo z^pODBkx)_BMwtzHz2tT}Fw1My}*oXS3Un6BGhd{&8erH6rT~a#)JJPv7v-03jXBahre-8xq_TtVqeH(?`0Hsa-tOYA zYhPFpvT%bpZC`wb$A7!YPa37xI4hCnSjv`%8FL-D*`b<6KV} z!SI*|y#A)lPS^PaZWR0*56&r^bH|rZ&6zJ%xV~?FnLlYD=CT1{AqYLPBo1Ql_GR_jCzk*cK?X zec8RjjIGvVq0&yepitzt%7-7jxw_%F|NYCM8_ZzwD$ejIZSMO1{{W{zSiixOgI$ZB z8k@puQ#=eChL9;tf^x$=3qiRwYhoy!2_pvBIWQZIQZJW4iDch);x+#FV5}iA80VE~#d#faNS_<%*HQ@VTK;MI_8d3UJ|k z&~PS1KdYJ5(|$j3A=E#(5QfsR$Hb8bXKeQ_!lsd*=d>_8|Nj{F44dfxl7jLQ?pnpg zs<()}|Akdsed2~LSH!Hs4b2$BMo~c+hA{UKmCK#m@`T*3rD9fxw7EA2*KBie(mEkM z6wS+L;-HDEZ5_(`o`UqM1NiM=zH`C$&<>nT%_D~@WpE{g!7A*yjB;@ z1yVNh9fj1h4C=7G7`I<%@P1suuXLZBxOsU2X3YqS_ z?0zccqLzFu0uwgUF~e6KEtu_KhIP{6^=-oDA)H)b?0kw_ZP46P-vAJ@%!H~+*2o}z ztsE+=nRZYF$OeVc;#@lcr(~*-Lh%L9xUNkgl`?v+G?LW~PkYmp=`C6P#EXKZ{o}^y zb3^pXLn7PIJe#k6>Y7ieAPX1Ng01Nd$6N4=)Ce;P)hzu27RiNjmn@ezkRys|fb6bo z@Hzd5EL}{u8)Z@{E%Q1b1XXk3z;m0sU$R8zITLB8!fyRNxLq3N-8xrSgbSq^O*`Us zCzWGTIVP3EeE|95r#*W=xk z=AkS{=>2&qwM=B!vD-~tE!@jy=L)uxnp4d@)(9j)CeF*WT_Da z#govvxj$0OoGfaLj}Ed&0QQzk1&1nQRB8J`$4@yq8+r}L-tXI<(J>3I3s{OSDZmHDCR6AWVOZ;tSnS6v>`J_z3#*`nI{ zPFMe9UWC2}TCMxt>p$zcWybl>EU6uB7|+us&zY=3smKj5m12n$DV)(;$~CEk=2?9@ zUAi^VyxR>fs}9lgvtwRhJP0u@G%;*xm-QdA6hRoV=#V+WF{BF99IUetw~;X3HwOVA|982?c0InN^)$T{DuDT;cr8 z{$l^v{PjPt@D_LDH&=D5%TJh2e8hD-$4^>BI-0Z-=|go50S5JQ#Csknh`-``^HK`U z?t&Tn?YGHVW~m~DOlKLp0W}|gCsgZ_F8~0c8?Z2uc`k()rz$p1gxCka>jhpc!1T7M>$R=tPS@7L!iiAZH+eWFgZzxsW-t8xfXA zAt(+E_M2+KGKfv;qlh{8Ag=vTKY9Kwg<83!x8+Au*L&rTte~;}K ztjsw=3@^AnkMBPmrU+nB-p<)@I$pvCJC_+x zdaPS*39RF@8q%Y-@5&`^+pUs*Y24GTguY>kJ+e*d(jtZWkJ}^Wjgm!_+Xu- zLp6s4oYq-{%c*Z$qb%sA$`zh)T|e5|CC4DqLxXF2)3>xj61H5j1OQK1v1U0dY1ZzP zs**LK>V>_(Ppk<(r$w(qDN;1dq`Y)iwnl&@2k*rPDfL^fKuJD;_YY39Te?vPeTJ8_ zB^(eB;K+T^;N}hN`5vl$C&AzIL$E`=_KLC+r_mhWa2CV(v^3w@!Z^CzU}+rZPpI## zqLc~i$I1BCs7Iv{AoJz|T1G*i-zor~@Jb#7*RfVpx(7E-Twu4S`~yBDQr z*z2R|*0iH9P*Fe`wJ>3+(Iah;9@Sxd!~3v6f8igA*TL3!C*cf4Y>u;CP?=^cfEm>6 z84xgq6thzHZuUyC60N}i?6h+M*#LIDczI4&j8N@%Qr|5mTkb=yNh-m&T6gQ)vsdIP zIWqk^hApzB9FVbEqnZHx9;)uruPX>dpN0X5 z)O(+XX5aTy4xNS}i(P+wFYLf?73*=q>=x|W8*h5$$=BcpZk>JHb2-x#GTxA9xfgoj z=HgedZ>fO<`8 z^%@^}xL+#pw#doWW^|VzoO_D!$&64^QjyBMhvat8%=!RSxVJvjCHk;SH0EatM36%tYS$b^Wwae%igTO-7-7)kxff-SFSV4w5>zB zAF&?HM)%|by>i&Yc!Nx6ww6i{&d1K2Y=7p#HG;ne{uWN40$UoqYl}V#nKGi*vdU63 z7pRI_&l|jFV-*Hy8qVBQ6!Ph6Np59%J%{{2%>OfkVj>-mEGVd6pV;V_8|KWJ*C_5E zD!+h{+b6cW14Lc@fu=WblP&Swb43o`xH&$s@4S0|IXrY%vy_UBP1-ZWekm4qfIY7_ z0(u9oxk#;3CURzK=vNeAn$RLVdf>MEC`dg$WY+At0kTbrikLn30mU6FV-Jv>^~g!i zSl~A9Rs#bRzZI~dOkZb4AKRtiW~RT^PjF?2V_^`u{9fGEv$BK9CPs)ot$kCjq){Wg zVx624+}WTGnHla^^4ikn39Pn_oT+oV-LHp6jN>^iH{|QHS3|aCXD2-3%SJdn#sCs@ zCFBeWd98V&!o^UwG;+|?T)t$*(|%2`Ukq+Z?m~LuLaj1U&b?+h#0YV9XpDWM85sM} z7@?ntnT^&|e#1@(yJ3RxzQOA?+hBCJG|E7dRjKP5$I3Oos2VsJ(-u=6GhAC>IG9Xr zRHFL=$=cpPd=tEYo{PA42d9n}#$F6BD@Zp`a0xs(m1v?`9F?POiOKQHK3@?)R@Y{( zj8-C9M~hGc>fL@8vtur6p9v+`tQ0IG1ubdLG{)%Xs5Q6-RFwWrSL^E4j zY0wbZXMGu9AYnt`uF)2FK`+hGrk2d;ulH4^R=J`tOVcsUX?9VtBvN$#`t8MO`$OPM zwx|_@-w92vKXOQRW{b6y*W}27g{-( zhONufLC2E1=P;c%1ZOdbWMRK;4;t7NEY*W_mfd9-Mzshmg{uea*DiDY9~k2Htf0x@ zK?Ac2w-J*fNAvgQ3zC{UD2JuwOaJ&=&0=6I3)VQs4h>CWiTZ&~BCK$NG84d&A8^sxCuWwbQFR1{F-W4r1qY z5Ov@C*f|qsPJdUY1lB2mb@!PPSbLu-ft5@NtYiB(C9qBjtout~?Szd9WoTZsz8V0^ z@#MCSLEalECq`%+3zRbei__7{eSmV>us9t=?*)|8hQ;X;jmHAzY{BAmkQ@t?(*ujs zL3s>NP6sSbcVEW<LN zq-RWe#{Tq-H7(O1@{3=+y@fLYUz@c_xw6Uf4)4BWP*P|)x5AcT7CfQqisV(MIePaO z8z-j&;VXB2YN=^vOXzpeU1HFUP0j|rq6v^u3C*DHmAn1*?M8U6OdTgXo zTQJ0keN-eyB;mSdhB*}#&9V&vp>c?rtW25;9o4_%Ri0~=|a4i^2m}nHjMEt%b|x4TeHBDkvK6|@t@cmp7Vk7SX6A@FZ+L< zMFipc?*-WJA86T->PALwvL?OnSbUQlSUTkDVR%b-25+aOgzN0 zkH7nQ)}W0YeVoI8d~y8Oes_F)bo9eFZ@<1gfBhpr{`m({<=6PHpO1cEuYZgr za(w*PT^uv_v_x|*^h}mBJUA~P%xSzPKpLQxsN2uI{P61!lz-3y7fe&9T?umiCq~lI-s~?H(Fig1Y?SiV)?e{r z8z72B;+)QQbuQBjqx$0XsVi$^fEYtaDhVT!5Er0cGQ%s}m?5VUwJRg73ji>4;oe3E z1;PU~A)^xJ=?L%(?r2_Q?1X43NhZY#(N#|cI;hN<#5vV{=u^anrb90f>jk^A4fTfO z`9etjh1<@7j=B$xQhFnKN-71$NI@!rzJj!d;M$O!Dgz2rMzZ>Bw3|2D`8i9cxf=}G@T)28iYJs5W*ob$nmC}rm$na zsV@gNJ7%WoQnS*G#7y~u^=4iXjEk?$R3EJTdV@npa7%)#oXCQ=uuu(8Fe?k=2Xh)^CE7%7EUOwI3n`p7xV`N# z{s)FQSl2^b_2lS!qG~XaWoI5iD&lDN;9FqYwq??$=yo~HRokxAmhqS zZzXUGw3ZtyJaJ!Dv$l{Bp4R41eam&O8vb`H@loRYTQu(>gZhaFo$nGl*7 zZt_;9;a9h%Veiw%n2$)U3baamAxM>w9B6p(j+Pg$>COFR%%Yx3CrSPXc4!98&7OtJ z>PI-u@EKKoKkPtrkKsTl3XWZy=F28`&6jBLfW?&RN2~e^@TVayY6Ec&_%w7ruqFBpTYuW29znBi-|r<{r7hC)ItL@5UT z7q8$YQgG$jx8ruwR*xDtmp_Z%J3aP(|B1f{2-xoU08Z-m(~e#`KSPWQYm!9B2X}M3 zfU+pM#KQX4A~sI1-$EOHHj--lbgacChy$$-+HfwIOrXyi8{Wg(US& z!PJT0^GK4;FdNr}QMV5PNS%AEFie;;I9s1Y9ViGZ7Ckt(u8%Rg5Wqq6>_Q!EC7ykD zqYDMrG|voHIpVbegoEu)1_gb5#spAoKP~=uf0@THPj}LBNv(il1oenzv4%NG3RzT{ zgW;{=>h&5rdV+b-bx>LSmsElb zm~I3O?@H1Z9(Q5nZR{RmJyjyX_%R-}CG0qN1a`0;Sj2a}#6;pKRg$<;wQggaAiz;a zs=O`D^i^*Jm;Wqia(xToH;K#(s%?P$3CdEw;3EzXbyO<0d$`z^{Ji0kBX4eYB*GY7 z!bzeoBfd3!LMttEs(AwTGQK2I#K z2$^N_Hs&n~W-OV@w*wd

fe~A$K+du6I*LmoVHNH-?sTZNm!vH4eP|;K67%xP{n8um!vYO0N6JWgW*4h52s91o~h=} zyX?MYBS{Y5Yh#Vz1gF!x+x{)BKJE~X_950E`eDz0RA>D{q%)?=fq1qg9~dO;<8r^g zE2uEK{8yA`{ZaM)rqk05)16Mm0Q+Iq$JEhSepItw4{s&AhLd(%GrG}PK3 z-nh24XysAnTsxp0+A6}JTg9O9Ovg&+%Q`}q-4z*6xXw1dzz*8)EeGt#=uJ&{?#fh{ z$(^}!2G*cnkA1K}!1lxLzzKrG`)#q55ihG=G^XvRwR>y84+L-lL=a5Ng)~7cJj4iz z(rhioQ&z%Gv;L~E+&)g&h{LKG9C8G9P2|q%M!4z3vWcWPkb1xULL(e29<-ls?w|>c zB0G#c6pJpjLv;+#x8u9dz8n)w?VfmZ8a_j4MZ50?AaS;zk&4^c@dnY3wx5USdZSlx zEILxOZr8$43ZNxL5)EJ~b)qLll8(bzs?n69m6T}ArAUfnp%1s8$I>Vs7IJ7UkuNif zZh^5D?CtM6sux7e&V@&%arC%qXZtp>CZ?H~1eg&=8fl+u3uLA4J)ymXygbJkaSAJh z!32KSMbgXI=gQ7;1lcxj4Ma*@oRPHTH>_+We8|$Kj*jzO)cMx5?(S~2#1cVjBLgiv zhi@f@H5BD>VOyFMRar zg{*|#!)dTkt$T)N(u~{XQr8XN4B<(%rnO1K;!87nmnm)Kea~mMWMvu41{Rxc?nF|_ zaP62w^Ng}8bWiD4lqn_WSQ#3;W-Ka{&anQag&ViCn9GzeZS>fM=YBiE#Q6xSm>+LW zc|Cn@TK9LO;Qux(3vQT*&TYrBb;r+j?zs_O+k1tejlTQd%5>~y*L+xm)3q?aqE8O* zl^W{enP2gYsg2SQvm8S*N2ZA3-CLD$kpTkQu_u}lbk zulHx9q!2yoC`mW0w9a5PdVh|D&Fy_58cnCQEU!Go#q;l8)|Cvm@RA0>^9iDeI6x(9 zw`NpvrZl;w8wXBDod|_SURSKMhh#hKze(@FSV2c|%K%L?DOQT7eMD>6jT(t$Yb*5_ zgTtm@TJY}>_bC=m@bvu5e$fKHwCGVRB`R67oT830$PsD-DZ(+xC0M_0QqTn}4M`Vq zlSpyHN)5p!E5UynyrZ!PH53RK&^5S!AuAh1JCMHs->u9ebZuAOO zEg%xZg&q(s=j3I`J!<#_!tX$>%$GE0*-NUrK_T|JtPlY`Gt2U@g$?7#v=;ZZ#q@gX zN>kPbo4QzK$uf3>xJCmLM@3ERgI(zUD=W_NWxri#81C?UsY*q1;J@K~0usj`U z&C4Pz)xY5GdW#ll%Vw|HMtNS{TFMBcP&m*PS+1zLwtz}PHCxHj{PRd9vao9jlE!hp zBmv5A>BiOpB|EyS3 zrM?bt4-NRtQ=!^OTlF1(LTIMfvRbV%xi~gBIj1&djZk+Vf|UraLQFft47x_VqJ`ng zB@f#V!O+~2X6JQQNKx{fmOR^lPz6Psma?=X3r6Adl;wrgpgh5rZI!X|YeT&Z2;1(d zFY~dxi$g8+C1<7j4_MKdjTSiY5xa31x@;TJy%Dk<0#>r7B~3IdxzapA1mUn!Z#M?& zM}Q)1)^{z-eOhd`&2ymE4oqvLkPr(e_83rNpCgP|lqYjzz_C36 zevJkD0w}MDZLl|>uMudE(TUQJlO(%qqlJdZ4c2fUE+_)s>Tl(r6cVRoa)|DL6e>} z=~+T*{iJ-#onq)|Xg4gx7gu!#am z^5BmmXodlj`k{{^$ajMy^}`-Tko;^}Lu_fpC@`dZ03n6V!Q(-Y;xGR3;74PDkLqcN z*d$@lqu4C=XjqFeFh(9cQ3PcYgfQBHCW@l^F!-Xe&_yF)i{c@R#!VvuGu>FYqR|)^ zV_}M7A&OeCL(v27g(upD9Ezs7A84Y{6FUx)C>oAvBxV;0K@oWv9P!(-ABJeVt!SD( z5JcUuLeW&a;D2<}+$_ z#zF~Mkf4ZeM8XK|0tCejF$f{F4Gt7%N*;m_>I4RgAlnTc^dNC+-_rE`gQ>=W0F4F! zYQg)&StEaC@Sn)TcTdotdjS4~6`IjtKVekAkw8BY4wT0M`Po0f&vF~B~( zuswk`xl-sTZ0jbo>FhXB;d~aM?A_K3_%hTXKz`aaDe0UJd2Vg<_>$Y743jX`(BdS+bZX z8SlbxmIXn4)^x9Y>T}PgG(rL_w*%5Zeji$y`>4IG)%+ggl)MYKWC`OpOpS-uL%w09 zR1iMlVr-Ad`jn!htkAKfn`^qpST2`# zpVjD3OacTqi$^(`KZT3c2nu-f`-UjinAv9wM|CqkxXt^*XCVS0OIdpDS4 zVfZ}mftZ1}N7kOt)?$yEPD8c5+^%O96KZQ212iv7DilDy!0Sev6*h5cySJErX4zKg zEb%2_cLha>n}R@Q?vvceV|8@l%$H0P8XEtCdD&l`XTP{f=6LJPdv3=coFVk^$3ScU zgM2*b=OrHxeu3t2-XEo@oumE8tvvF~ZSU2O2VMEDkWRw%3E{!e*p_|I@-cK$a4A4N z7Dh>e5jjaIK+}_>J_>T`vqf8)$;!>Fyh7{3r1T@?<%M{OG~Od z*&5FIcP~TsFBq%?PXimW74UCBVD%zFm68dk;b=HmE4TPy^0tostXNYoi?}vnf!!%= z{Tqx_clNCLur&El$JMH&qRQxyc)4@RA3-mT3PK$hfI2Gp^s@~-eNfP8K@Z zV`Y8i*n~Z+iZI8CLwmuODQZML6t&&v`e!9IJ+a}0WteH9ixSsq_qdu%#@s_-o{v4S zT-ixHTXxa(WvkJ&X=bE)+XCF@Fkq6TF!$ymm=8Oc0jubUbld(E@w5aKX$?$4Li<*rv86Y}R)Z9nf<9g)`~g#9jMnN$6Om>Xth8jSd6RK&6w8QO?Vm+n0|Z4#{+iml>Z&x=twEiof68Zp-h zkZ4ZCU}d;KKa2UxnNoB$e$woBZl=~%PQ|QbG=+K!_SdL+<01u+eX-g)NoHNVxr~{v zQ9~~oRYP@vdPa_)VMIKl5Q$$&9&7=6**`zf%WEW8eCW>JGF+tB$CBPbjCIYaJY*R<#HvT)b7;-3ZxLe1IFgWaL zfZa3Tbi0Us5ZuqZG07vA{_So+^Olkk7xaYiymIM9VfdsQ6OMa>u72%-p*v$|%k`(G<# zIgep)mP7>cx*#MT6ey2SL8x*ia;95ZY$ssbk046b7BXV@F|U&tBA6UbMB~=?|FdH{ z#*~P~!{LTMhh~QT2j3hpRpow( z%r#4{BP^l>@Tld9qrX}T>Umz;m?6;h;rsay=SD+x<+$TWIa;2CBid=m6H z5DzIbks{&QuTgFw;$n>y8H+VlW8^(}fBT+ymN)#dPbmK-GBXBW82Ax)8~_UKPDX12 zE}J*&3?PcQb#FGAJYneGeT3~)!^_yj?4*hQlUk-s|_u4hvE3jrz z;0%QD%bGaIWLnc?Z8JFW7hr(hr&?32I1A-EvEl((!b)CbjQpie-alc>B}?>wqQ=pr z@%Qc2I)VN#?!W%CYww3dIBd1=Y_^NQ!mF*jp4s6c6*D2w=Q(*jh|-T14lEE!v#do# zSpnS3agIc0rzIor~wB!}RMS~LAV5{@X8{%3*Ux`2l~6rtDutY|hjET{&x!$0-XwI8=oILv3a zJWCT=VhDsSDz2mrt1;ZCm!gsU9P4Rzg016N&{FfH%4q3|xo+;g4$U^bFp@~xt1h*f z=k{0IcX`1|E)g+y$OON2P>!&;kK40y>G)24s6AA`0iu3{FuLv2V~>;9Cw5N$wsDE; zgCWFzG>q+|JvMfU8YZxmC1hhBrIHNQ7-u&;(Z_T0zgQ{lZM&kHH!)c5I@h+MKUL%@ zIfBpn`n5w8%p|4$k|rf;)z{ zXU`iJAxd`HSxcGAOs=-qdBJF^HMz0?3NS)$%4K@293-Lu6a?TISo_LRLn22V!~rFW zu5E|tCRgtn~7@Ew~!h6M!XMa zvFb;z*R<#({euL%v52T0Z5)D&$}1&VLNg5`gKBh9a}c%zl+_BkNMu{(B{9CU5+({b zWtg}Wb_myMj-G(AVeq0^hNi~iRBkBm^=!u2Gsn~jeVpo`4^bC1B6#C*YJrZf0%|LK z4#FK*|1;in9aZ!5ajJcesq;C)P8MI=^RcRWj;QBp>QP1j-MfpQ-uW1HJL4;M zMig=it&~>6&ZNApa(ku_*r>nA zMItvw<~DkaMvG8 OwVp+bw#*T;&f15Mq=Ej%cCW0yayAiQVDj)^s%R08K-k70`y>0!XUnE3!CnP)QPv8)<#Ve{E@)=^+Vu0Lo1JIf z%+3F-EeMSXgzhLXcD{m`6SOGkYxpB8q|Ml?fwmqDz*Qt;5DY*CdCSF1e7;d+Q)pXa%H-JUf4O{`9%lK+q>d z6+7!V8U5|2bMp6Ih&&+?nB7Y};C*+D;5&ysXKjiOZrZ?FvEwr0Y&zgiZ zp#mwOCfo+cfz9G4f#f7I;BL@Me)fOtyO^0yq%aVkp1=}L$vs#OH}n>*(if2Aq$hjw z)$D11S4kYJVhf+_rw5i4T!`&G-3}52 zE4QrpoogSM7c60^tFDRnnA=D&{8&E))OLfvCpS1Se9<2-D2$Lptn+fR(UrC-7_ zz1sR>k365oSI_qLX>8mLjucDX^PUHtc2FrxPV;G ziyM@xUgsIPk}0v=o(!;i@JlK3!%>r7{BQ)GDyPtxqT+Stcc}liy%squ^dVxk?i3}o zJ*jeIe_Hzh$=A@EWCc#PVrNIfPDVuBHHElfhkxUx@8tgFdkPT`mPquQr5y3l!ky>wo;v|W@v;H} zaz+HZ2d)?_g4nZZbu_kPre|7ipp{~_V`4wiQ~F@7Cim6JS7b(>oN6o=c#Yr?ebK}O zrH9n;a;=pDeIg1P9RT>2@X%E3JO!?wN$x0i?o2wAx`y%=x}N(gbq~Xng-qGSa>)fR zw^uUbNnE5B6q9Sd=ke&ZyeIbzw;fQgcl}r3QTHctuK$O_E#Om*)fuVPvYa82T){Da zlVgY@E!cB1BY%=Y7SB=Y6;NK=v9jK21T9m3QTG;ORuGpD)`{o)LU*=}EHm>R{^$?= zJP~{uj?wz;26zld18OtqO8*tMEg(>944Aj>(=20mEOQsZ_OtMiJamaQhF$7qDlP@0 zSm>H9GrH=%D}xpey|XtE!ZG#|_-}=-0DPjK4q*655Zyb`I)pqPx+CUj>MpC&eC-UX z3at!7)IYHl9}F5I(a^GElf&jTb-dA815f*{5zPx?HzES{`$7I7_C@plYlEd~_QCUh zQxry%eNsHPxb)}9h}@g{d21xPWx#*ow9pVchc&2hhf*kFX1m-P8H_6zpukfmN}kZn zms0D}W&;e=cHY;*iCyBHd|yK#j3#Rg9mfy+uK(ibV?A0_$t^3b7pd6l59L)!7Br(G z(HEs1WfSN;1RqC;Fe}VYAkHx}@&F%DZt!N2>p z8chqoRVnC(9rY`Wu&WyL@FrTT zqiK`v?T$##v_R>7Fr+aYxi_0L(&FUN8GiLgQ;p60YCsrm%;vJ}vfS)@pLgt2amMsr zwXMmyAl8hUd)n{y&~zVLN2QV}aeVBoS0FaS`O-G zDQo?k1LUS$72k9XbfGmEGGp^t3Z1Wm?Uuf z3*Q<5`3A1&<5A-vkFfPe|0QoQZN3EhhGpKCgx#@XOLD0&cN9oGRf?~~Cgj^++PE~( zAYpY@**Y@X))827WP?0Eq&>hh;)m^4LyAB7>ip^X(;WrB7pSlJU%~&?R(bp1%Vme1 zBoNtP*3`Huk%s%APvIHZ(Z4#a?fT@a^JnKzPxP9fo$A$lmicG1r>7)PKe7SQtcx2x~E;K)ZrM{B;!n!H@%f~8XNNh9(E2M_A!@*NmMIn!ql8xSGlR68%)E~ zlJ19fS>_*zEXapdktZLz1%m!a`5CnC>iLs~*sTqOaXXP48=4ifANA81z@^Em+W`+2 ze6}|L?Pkr8vZ?0oX_Wb}`jO{*?2UeHpABFl%_SE-xt`5TPxB}S3K@?)A_8RAT)X6n zypVT1?Nw>GFqS>?TP*|s?)h)^E_5cUf|2(P6RUFloIKrUfon!n)|W;yCSq|=P;KXV zn$RnVMKnqr+$5^(q=wrD07GBBzj~qX&7W>=u596o9^@K`x8p*wq$*xsp!6J7f5Fz6GOZECSsTm0 zW;Q9YkPEut885eln&usV^mQ3ckmGt4oVUhXM9DIf)D%vCLdta^tJRu3?SAhws}H+J zRoIemn5<$5%UcsoWa~g0Mq?XS-%}15lE%hB%idZy_}<`#KKZ-dhtqAnC#Uz`zf)@F@{SdEoZX+_%R)=M zGtCZX$lcUAw6!?@WlUEPJ#M~v`OR~3k)}prs}x&S8LCEVPH4`*!vZH~F*!3mXXX+8X39U*N~43y(eI4rc#fpp7yD@xR3B0K-s zWXIeEq&b6k8OP6_=0t!`=xZKox7n+{5!BeMy~56`Z#RHsV`~* zEgFy{V=(8IZ4cXMtQGND?DH+#UJr1y0Z!Jnu3@2gaPDz@Zji@45`Wu+w?$IhVV?Fm z^RuzMEP_)-?pqv(8qb}8_Y7l03}=etO2Zr}%#ETr(LP)Voj>6c{Ehwn)`Xd${|L6% zhR8l34GXTfe=}z;Qex*%0f?q#`at~AkwJ1OUtim2u-1Rd@=R`R5gAtks8XGm3j)sd zMJ_g&H(N@wVxq1$>%qS+X{oQWE|*x>wl2lpC{9F`sy56P z!Yi37NtLUzV6?06$zA8t=^f=6@MT*V8=5HEjNbaC&MojJb5E5Gr%F3g3If=Iol4%y zYPf#t(Rbdnm}stSdTYI>P=?vXn{-UvhXNbj0P70%=Q$ImHYpNK|FI6RMrb(B-S!^W z!ogYHuHB+Zih=Z)e%!sq*VvI<6GVges$+BGO6q&F_X?j-veCcj3jo!tP3I)dXL_Rh zX}zd@D&E8AwCoiXT@FGTPR1Uz5w>XQWe=HtdjpNHp~nvtKsCYyTrUHX96#^v$T~>t za8ZTK>5BG8*0k%ZwO*=AvSQ%;#-Motu`%#esIHSOU!_U8##wKxObD5YpQT$u3t0(M zsHPX9tEv6AHPLro|9F$X--s+O>FKShxT{~9+1U89sMs0)bYJk2MU+iHS<*}~E&o))F@)1*9ej8sysVIr zqpHQ;_tIQIK*IW3Iutzamrom4w{b=Nr5zoS6MXVmkwVJybnoLc%Y5}Qk1Nm&ZP_&+ z$Ya*ryH0N}0&D(*DbAqyhG(?MwxKxuYg`FLvomqW3kkUu`gemgBPQJT% z|Ce_cZ(p5iiqZ>{+11v4Q2d#DpJ^XZKkBv)&dyt(0MI784o&GcRawz2+Y(irdd zy{7}g^p*>C{#DbNE`evi2zgxokZglQDORenKs#n>DyPe;%&W2?dx9=xn&sNh)h+6b z+|vPZR}wN`AC_r+38rhVMy5GPq-hpR)@PTx2D}10dDzf; z1}Q8cG}z9~t~L<+m`fE-n6-N|*0P}xFU}1n{gU1*hRa>(WF^zQFSVp1maHfcTNcc` z5QNE>zIgH|ND=%#nz^*_oJw-9DGF#mMDLOW?g0{wY_t!Hu#^SMQYO#3vUU`lA%& z)gP|Sv`&AbO$$<VXbSB@&+wn$ z7rbOaY81tSG>Q*(?rki{zF-&Q`?w%*Mn7P9QJ7g|azPBTKg24&eSbM>5#PSQv}@4+ zl*^sz!xY1W@?oYwQc$hs)YmN;wPxie5iU4;x+CL8N|!;XuRhK{gVo_cr7}QU-*rOFspi<-|!=fqabK5Ut5CZ{S zJAu8$E^LOnx~6Oz$%i4HpqWy**Nl@0Q!^DSPv=%T#3l}~4F^-B(IStNcy?@{6eO3HvDxIN3UP8{LuB&w^2BLN4eml5etmLk6aj91hDlPhe~Bd zk_ezQ2u**Y&2tSj+#fqHL@CB|nxc?D5$9*qx!t(Vpt`53>^!0VqV)}yRYx4Tq zN!XqjF=R)8X)Cm^9xIaZB@1*Dj_Um=1rK#1-C51CeM6t_dwUk-t@kS>iOe#rnh?ZZ zxx|fdAUjkyQ%OqAZ~qmPH>0MV;{lw%e|7Ql?W_4F{pDKT&q_I~6r1rf#2F(RrA5YB z#k3d81MWT2VQwnddJDg?5nLEJL`{DW>)4dC3~Mh1HBnsDnBS@_!@VvG#?E|8)A^^D zB5K1)W2pl&#h|(d5FeTikWYh@W4KK?;I_Tj^*hbozHaP3G^vk%0lC2 z(ZnTICa?+Tm14ylKmECE?hvDc$-ZYjby)hh_B~s6?X3CP`Gdm*2PLp(j-M3=I=33z zP5$(I8=Hd@lK{RzLBF5!IY;w8N<8mKPBWNNv0p-kzN_&_ub4r95<|>4uBXh8hgZm{w%jTYlss?cep5NfVxJ3XtYE$5zq_y6;895;<}VFM9+G^3pL#Emk|4NVq-#khPAJ~#3Sq`%Kn z?~me}2W)4YjJLlWQqyIK=vTHg`JgC<)XGn1%4tD2tYk&q2hG&O)BpS?XX3h~$!*Xp zGRX&DynJwg!`Gq>wg)hYFLj_8UZ9`yHvYtZ+rcs%O-quiQqYtI+|Ss65JOyZnL}IR z-Yvie8=>&3n7eCe_&6u6_fnH&ZS*e-UOIvkb_cQ&yR>5aaV^;-rc)L$b*~T8ye?Ha zV3SHp+r+{(dwufc$rEhE_?zGShRA|YlCp$v+7AW+BZPl>`gBhI-^JTE5dN5#3L81& zSM>u8z=3vqdi?wKH}C3}R?Rm-YjoS$2pkSi^VeU0Jtr@D0fp7M={jR0GHuAPkR_qe z-Vem2ux`7?I`1X4|HCu^zFL>2@rTCVj3*_QgGJIQ2b$v$8})~CaAF`V_>5%ymXW3U zVhhE-OS5xQ-|63X$0jd zR#xViAPiH_nJI-mZG$j3w7AteqM^uh$U0H3=tRLXTJk&eE^X-6&*mDO>+yVkj3h)B zM3uBKcV7P=haSacdJU(A@$AghViQ&XAz>d&R_L>M zaeYbBg5R-1k&{1!QrAsKp)d^bu=RkdA1keb6kY>iY|C|UnC^a|(Y4E^KN>l>CR29B_SQk! z23*{A_n@3AV@(8C`A~Nm9Nw{^iUrbl!WGa7IeNx?X{38yQ&r$SL^H1 zM*pstG5E@<>_bg)iSsse*!~q0ii2Fzs<@+%VQ7Ea7e+!(U&c(G9{s0x_58mZPVv;T zqJHCBk^AqEgl3s)*Tb-6KP;nbkf6dYET(adgmyZN`n7IHUw40nQU%azr|0rs!86pj^rB8-RU)94hBFnJ$562smM|e?| z1z%KU7-h%wGV|v!c$ST=Y|jYf8m+nm8Jn36PP}UtVWD;xcb72MLIKDF_+y}M3rtuh z6?9w#dfaT%ho*($lL?}3M1lm&b6>GGmG|PF7U{)RZ~j1BpGQgZGDQZGR3(xos%EP0|KxH6qt@R zAJ-`)*vuQ9gPsR6mjh-!93bpaAtZ7}b%q_n)fsB>8ec)O%xS`c)wt^67;A)iIew!E zRCluCY0T5@3QToX(b9Cm-%$AHYrSth6}GwtS40bQCBtGStk@_bmuHQ*yra9P1G>8% zBivHNB21|GU32V~!gHo_jVyKH>WEbMdejbBxYoF{kFeZ012qBhY8ics(S`QO0S!LL zo^Hee`NNknqON!CK0I@l*rxliAzH-T!eQZ~EnvhzjS@kcVnzhm826#)9E@=uMYs<= z$s)=*8{Ko#U$pzsW13gSlep_jjMxOZ@iv zi7|JzXt)oLkLUAa4E(4ShN@i7{*^G1$TZeVt>;*%62z+76y-ZlH+H_G+>$uI2W=5! z>lD4-Qz)uMAy@IpEZ7xEjFyQk1Vu8w;ZQsvb&b}^3H~@w@+yXAn5Q>vBa7`>-LE~# zs|H;2#Ayt@EEq|uqF|!TwvQ-txt$wIX0J!{P-=G>m`@!~2U={>l4ACdc&^JuyH|wVe1C#DqsbJ`!C7fd9zeXc;Hyj2hyS@w_pbBeoL$>@;npN-06lH05NL zB$ElvBFg2FrW6Cn^UB^;>j=~+q`!#LADBfm6*hR&JkNLn1@S>sg=2PTa|1PfKk`kr zA(hfdE`@m}2-VyeH^F;b%O8rPZ~*|D`umVfaP4-oHU&XOsZ}pk1ZG76r>-$q-&+8shAOd#4Pxjvtzth1+tGtAs<#9I`1J93XV*_&4x+6#*7 zUCv@lsEy-t)ZaRaZ=A8lu)SoPn940Ys#-iw12F~?L1y~@S!0j&@GaO916KNPjfMsK zwMmCbS;k7pU}3IoKSs}NT&*Sax=UzI6J7?I$`v%Z)vtoj4yJcT zl2a4j>>pb*#iv{svRd`aZ`8e=WK`MR?SyNzWJS`gpu?%V7+0sbG~OSW2chK3m0m`F z(~FGq4LMPakyjR_@h{h5aT%?VMKe}Ry7o~5Yk_%0dC2E(3h?V4=$z!KL#xg|_oG7r zaagvu028V7;yNy_E|ss%=IvTa%h>_p#YW%rjn>?dsd@RS& z_ny_<^Sov2*tXwWL3DXM_N!Dbw>#z0NX5Bn%2E%(3Em7 zSYdC))psw>2%odL{a-oogQ z;CCN1$0_s!5iLT@8-853Yu@(ZN~YI2OXj9O&j(bD2@_>8P+{vzrd}syoO8kcYNIFM z$ei6ap(PW80n0WUR>;yc)m?&nO7F=DSPQZE!5Jx~1iAyXFbMNP5cZKPsDy%h3r9mS z*xU&y`jZ+HY~h+kZkaD>ss!B9d%nLTN;?NNCcM()mG$@xQ zid}zcO);`7PX`wdXG4X#lI>V-Si2NSEegTG4ivO`2NF=WHBm!oEiir^!4jqvEw*RS z(~OrkCK@AZ!*kUqu%QCGhZQW5LX}08z)4ZEDC8{@E;T`OgPV{LNNiL9o{WuPh0qO2 zk~I}8R;O}rXbL#;XQu%SSu^Zu(505Jc?rj|&;p6`c>N$;P1CdCZpU^DdDhE#$Ou2h zhLtp>B^?!+{g%4UStjBIHq48JiObn^nt;qXq?LFZ2ImZW1htgPIe8}|J*^m!4Wcr1 zS;HD-?rTUaSxV9hr16-x^f;m|vs$_vwWBw6;(t;{*}`6Q$(qB?OAU~Tk{8U@HP8rd zHo6js*v%~;HcV@py<(-RgENtzX~Jlv7h|~rW(UVG2$dRRMYm#rhp43COPc!*Eq;uP z4*S8qm1&$muhqJ(ez{J*z~mkU<}G|6%I&ZLC}q3HEm_mM{VmP4^g45$W?)J)_}uJi zj7uaUqR@_!p0=2LRXa3|x{h`WwJQI(cMYjGaT_i&D55=;9gJ%d^ql;vA>bHc*lF~b zkaJ>H{JA+0TRW=xc#OlEv_1%uS+|yI5{KfPiB|I0l6V0ob(=D&5xS6B#tPUjqy5LB zL8FQZbX-B#*O@c_I8{Zk`zV-JKBEe&N!qvsFnigkMElJ+$6-A$Y>*5vF2oi$^QDJ0 zXz+=Gy3)(iDvzLxOfz&3`P_V)d@pWKr3dLfTN+D^|5Fow6x5pcHsB-kZ2+IUK-MV0 zMzZ(nMVfTBIv&=yQ^kcalsmKXE8%pmBqE|?Y)l74(n35~+|w;;ba8bFFI&BE1^>fx z`)DVz9Sf$(L0vHkHroHj=ny>|%{%;O=-Xi&Z>~jq!St7+LBdMs&?bIKAj~)~EbftBlFfw4<>~ zzx9T-xO}FojE~uX>K!~#-PnI{FooWqY-Di9VFA>nNh=SLn>49a%|NKw6_7^bbm=F- zIawF%5Bh6og!fx${XvP4VrG){o%6=X-8O<4Z3(VT0{cN+!nd$+IF4=McGsnd@bW58W%LVy{AR0puhnwId*J2L#fRd zu*C6k+!d{lnTxiZ=`Z@E{#)ENZOf>qPWDyI^#H3kAr5F&$_*`f0v15N^t@{u*QLV7 zQp8i(qG(hEe$nNXDxNWz}4cV zKFVaMkGO*f;Lh}2zh_yd{~My%1E%l5>iJ4wc0X6vG-;A+PSQf=d90wMas9NtdI$q4 z?wEpLAe;0PYR}-06%(wWB{)7!1y8;tbZ4rKtFZS*_-KGmW_zR~DYbX(1_GE9H3rzr z8+lw%80wUKYg)iN{^Ihod+Y;oLJ-;z&^~~k8)G7vlaq@qUwdDUWGs$0xzyF>uK4qg zg7~74O1-mS`^+GA39$wSyUa-wpEU8VR3dN;(O?ejU5wfUK zJB6TCT{Avmq!-c=baUJrvjL{|mxgHqL-T^&aaqOfgGpnkN|Z2EX>|29U%Ae%HdGlx zZ;CWt8^(}gT2`uAYbAkzh6i+UyR&}`i4&7CV%gRB5^{~V<1=H)p4qWvzYJOOxh7{c z#{(pzk{#Of1~c06$AwV~ZDtV-_rq@wxdaG16HISDCf4(msl}av?0UBy4Cc!c=s}1j~WriNOmlR z=G?m$5e<>x_Bj|pDO}j7v(}Sawv9Iq5)*iBS_NCMm~qf(Z#Dwta7o|h7>k@ZqS{ya zqpy_}o(=S(iE9+(4=k~6js!Bu^|s0Q#gl#PUO4QT9k31V*c(Z}w`@C5(Rp`ZRVyjf zn&-BCFWhs34dgqT@iAEBuMm~pOL0ct$x{F46=aZ%#7jM1N~Ye)65jtLhlI;N;E0TW zkAq(gv%H|v#>jcpb8)eh?(n%Hmx9Q`?Cu^{z)TXSXWa|6H+&?-j`^7bmqp_QA8e}5 z2R@MMvpWzD*iRvX#HEEmGe26gRYagwb8J|#g6eQdRQs!XjO0-`_QdOP;rDioF)!j6 zZh+N%+9FN7ggjPB9lvrAgx)}+)2GpF_c+sC$qgrrQpbkon(F+w7B1id{m%$JmKSPH zE{Nh{l`$WNHS2kwiw3J_r1hk~Rs4=-4D%VOa6RATa!3A*rxw`cRf_~Jx9{kdZI8~{ zbSWC*(WN-5>wh+Akdxscrr`+w7FoWSm`ts<2s}=-eN4US&=6E(r+tx8bf=8B^|v8( zG|GzFr)v*R?JP5yvpYuUD3Avls-fgYToTqOg3xX$a=_;Ig7PwUqZ3#l+-=QFwp0tK zm&hD>EzOzjHevSrZ`H|0Q7~m;5~g>;lcE1t-KAx?A74G{FpYw#Tw;iB=O&DhVAjym z+J^0wB(I8GD%1C=eP`k*CwCQ%ZF(qXExQW>DWoQCISS%Mvqqb7FhSud<6D;98xaNF zhLWP%=6c=WR57vq_!#4Ej7>cjQ?)8p`Y~XoY%T=S;fFh0+o2dX;r9BaJ6ebtt0&aTdHq?oMag$u?eKHjTtY+Z(FPE$+@az`bZpsz)nN%`7 zY539^o?Ukg=a}p`6QN7V6~k+*N?Nir?1#9_Y}=G*CbW?$Uz!NQYtI>ahQ%aIJqW)D za}-#+y$|O^#sAg>FNr>hb~Iym9OeEv8i=HPBu+ZiVabk;zFtt>-WEV>l-i6aAY%vb zl~=w4dMcOXbzB=s4RWWpak1gFx8mV8kGN8?OaiK-A z1ip*#LSlu(*(9#c!l>0pO5QO__6$xZ@Ib`31|M1W7O`kHWKEjp$JcyOF!Ewegq5zNs~450!lR9%2;`<$XcqBY-qCP!j;GruAKHj{zNW&MU@z{^~lz=$EG)} zLC?CmD_;A$W$oLcGB40ht^I!8bCu7;j9w+ zcfSb%8?mF6xqevZ>HZ9+S?*>CpchnvZS z;?O%RwykO@p_8{O#RG5wbi>MZYzp$AV$*tl_4%l%4&6F|G%wTXDy2A}1*-A^Hz3N;IP- zRjeE`$2H}%`W;O@{5xX)1pX(M5;{OKOzl}R`<)T!oQna72)v%HxK_?<*Q&&;@@qKfpyp_2XE0TVYSKh zC0Dm`ZMb$`YhJBCzIa8}j(~SnWzlxw<{1wb%Zp$F{y; zJ5SWDVS@O!l@+-+1;&k@a>MiI>zPb#)K05X9*7VII)Z^>=*^KgEWH?Fqx031ZaVG8Hf1 zv%ELluq8?1!!7$SFZB;nacE^qOG1&DBvnR3ej~L0&dJMHSMOiFxVU-s@;UjV`2zQa z=Q+8NwYfN;@VJye(@@0C?F@-9p`z}i2%QJ0AeVxduFG{;_DeD;O!$a~YOK{O4&gpb zn2+c`H@*@8%E%Qqb#+tkH8ezNQ?>GM$X^1SaS)OS% zKc64ZAt&Q_eryG?%mzyafg0XaPsZ=x)I_{FA>Y{K*099Jcj7f&m}T9E~ixk-@iG09(Wr^%)+$zGIL1eW`o^m)~@7PT!BkR zISyOHFl8M)`eEvdU4WLoSGGl8?c*&C)%|Qz(!-$(pw}WOr*KwB3z=O)koXDVsXd*;C1pc^_RjjliSBq<}#D3?jFn|HsY5U zj69<=y~BsAHRXDYMpLfshSV!_(tNp&&L;TD7yiIPVS6`*{2L2~7Yq6WmTRvISI&wJ z$K0QWv>4WrdL_xn+u%cmNO#5h#mxRL{DWjrQ>3c^ayQS`KHVgOdBJ9``^=JO461iq zUfb`x6}73yTIQ@B58uwVtx!Hs7V8BICUmY$&Dt#6){Ov-b%d_4GfZJTC3(T`c*a&C zL*SZ=l~+Z}ugzuZ)in+IZ}tDG>s<5#hWqjEn;U4{3;yDR6&*d107?)to6W$abn;g% zGpE@p`JS2YLwZA5@T4U7Yw6mg!rF#U6s{`vdoypJYY1BYME2N#!!`fI#(L;IK1jaa zgF;(tyORn%jGV``O0smG61ClAT-?@8jF!?ANn}L-!QyYhXgI&1%^eQHfP<-jIm*fm z24#KE{$$C73HI=n`J*FP-GIdY6R$i%wLgI7a+N3^M{wcM2DKIB^7_T~Wz#RRXZ&WEF;iI0!^o=1n&uZ(xqitN6k&`PPm*nl_CGN(S{BuQA7Y^a{;;2dvLRc#PFOZ1{Y`k2PRa~4 zgyJ8_Y2Zfxqxi14;3cP}EJpba8gDrhN~KDn*;vE?CE|%36Vfnd)s zh1N=j3O6_7pU;S3_hwGqzE6H@%!<(MBm3R+==SAxt3uoH)!2#>DFjP~n&!qG$=~A1 z9{-zYSxRxK|F@zUZr{6yF!MFHKYn(N*X%^=V|eH6C6yY8_M*b9tSqQd`rMlSl4IlK zJSHb)k_Y{J51s>)Rl*lnaGp1K9B*6>OlV3E>;wd?j<;x;=iT}E^S^?>kvCjHXwFbS zdU%*C(-syaw=1o5^87C}O^fHqgO^5 za$i0B^^@S?f7-m0(Uzo7>RmiooEaZm+=wBD+7w#Ky2kKYO-s!s2jm+d2x-kV;^nj{ zHuejTjkW1*NSLA#&wmNhy?2QmW_sYi!&hq*ee-Bo5bDFww+B)5`QQ4cI@8Rq_iz3t z?`tRsUA*Rn^FvikS`+w%F-hpwZ$q zI>_KVGwaxw%;#ODh~g+*=c}fKkgcLr)4#yAG8t^Ppix(CBpXLVH(ocbe*Av8dPD6p zQ&4{8AmRxS#zlv^SK7(PJ=-;;knCwQKEsA5N@1N!O_$OE zcl76!R{xMD{X?&6o{mUl(+}G{zkF#VOq+?O43d!uEF-&^_UIfvy?nX%#gs*PFN-ln zI-0MvB52It+DWAS+Q{d1_pagYwfgP#-NCR#?3C7#S1{}caelYf75u!e-u_&2pelo7 zo59uWRh8jj{R2^c1kJH87(<3WxSHZnFj{$c>9MJZoz32qq>z7wQlW7~#6FG2du&pb zW%^X4@Y`US!V3N2_Sm9nLvw5cPf~d=?rD(*nng#YIUphqmBnl(I@Iq#a2@z`_5WD# z($s!76{0;|JM1bEwNp})5lu48S3J9zATlXwW?Ngb@}9AWuvR^*8V8I;rNbLBOUGy~Yu@ zopbWi3%~PieY9K;c36UP$;+iz+LkLz8EnB3zFOM`%qeIWwl!}s%o4ui zI0-D_jAZ4fy)r#04!b3ADR&Z(q{t@X18?n@7@ow`0k4=2fs05?4Z;DEl zf2c&7MHG&0toG=|4g`$tHK`f4opn_GVSPp)=1>{8tl@pATiWh^ECQ-Ao$LEV1))_b zXHbUT23ya`4}w5l z&NQ{P1&*850u^5gR-ip5u4xo)^=yMJ{4>hXn#br{uLxbOpu5Oj%qa;?C^G(IUBhtA z$knU25CaY(u@=QjCNQ}%t|qufz*-ER6Y@&Q5-R#>$mW2QD+~_7B(p^(lW;i!9EseR zo+2V8@Df=wn%eHuS}e+Qmd*)zp)cc-C$x-eW~kpzSz2ki+T@6?8IeWGusr^fBpyJ+ zgCegNa00GpxZq_$i!BN-_UtsV!u|x2I}EeJZjF$A6&Cl&=!SNHXV)Y8OSb3}z6v7V z?&VTrn%Y6b9AwySb=qw|0R+owF!~is8fx;;y9K%(>uyD zy2z}a51~$I=COpvO|H}a_|BLgn|LvPzNIv# znjcvqyNo_#P;JQfT92V{;(9dj>^*26cgB4Lt(d&3U4@j7JFSo0K-}ybpg~@j_`ZK)3z%C?1Fo z^0zb}=~8m<`F$J+UC64m*|GTdi0y(3=%uvYw+A*(k5EmTsjLus(2RXKBa9|%ErQKJ z$ce-$EpS@N@}*v^^Od#v&WK$j9_Vo&KxZ=N$x6qe~h6dv6+ z3T(z3a*yZpW8|A1MLDBjjIu2>uJBu$E3}AcSqLn#5{ko?>v)G84jR!%%OCsLpGVrB z`|QqUo#r-;vE=NfsBa>B19wjQ7-Cc!O^006$|!?SPy7@nz?qH!ZxE-+}U^~l-3JGhEcs&e9Gdk3(J6bd}vr!&l??^x!%{}5d zCvOeI8;Lwr%fL!zRq0Bg?#(%Iz{0En5i40+kTo|Qy==o0rbhd!ZF}O%%rzwW$OX!RXc`U5 zj}^DEd|e##A7}O&h4_b9!)t%g6M*NA^ z==WEicBb}Pv&uS-Bbsga2U<##vQRVMx;bdPQ^@2G1EG0Dg$&>i6gz5<&ohWhx?*&9A=T}u*6vF%BWKcSz^wt z*fUnS3Rz;Tb!8mP2S{z(ff<$RHVWjBB?iuF6vSamjBi~T1v+4fiO!eXg&CDUH`*c< zw#1Mm%B@_FpO+=Z6K^cnY*=N+8#??7Z`L$Z*WQh&YtbIYnd6KyvS>8eIzFDykH?&j z>s?^fwyIpt{*^G1$kgTlEqO6=l3GZwZ2-PS3&^Ci^^CtUxV|w9+>5o^jO^D33(U2c z?5h4pRM~0|*OJ7@%XP+h#y8w#gK2Wg)JQ+W2?CfWd39!h%{Oc#i|v^MsQ);UvDLFN z&>-4~Bvnx`QD)muESbCgiw_~GNAo~kXG64*IT>uU{J3?nT&}Z|@ai-HZw{A_CgoWmV=?X?WG>qco(>_TO-~ z^UUc>qd{u@>gjM^V{_!RELkC*lmGw85C8o4+3D|3PX6=B?0^5)KcDc9Bu-~Uron_K)}e?R$8_UgYPft;TH{(p^}XhSdHqMXTM zhDZ7wA|OYBv)g^xy{=h*-JWHffE0me0k7G6Fc3 zeHm$E#+Q1`%jSV^8WE+2@n?}h zW24l-#T||MQlhDuvUtXkya zOn>F2f8))j5j6x`?3OTYqBQ5G_snC^DfThv z1t9x1OQcBmFUT8bDKED!A`GvW);$I*v)98SrmD}J&^#|B zP1a0#Gd~+~+NOlPD{h-rD=tte0LWTO%(kSEB`u>dXm^P|R@qPLk3>R5k*zGY(4Iw_ z>3I9Vy^9cUT9R36TYLWN6RrOy=C*4d1I@-tNwWC30nMG3{;V3Gc=k0}%c@ZGm*ad& zr6f<`O*AQ4{kkL>qpBoNpO6g~RT*2Nz*AX%{q=}z7_Y4CS=7K+P~hKAONp-BNf~~kJi!nQbFEo+UluD8X*;HA{^UUo|a1rk3twbP6zpHv54zrb6hOaCJlQ}Q>n-wpj6pQ z212{fYkT*l-NJXbRc~j5QTCB;@{FnZ?Z1Mp&D^Q;Vv(M|e|7Ql?W_4FJ*3Ix$R=9H z;SP3<=FVYWr?+A&D{@b<&Z6mjcf<4NobFL8jzB>K6vDn+1#*1`3bt= z9oky9ScLU#G@-8LJxL|GhmeCit-FC&Us-F4cbLmUhPOJAo4k-4t}I(I?c5IT!O1st zH3A*sac6XZyNN%UN@Z~t?oqexmuq=HE9IwUdkfA9nrCU1G!)#v&{60kxTM| zip!xX*f0uyP;;q}^)WyaM{U=V+u=xn`o|30ij5aKNEYy2#}cwThIb z(EFy~DO1GJry2=^AO+7p#qP$sZ(?rF>iw%_s{3kWDxn#>{3c2(GtV{^vupFC*PnSn4e9F< z{%40gM98$?Q84ufZ)!vy)??e^Lz(u9mh7I!^KA3H-9A_yP1AxYWhtn^lHd|At}e+R z5!B&*$_rVd`1w+9C=UiIw0r0_UbKty8EnjAt{fa<04-(VmH4f+-fHy8pRhIbj!5+7 zw1yrpGSw23XP2`n1{k@c4=Lt&f$6wN2BMfy9Grk&EQ@L3!&RLhfMQkHGW$lZxNQWc z(FZq{@?bsMNe=qr+3h*NvVX<1%kT( zz6C>nT}CMqg~RVE61q{1zrA+|7pZLs>eY`kPYedDQt1-auM zjwA<8WiJY(?moJ=$q0_Ue|HSK|A~-yVp#bJCd77`TUw5F{LQ$RfVSi^ud+}Onzzjd z`ycT8hq$4<9GiF@+)vEu&hCn;eASW7Ww9DGD6K7-PtCoAtsu;pLRWnWT0y|A!d4Ga ztSYKFl}W0K4Lo`B>#x6_lb5`J21DF+(5_t;v|u2NVwMx=q{C#v2=RfcR2|t=!+ywC z6{VOviEy-y%;=8H)Q(T;h|0=-s9fw3 zVYhL4*U{P5BPvk_bF6#vrX#>UC^N?K2rL=L7{-j*&N2HsR4cNX;~W8T3@?stKli{K z8OK3c%;R#G$0Py7B>+TcCyyzn8O%xE%`=XG*pZ1mm|YwJ))@juyqlF)HLvswPO;8 zr^7Y`{Hp8~HM;C{-(0txsP1PgXd54CzY}%xoj!!`^dX~3e>yK-`t$VSsYY#0Ym5Wp z0b-h5#B|62>g(>`#nmNA3I>_OG*j3Vx>e7{Y*+3~oG+s|I(V%|zzJ+*M?)VEKcTwo357RV=1R8m+nV1Nuq;-X8md(n0Ii zCo+%P9m80S9l`CDz2Khs;Tv~`Cz@Cxo2ui?95zH4nNw*Eb2I4kzXQ%4mo1n7$P(k7$~CY3 zE|_B0yaN6kP08`@8veAe+-RyCUB0KR9r zk8<$>`gigNss_m5TPo-ZjSMH(SATeWsxbwuN0%?#9%x_M!VkPVJgu>{1*5p{&YO?0 z4G)mHY5CVsYpM>;dDSjhfx!W`Hjc6+B!+w5ZPMGr7BUpFN8-ALFur9a)= zT#=j>bi+zkpc^1Gh2d$yR2WTJD9Nrdmp7*SrFjc4vN5jHG({E!6D^p^rBI<_J8{aU zapqpLq$+s1MPfL@r3Z2#wtCfYW){%QkX2mDuxF~ZD)OyRtkjHjkx_9QX5*21p?)AK zZwe}umK^L%_gL&TK$|JmEiBRDpb$*gN68d;u;#}YcL*R= zwZQ+SKmmbmx*l>zSkp$z?pUUAe!h`E@+_lsS**^Pn0OyTx=#Js>+JvI`g-PN57mcHvuyvfKjhGwGdg^mA}p|a zy3ZoTB20mRyK(-^D*kiwPL`&WiTMjxVEzn=0M2Lt{PE_^^_+aaWx#rRe>r`X;~W`%Yv^~tOywvhFvoQ zN3@}ueF;Q}9un6p*>C7azNt1uRGS4W^uiV2R9Q&{la*GD9N?Y>T#z4GAhEKQtw$D|W-ntjBiI*v=bF z&G2<0{W>Z2``z;&xS3_9M(afSFmK*mx7DJ6R5q-1B&-NjS!+zgv`DPyq2J|8NEC9} zmP3Vnqt&1#U%jQl%8I+w!*{$^{jNNl;>rp&VQNT zu4ai8s-&U}^w%CtkQt#)heDQg3dPd04FB&WF$NLC;PpaaMUDXBOVz=>C zmg#>d2*7w4{J?GkK?&s5iCbw*RJ-~Yh_^!47Yc}_=R{ueE`$(*I4>lV~_=nl+=ge9kA!{gd-{P3% zct#0qa2Ok6m}G26sUhYVW{goxaUX`bp&!lRMG}Iy#*6G3nd-7<;Pur+wXU2L=5#l2 zuAhE2OPmbAk{yicFzT=NQe^4bYs}}AwdCOaUa0SC=rVH5q z)V$Ua?ww?nDz&`y-gf3|?_md5TJdsSE#`^bcns>?lteAJ=Zj1(&Noy8*6gY*_@XLh zp)gA7T=CV67Rj2IEGeskoztAp5-IMq6fK3CZ_;0?oFy|V(izUY$!+?@3~Hsm_Y7iV zJ8oH1xR52yyoee1KFMXOFx?x)AJ=hp!-d|ft?3Zll}do-37aFu8`1tL%QLyvXuON8 z;aHeZpo^Xg@6ix;q^YDTw7>yK!8&jvG{s)|wN12eI(||eX8(AfEEv|qR72%cU4~OV z$un}1v7+2Sa!`R~W~q?seU-7oLGGdlLo#Yc6-KMW0&aL@F5&KLHaQ4d37aMBr%Yg3 zKdC}XzJE%UveWLBS1s;n(6d0Z(?O24VM+yhRy7v=r)EaAdDWH`2J0Otvue=z?-2M{ z;U<-CFZ)(JxDreK{>X$W`W>~XZ+2d5WAS{$N}AG=&R6c4{^PV=-`o<(JvIFysn&O^ zTnbeGIe4}-gXBx{q3bIj`V~uMF(%uRr_avFqQcL9m?H+wYZ~#N|25y;n;_`_eb!n6 zS439esWd^7(A-s1?XrcyyI-31;TKf8E&&up$6 z{M$`t?@yNj!&j4Ecx@2t(_b)sw+n_Y>OE%>PAJ@^r=5@C9(Mjof4+-3@0XI;Qok~; zO3=Y>*|y^dd%nI40zbH+?Jmzb)$;5BGk;%`C954!u_Y9vd~LQ*B~n&o&^#a6Wfzk* zW&^O2;pyhA(3g&-b7G3&WUOQkxGNv(9_!i6ZkfB-Pd*b;%t}^lxPa7W5YpPxR>$f( zg&T5Gt{E>#Mi(qoWJAlu)MNv-N*1RNDWO@Q5%QUR)K-`Drnbj|`(>fP{ZwmOq)2qS zClle=UZ_q%UnMe$havy?JIO9tLwf zIMMgnTm9p1FzP1hK?@up(g8`?A`uGBYkFr3;V2G$opP2u=JLZ5${ao{eemV)KZ!z&g4nP=UFW2ypRi-z!bt~J}_AM{x2_An)&K-5{54-W<3KEZF zG8(+;pH2|#zU|g{-+k-%U3m8$AG+N$BapX#?7Zy5S2=u%JFoBVv}dp}zvz7nZ^m;l z$KwxJdB^{yS~we=1q{t7>#u-td~F0$sFFjhJ;92p?pZw_yCVCjK0CaU^<(gQ`#}TK zf7Iy5L8YQCkL>;b;<abkvL-el4F+uf7-Z=TJc z*j~R~&&-k{JvW_oxme8;Sum;eSN^htdB2#=X1}OI51td6bM~=h0@5B_BjKmNxaA^! zPF_^1l$-a4nn5Nl7><5nji{QWQ6VG-^B8iMpmQ~2#cajIT+1zsif1V+0J#11-IMv# zujZJ3Y}YccXWV!eci|T%CWJKB)w%RvXd0=fvnphjKPTM}Av`;6TCx?iA;SMXBORL! zf5FXZNf%ls`*Z6LZ@5ud^QPX6egy)($9ZOZX6|wv{5-jJ9DHKlQy+<$_U*8r^{|P8QoG*2$CGw>IaaIH?L;J-4O)QPi7gA# z8CTBAbl<58k?^>v0TwaGTK0>K{yF6iX7?e=nDSoB;W$&P^#TKt<*EaMh^`qEs{ssJ zHNt{<{hYrSi`>J3!oO@eBrl8Rvcf>Mrx%L`Z30M<&THDsyZ@20!sc&Uy;bl78bn## z&q+Y<9s8q?fm+7ihH?nHReK3~WoCC53!dxpZ|ZS)7}xJ_U?cuE>>d3*-;SFeNa556 zW3lu^rc~9bK5Jz`Qw{Cy$wc^#9T0G!kFM1!&_hG86ox2zUP{;d$>K1!;ym{95vat9 z!~K~d5m%ACqv8~^&y6ppE9j>^(@YxO5xI+`TRMTC%uROL(xvnbHr4QX+hUoysWOGS zglYUJ;Skc97RxX*k$u}Cqz!EcD#K5Jt=Ljx%Adrx?=u|n8#e>-s-Gvx^Q#%1TN_-) z4}RZQkgP*L9X_MK-J^6~mV!2UGEU0B98JM3Yq)$*&jyFD@ck9OLCgkmh4kfixa7FM zd^{KkK9Nh}Af+qN@(y&M1u)_X?a z5mJEZb`j2F-hOZlzVPXWeROsD>AyUN@8N|nLIylOzBsnz-}u4DQ^`NkI^1V^Gu!by z%)38lFQH7S9=A1WspEBmw$m_@jnsVTu^&^s_c`h+<*6&6Z%DI)N*M1d!%Vm_W++AI z%k+=h>DQW_@oMsh%KHzqj7%71x?)C{y||mHB`ZpKAq16M$>Z`&>qn0(*aL~S`Aglv zCz)jxNk3)Yc1W%PrQ1rEd`_E{Ct`FX%@T>Cu8%2^-wb? z72v)~JIKPewyi47NmZ$mooC-#E0du+^vlI$BIXrFmvKj1&nq#RP*N}?2(2dvN~WK{ z`kdGl#Q7aqWZNW|zP6^JlEu@fd&AvC8w$E#%V5!MQq__!*4sDn0ub$W*&_e>Z%7y@ z1&#SU3Hh1H;-e-~23Sa; z)0&vtF)q0nxUA5C6b=rk(|LfTMna73M48~s$4w*X+ewkPt^#~WG_IA(=J;inlHBbN z)*?}1fc@3xF|2x_kcG%&;;^V}HoxLLXH6z!G@u5lkU{i_H@v6Cl3)Ea2;Pg8^OjZs zWTkn|RV%l~r9%3!z-StwmIR~N+=wtNxdG7pKNZk>-L3Ap42wTyJg%00ss2c=ff=rc zP#Y@P1rP_;{4$!gjfmm#pit}-#k|R5@5){CKL@>8rHU?@fUy?8xk(PipcRZrILD8w z1eledOG>eU@_y&;GWm&a6)>xNVI3%*8kOUNeV^B5g(0Nl8(3a*>!(T%EJko#oVRiUBsR{%w|JIT zn|Q7%)CbHC*j!RGGlGs}NIr{8gGk8@GJ%Xn>(pA-#-N3@A$BEPXHnxdw^_6=YGyn$rqkK-8DN9o7YkE){5Yz9hZ#=LI#_(fGR{ z6FqY4Tb-EeJX94QSy5vivUSeA^0966>^;#u0u>{IC ztazzN`7vC?wm{kZJyQ{*X~w@_WO$xMz2`GMhCk7OmteBviY!00Nn`q{x7+^06%Oqe zOgFmyK#0)vjDL;2mhLjJT{&7bTb{Tkwt_$#ow?U*rcNRyMh~1FsXX0T69q7q#IDHA z`lIwk0Y>HF49humeRdY}T?ZpMesx!DAkZ`?Wx$1AVF{yxt#>>53SB9U2eaS?=oX+f zl<-eqz(Ey80H`&KC>X+oNsmIDe>2J$0;B_VA->90fTdE6-{i#**T+SqmI=p0vpMy? zW(rZYeJy$qY5FOlz?#qT$mHj)RoS255sR85CjiFKQZfOCX^ii`F3TX^j}+51-fxr` z7?u~WVI0x-#p+lv64WA*{>r0~vAN2GpmuGtJ~++NR<&{0fJ$N0*KE4=7@ATgk}Heb z>)sGT!=M~=jb4)pQWPe5iYvqLmVWar)`|1DGi0u0hS4@QWo%aaMAMXw*3(6PwzSkOM)%?~hjc_ImkBGQK6V3Esy8E~{YHRg;N{traaBFOf% zruCy#8##^o@`ACm3ve!dEkzXo0V?U$9m+~|bGae*Z8Cr3D;i54b+aBmUWY+`rRgtI zRL6n%*)w82qgz<_j9`XWqs;|?i78}hXy$`9Zq*_}2yI95#U6ieq0M#(wx$R}Z8M9C zQQEebeFzYX}ek!5u=--wswIKGRQ++h5ia(o@Gq#6>il46q6{Wg34 z1Q^3%Vqj^8Sz_Rz>`yMkT-p7V&@>sTh!tFe6JMk6KJ0Ed-zlSZRU|3e1wJ?!tP*6g zhASIO@M1gIQCAC;t=yxp$WL-m&VWrIv9?5PV}&OBwCvJ_Y_(g8kYbOR5}b635K`I@ zt*e~q@@M>D-YV%}m<`tAf zoKi%CXs7i0#KX$eKa>*nzqD6saa%#N+at!UOeGxSW=9??SRmb5Hw>dQTV_1Y-I4{^ zLp)%Jv;IFSzYM>JxK!p>(@Bf4Tbu3x@CQ&4C8rTU{_Mo^VU!coQmQG4p@tz{=CvgP z0t%LqksQmpwJ{M2(Q_?xqF8UQVsG8ZP3>s(G698l z%@y?Nv}$d^SE@vO(RGe-U6opfl|vDYF+w%t5LnU&E99|-dXyUz03y%IpfB`+anCHR z27jF^^@&*fTk@VXhDCs^R_C0n42bR&VuIPP6mjO&fHBSRGb8-Y!XTY?qnetz?^o&4@?4E@chzu~F9WYeqKmM@>CY zFpUxj53UppJFUVm>;9_?43xX<|Jr6f_>Fp*v{SY11DCDNZc!cVAc4g?3Kgn`@?yE=&k-RtHX0{3*qf zcM!OTDm~gO3jzb+>wJ-%1%)kA2(%JqvPyS}ddkhMigcizw~v-4H&Wvplr5|K>-l_s z^{W|I$z@FUSdH0w$mk0jnm$ObGMRLs?I$6u-SuC7a>6{mQZ%EG0K8vvHmaF10F$2e zKmr3yF05$8OLZG|2p%$n$wv0rW&EO0o$j2nrQ~mA4WW~eyJ+MxWhf&g)7iIdAu1k( z&q|pYFjaG<7sZz171Y*(cah9djr~#d71&!Q6cM1wy)+~;?8~inu}AG*O-yDMyMvjU zh7!Ybn`o7T%(rtcPu^2P$c~#_Njy+NMy_qhxecDu+#02MnRX#FAX>iiE$lmzdujHX zMFXfb;Nk13Zo|-4L~HMC^hGuF4%B(a=W_qHg6ZNeM9EeJlBL6OLFX^cxrU!j!{>_7yf@|BUIoP>?qSMlWuXy(ykiP{ zhC9?DYw)$XDFi3-?_n?G5AyDm476U_;Tm56U0I?l_|ny^q|wYc?J9%QhqS+83k2eb zuyyPz&B;X{+GBT1ZD$gGzH<~$!t>RSr7i76q^C>AFqyeXO;93#pX>#sCX0uW^#=c+ zV+kjL@q7=DcM?;%g1sQgQ0L&$y`ellnJaAN*fR+02X#61fa$PUg*%sC)bFurPvo+0 zA%(k(Q-;-ggW?!I7l#wo1KeHB=39SKu&C_n{)FwT5EaQvfno8Vih4N-a>`I#JhFup zEGXs5fFrRb00K!4JGoP;E$spuv4s+(1;|f3HBt(Oyn!grUYAZoew}u;We%%in%+t zv8LZ#rzQ#2iPstIk)1P9J7UDK$#|}zO8eJ#3??^-+5Lf74MRfI@3-O3e(4}sga8C& z84H6jVoS0WrGwzYwvfP2YI(w7*N91|X_Bp8CZh>>m7}dgysmZC==~<~f8WwjjLPNq z3hsg{Fa4MM#5T(QbS4Q3ildNYH$~7}_@ybC(+)5T+?Ig1orJO{!l1J;Aj}~LW%}6v zu_ZeFhY85I*NI7aILz(-p)`_leXgM=!Q{}5JKW&}5r;G8crncRG`E0sJs$ULGVB@p z@^5EU@42i(pP8oN`zZ_|1ZYg{5U&K=dqH4+43r=^tO>$lX=ENb4@MTv;ng+4Y-;Pn zbm~3-s2BZ7pX)t;DfiuRpW3}%P%G6k7tezU0uM+Uuj}$m>!=+%Di6drc|c9I@_=$BLou*aLMr!;zQ&_x0&&>P@Xkit0^moDS|QG#c^z8 z`k0aK$RPs;M#6-ys}ya(m=RqY3~37gNUA2-ub3*#KLdNr&w*B$eFtulV>c$HI<}@g zzY0PaC85*ttx$!uc6&}BH|zkq4&|D>%qafak0ZxVN3bA7cAS`m9(o2BBHHr=M-lJf zdltG;hWfgc*Nbj`E60; zZ{M)io{=|a_6v+I?qqHf_vK7ioFb0xXMV4Y*FFnu!PBH{fVr^H2dTf$)btxEXhxuIzPH${?9_i6b~DV1eg91ZpKJMs)WbpLfZhgr9b zEJB)`2QoVKr)A0~FhY$SMy6-V@1e^4clxjwfc@{pHVE1O96#ZhISq{F|GS)H?|(r5 z`@}M2nccJ;J+e%ZYFQi3%vnNi5%zps8G!t!A(*Ctvn+RjXPKBBL~c7EUF0ie>RWb2w6(s^zY6|n8@{!;M@R6~)#QiaCK4)*Ht zfl$)xG1u96W)fbGFVv$Gzl`El<4x4BF-)0A3;(X>f_|{%k4IDLO}F9h1y?_x^7kNE z;^AGjB)au>pZ>`WA%C^On;Z`K&-tVN=5L0ra2eqH_Zso@kaxF#ybpYm-|AMDQB$am z3yq!C4lk}!9l8#)fVB8)HWca}MZ$v^RkP~^RHiTr(f%^LpKAZPkCAoQF4 zO;Y$kedJJ2JE6(V`%6%i$ewaiCbj5BDBORqkTvnY<;>oHBQ$7L7-_sGV_C`__!1v< z`hD`BFOD-(cDTsU&CLH*D2LNjOghq}dl+6vZraBUQzW3u`uKd#deAyF(nfB-Sj6JD z{L>F4Ilp3Ek%Uew3U8iLgHDShDo|~WWiSXZ{(f~_*?aRws=~m-5j=_ZY%3EF0l+Y? zwIggDn@#)jx$J^=*lb>+`&6Z~K2BORRc%gqw&M0mmp5de8%%y_NIaY(Hj5;w>u?nU zq6VmL#Qk}-u?2|Tr4{#`r?kV$XN;nb3u(p4{0L$YPc00u4m&0}Eg{n}cfMUnkeC;) zr-q`{ug~6F05~qXd+V9K^)2{(EQ}G%b+wGdf528MV0>f_q6_TpXU8Y-sf{L zpY0kg;ff6U1{-Pc!^>d{NYZUW_;#Bq?9St5{Ug}QI08_VJT-|4in0cDeb`J8#m=zF z%D}Lx>$gpzey7y0DBdkZ!T4d1)G{t`n*Uz3^-BJZJJ-@zsV;v!m9y#e`*6v9iOCyppuC89;Gre?;jeJIs#S5AF(D( zK*hBEQ@v~FMI|XNCDZGl?n--L<$4+im=-~{tNLzt&;&9P(qM-WCezcy5qla=iwTF+ ze-oMuZDRl4DX?X;(*@@5k#|lVBqlq_`vYE;Bg^}<@p$IQ=5orKVX2|+%MFveR@cuW zgG6{V8|tt4k{of#{q+rnpe{Ljst-8aGVX$({Ky2+zpP{nwkK~vO@15jQnI7-W$*6n z?Ctb4!Fs~`X@7U$^=*jRN+IYB5OpqMT2|0r$1UPCfZxOD1Z+A$m@Am86$ZX-hayX4 z4&^ul+IQJx%7tu*c{_VUeTv(;uV@Pl0`bZlzVnt@ zNoOQ)$mu=AMO31{8$ZQ~e};)w&zp#@)7H}LRtI>pYG*6X+yPg6Q1Q^}56Z)p=r?P0iR`O@|-g1 z4Bu4r&eV$PXy-mPxZq^PoSw5Qf&RNQJl3u@ za($Ha0W2+;VkYBD@lAzfUH>+{5F(Bkl^^ykxfKq$AwmnoRk$>~1qmYoDqZs2wG{oR zC6QnLjz2{Kec`2 zVT>FGoRM%z zU|uiy=!=`V)d2CNFKk&_nr<;2T=2-zt@O%)PuvVo^O8*h?qOV38?ZxG0>GNhHoM6p>IQa%63+UDKy5>MPcWtew(+;| zWeLeJSA6X4wof&y?K)G^ck@OZrXybF-d()G)M;0XY0+G1wICSE^B37eow;hN@Mb%b zm^EyFSjo3};&naX<5=SV1`n;qsQcnlwu^)o#eJZWg~Oo>@b~_zmPEc{lDP&8PkZYz zN=$af_?~SYdRy=rBw%A+u5O`uEkw&}mn$%$30*Tu=3nptvn5t|-AM0yy?90RrhOCY zzR>(E?bz$u?)}wRtpGoqjo0bRg3Oi8?N#+OtMOu3s!{;$h-?Br{$90eORO14 zg;AB{rdQa^=^61)6QFT2kW|df!S%huwU0k0{#9;UB(V?p8W|mHC5J#mk`IoL)TNg{ zlmR{G`f3Bd?hRK|RxWea`|CR(U(^|la>pRTTZ_tZV1RDt-bksfQC!lceCICqRo%;8 z*?MWa#KQ8EaenUG?6;tVKp|~mvOtdYYIt;cx`;|^yfguAogSO}2BxPX&uB|VpmO|B zjSNFg5!-?hSJzskw6)^Ano#1}10&3t$Fsyn$x1IX8ww+ghZBf{n1_Bu6!Een>UM*V zWy#fhfPG@{lNyBg$oBojcJuC=75!2P4+RwsW8ERErqQXvJza+9Tqer7cyl2W1NnrD zcldFK5cUP~y2;pSLd^NIWq87~$K~Uijtt0?Nb(}~W+(VR$~jLa16x$^)~4AT&#NvLPphmizm;QsEWsEOlZJP(663r9f!)ms)v!GV4JbIvbgp4dn?3LbY#md=aO zbtt}~JBe$XrdAhZ(T!&hAb`C7ZB6_p|wR;5u0 zQBVaXWR!J_3#{wH(qK{eKLvP7FR+kyWYy|@KE1aX`){{lxW8=6oo;EY&IBp;bFA9% zmR+%FbFJv}D2t`49 zp7C0Y^#meKqp`CI`j7fEiUZKaKk`=5@7kz0*VPLVA<8aDPf?P%^L%~&T`1urz-Y6I zfG9m@$N#=K16K@wo8gu5;d$>@GUVaNB?lphn9&XuK1y*I?}-uRc3-QrtC@i~oOM(B2-d}B1QCEmDH;*VKvvXnsk z70=+cjv${@ya;x(2(V$`X`j8WhAQ1!dt4#e|0qlRq`eFdNVBPq`dZxjW{XW)Od%Cu zRT72bzhgyt+^@(@VG7^eZ)rA9abg&>(&$v}ska~9e!?!i-JJTBo`Ay7Ql)OTVfmp) z2sPd>vV+f`u3ATAhQVXBVrk2A6q(CB1Ywcc8%}MepAZIJ8ILNIt<_;U@Z97fYKBzy z2T5|KB%KHC5cHkVUFYDz<4ZQ+8JV9UT9I(Q^*m?Jqp3e~BCL=?yw!4oV|T^@Ju zmQ;!iug&q(Cr1L`+TE%yWdU<8vWNG`Wk`oWiaJ2{#aLzdfhdI8s>cBl?R$lq=LA_@ zQ2#SFCu8}-6;l#zcXD9Nn%gSLI*;i%cJE>DAEnF)a>?8avx8yoaxvzLdm`G%48Mph!v77m9TmgK&1bnc;3yGgVLAA3 z6vFftb3GWjDCtA790fiPlW-I@x_& z|FuT0rG~r9fXUnNJ6crgLJ*p`X4qg0;RDy7mm8`F>rQVMv{b~T(eMivpaNq+@UQ^l zdzlcqT)!EU0@4-qH-VOf?T#6+!Y;3NfvxC;7@hG0k}^&2uJgq( z;1EBC4-fZ!H}wYp!^QU6#u&1O|MbCk6!p~x2*Ne1IINgICI}!2kC09d>y|UPs=(8e2KSUTn4*Z(UUF z8lk)~lxIjMip4BTV8XQqPpSZ?u>7=4FE3i-dR-3|i+c8Rle3A~_SF{&FI zB3eNI4@**kG{T8cZI^FOV2+wb2&=_qk;9^Si7)>M94e0+!j_y(R89YLifp*<~|xx9UL#)pV!CJ)7#q-n{7`gZqA=HJg?Wdq_?QIm5>4a zvA3BxzF+#pQ;UDvF0}k7!k%vn7)LRez%BRB2C4qmlt{^aymm|N?N|K?4txpOIbMv zOL9CJN{27>t7LGI{Y1zbmFEI%BLeXs3O&fQ{2Qi8_rKD2O%Ba0we%8_2i>GyF2(Q0 zz|RJ4Hq2Q6N`g*?K(Ey)O8&8(b~Hy3oPxhwQOOF)m9Fi5HqkVX-Mt(VVJg;h4v%EU zYaIxM#=r5sWQBj78ev26qcxH(LtKH3xhwynFkGM}NBLUJh3fAO7qdlT=rlvJT$8xE ze^@`b=Emw5+ZzZQP|l#GQvkF1>?02l4jrlJx!qWih7iwd|e(7G55 zME0GJN?LKkHKDYSI6_jN4X1r#|5Yw$pki2+is+QBR@z*VXWb4o(N&^=~_qIpjC&&gxTzZIS_JxQJA08^SHig0SdpGi+Y`4Hs>L8qJ$0gmz@ zJWgF00T?JPjE|;XvawJuXmOhI4pw*`Az=wjiwV~9z-ty?#G<=tye;6HLb#w*@Ry>O zfv>WV9IByG?s20f`e-~w`r13dCs?|gv2Ty>_7}Dt{$h=HrjjeeoK7;CPx%i>&*{NL zT|Rspk$5O7E9x|7ysH-<0peyOy%Ns#01u}m_&hxanVa!8@T5N@^UoxCK!vFyQ+}m& zbA=HwGYpVM7;KSJtdLz@*m+-6`Kvl38MkO(n?MLS8Mr7XST;E_4}%e^FH@UigE=tq z6tt<2NPNH}hWyeFZzWzjp_>=C72X7WJPw6lMWN~|O<`liJi;{jX$UwzTr{Hm7-IX; zC10Pj%v@6$hpLz5faIW9WsnlSRoq-)xh1e8%$-77B3Gp)9vnwc!MMnv5dRVMBMRNv zEK?xK_-T4(CB*dNINJcGNx~08pTSGB;qw)Ss8BE7VT|AMZLDkCmti1zW!aW0Fz2R^ zmlyX(cqvVN{+~8-U^$C0HQTtGglLqF~uq`qC#MG!{Rm#r} zRWe_t+-`Vi3tAzy!ffyQP2p;{T-___fwsx;@Gtc|VxcQ8*7T%56O_>VcCeel9KjE< zc|%WXlX}*)Y0OHsktCDS?FebAod|N~+)pe@5~vAFC2%UVV=aiP7@z)zR@P%Z7ZV}* zw4!P`@Hs>(^PnCEmW-^tW_=9rdkdy7_qOFhRv`JMk_8VI?*o7K&>t>n0jGQ*1`P|s z^>e9a6;#cc?`83hhDfe({}(C?YawuRSoj4joWiAe4>C)Gd=68MQg@;Q!z$dL<08Pb zNY7|x-R8)FY7=)bpRWlWo{xz|sRjOi7_Ch0>~H(0Elgt3q0I9Dg9?I9_HYiSs+`+& zo%pqJT$Q(|D8Ge!8O8;Pf~V{|_T*A20RCqZm+{D0zN^nG=_MP>4k^VI-c?K=LPsSN zx>}JxsgUUjpy(h|kT~*2+p8syl;ZvB9GVav3^(jot=)}ix|N?$&{6z&$d!f4E;Ug>^?OuN}CBmt)H(`{V&c|&P9lcl>5gx@gySu6fasxa9nKi*N`P;{^~x=ZrTiq%0vkj8<4lQ8_~(v?6xY-Yosh zz>eqQ)9U#VsN+$R(q>sjevaR_`!r7K&y~IimqODo-OF{PQaA9W(rZD+0ykskg+^X! zUM&|5f(TT@Uly1w^=l2mDCB1{hka=K@eya>)k~>NUyp<) zlI@z1Ll@Klb5nt`FH@1RSDv%cu511cqk90752N39SBkY$wi-+J$;YqmwHMBdj>jD->2SKOx^7#RK8$&?0;;5z)glR&Wwk#kqX@PXQ(xVaC;o3 zCG*O>ZSn3i6y}J*=!69!s|aZ`5ye;L7Qz3eAHnicDu^+El#k*mucCi1{hgaW|Icwh z@}CnwPTA~H{>|V~zA17}l=;6Erv>KGys1pT%e_Z<%6EyT=YVP84;Y$_b#+U^o*>`2 z11W8_Ckm%Q7mv_1vm>597rTp>BlEr;6I~g5Br_7oKv{oOvER5ay6b=(5rZ-O(HSmo zW(x5oy@{mXV)TabP5WaK-wCH)y3KfkZ-S*>G>qSB3lXfEU!;Bm$MmEh#q2zt1$qu& z%@jN$Qs<0IBN=-{RYAb5+L@R{;nW)FZ%+~kZTspsxE7l=tci1-lg#ZBUP-6z_bn3= z!65pJe<1^K6@l|8{O+StjftDmqNqB9WlW9Y5rDp$GQTl_-q#w_^^`Zh)OW<)+=oqO z0y?meOp^0MOY(HOGP2y5P#8r8C^Q~moB=XIm>9Rg`J)gY(w%8&_?T&!BvYHp!Y%^~ z2;y5HWxI7z01!OHOk)X6(dkVEvsb@|lL#-xGphGQ0x2~OTd(`TINzBOk{VFOt+&;G zmhy-Tkr0!Ws>a`#_TIuo7#@5cB4=dnsZkOCnb$H|#G8^Q(v-NlqZkwL(kGy5vew@t z&GdEaA%A+bE3$xFDOt`>SeS9I;>0P@iFm_~SIdJi^pm}@eUGko;1s05J@bo<(cgWG z?%bdg*$=(q6}%k*8*>-XDcAgevxIUbetZ${!}yPi3;w6;Tc8+fhaRMsT#w0BmgJ*XJ}tK z(BK}PeEZYg-#5kmA*mNKZ7Tx0twJ9BV{x)nz)?huKt*ULoDviY$q&sa*(&mTO-!4S z9rIS~C#dv36e~|``_g=_2w~GUR6DeJq-CWALB)W5LMGi)0D{&jB{ZB%0Ud!(3smDK$Yi=XBL zl6%MORWOB4J!%2( zp|q++0u(()qRa1sB>ko`D9=E@amTn&I<6UkKx}>JY7Ka38VHpl0@E;5OzNi8t{YKc z%8Kkkl7vE4fKb?|2oxt%8P~GsVxly-O();L;GG!!&P^yfFJt~@sFR+s7N{~~a7Y^ML4Rv|<(UaAC=THl1jk+PXkqWG%Q;sIMyHcM!n6sMk3V8AT` zQS>mm0U?4SWbWASHK5o5+#A_zY|hrA1VmL|2ss|6hNPsMRQY>BL%t`FC>&E-tQw4( zov*5yySe4y)zsK(w{#6x^|$qhD5;+K{%~!fhxNKh?QdfWafm z7w}e|k1x{0igy_a4-<|n!T|;N|_N70q%<5BBt?n+uzx^mq^gmg{UZw8C;ko%}?>z<@Aj zjX7T`=bus3XAc&cYyG*d3ypF2S@L)eysM;ba#s%KOD$RIsRm znVneTRT3L(>=p=Q2)3Qp*z{g{;q6QT+aY8;Qzw1(i-_E;>a*R3TD`3xer^xBo$gFr z3*FhFy>seI$vfdavK5mCzL3Y#rIE%{i`x)u(yv=u3XNnUnk~MRWbFDgLtVRt45bj9 zFFnRJoR$qxkUokngTuB6#ERYLP*P6+bOLsrgheqkI3zu@zOrQ6&(=qo(yzO71I>4g z6rSOt7`wCoYmT6;Fbx;E%-BXs;vdYr4F9+4C5oQbD^jw>;J?X^9L$6H58rR=0h-~~ z$1VE*P&j{mH<&%VcgB&DHrW3c^FNFhu7!amkhn))!(qmB%X~tk?X_gh`R;Hi*Oqx? z|G1hLsBy0o=>_;{C)oZ=!*}8>zlF%8s4W=U5Ku_GSG(^eN|YM8?D5uX6VnSETY1C8 zQHpkZ;b7SQ33PIbaf8H!%m`_|(CFq~-}|K)Zkh-mlWhmZ<$4QwL+pWt3#E|JC&iF( zirZzH5+Y^ToJA7Vo66Q#D&f;Ni1{D2u9Pj1XFH&`OV02UrZYn1${@RJmZ%RW z*edoG6C8G1bQh7uo)q;zL`uzp3K0@b1KDXh0DUW4R5qJzVE;wf!?QB-sa+zyPos<4 z%|sMRMZ)%?WX#q+3ozrtwOtzi;5zus>WGfx!0SP@R+!@Imleef%xTfhIxu3)Hs^rT zqA5r(R6en4Oorwj!j|K&EoUdvHb^%!3{5|DURPRmMzo3#`%U<2#RXU1higL&88H@} zvNt_s_OIVzzmmBUBrO{i@tY>J53f28P+XLtG{y1=W8Zj_{pCj}A z{ZJCZE4>%Oi|=Lk*!Oj?P*Cve=y7{kdb<2Haev$sRXJuo*1?DKAwJ<11!CuT-%r>! z>pe7IP%*Vsu1?-k(N3m1WdKC*mnN!Uj_NsGmOb# z18%FCw&5%a=Jn`ckcfXcYn93@Gy}apbDJcD_!Tklb_JF}9|Tq2#^_;}K8e*ClijS8 zmMi7{7G!c_k)!kil8JYHzT{xW`@Cf|D?#T!z$WCvVV3dIoG;f)seLeClBy=ei!n3j z2-bSCol8`7UB4}$$W~hsN1b!)@|mr0(+y*01E(&Qz5Kje7a%99ssX~svjBN*cI5&$ z=hkiC?ce_mmKB@Gqgbf2pVnP>gD25ZeSV~;d>v1#0He&EaE1kzu50doOr}`)M%)qg zU=a0W%;zL@&VlwC4EKpfw#|22GE0E{f4LzL1afg{}YALc*35x8&wPD2_Fo?(Od2NI9tK*}l0MYZs3LvFO~Mz-AT zkZ2u5(jD=q`xT6UKhHJ`Lrr9)7bJ-HEu{P9v{w1)#Bn|mC}P&uq~%(q$C0q7Ku+`d zhq|CBzZUtJPA`&>IyyithnP9y~1FYxo860G{+*1wk*`OuHE? zasX~!by}AHJ?Npc|A_dG7m9%SJJ5H$T-mgBTQ|;aQe)M>h5h3X~hH6*ELS;r&cG=;@5Qmp!pgVqqa;bt) zFr(m2<)MWc9DfS@hdT)dNmJJ}4MXnMS%2#b-H*QLOq|kHkMzzuFY57oY}0p^izo|M>nGwqGnJIs0s>&>?aS=(HhtQdpgonzM@zC9>!P^Nr_# zBpQB_Mwg#Le`|ew!8ewY84?%&Fcj)4XNt^R?$rFCAbY%Q(Z+1rhq*8RykXIw3Fmm7 zjiSdC&_1)-6cybj1`f68BUtgNzm@)~ZGik^E{dISTD&KBz;z>o5drw}7fsw;6e4Wy z7|kGGpKM+e^;4YFOBLnWB234gHpKHZtkj|67`n2(%>{pzf1a0$xXNKbx|Fw8u2eUA zXPuBqIFocBwV&;CaRDzXB2utq8$qC;5Eb=-lN_8S41+x@MC86zc4DhI-lq0UzMQwk>Zh(L5zB#NZy0` z_D<_}dFQ}d(`bnd{>QeNfH2^q`XD+iIWg5DJ z|EkRlpA%!ON;z~ALsUQsqKXENDXa$ETS+;G?`U^kg1({Pp0nctTJ!Td4@G$yYpiIn z-S(CH+o~g=2jAil4aZR@B<=8RV}e=zQzln)x;lJ6e`jVTl zAwdiXJ!Oo-V>E8_Q5vhW8FsT8jWn_~7u`l7EYgSp+#Vm340(nPkMj4_dx z7oil74?G!?vaK?Q+XsAJTK6+P7lDD{0b5@K2@FaJ3{Lt}PK$i?BQ(y$2icq!_8~BE zQk=KW`zXsC?`fSd>diQfH%2CF3+6O35E9l5@Az@V!9L?Uz3W1ccs?pNi6MnPtvl)} z!L?Er_R+;6uKnIdAe5q7+Lre4uNd|;d}BYi*_IE1I2e`yM)Mj}rNV$%%$Nb~Dyoy7 zV+)X=;mZ`Edce5FjFzsO)(oJ&zdx3baxP)T`UaS!@Cg7T#ir(~*JYB0)O-?T=F?c| zJ4{_Zh;3NNO;FwZ;q%iAvbyoyLoaZg;vH^$#Y1Qx^gS%k{9%@Of3$^;J+O>F6f9u} z6LSgkl=KU2jX;id>vL&U*oO{SOf;3!Ooxu%X`N-p;JfQ&D z%L>Iv@~nc2Z8zqn>e=c&d|5hR^fQP~_$ilIye5M0k)EU;{-*nf5tY=Kn(|7n|T>RLX?!6IP9 z$qaR|ild`1sO35>&gw-~bMcp-ByJ!)!dt5+L_uF)2yzf}FAJjy!+W=K`xHRpS#1`) z+wYKR;&ECJ;u7|PA-ksSno9w-mMO0JU;n)dQlQ!CvYuAtc90O&HAlwbqs)vw#9~#g z;_%ww&i4C^sG?(E`C_a$Lqz1&0{EeSbz*U#_Ycl0e&(2Mmx-iglM>~)7&Q!LW}J;9 zp66EYubSMV=OF!>ih-l87w6Q?KiK*AxMFz@5kxbrD#740j8H4mUT`&lxr2Xz@);zz zs*H3X_;z!at7@es~LBWWI$~Wh|SXm$`ZZE7Z@!PNvgJU&p%?oeqYPqdj8oN zjAapkOHb*Cbs4TPpBL^EN_Q(i(1!(33{I%`#;9ugj0zlnq^ZhB>8LKG=-9nBz`9M- zQJ~Yz4RNE^cbU`_ARNyA)olNdy?Y9>B+qP}nwyU~q+v;+4*|u%lwrxzk z@AuD446f#GBKjgvp4_=JBTmG@K6|fcts(zZOnd7Bfiij7R4O8CLKxK4I42)dY}ad` z4Sm`fnqRF)ZtNQV&!|jEDL@{)xi7csf4TDTXnRZ$dXv0NES z2CwXVbMB)~jt}O6i!loXRbPQnEvWP~d5a6<2(ctw)ra~$!PT&mpjnPW68^0L6FRoj zQ~5Jj`Pz<2+WEeY)(_$2G2W&Vh?B9A^2gwrbF}4xZdIR^P{Uv-OsT6Rz`mY?0OmG8 zJrq+kq0U4UK^2Ou+*a|H2V^7zwu(sP(yjmrgKU6>j!7i)Y+oD+ zLXnNTZjC)mxU^p~c2e&xW`Z`h{<1A^SpJNYZed7xXjSo#AZ3zq(!yftBG&z#$O{g*yf8Y>K9Ue02RhM5^0!%~D zoen)#x#?mx4d7)An-icU&<(i=*`R=O!$n?DtViG!;eM}LjeV8*q2?=~lnQkq<(d&e z4=io`QV<1oKu^bV-(PCX+vZnmkP6Vz5mcEf8GjI)7R*2jz|O?jB*iiy`Ojm&VBMZR zOt(;QWEJapZ>MY$ZVwv@pXWOxL7Q{Uy(-<*s30m|DodZ20t|LxdVf|gO)G|~nN9m+ zk`peDP8+t0yKtOyn@eJ&3!Rh>SbuG5LbJf0lZwdZcn8;8$0TN0eWlYKYPryYpiQ=# zsVhCIjM~0Q9xl(1z07&o=#)6Fhoom3jSG96+B?l2#4MeW3kgpzm+*hNwuje<=yz65 z=lFcFImsZPo!H3{;Huza1%NQCFil{ar7hJ3h^h|m?yMucZ@E?O$+K$~baZ_Qj7M)O zD@Y{J#K?pQ7nb4xAam9%z$z-bM9yz!bXT~m9vzTq%�dMYCllljI|n;ZHJUZ{#j~ zDLgK@mu3`Sww4$2^^2Z}#Y6_w6^5sxy(~#Lt91jTV{Q|;y4&f{!0zy>Zq-%2ku+a| zepmz5gD>U^I)jNdiuM~l;P%393jHL?&rKPE(vZVqiGm^|PEJqq1t)`vo&_4Oka2<6 zla*#0dyDeGJ%c@35rUR<0?aiw-^j|oUFZTyT}slhL7dy$yUCZhiIe)&=`1?RTIw#P zZTv~lQ1a}XYybpo$GBm@l}s9H195sjV~Mcxsl5qA9Vv*uH>N>MTrX z0h22CdX`hEDe*-q27ty@8)TT;q`zkKW^%sJb5QlGgq5kN7A()EboY*pUf>g#(E{4l z%5=_yyt`_eki^dARXtsJ0pbOPl621xC6xg#1CIUSopqUfH%}Q>wsEapAGnLK)GJ-v zf)F1heC`2olZoJ@tCR>3c1Uyr60AF?Eh*>rQRg zo0_o)#2LF*E07m2H{4dnz6>(YOVdWI#_y0jsc$aA%qP!PCY2kOn;i?YTi<1{8~hUQ<&%@JB$ z2<=*%(8N}##TXjE29q^Ko9p@fTzpMSN84?=zUtJi;r5vFc!xCkj0#4OR)Z2DJYw`Y zn#5n-fI_tq67Q%|mbA@ir#Yk-=Oj?{C06=CZd+zTEHkIYPnv$gK3nmd1!tb6QR9|f zrn9LABGpplPhL?G)<+`rTTa!MlvRYxVr_f0+8P!J)P7bRDnv$*-msKpYSpHw8S;eu zSp0!KLUjxMW(7Hut}MIau?|hyWLZb~W3?<}MVK<#H*NTed=qysD9>wR!l@U~Fl#TU z%_%tgP`la?&URjD!){in3+~wKCeZ-4%IY}NFDT*Xk-8Kh9jgVU8` zqhF@o#~1e72IwejSZdiAcPb>?CBJRfE8%cMF%MCkl!hP*=uXxbSg5q7tz$)W>Uep% ziPM{;|DEa}xD0d1- zV{r-dnrrMY5|tXdA~Nat-GNziF2E^2(^27_ysR9cN@~!c9QSqTCXA>E#VSRey? z$r<7LI%=YjWao8@r@(9(Ler^<6U(1jDw^%D@Kc(yiS@0#uU9_vPp`LCv-x70(b`X9 z`(6>jIXd^Y*1T_m87GdQdVJWGf{lF_1fLy7O;fL!^}g$#(p{uK#Gmjp#R(Q^f(R4> z)jx~H3Ar3{Y#|HEkF5S5zvKHooQh5`BS{qIfC4$oSg8?mP2Z+fqH>lV8LUY3b{zeu z$yVIc006@kqDouMv&6?zdhc1nYq62W9M^XyK20r;k#dNj{4Q` zs`xT}|KQBh#dN*gnK$pc!?QRIKfw|VO7%tt6u9JExqh_UdtsYvwDm@i!nwI?{Svk0 zKO;f~u~fwp^wY9P5{br|OeGiM7!?RFA%Z=J5St9#5~AOnoSN8f4*xBDDzTFE$(E@w zILMHG+wg^9xDZ88t1!w)Lk{x_mMjy>J|H4Eh=IWIPz}>^CF8p-+8`LTfB(H65%5{- z@4ahg>r?&c&*N9LaJ?OvyD$RcM$|ZA*{Vt=+0etxm94_$&WDb=){-5I&3n;HE^jy6 z8T0j;kNZh7HEI<$?M9AdEl@W5H~PML?iUuBVm@%ou%<&Es43R&h$iV$Fb)kqk%ZX0 znx+uwl|RS2A0IDvfc)KG@q%D+Rnt`mKI&zQ_b+kdzyrJpk#%HGV&=8aiLM)S@H??! zam;ZOe`Fx1=plTFI}Xtm*6|WzIrFpf7TswkCHJK>1RF)6@%x0fBg zH-PPam+tNWw&@HauISZ8v%9K^k@q*E zoV3`TD2T>;*cY2n4X5*bo}8BrFGJWSJz@zkz6KjgJ%GCp-)h8GU=_f#VG-{GvLZY4 zj0^c3hJ~wOp{2LtcG=|dKCj*l|8`<3rGzddZa%h=6y&AwPlW5jZYTX!c;S3@vtxe} zviTt=TAja>>IQ+_0F656Qq(bxAwTJHw2D<%%QdwDMZ5rxp8>eVznolW(y7-TN2Ul1 zY=pIzm?GoB*GF8~N^JqIks#Jp^$<^fQC?midKMR1>ZGn{N>%Qf($2t8SLmnDMMQq*AK|EyTN(++;ig@L?h0LY=8P9we9)Npp0MB0T@+r7wE@hthPbys4 zoj?|NyI7AFZ{%T3`ZBE2HOTII7yut0TGrD*An0vSCE*H(@&f?VXS9?0;`a4TX-a$k znamO`LG&8YYa(ZiTeOo=Gr7aW&6c|{@`vjFVm!ELm6Q4W$yv0;>E^BJ(&ziJcaCSD zl&|ZoZgSGiLcg2RF@kH zG4j00)~j5|yX;n%W!u{rThmZX#x43lcOBDjs!A42Yn5QPM!8&0$Kd(PtzTA{qQ%dy z_Uy2&MDFpsO>4R9!B>+`MS(d>;@Yuv^I=MHl*?*yQTxjGf0~@0C&>}20Vg09_}J3I zm9=+B+_%GIPf#AqQ`p*b&WoQoy4v%;Px&uNw|xVz@qxQjV#~9HJeox7S#>kAz3^m) zq7ne3+&z0)u}aE#4fd&|d-T!TPP6L0HlU&0?XZf{QYIF)qsShBGL~G$!ED6 z7vrfMFIKxHB?_`#bDy(W;4?W<4Gju@({dBw_!t?9SU1Yo)8RNL_M_szLBl?5U^o^R zs$}|;F9dKqcg$6w4SIz4$^t4{r8P1ryz$X)3|JOvvorldiI;H8PK6;X)Vu}u8nmTg zW#-T82bHi{k1;}XHeh>0&$7JXFm;mO`XFLj^ z3GOn8ac#i1giokyQ7*<9WJEVnO%8Fi@PAWLI=(V&GHJ=X@csKsVH~L(gxEa9)c?ypnOzKt*l3AZbr!4NOLo^l^f}|VPxa+(1iB+_x0SH4bJf`9m?mo{i*HFX zvw-e9(M3~r)-s!l8_8cT4&8IC%0kTGyasPaGoS-TZDp^j=mj={0|rrJ2LW4vUX}bf zr;yD(AUN~uc*p@zjLWqK%%mwve`@Aaf2O4T7{PPha!d-9g*)aP=bn`-t&e+IWkaL} zohr=pR(pH++1iY?<0LCP9H$CjdR+UxR#}6dr+`yHVZLOfqUyEXdHo^~`jln%Z=+vn z%qmMsnZck8I5fUH5L>(!n8M0;* z8}=QqAFU;MZ3*tP(ayl`H`6Yx_qp&+Y0{@9x_D?GmndU`{gqeimHyTA%wwp1aUm`? zWqt?;n9-4Opy~`OQXptzAjQ`>)dpLeiE!-kz!J%^gx8--j7+Vy?)O|nu6|W9QCFo& zDMJd`qL!p_6qd064BnXOZ65_ko^NT`L`q{NNdJ=ew%G~#8G?Xmp0Hfy}*%{JHD2wr!^wmKrQ=goB6UHrusmRwaS z3Djd1dnhfr+Zc>)QMTk#u?iCNbXOWUlyTv!aoUXwR)@xePU$cRc`O>`><;t~?|HWk zRuaa{Y)VJXx6f0!@=V#7q`*gbG^h6Bz!negB=d(Te?b-GF=jSQ^4q?vCFW05ygpGx zU^6RGSp^F-Ng}AIqy;o{q6tZZM4Ds@ z9PcPok7F}x(1e#@b<5f8uZIXa&M5KhT}(=KkJagwFD;H|=UrcEoa3t>>bFny4g&3# z)d`ZVw0Sn*Czx50ZL8Hg1t6J-lu{d=*#&`hjwYa*N|C4aN_DQu@iO>OC!~bS$hfrJ zg^#zhMdc&Yx@j2$rFqkn*saHH&<9`Z;&YNHJscok%gMc&X+cyK8|x6(v_(-8+J|Jn z$QPYx`(o8iZ}d6?JyNDkof~m2%@O{xFpair*45bF5s2raM0KM;SF?EpCFBU{yA!_m$7a>hHPriHzgtSts8tNYw?J%)IfbBbd_LZWXnz;Y7wGMtYTRGCj61TA7srP#1p&60Ps znU8YAWp;i|f6$9n9`P94sH{TCIYGPS%F@WSt_IrepSDHAkMn4rIs(mnfc0Rn z6RCqKXg12!N)Z|Fl={(TMf*?@H022K$A2u^q-lfc^F_VrE)LDzfT3;2O>!Y!P&Rj( z+p|js&BCt!SxYp7({Ev0uZdY>BmVj0?;N5jx#R>~mF652a~#KJ81Usi9MI$?IQxK`oonH-(Om_=9g!?NQ zd7|{OXr35Z@BOHcNzznjL0L!+293X7J@$u@qNvUiT%sW1Wah&$9mQmVrCBZmJJje| zPO(C~O2uCSRq%=z0lLLwt4Qu8aU&RS&N=Q;h`4aZQL)J(sk3-ffhT0P=uDXQnqoDJ z<-?DTaswQ+m(?un$B6tvl1Ew9q(bT3#5`w+!~O{aAc+<1A7NcEg|QOd^dj@Ps}#g) zd?jtlh==5;eWrDMAJyr#+2nINZgT6df5J|=+&z@|emD)4$lCXMnO{hn<9gdW349+e z7845%<@32eDnDO+oqjm!kFA)no9O1n{t}<^jRA4|{V-_z^n2ZnzuJmT5F>bUwu9~( z+pZ+sXmAo2a51zdFMFe-3F8AQb3{1%LduA32GasIp3+W@*U0H3-LeZAd@{k(-JHP9 zma9~F>x~0&+)z{r{4xZjg4Q-_C~)T`5DQ)!oqP{ymb?=sEs{YK4)-DLk1cBKW_Uo2 zbB;mlBK(Q4cel4m`Tt==>SdoR`r%-nCxLaVX?)R?CGsZ_qya9l$pX+ZHVgM}sC8ezoK_})MypiEhgNluA&(#mg*TI3yYUo= z$mC`SFN#q;2%S>OmpRo3UeR2Pu>wOUWP+2#g}&{o0M>=JYSpwpPdOq(AEO~p$~y)_ zBXJ@EhP9CJI+5xxPl|=;fQZVGBbU#w0*TadAYT;j@ikvriN+3UhYQTu<(>}9ZWdGzElrl4 z9-t=9p=qM<8`~MdNqHH#~ zI3Ga!fRrpfJF&E}!#0yh5Bvf8aJ3!wp6m0tU(lBr{#D2Qs`N?rB7&zEgX%qkBOzGf z);^)Vr$Q65#}YpI8T^jGSP554_`fp{r_UD}^dto^wb>$bVU5j>iUZ4cSfFcWl*>}e zW$dra2W-$|C>-2OZ0t*Pw>;^Wk?U{Udx)Sj#$9@Ryr9Z!8i@!w$b;#q%6%`OqoNID+RXaF9|idTulS?H zI^uY>Xv`{E_%GR;lUyIetBZgnsvdHt;1PXd#Ew3n7c+`YhaC*rM4>Xb7!}u`DO42C zIs10z5C)nnowz|&QE>aQRAXSd>7z6$L4j0gtyuax~%m2k60{z={I};(!08SD$#K`u+AjD0ZixPC>z?K;fa1U)i zMV^)rLJiE>c|;sWdB0A!kE(R#JkxDD5)p#4EGNbG&3H%Z*@MgkN@SJ!k8IhC_Np-I zZQ22sm5^#CZkXID|Zbv0|Og09j}A74Uye4-A~+VohgAjKoPJ_*HTiZ~y2Fx^ zxhn3kuz-sXvrLPUMbCNmov0Pk00J(gu97eavI?a(&27tkjMh!k+Z1`HUXcIha|Zge zK=n$`WKy@_7gc4O{onD&iWRx>Hv&a>pP*43nKoz#M+ic-n{|EpI$B?8LCfCgB)`gW zp=k5K%23s`DOdn+_;~ZOGx~4)*j)E_D>J;eHO=J)QmD&tox443GPY**0{zCmh$#zE zA&StEqymH@7Vv#i7r1gtiTt{Ati*^cp`fZky9%$S9%kVC5*so70Fi_|M&!4llWpy7 zE_>(tc9m{%mORq->@zgvqbKl0QfdK4g6DkZSq2|yUaUuTf|2CC&Iz~iPkr4JdtP-} z&YpMqdab0Cf!`K>N#tfSdtGYy2%|-zKY=9m?+k>8kS`cxeJAJq(~<%)k7E+LElse9 zvqv5hL1}1Vl2Nxy`2kVomQqb0$e;G0DOB^(0UHX-SoTBnfq@>L%p!VFnf#FkYl_4T z9PORzpZ3Eh`oFTnMV0M8yd#tFJqa#lp$W}9rPezbiELN|h@pPy+DMx%BG6yFRJFTgS{@rrr{uUl-B!#A68r#|D< zcqu4(lTX8Y)vYLJ_^OUd1xwmEPEC3;9;Tt#4lN!~`I3 zRve#9AP@BxEd6Co@AX}YcMnn1+2F1$#8cEFh;l;R9&B9q=H#X^L5fXB7#9Gn@D`>W z@Bx@aM=Y0`ntqm=<|*`6O315_Vgkf=$k|@K?A>6f_4OAj4iL5&P z(GV&vBleqtur3c)_>^W;2^$^tU*$qSxr=ob8!31~nRxj|Yn6q~OMgoE zNaX#MZ^}G6Rrurohjq?}h>GqO`vQx#cl!wsCWrO^cjm+8{-62yXFmR!kALRlpZWM_ zKK_}Hf9B(#`S@o({+W+|=Hs9F_-8);nU8|ahUk5|s;=_6PdL;aYpxF7Hu9>Kzy<-&CKCU{VA|7oV`-7Jv0}JCKh`_3 z2OW^)7Dx+NQbeb7E2C}HxhA)aIV|elFh)y8iCqR1rH!H0z(?yPa@GI0@3xLv4tK`w zAyqx>=egTts!R8d`2(K}to|pRol5qhn?UMMZsM(Y88y*@al6%M0Q(63n{KMsD^$S`b!-^*%*1;yK z=0bhlHNUp0n?FOcoe$~!ZB55TZ(;R(U36uao_O*e)^QE6`GADbmHalGy_qYv_LDFk z`?)pOP5h6!ka?25mSnQ1WG|nmR>b1*yRdf=lgg9i>&EGAf8+}znLmPjy(8dsF1pZ; zLd@|@h`(tW8veYT&r|B-;qbmgk5N#e#?x@>_~OUd+s4M2s}OuxE>TTnrNWS$k+X{MfN|03V3YguZ-4m z?fbmNoR_9@mAQPS;G}jtVmdN1wiAxqbIS0s z&Re1VR|x%pymI(&M>3(R63GZ+`*uW~;c2YzP?0~#=P;tA;!o94bMCjb(;hU~jog_1~ywgEaRpjEk` z^Uq+}j@1V`Z07ZCQUsa>{^)*)7jQ;^@lH6oGypH z)O=KJbv6u45y}A)(D97efW9UOG07PhG0TuM4cM%UZJ0SWku)G(2QjkzOHrEJq#Ir- zi@5@Sbtx}28JYkdVUX|$Mg)_mk)gC0tBkJF5H)Dul@%OT`A>n#)&NHokxzA7s$R_$ z$QSsGxL2&_hySu~*Dml{M67xbYwwpZXf!_b(z#FKX=HU^m*w7ZoWyQpd~JCDjGDgWk7 z(Nu`k6gM#6hQ#qx6}Wx^(%I@v-&!hrKEbS=)CROSRG%~tpWEbBRk>p335e*SM=vXV z0BXho<$UdVIksya+tRjzo$9c=H~S@Kz+et9opLJ!))foz`KEfu8%8wPp5>;VnMP8Y zsomtzR$vOs3>SE<=jY8`gnqU*Ns$EYC>p^zj%~+v&|E+Z*w|?i8fJK+{*p%YaC@YS zXnlGVx5+-XMfT^|2I$7qX{MRVY%op&G=~ay&wLBG$m!{;Q3~fq^wK57;|K!|uKhJK zZ!m^5|NRtbXa_q^qX#x6bk)gO8smKV#_a5(y!+bcVHZ@@xKGGiFAPH zc|Nl*z3GKv6vn6U)3Ff)jOD+0K1|c3z<|K8QCUKbO08bQ1#MqgL6rFg^pU9Q-(*TT ztZ}Ee-5H3kMGC*3kFv-4A2Qa)mjgzh^8%XjI@sA8P6}s%f}6&YNj5bML(`7jTL=~F z2ecFJ1H$Gi1omZ;+T5{{fJ*dguE;=yv@ZAS%6Ko5>)Y?>+|AJ0K^kQC^mjb$3o-;VBQfwF3oF)^>svb_|~H# z5`KTw0!t>(Uux$ROCET#)h4eI1~=xD*jwY> zWyyJ2El_K@U^M_d^3n>w)9iKV0i#DR)p;a#Pe4m~h%NCsgmpSkVQmC|?))8$Ia)RRoS*U7 zqVLbNPF+t_hGdMJ{RUwzt$a_4TxV5pL$7*Ep7)KILr2bdkP=3-yAW=ga;XyHM~XlG zN!J9c964M_vXQ#f8@~)o|Dmq6QScMiJ@lW45Fb+Kz@O(Uw=(}};4W(OFV#}$O+wpj zHh@APOlaEr5@m1R#!i6DK<5hUuncmSPPiZkzWT#VCKE=V4!>DlNnXBt*0bjMp0H6{ z=o=qXD~K`Ks8sIH8%iEWNMcoJ3Ycx9R4*;Y2|syo5iSX)8sWt7|J-_T696d*IfDCC#7S*q7SOoFWj_ zMLTo4X~Wq14v-fMEf3PqoCwH3}b|yB0EfWeXmfo>s zI$U|cuwczx$=mwn_HcXhhN6rZd)S@v+(`HMDs%JJ5AQs5knv7ju9V`V67eBZi*zWD z^S6p)YY3Lnaa*KVC0~I>MH;)$!HRJB(3l+{Zd$R|*uhKNpS#Y=>3r8ZmD;GefAPwo zR>5y+vM{+?}I&C3Y=qh?DmfE4Pt{>jJ~&H5QO#OB1IYAATt9kwI8a z92f>Z&bL&jQ>zJNS?`d+((GaTYklAt-|s~?aj~+FGz2!O^oLn zgE$v53+jLZBfjGXgRE5{)V!k}rqS-y6S?}i9a%7-kywG{mvVa-@+I4BV^znY^jicEo<>~tHR8RJM{iXc)Z#?~cv*9P|bO2%b zr!BaCaI!RXlnVvB2=e!`7P!Dc8?OWDlK!WNR+9yYp)9MqDJ;npO(iQ+->-8}jt~cG z##&z7I)mqn5x;pbHbAR*=KUZhn>F7_Dc5Ki&G_@;<_53y@JA(RBcv#lq>3}@`9~*X zCES%L=biTAuiIXtlavlYVnL!*S`7VOeQvqo z-&;T-?K7>7OJ_&`4LNBXvSTbqNOuh6q8s~mKEtp zB&#JX(h}~Ax6jtnJ-n;`l4eyXDyS_pJ-U#FsKe`;>*C8OOh}(Ifdvn9GNh@W%Ywly zWP64nRHm9YK#u-uQq8;MZWrC}PwB9YCb+8)h2_Jq_WKK&YkV02D0eJ(+YDU!b$NCc zP(!#0r3*9SOS_`+xY|=qS0i{tf>MOr+}2vM0xl}_$4Tf7*9Ydfr5)FZ@RqaF%RB@b z0inI7tG_{OXCx_e*JgrbqM?d4?|{#J{NgvoXWD0Y+tj4;0!a7!jq3h1 z+l;NO(=IO5XQZzlcVM;F2B7nFv<9Bvv_E9~dWWv!C^E~?IR+w0>o3a)x+?H0s|~19 zG75}bboo>rd;P2#@UqJ{%)jv7(L;|%3fgPnBpODrfkX>%V2z7yYbCy9Q|)}FWyBZZ zs~KmgZ=ssR5`Rod8i>GR%4yM1&41)1^IMzkaR?~&-R#imA4}0dbRrMKHaMw9@BlM(XWQ+d9HgEsoO3mYWP$YwGE*-ZyAt}FsG?M+~gpZdl4 z9VlGML>X4Dr8vw%N7nX;ZpXbO5+%k?y5g$hRR=Qqta!0sWo|>mL`s4=NE&vCJLnmY z^LXQ3m1~9##)okvo(q_u@GAYsdo53Tljp#MVUx}A_ngC5M0QPlUoep!P3e2gfgepN^Mh5k~o623Kwoiv=&YOO{EWz^Fi-q`-<&hpH2V{XKON~d{-E;Fx zr9OS~qe{q&`HoSqaf^g`6a}*oeFYY!BKQx|A4Ae5s~qw^Xu>fRi|}yOQUDV75e283 z;oVofz|=t}kJ&cF=B4fIQdi)m)cy|A@9t1fnGkEXO0q+PDP({?UOb~NhMbp03Nv#C zUU84@*~&uK-iSY0ezGyKBpkk<kijp(~q3w~33=xmT}@8!!*Kj2Ankx6~B%k5WSz zg&>I+ri>sK)F8fvBH~yP7Xffqp6zvr@YqIT5w_%IQ@leLC37s<4VOMRbrsEif>Ber zPdA@mp_K<{he4_p*hS#GhYBMwn504`^#5ahnONyv*Z7@U+n6{M;U*w;Cg+Z8c0zc& zzC~oViKw&gOtCl>Kw(|}HAH#DkcJe6Dv~LQ+!NJwdd=JMI9FQ&8j56mI~zN^{L<@8 z_Jgi9K9mBnPIHu)V2wuZWe!uNjmg-nt|hM@FDs#Z?shM?czvIDE7uxP%M)beH)}~yBjF7ka?B`}R)MqH0Bqz|xTbj=QCglSSgFicFSRz* zF_bz=h~q{Y`Cxokn>|X%qfzkuv%AbX6V85{HjZz~*+#dCs287C;7~5r{(kgn3$K)&jMKrCz1xD zN&{K?G(%GFRLUAr`c4B;eSeLfEn@diPG0OKw3>_X1bZwaYY#DIfdzkD5u~C6Adb-4 zy19JT=ZntXwxT^{WD+_JtK)ABNU+=pZ!>5QrRC})^%X7m>31PAa(Lp{o%t{fx(?%z zo@G^PnCxG&i)eCqdJNCAnK?WJ79jn6{F3e8_`400;z?w@c+kg(I|x115feq!GPjEb zMy$alpv6WJhj2{}2O?GlPDBDH%p)px1y52fsGoz2(#iGsn%{TQ>j%W{k=zplzT5(w zZnk;|z~dM&v+i3(qPOki{`0kTib0My0h`aaNWvvWrrs&Rm`0okUcF76#{QnJ|9G+Z zNB@zRr6SwZI^GLg*XMG7gTaHx@rv#neZ2K~u5q(sW3o4Xl#jk%YvI;n^NhJSAYSdP zJvgRYq~zJ?WnAQkOE89cZSlUGmBZ4VFi3;?-Zt)miA+VDt+-Z&5R|T)$*vvsS2t#b zY!8m|JVcx)lpOOj{}gOlX}@yCF~D>^H7AhY|6ZaiyExu%;h<2!uHmowu#=XDRy*Z$Bh?Km~% zG7kywR;2zFJ`u$?u;(_qvo%(5$-Kh0^rhB%7J2033xy2$!PxJqV%OL�)6Sl?S;q z?PT(XX%@>&@8>)SW3?179KghT94y=q?$v{@_fYn8SDi5#`f_o0Mhwle?j74N_KSGf zvj<7kL*4&4aS)hd*gHcL<=|TwekFuyz9dlOC^wg+;shc*&^b=zJxMSYDU)qdM}!RH zvTe?=Ee_Spk9WZhV96Uoz}u?&)+95%7 z)Crld*zLq`BcLzWSWloYJU1JluVFKwuV-2WbyBewqA#^~-oF~8SHz`fg6;Sruys?P zRu=t28}L$^BpFhw1=ScbOu7T$1s{#yg^;|Hr##c=+UrElU!BkDfw6Znj^cZ|OXbP! zZ#3beT?Ja{QKNAlg?#k$8m<0Wd6Q4)O>#BN+rRM42eha9Any{5l6ah842IJu`=g63!_Qhp(EnYn6;8OtHJt=$b6y`YJx3 z_XZ{qn}hphPy`wlaas}MX|!GLijOet#mm-<#IrRv1p*io3*29t5MhU+}$VS zUKtH7?w;;{29RG+pk+VV?u_Lu>P|sbH)-D2m)5$I_T;(jZKgqU?x}gJ^~6R?Rr^$W z+#5}iOHgN_nU0t)Az%yTaGcGUENp8*aS0FZ@?>uuY6h=G(AD+HXe&XxSRQRkV6Ah1 z5jt)pLHpX~XCw%qt|aEkHT-Qt+%QQ0lB9JS7W$LUR-HY=r{M%NomhIz?t^s0_LAm4 z*r36TPdlML(y68+;U3j5ml=@B$2OARvAF}qk5<6eVpJfgllOi1Zn$2pdP`gP>t!lMPpq-o%pZmL=JN%ZtT0 zh%49YJ;^kP+dOTCw9WQ&SUZgD-e(rUvovicm{EjOz56OnqRU4|%LG}md+a)fNb0}n z^Ruz6{QSU<%oEr2{pS;`6uw+KaNfi8bKef{TBY|3_PvA4cOLimpBo@}Tbe-NWJ-cD z6yd3hrcw(n($T^^jj04R4m1S}s^3Bq%e(YTbM-Ut7M)i8BuEniog$1JqjrJYc_WzZZhu)a{Wz8!>ujVYch>{l?bW z#V_%{90 zNBg=!?&7DQ;4uN&kL#p4a?dN-D?PBZ-<1H^_;nEzr**qK0to6ddx%2p5?No z2b7HW(6C6u?3BlEdubWV9AX!i1gL>VR1jhu5lFgKR>yPs1LBE{T7aH9e}9<_P=?Ii zKsW{4f&w;6uxbUh5 z5*7VkL;&u%n)?HHvt~rj*?wFHwQf)g*fV}eqOO`tUV2d>mGQoOWDkl~uHibuGbpu? zxo)d;^Z=S}JzeDcNVKEoK?S9CquURP6&XQNL7<|-X&`?(gq1drgBWpVsfEhX?&a(w z%c!bSk;IgBSos!v;+WWl zh!Kc9?ST#>>zwN`Ygy#aoY_o~&vV`cQ3;g*$9!de2<6|1Nf!)cD}U6o(LEMF{#|4o z$*6Ijvh_1vSw@eZD><{oDc7R)=K{gio1Jdn<$s^cUJ421-jUzluE;rr_H(J5;hN5b z?b>yFmc@oGacj@87uj|Gh4`xx#HBdmy6c)i5+pe8SsNHBgAi8f6H0u8mF;%@_xx02 zK@m{mU24PTA77oTDK3Hk7D}7yXj{UA2_JulKZwshUDQFGnmPQ%@rAetKg?Y;ExI)` z9B;|!(-HmrF)E^-krqefs#eJ~IDE65(vpwlztNe9H=>gZaZ;9akfLCdq?1!*8|tnz z`F0HmWL!V@!%S|&hnJIyNfGFFZD9SYUrWK+fkeG?@1=2!zT39fh<#iSQrAzw{BaeM zm9CBBy9bDF0Gjt3OH5Iaf(nM_my_3M{@txF!0cQ$uz`kZx%eyLRwubLQs z7-L6FVOz_^l$-5RkOfZ@w0v}<+Oh{`N@Y@LvGs-29?`3Mkq^prekWIuMlNxch>)fj zTNgyHY$p=@RpK`;XK$Owo)%@;G75@-@8j)N4NQlxeuaQ9AEgtH zm%2QKubG_>(#E94#ojisyBX*E!ps=wcot8o%7viO)yMH3v7X>w-9~!uH28uPhl`1Q z`P!N>W~VaxHOV^rRgz$@5`$Ui>h{&&=yTG+=}Y>uNj z>KD?KQlTfCaIVsvn=YiNL2Typ@!b8|T$lVtw96un4XLwd&R$=djnU|XGXu4ZuN+Pi z`N?=Gs9-UY3^v3i=vPqZprD_g5BrnTqomCa;#PB?Hlw=F>FFO7?(oila%Iru)jAM_ zW3`d$0?3tzZGyhP&KlUf2S@gfe%r8)cL8~yu?PNl&%*-p--Q)p;~WCL@2?kmb^iYM z_rp}*lV=8B+|I&b`x6y$0}b`V=~zy9B=DE4u2{DIM~}bPKjv;ojuhk3z5^ zRoo)WI+*dIDaqsTv9%qe<}P^BK;S-3^1=_BnUFZkQE54ryV9%vf^!Yt z$-u^$vBs5C6{No4?>`SBjGxFG>Lj@Pfh!wyV;X_25KfSSq&DOA(Fe_iq*AGWZ6mU| zd#jfjVR(5Sv?B~;WkATo=t1A zhz{FjPl4JeKN2MN{V*wd^3|T(_fsr*c<%R@!ss+y7&Q>9;K}9e5K=R=;GPDc^U>FubSO4GZ2#@FmbzTK zz9ySQ#nj3lggb~(j-pkk885fx^g@0|S^Y1k6(Z^XIIX#?c2>&KY($HNbStv=yl6@* z*HPAIT||YfHu(sV*I-#3?NDKq$CS06xFbD0|J!XfEqcM+3*N()lG09(w4{7ab8z=t$O&^WC2Ag4+Ed{3KxqE`kJ*|ACa}q;dWM6ES0maYDOE#}bQbwu ziqmE-VzX7LG_d5%d9O1QT|ZXpgT+vXUij*sJPUZ8YR3EJ3o%QU$CW3=Px{N3-QD-+ z7`I5E&Wnp8I8(ppsCArFt=!-NRBPTF1C6fE8WXQl0XT;Ulyu%Ex$s6IPKA;az`1DL zi+4(KK*~25*R9&9U%lZ0bIC%l)X%`7#8FG=mdI&L>V+R2#n7FGSz(+u=ZGQ(>ee4qTG|kDMXm5vrOBy+5qOI>|*IemPb(USKc71 zerSB@J;t1QCEN;J-`$>d)xE7o=lJ#%ECOLsBg}vS+R30>cC8v||A?Cj@{kTJ9jMz7 zUQD>H<1o1(d!|93Na!w2-`CM$mbNVLPF!u70Oc443?Pa->LO)`v-hlTzuV)%*G`Uq zbu&B3t=bJsHYtX%KSj}>>P`#NiMnXIQr9A6U5w+`rNFYgkC+M8u%hs+*c!|eRQTbP z_UG;|*ELH?PY9^`KaF*^XD*xuHPb;|$?c2n51VmGBF)LsKZ^Gm`Pk1VJpT{=j5rhC znyFq?up57fr;v)Bc&3*?K1KoqnS%h{Y;lqnsWWmyWO9wsIWvZ~4uzvk8Y_B{V>MzU zrtI7Xs`6hj@F$*NkgPg=(_P*tc|hgf#Gxs9KRer_WVGyO*wySe8% zg8J2GE%2-F9Osg(UjEPuMf2}}^f72{pWqb&Pk`kWv^I~sbsoM%c78L3<<;lOedx;5 zfihP;Jpx!MPDIP!Q$7MSkqGQSm{-k=jCA#^?Jik39(>K}h(oi?)`+LRiOK9c7RWrR zuib7ul@9@4m5<*0jQ#I%v~K}-UzLwfBx zXTdhyoI0+#%bij69zfqU&Idt>>P=|(prbm=r_iC^hat^Eclvambb~FK0WPAYWmp}g zx8kUfoDKWlW3w`S8<|^B|0)#$TU~qEUm&E$DHttK!-6y+-EKUYnhb;{sIeUsCKFFH zu9n1sM+M(bvkIS`NTDK}?iQGpOUqok6#@8~ z6L=Ol>v8snG*V38WNITo0lG|HA0R{VI{9cHBVQDmE_oYKj=$_1tt9G<{g14*R#aCI zyvQ;h<&CfiZ#pfq*G#mlqS5LX0WUq>?e$ib%>b+x6L$(OQw83yiJ#+%r2=opLYZr< z*f|z|x7b;hdX%&Z??h~PVv`tCT4nQJjK%x?sWnL{a%XnDsg&NW&Wh>32Wbr7TUmr< z%9pKrPNxUVl5$a!@Y-#D-=meTYWW{#sc^r^7{e+QAOxtmr6ogqr6<>Hi7)00)1yF)gs(mLF5UZ@xw z+BalMgAVi>oZTqISb!G(`PJvqotK_7eJV|1o`G|?82H{TMXQpP{HO9TN;h%RJSyvA zMn9S2@NJWhX6LVt#$}Ipi)V``fAMu0 zw>so5Hcr4)4C9kv-E>>d7Yk%M_KCov+?Ca!XI`!6j3~aenEge%WLhLisvL^Do+_`6 zeHpM`nCO*ISPuabp8(H3_(%xSyfCVWLZ_{N6rT1%wibVDzqcBn3`Apj2K^=(2O7Gs zWDq2{fyAumzq2sOqu{QB@sXF)Wy*s1_^*$~LUf(wb@D3OxID%3J@Q8HxvDDj`rht!f$y?wNzhd#<4-SnjZ+Ius%4OtcqHnHRF_>pi`T0 zP=Q`_19wQ1@Sn85Ui3zi$6S?ItmQveP>d?2Etcx~lF6l^;$N*m7n&BfGz;MEXk3es z$So4aS75&{IUbYEkg^V~kzKBT9={4aO^J?Wm-iy2ruT9lZr{He-1FEa-ohZP zg^hXc*W0_1*9XnwNG6Q*(5QPlZdbP1RK!tf!nE@upUv=SI`4pf%6L{QbKIbr=-`YgeTilMf#Y7A-eTnod@;@<9^6Sfa`iyIMptzxy^Xa;1OXlVJ}*6fgeXPch?( zG7wue2OEhIR%uG+FYjfuDYjvCK z=l$v-HkZ-?cNl47As$Tm8_{CNpF^6(n?j>1(G*iLmT~jwNZ>eY&>Q?3oYGyi%=jMo zTxoB}-SDTaP9218h`iw${9Raz8=i4I`rmvT58FlarK@Sa@b-H%+oLS$f1-s)a2IIr zga1$~eIRr^O$?5ZcsS;3td7U$TjWJ1`H^@7)^+rko_7a=qMXP=gx?|f2ljWKQ`K=5 z>!h0XO)$${Wx!XEGaeC&jDj8iQK_$+h9)OtRE*)vJxP>4K~hg>zq2sSM=;IThv1^U zOY#o38iMULu5&lRUcRyn{8e+a<02KmlgY-v@UQFpX+pql83YItlP9OC4ClQ>at%4Yiwo)$ zzwNJBWe#%^&7xp$A<$wt#@}Vo*6q5LIt`zjhEh;Gel<%sYPbC{r##5`N(Z{8qeM1t4*HW`Wm#tOC>sj0#hKE9yHfv zx+QPQ-^^FOI9`t5t4MQ`Dy@K1j1Zn=AL44q0&_W;m5x=kFf8KQx8u{yIE(8Zt^v*l zXMncKD_s_{`y|YO-mf}#+lcvXVJc85^)hoy-GTKtv1LIASshIYIGd;fL+18hQb#ak z7v7+^ayKB_^!p?D*_kbjPXoO*MBBQNXv7!GNw&iDvNRNaQR8zm>2**)wG%`>Rgu09J&LzeG!|Yh+94^ zPl5ovzk%`z6!zC_z`TsvDufkI?%9Mlq=QDuBL!caEqJ7Y6_e1yiPKOAaQ5P?BG}^& zP_9}AykKOuP+J@uAasr`TpLTtRG*KEX4W|}ZUOdGqZ!%`qP+iIvQqKPeI= zuS~zezGU;m`fP46i!-stxm|Y(_(PbMb0O}@FZLHDIuM;(eM0y-blOCW$5M;vPc@-m z{ULx-bbE4u*vzrR6(*S)FJ(l{Ab6Y;5t!TcrxYiLqBfHeztL4&n?DJCN#h-o#eFte zbV-cO*Vr34>=Q9O((Jh=2-zm4(ygaDP$d%GZE66Sz~%&B0LLDo#hN+AjHBDzP`N+! zg2;VFu;|zam&amY$_dd;+As%2^Er3v zr1C#k2e%Kg(}K6B_S%;h;*vhl0}HoIk}`Wioc#8p#J^j^7X;L@$v>=!b7{mRQNqtW28O6Vaqvla^ zTJ!64{`Mk3mb2}})U3J|vI<{4x(aV=XTY;NEs(x6XQEr37a3kepsmeN639ca_s zU5*oE7lm<_ZC+@2xc{!`iZgG4CF+_IHfTr$cnP0DE5cRH5HPGe%F=fe-Xrb4cj4aWONv>o{EyFTc$RY~eONt*s z64zw6Iu?Wqj1;mGabPZ7K?LERS$EBC#i@xU0nuy&L%HauRw^DSt4+@RmdL0kh(&^P zIC#>8e-KXYi;%C+R9 z3fd=S$v9}284LDkY`xoFzEZz0ZI+>a5d&?>P$9U0rj%hM${5zCiskxQG->_Oa;fG( zW0`5CVuisB!gz~*RSe3KLz%8d2e$nRqF)C0s(aX>-6V#dGwu`^^B$M%M*)!2dB{~H zXJ0>{X}E<$W5lU}FEpB|twAhN0fe4dP9mm#3CiMWqk7D_CY4_-*sG z#>AGD-t=al1(?)D-WB&J1SK%VQ+HysyWPOzL{5wgPIi$>VaZ>{qS2S1=^PN35qbuG zd+E18Fw;-pt$-8!y%vVl9@^4XZ|=*^M5J0MC5Nx!WXoXdubGD`3sG6Zr3j zrfiB$V^rV@;WjnmVB3nacFzQ1x$;I7d#7068Ho|+8n#wGZ1`o+IvOeNBnx{tD4r!g z6OB*5Z*!5anW-n5rE8NE{i!`#gI)EPK642Ig#E^9^R-X?6zJ;1y6!QOJ?raA354c~ zwc9e8YyWsB$eAq&Q(YV?Jc=KCz%NH3dI zGlE@;#69fq7J!+)Qm^nf3KtnSef7E^rU-Z4`uW&*Fs8{#nB5bHT;+4`R3tgftLNPK z+txE( zzxtD~J~x^Ferb*wjbxq2b?H20Lr@<6MHCXu&u>B^^rjH;g-?G^xo>69ZC`<*n37M> zK)1n2F{W;NF|`NH(e}O!0l_^OdJW*F&utgUfJjVoo5;2j*<7TD)LlI@|-!Fc35uX-!EoZ26Lz6fU_IX4w7K0m^QAwM| zeHx8G_ob;H?Rv+$L>esotzi!U2hHAC#K0pmSNooz_LlDX)m9I0V98s9V8>Hvr9-%? z4_D*JFgC8$vaR{4ooOLE$tc=zUL#>KRVPhs20DJzHkkLYBIS_tuyTEGpykj^mIYe& z1JLS`X0fe<)|I25BRJFIi%CNNd^qSLLApDOI>mz8RJt^AXW<&xRzh~cb6`T1fTCKp7{=g>=uDT4|-5kU<4 z|5RLPiFpBZay{6l%t*5y?wcFL9fK#WZd-OsmPd65=J|(?MR+dP$&@$7kya6CtVkdnQVtApDFfQi@=g+ z>^lD~@7VHlg|n?$v_YEl8XpU<<0x$pwYM6efrF=7)C9->1+Y|`RQCm22hF}oI4!nw z|7a)?joptk5j0uAe(rgU2=~Za6TJOfSNO{C%1i3qIV*JA{aGesuw#74{^eYuhKm{) z4}9O?>j~~&*J0fZVrBnzdax_h;u&p7a9_Iag=_T1>I*%l3ChRR+*a0k)7b$__)z=o z5OUmv*b5)`6oIGlj8VfIovi`E6zN_mrJ7T@;#p==a<7I;JbB%Ut>Bq@WgCRww9A7) z^m{%c^JZ&lQ!$2z!wtJhs5_&xbwTLc=y?yU|Ln6xaw0Ol1XaEA+g08nL^oKH7052w zsyz`<9u|Q`tW;h^?lmFJX|-X1p0+s(jq8-pvqL2jcq5uyZB(8sq?kK~!G{I%Nx>Z% z9~=40Xa@h|9o*(iy3;e-@p#5_x z3?G;2iN;N@b~b>=RS+xQj1BB_NDp*m&kO=_ z<`muz8N*tsB@n7iE4!dyWn$TryGwlHEX&yJwLfk3@?x0&vE8ocqx!4UF4}b zR?s5aGeOw_KB8U!zKfV*6segbQKh?C>$!AzS-!2YlM)<>Xa1Cq=N_LvTI{Sw{ zJJ#K)X$_lc#^Nt0aCc@aH!r>S@8AADzOhdDX_%4~sP52BZHpTj05Ukzy=c?rLJe(t zWi>Z|9Leu7<)twGL96=xTCWe9ieo$HimZ4CKTnh#GrPt57wPF8Q)wHo-Mfq$O|6mS zpd#4r#Nr7Y9yg9|1CrfrFxOwS=SWcmf=r$KsxLh%iF9)9*V`uC#w_1Z?}x zxPKeWb42izgqOmcYVevkQ_w_e#al5;6h{8t?nddA1Om3ZyT!Z%swnuJ8#CCb-GnF| zT7Qm^1cdf*oUkY>N9MYv6yvIEpVZ^d2uSRIa~nj8FP_;>+Q2&9{tUQsQX>TV$*nLz zLTJdK<;7PkvC=c9V%MiLfs0C{!UajIs`zfK>+tUmi#&|#b4_b3f{U_fHsdN9Mevv6 zA)*UB(g-GwYY#H5A3j5V(P5~(ot&!%u7M0}-e@vWvh1{F7bL;Qizm!QY2R^PIsDFko<}=xqVSG`OOCQy}Vvy(32zG370B6^?ia1XS9-z0R@4-w}!9-$!NDTl_M8WK_>xQg`@>Vr&d#Qb()2y z$>^t|1u{7q1XYE}cKlw^C^*0+Vppqz_?ODY#j9;8vQ13u#)d7?EyGrgu#}v7?>4RS zw8V|#@5Q~4nJ%VT8PLD@h(I@&Iki)t)6~@wmx3%_z?e@{sH-yElAju?Zc8WqIE$JQ zS+gUP$SR*j2i^1$f9+bJN?Irx8wClczPC~%Pv?dRbc%29eW=yf?UQ1HBp-Hqjfw-= z%u`MO(rdkvw&^giDT2!1s#;FJpgwqD5o*3HzH|W(z#N63I8iODMz($-){F{7H{L)t zpKIPn-=?nAy{rF3f}6&-a-TO1`>*6Y4^n0R+mdvOw|PIMZ*HS64YgwK-kAHj)4Eu# zK6t3zJ?oV3<=MXClHafv)i=C5>v29fz3-6KgGl-#@U{-|CXpdIuJ%H)F?l+=0+JAC z_0mO1&>M0-RMqEp%9}btM0^ozvKtM@x%WVI;JG@% z7@t6s1h5x2j7!-3E|U2EB4EEbL7!OplrL7Pg@in!PpagiQ8MuaNiPB;$OIWTf{bHc zz_7=V<^f8&1b{9exnV@iUyMvioJ>)CvFJ^;nCS^+5L zk&38cWYEa1uNI_6Ld56_R4>dl9tds)X-v(M$`?sfKCo&}?qSnGGeV&qY?8*mV@t=^UsPb{Y+OPK_`|J@>*trG^)&7TKZ6DPaOsEvyqx z=MQsJsQ|3|2rXD84x+B*9H58@AEL%*xws{$_RgrPG{gp`%mx8ZN|0aKkBC)5%16Mt z+^;0`E0^s|>sKy1&x&MQnF8n|E6A}GtFjO8kczLd0$P^}W*b%EGEmbZkfn-f5x-`c zP+w^zJAR^k9$M&4Z<8J|O5z!65R)GXQE3v9Y{LTyPGCitM)W+(L0@{uoV1eiW*uA? z8Uy$+2t9tLD;7LG3xTFNYx$G~JmtM1V}=57<-%F@HPIk4ts4nsyKvy-tvLzgX~YoHcB2H6%nw^B5nHPvO2%T)z{G*$y;~y5+t>i! zmwC;-B6DmK1SzZsO7<8&oE2lZ*d%ZTw0UM|VXP}5RI%na#+Sl|IWbbhn0?}y5js*} zmTk?p!g(xA6QgKj;%M%Ak$F`20n{rI1hLjv&NU4c9CLs6r#nf>ZT~y$A^6utFVQq=z}lwE|G0h>`w(u|YV|Vc0m}Os+tO(o$51);iu5_ zlprs$K_}2Nxjg%6*>k)F7l?g)2Zmn3oBFqx_hE;xu8zErklV5QqDD};pnDzz)+>~! z_@LFOBqoSK+t|&Pq_1vxOY&fLUW`*SMP^M zQfiB`95G(WxAY(GI!HqnB7~{(e%I?M(vpE!?1DIw|3r9PA~!bIrwJM5$Bo;_Ww3h9 zoOzoF7U1bhf^VQ>8+b^sRHEY>g-4Z_8_yupC5#cW--_bD`(Ae20FR8RswoZ95|A5= z5V+dK87Qs7)W=3`h!QjS5|oJ8nAJ>ZHWaT4+~9-*JU{cxQmS^M!|(VL_y#55TZ6_z zG3m(^-^DGi1JgBh^llFIZOXx-g|*G5?SIj)Ev7X72`^N4ZbJy!>X@Gu9 zDtla!@t-c3OrU}eOL3&8VWAnR2>$cGA~GZBqsj|FZK1&Y9DP=)(bX&xsnMN8gfab{ z_~5FApp(PWd4C|fQRMrpj$!-*LwxmhQ;hYPv5Rehb|DrxB-uO>Z6^RU+8~@MhXPjW zsAZD(5PPOQ)0X`hpa&n(cd|@2SYn~b{RBk|xf(a!=hgvx(P`nem=&5C_rd1Bt@6WP z^o!vj%h;0AyV+nain@dP1iku?AQY7T#KU7Eir3Vfqe$A_H`4+C3T|kUlB5WeT-Juv zwk@;DN&T`?Tos=8;ss?w{WnwHnVU0B(41YleC?i+p9k zK7b>9?jQin`#taQASo-&m-h|4W6>(i4p5d0bJbG z+A}M@2n9GMc=>3UaD*TJ$`9ej`%uS z+#UEzBSj>eeaO((3#&E@!y`_y{OV62yf728w_a#B6$Uu*eeDL24Uj(W&D2B<3P+k2 z7NTGC&i{)1qh5VtFzNeP+VH6g~!Fm`qVn!*-Q|Ye2qime$#YRc}W{Ry*ngJeuH76_O${2}t;#mS} zfK=WcGglNhrxee zWt%8l*Q)6!N_S@*xyOBIITQ6GcUeKlc{faT5Me_dE%q^0RmUvRY@{IwF__P?KDmH! z7y0ys51#NLQpPDIepwnxoDkFMF@ zu8vXWbTg6GS2eChA)3#Q?4Rfp6o`48%t&$N^Phk4FfUx839UqPMumg@Nw^DiZ*P51;{{_#$N+@(6 zs4a|-3~jv1EquBR3$)J)hlF%Koxz*tGl{Jem!+YNn|fF-l9=GChG-Yv!eyj7Xpdm* zIaK7}MJamLXa4u7;bd&qzt(+o?4oGqubmBxa`ou$14-i*vJ6sI0ZT&6t6&ftH8nZ+ z6){>~F?mUU{!~(dUqPw&fR{rU;s#c!h_D}h}(yU zB}*uSqo?L*3?t{IP2eDnp=)QP3li__JJUF6%h$HQpo_wIB&@fvr+!0YA>(DK4r+W@ z7OoM==I;GpbvA7EO2eL-fl$vBl<&Z{lhKLlgd@;TH1ah?dxzrojKOy_A+wqsUWWx? zUSv^ZxtXlY_Tz8u5oQ7ri!@Qu134kdE=om$f^fy7jxnv}apSBYJApzaYM-&l!PPD) z<47XsHl!Mz-7hCYu91|;;Yp*?-avcx*utopw}vx7_)?fr+XrbZwU+JX5in@Q3Vt9p zR+VUfjOOn`(-eXUAdQK|0ELpCPc|vIk%rYtBnR#yM_68nJOA#zOuPmKIzsOZNLU8n zgHICB_5Lvg^F$S;#o`f$Sj3&UPRAzz1M=q47PB51BHYUhZ3bVfUHPT@@~RT3eGwu& z_R_pDVE)1Pk${`KEcdJ*GAW7f+U&Dk{Nw`z+a;wt@Oh8d3e4%`Fv#=6s0Uu`lCD_C zkHXX%*HzI7wDT}y7{1#)Aw;VTJw(z-O(^!o9{(9|r&+9NDgJnr5JS=SnqhjWH9EcW zSd8$lu8B0336i;up$Hpov5OLoL!4>rpJ}Oh`%ui13DajkyYt8lFnxznD=CpZ(}oP; z>?>yrts9Zh6>yj;O|grd$LoNwc6b!_4OhPXCNUQbQ@u+b?jV1(`*EN+>&Sx zw3lMm18x4wS@%G!UZ+VzubR#9d8&!joSl=_ub>rj6@&e#x(oPCwd4pARfVFx-0C;N zMLqGPOF3sM@NN;Pa?rr^5M_Io8u$&d6zE(=F^>aZf(cq@I9lVD9e zDo~)p1K;qlfOJ_4Q_n0*OPEY8w=_A9p)BXPqNUyiRWjJw?{6!q)Go^3P0Pv?hs zGA4Nqg_z!ypa^^jv2+rO^f=ZUNy!ppQe+(0uaC(m!Q@U9(>~J7CHPm;|0^ngOzw5< zMHej)tVZiCJq4tDr>o98dEZ{KpfT>+PC3MjIzAn3 zF%@xx$F!K3eJCatk4B;^-b9{y$?NfkEG#TyppS(y2x?QWdf_=9;2aOk{AQ4hIS6Bw zzWb@h_Ox-siJw`JstVPtpHwxu8Zazh(xGK_1rcCyO)H+jx&;Ks`pBd&h)r2794iE9 zq{eouI856!yrCwxJdXH-_-}SM;4`^Pd26xQvyV}#^5lS|BT`xlRaLaL8mwX-$oeYkCh(73RQr5}tzW*@u#?|b{YyJ8LHqkGHcu>o4~o3X?Bi z>ymDq2mUga-%RM5(!u%A&@XfA#ypI7VBW!Pd&2GxBW16ZB#K)~!g7axesaS596XW- zZmx;(LpAa$8NfjgteCThj#ecoZ$h~qCOQ5(+@Q+bk;5diLP)QVtX{0s59@*dnh;hf zj0=O|)}q}|tStQCwj%$C1IKE16)i9rrUR4I(9g@#?3`ZRWp<%-l6eRZ#s8> zd{Tx^V6{M}@%a;aW0tPE7(D?I5tE~r4LT|$vrA(_GtegXxZ)<6;Zyh1j>!|P(k<~zYKh&c>{bM%~ z5|4RO*LU$14^c_vR%YD8G(jM$F&7=2qu)3d%V;;@Nn0m_rfauKR%GN#Q=nu;*larw zToM>>mpYn~h7EsEfL5q8^HVU>$l*f%G3BP$SeA%>`><=AEWPCNcy-~>|ByfEY%{Lz zR{C%5|M$zd-C(QAM+p_zc)g`du!Z4X%#AJgETKzYv&*iw7_ZD@lq{T*mKMn;1_i|p zYaz}bJVnv#p!Sh}KAgtVQe}v!GAVex{tW?we)WwEt+wyy zZFL=VM_pg{@wYIQ*aa`eqtBK;+Z&6FUO4;OdsA+b&o znmy!ZFfCV_a!}^M@ z4gN#x@VrwCw1G1qxAIuuw&%mdJ`s>5ei3|;60AED>58p+hrK=}$xZ6PvY+?}>Ju&; zO?8ZkKWXZ)w}avOOPN~e+zwDKgVb!|L1VN~BWKuRObMD9Vnnt7)BXEqP)agMqL9}i zmJ$R_Grc~31`l}XXHwAjyHV7e%j@D2M~s~_`*_a+`(*+H}d-@s5jfYNzBJk zeen9o(`t{obTM-78bA3uhGsRKBl#D;;jLtRAyqBXDG>#pcG_qkU}`-)DjiU-&*2E` zMd80;I{A)@&{PS7!ZV~pGd`v)hL)j{+~+_O+3`S{!V~rrwNWvFGsNK?Fr73rTK)`~=WXMA%?p=lL_2&qHpPB`c%w%S1_LdC%Y< zOy>Pi;A<{%M;#&Q8vRq-sh%cf;P2Jc1!CMoB&hDE1-o%-P`L2>6awBQqGIEJ3)-+- z7)&WZ{-e>ATS5bWB+(JIU3Z?|yiNGJne^vAFU1Vq>{e2NCF)1R;~h+%K`dfG0}gon>Tz}!Q#$^?-yY# z;|fn|muFuEYZG=hf*hhQO45Dhf*G9UB#Q4mD^vss)rhq5iU*V+iDk(=4|DDf_2&8) z{9NF_wpMZez`xwlk@=st^YQwm2@UU|UgZ3EdnaDRpbTSiq*`{i5PVj|9+DHQQ)iui zNS6rEu&^OS!jRLHfyAP`oWcowWyydpO+xebCBmN-ij8JBz)26RTg@=7~@99HgM%LeET#!f7# z4a-56hr5pQx$_1SPNc@C9A!jf+5TOfI$X;RK1u=Dq8b0)<~Mamm9kg$H)|lI)=h|P zNT}YTad{SQ`WGyRhpiWN;k0T`MYbvqna@HnS9-n9tn*f&f-w{LVk1zUSSr+xp8RFN z#tlr;GAC!qZ27@ldC+8xD5t#%x*^(mW|3x*eVTn5UhjkYe$>LLQ*z1L0vlfUY5S7^ z8P%ZozAQ+wYHfl?KF*qYtKs>d-@j!e%PiCD1O-LVRIfqAu&k)gNtQh?<(m6Fx`BIJ zt)b}yF+FkZVn1cnmD~t(8FB@DZtfC5UP1krS$^RCX}19ezuHtDP8jA`*G&q z5QALZrXr|^8EmI^87r|v3CpXq{sOvE!-PNL)=OMXTL#sKR$xI}roA`{xo)9P0f}ho z-k45uv;N7J)Irv^^DP&VItM!t$SA`o z&}`1>0HO}LW-YM;yKBMPhqRK+^pMrJ?v&ARxEl|N0Gjgk-|lyx>k)*zdEGt-Azn-5 ze$!J0nWH;!)wLUOuyJmND$_5^wLmp2_d!VY9hFuXnxO7^rpGrKlpRMVH(R4pW8PVbdgc&=HMh2s!n4BK=> zdps!;6EV^n?#d56^Ls#Sx3e6!93qygu`S<-7l{M!wR-koy8>BxUnZwZp`v7$Y|?E$ zMT@}I#)z^C3KlR}ebeAxza03?8XAVs8wE1Wa>R>|U()sq^n?p=cFr@qG1IMW&SR&? zz!PJhI2q5^D`-y8Vf7gj3ab9-?$+z6JR=-KazX>8T%t083JXI}pgSs-&nE4U3`nYlnb0}H~hFgAjotQ_W2UQm=H0t5ktm)h%GU{$2{`vMU-quQY z$0PJ(1|B17C*uqxInJlvv2PouHF^*X7t$%0Kb$BT`Z=z}ugf$F>CzX0YP1f;ejJgd zM>dx5z)Dk9jXz%E{||XUhQChWGxs?2oTXTr<m$lI z(qd+Cr-g5{*Mx>zb#Chp68Xz4J}=y zKS;1UlZaZ;&L$XB3aIG3NE;bc9T&L(VcSDlX=~Kr633#wdZGkAnrJgaC0a|9mnXLs zV$=Y%1(TN~5hWJ#M-W|eic^tO?!;k;vKOlHD;17O87ddvHKp}F#7d$Wm*6Mwu7y_% zv@5=$U%>tiEaTbx*Zv}c5!Ji9Wg7aMWp#-Yi5j{-1E$os=2bJ4#%89RM?;Zy2xb;h z)6K}XstIiM*TXZqX(fAHwT@&TxlroSlE30s!M8`D>iC6c$FxMG7#%S&C=DPPoP51o zgJSi0F(dCqbza1v4_Pw!Yr*7uQNeE>b?I=hBNt9fWIB9Zv1_Pkg$1W$4l6zb^2&KZ zM5$M(6u+0eihtK%B)DesSSm;mlPz5G?SV54en9&0h1Bt*NvpO=yHF}-* zIVuhYc~DNb>Sq1F#%T7RWJ|fblnFj(MDk*tvCene_HB2eTJPxSAsa9)2*W^2HL5=U zSE`_pOrsZ}y6J84=!Wf%W_!2u=)5?ZHNLaXZ8Ln<+blS9j^MkaL&b@R6BhBxcyyR{ zd%p1`9&|%3-gUk)!b_z^$d+TvHss7}fB?A@qU2DuM@0!DP_zo3&@8Ey5TnC5RO4PN zrID~RA#Uoe+0({z$_osKX~zpHtMC{#ZrE;)3N60hZYvf7>gqOatesEvUYDYV`({c6 zk6H?6vnD{vo+YlEq9C7b3mtsL1Ypx3ceaePhVzV<|Z?KP(%2+o>~;F2LF& z7)_WFx-yl#r0V|ihTTQcsN;9|axX<8W$Th*F(bHMaNTkbw_Y+*7wnd)ZBMz>RfkQ? z#I-uP-xZQpT&}=@`afTb_o6cX5N9D7NKxpe32&9I4w()qUuiF_Iw*IH*`-|F=Cf7T zlwhuc##3w*c*Ez`nOM@yWNo;4)79R)&B*mXTMI&G0KPj4jGeEc=r~#w)HVExl|t9@ zv>NTb=xJAyj6pSMT99|Vc#ZG(3Z6lX;}`RvM8Qas)w1&Cl9*!z#b3dt@Ej!tSYaUi zm7%L0Lf|%wp%suy^5W@>r_Wz14Fr8cRBGI^{ndWHAU``>R5N)_eun?Q(EtB}{7il( z#;STg~UQE+Mrtc8Szy9M~*&2_z>G9(RM9@{9jrzJ)sf6%mDQ z;rR(n;gsBgLmJ7uw%}h_8pDsF6ha!gU6rJsw zsdr`#qo&0Ux^&Y6LkiA{-5uQp34)PZ*8G-{4PS3qX-C_FKOF8cH~F>LnVW*~YC+Dc z<6{X1b<~=4B7*U-wIv;&!`XhO-7RhjOJ@+eg4`{i#uqOR_G#qJ4UQB`oA)gbIvt== zhM+j-mm0$H!-ne zRgFTULoBt}h*7i7HqEurr8GfGD$+J$_yDThS)blIuz%GwlZy1O!O2$ct%%>rhzGX| zrYH)FzAVTY9()NNMl}zwD&pB^=bjmw;HLR^Ah??MNG>3=UNxf$(S_nB;5%3PY;*k1 z-5b?h*@9;?6L6(HCuFJP!YFu{?0cb?+;dD7uR%j%qX;u|O=WkRp6Lq$FNR#VFwP(P z^szB#b>t?HqQCMmQ7&N#p;$@>qLJ2T$i8 zPukg@Ld1h15^l3pLp%U`wDou|8UaP}ss;iwBL%wyuBaBrd!GgtO((_L%yg#Y2B_z* zGltznPpXYIn%oyBUywO@cB)Q!f!7E&(H2bf&J4=Cc zGs!*0uGy0TrLLpA`L5@oO5Nk|Q!CImX$e& z;k8WhBFXBMrA9ro7_(I3={Z;@ULM+dW826wJ>GDqAKg3=d>IX+wb>2u7>)+iZqOBA zA**Hc2-NBXEPD58uCYM%MKJZotwZj+#Oj2d7ZBYm3Or)IYqrX$D<>_&!l7373PL!> zUIP0o-xYu^+;jlLN4&><@L2l}k9FMfPq)EBQz6Wh*S(w{feddqSp}F#{73w zNhPyUE8b|vgtPg&_pVFm`OsUQd^ zW{L~)L)!vjG}&P2ICk(K?u(@&VBm@)R))iUOj24NyDE%m#Y8Z1^JZbjh@&re2g%MU&ojkmW#_Fg+VXW0D(wPz{tqy$} z!%_2PcSK5@JepyzKj{=Yt5=UO+8NDxHDtLt^gi#|r{aw1`)XU4b3v>b)%Vox^-y%5 zuK=A#Fj*WQH`Xh>j{W#ZB@T7;LTAe-M*Jle%QB6aZPap5KP%Dd-yB-*TISsQG0>ph zn08>AM0*30I3vs2P2JY~Rx!dT10NbgH!RzdlBKmXuu&^q!0pd$YXE{oYRX8CIvaU} zuuASr-eKB&1@z4%FS1)!?#Nb138Vu^Je88KA=4}1o}?9OkT5%=Y#nKB>j4V5^F0OM*{H4fU%-E@Ro?vnd^KPv@kBQ0IdyJIq~YG{Q+NS( z^e;|ZyFU5i>5He&Pt=@WoT}MdmiZU+=cmL24%%r^&y2~AZT`G4iOZr)bqu-QL}#;V zW?0BW@%7yBTeyDKjlB~o`#_c%ErON?zf_>8mJj27v!?us^5ju@RXzrbxW>^D6y`| z_4rB|ssYEJRExvffd6bKA~&H~G3(L3jR9PWtTroPwFTWe1JD4}MNu}{T|ISVuGc^D z?E!0}K09OqOzo$L*}u`RVxW-r$m2nP%$mz4d170NTb?>q8eW*98-G?Z@PEAgvs#70 zL{%{IzGcP#$KLyQH*Q;b2!j_pbPbbfd(C+VJ~n}JA3 zLQN710otQHdEWc?;9h_PB@zTBdB)Qr{%9u_2?7_uy%+b_XRa16$g^!GxL`D9ZD}NZ zA{ILp)%8426M6-)h~^RpCy7csD6M&jh(j0kAGxYSf1b(i(4Pfi449 z`*KENM7NBsR0C#UftetFPWPobWX7U)q|2_F4N~htJe$AFY1Ce(#8VJ8q!emE5!5y4 zNt=|Iiz%J*gsT;y;N!-!daShQ!}!~cHKt-6p=6c_iiPaR2~mq&6!Qgn)>#(+G1Ow+ zq6(`D1|oP}2OVHW(6Dt(0kknt)mn=J-}|(122>)0OI+T(niwE;i{=g1RLzXRPmMyk zdHUVB0WC#QH%4*hpxn|zEkv%*2Cz054k9q>0U?NEYp=uw%-tECHL?ivJrbNMRC+&J zy0^yweA)0>ke#s9%MjiH`Fx{&X2|E&*2a380n>3RZhpSt=fo6_?E~$*0aM+;LLLv3 z*d36XYzInB2DbAr&vrIy_y)}9v*$Sh-AA;&A~0#JXJ?`s&Q3wg+C=}as-1ae)R%_^ zY&t!fP)!rBoQ{?VK1xhOMrAV=Q1KWZ9BFM7eXQ-bQ8Fl$0rMCX%C6CUcAR}sCy&X1B;U4>ZGqgj zTc$l`dA4Vmq8@Y$lTcei7Ip825XlrKmAWO8Um68TqHUxQI)C1T!)JYG!c5SARH8)y zSn&RMn0LPI#2l585*t7HYM0A++N1bpUS!|{%~>L74034$JH{g7NR~I3ht^kxDV2;I z-b~KFX+Rb5vr18@x3jl-%<>zavLARBivoYX?x6Omw1ze0_~Y$SAl$L2q# zqR3if7}1$-Q`aHWBk`)#UselG67;1xRr_-Tpg(nlxupL7tKO1u6-oj!;>V zT+wvd9tUmWE={+8`7d06@oV~CQ<97tN&6IL@U}JXR?G-Y$_&#Mn{N&*Lx?6-rtr1& z>DU1O@?Yfc`S^?f7@xF3{5?vyx#!5W!f&-gZ#m@H27c?|ptp8_-6{dL8oIq}oO&4S zR`c%f1G(i{S3lI&+WIx%Y?+MmHmt3NfxEo`p=;f?qH1-ac3DKNWl3HKP3y>%a?e@0 zO%OiSWc)Xic%np3=S=G$X5Y&O;)bDE>p1BQpa6 zF}JqF+j20qS|`^BYpaH*)!JhZoUJ;RR%?$P7+c-RtRA&;Z@5~Wqv*oZdJrtFopH3f z?-d>RSr3GrwQt<4&gBch%i0nv%WKWp5+}=-`m!ZPmKVWq8+Y}E2P9!qb=Tm^D7h`BEMbr}21-Lp zqXlMJn{r(2{ZdGOj=H5yUt^-geQ|7d((mcLWH{Zap056xwVC`vtiChZp6%2`ZhYRB z0)RUhfZ@QowicKGm{*|mN)+UtV$KY3nlZWI%NOJ#lLZ#kQfr^-GzmgVXqh-Ptqf)6 z3I>7K2;W04FvqmR_xpssS6@CkB2V7Wr2fB5&E%6YxlES{kGRsBN;YG8j=N>b%nRN= z`OH;M9%NH^zmJw>tvttq+-tp=-D!!)@?7L)ol$3Xwea@#27I>;Ce zRS|mO^B$D1*lp_kp{9~}>s9F1ut6_pL||3dBU6ADi+yleKIH9ky0Tx_9N5`}P}9`> zbR4t0V;R$DM=%83DIGr3+B~TwPo6y)lZn-jj@)^d&v|>CiM$*L)I~UXLJma^BZ_E7 z5=QTs%>vSlW`$LkpSGsblEA7}B&c!~*#@)sz`kKIvhu>@D$PV5G2{8cN4(ULOevG@ zTDOyYKn72O{UF%;<#{-r>>F@_&G;_`BbVov!7o`Zxm22E#9ZEb31x+@rj(Y%ho>q1 zRb=Gin~53L(~m^79r0i-g}i#V$8|)1Epn#yk|0H=e0p8E(SIlBR8iD%fCKr`_Dg}N z)t4#YRlzHclCpz75dqtuzZP*1rNVj1vh2U=5uV}SNveAzZ$!rY@jBOx#zddoBmCzN zIakcfje&sq~`0o1B&O!fE z%r=G(L-aGs{S3dSqFU*xFV}R`N|c-2Q}5{MhK?JkU3z=Hct8FSdgZP(zvY=XmaBe> z_0!;Zi>itTaOs^asAi-jEfU3-o}xd*Zh5`Ipk%S#J^JDaT#B4CtWy-fun;w!I}o;O zV`r}s_|S0I5(iOl^6nlF(M*|@uNfy1W1 z47Sy$W8j&cde4I88w!(su~jMBQGe-{#g^lGANcP2awl2si&NW_)%xj1Sq(o-SboPN z)^TAlHmZjc*V}KV_*30XHsZXOM6X`4@(`|XfLUh(-&_z!@1$`T7=s@Wk)R^CjBtcrMoUtR<(scac zwTQ}aQk!arPSJT?gC6%S2FQomlzpU)KgNBX*3~<8YENo^@47uMO|bkeTM>W;orzrU z&Ah&sn2LZ1$Ax717(e}3h}$LTV6fM$rSwrdX|LI=iL&O;r9arua8LuQr`1|@ppn#w z@cQv-8=CeOk`nfq1Lz+lgubVw>C9rZbqo$6~QO?bAhK$^c9Crs&M_Epce+Jnff7rO3)ZaiD- zRX^?F4H{os?= z4|Z_)+HHe~946PS3KYWw^f6E4NA}xxR=%lQlFGs$bzPwRGh%*Xi0e`;0SUa71c+(l z3$F?}yM~ANbJ995)mc`m|1#yOWH^3zAX5m8jEMhQvxy8i9SC&?`fOZPhwZRQMatsF zl=#XgCnqO}efrf`UlEZLN@5oAbp1hZ$JoQaKYKPN|8x5K6@))7l|<}x{HlJS*0BfS zQn&vzdHbe9k*mK6x5kp3wZLKjF#q)Bmt%6ybHFm-2I$E~WZID7OejJDQ4`c8|J`xhR$;(y0lM z_4a9!A#o7ZUX)CIz&NB%`(*7b!P9FRp#he;tyRYM;rA00%K@Pum{(F4tPzWyvP3pr zYnh9G!WQs5l5Mu9y{t&1bX$xd$5mhvoU{_+LkS@WuLHg68A2@GTGC#^A=wn5WVak~ z6{)RvzQny$WE(vrWToGLhWr`^N@1S776}f?3GaLPSoaqPaQq`tLp+ zAi!Y1R^@uBw-}0cTU?FSa!IJ6)W<97bXx~OgKB%QH}_hIDeVx;4Ffw!jZ}WSrBue& zAe$9tttHGA#;(8(=ytCn8QTL0W^8@LGv*zK(6!Fw%z<|mD$DC2RWP8(J`%Y*%o*FB z=#8Ef*|hxd$KK21DAMw*}2&gUTGS92g0EuUM|v;%stB;+)^HT$00Y ze5vcE)=I1AKgc`*UkVRYhQ}>y&YUAx4I5{PQ|!8;mKK(=-fxhg39N7>iro0TY{8{S z_#NKox^mu3yrQ^F746C-Z`1FD;CZQbsKPTSAbnhDvdP} zT;*MWPS~7f!z&hOM`hKATG~};YM>x=xlFvy_aItlB4$^`G~seF>236HdKrVS$}4*p z4mByg4Gp$`$uh}7FIlU&qnDwJKI;cYKu%x6Or0M6)0=Ai-*=~YYF$yk@m;ob-ysoA z61iRv!?Jz9j;_0eE9}(nG_E(Hr3i#>pjI(P1aQ{hi6TiVF!?kiBC{PF5t=0SJ5{{^ z|7de&4uEgl4bGNN-r2H!6t%q>A`h^oPr6E9Rm}<_E3mc?$Lo_vc&b#+r-kxw**@@r z?if1PvPR1F5rJN#ukOIcMuxzVbIbxP)aAk5WQ>(k0Qvy_(Nnht1}vfiYS95duH#~& zX`y>%yjwRQK?26Pty&w4`|O_P@#$4-{#acltDbk2Qu}A3|4yf(P&S9bc0KLO(Zstn8DS1>wTf$%!I#2-mG8*|0)2H{U>fFpP^FL{em*(} z9S`JEbeQ$9fwDuhOrT_xM+nLauS7HDWT>CcA8H+<0b$?vd>(xiicoIX-YF=|6 z9vqCv2N?KKT^I^A8~vHEED~|3ms*c8mkQLX$`s`~Pg6VIUT#So-(FFfY@MQ)TPj7B zDpXcHG7~O~BnIn5wg*WPo^mK25Ch|3Qb}@%e;h~4B7|p{r&E@Se05ZIqhGJ}Yo2%- z!Y^}1q9V^(rjpeI>Rct|x{}$eR$r9LT?XdUQlAcVU zrEzR?ARB)A<8Px1ZsG8!G5q5XPrrWpoBfYJK7D$4_~YMRe|vLv@els=w;!`2y~Y3f z?eIr-@sGexo<9A$8OUM1)igeysgcM>NctC0t1h4uOQ3DeOC_XACRS?c53tqysMwvw zTk9iua1A>&e&xeOr*w7%F0*qUn8IkEt|u-imM%q3^A(AWX>8S_{)JNjW9OoOFus`Q zyJRt@EapYpqmTwX?^5I*iqx9ow>G8%AM-nLO;K|i-Qq2&Bqsu^IC-{Do)|xP%4nA@ z5KYk#2q9vo>1{j0G1bh!=t2!RtB0nOr&>>Z4Qj%JFCPgm0>FP|ue6Tibw+hb$#B^i zED_fd{_HSqq>2)7&ot#^W+V|2O#;g0k-8QGo9DuwRqF`UTS$KqwLdV6Cem;4rpsl* zBPfUuH&s~Xa@*WMP2cxCEmBfQy~#ypp7Hi-PE0w#dt1vNilguhI;`K{yL5s{Ny*9- z1R16FdMOLIY?v9K{_E{g%)q^0s++Q`h`wMq9wTT!bPsfoLD(_&iEoXG;yI zTe#VXw;<{EqBmmV~su5!#8C!3|MJDHM%djUz>E8m?ca>1`8|8_G9!+D(>{={3#>fNp!TfyJu!E z2z^@14wZO)%@~U@KPF*i_j!@YBQkwzLj>D97BiCR3qpzBfL5fOX#ztXn_Q$sW^^eR z0^lVhx}*_T9vsJgg3j(R!{L!i(C}*cKwZ(Z$}B0mb#7onp?6Y z@hYR;sk<0gC%G`*9~cLr#Q*obDJOoCBdFRg4xcO(sS)EmJm6kh%rMWUoB zqUg0iD~9Rsch&y6Vd*lVDm<3s$q$az-0{BZGflI2L=(O3uD(C(iWDeMYAqD&#n_3D-d%}!vSiWN;Pbr0 zi!ouD$~!7-U5VHMFUK+G?4LGz0+y__+eTEetTSNQX2WJe864{wxTiEt4#8T8#Se~% z5(4-R@Ir5&rx{`Ixr9n6NLyGMioxckf}%gEcY-Z^CAz0Xs zf;MkJ14>pVY6z_b#;=pHh)GHF)e&Htab;toF{0K#R=ol#%@92-XOYOH%8LkAijrwA zZdq2QCTMAJ<5L2OjS9e(u@S5gx*6nbxqmf@PU% zg{1U&{gjn8O^><(#w`hX(kggJ4?ji96pg8(y&|(;(-Jw$M7#jOoJg3koXzqkAeP*P zXa;_)K~EH@zR zU>Q1lrN&s%d@;a9l)}g*Eq%Kt-^WFVcyO;p9OlnkgKn!{u97b>xkrL|3m*t_JEYKy zTF1C03wpP`rMc2xM3~2xC~?s>ey)ZPQ6Hz&Q`_4x^}9t5C*j|Dz_3&<4gh~ zI{LZB(`nJG&`s&CIO+{Z}bk)!&X1Te*@eO z<9I6}+HfLw@TJ}{Aze^HZmX>~ zkj{F{jGMKLO@P%4o@cen4>ik>98KG5vvi?1ti|P#&N4h^1FCm$PjzGW!Oj$Vcd&uM z9lHfklRB^5MQ_rqR@MWdLT5l~!0EzGf-7}hus`Up9TDDcp>-Q2naD;aS-*7N7`@v@ z5Ti|jW?`D0QQBa%Zuh5Gmxf6nW+FqjhtGy7ZfHqsyxd?nrS@U-$wMWqQh74Im5eqy@#%$;!Ay*>6kSC=oS};kZ2-@z1@JuS2Bo$V-*#H7GPUA~*711yj z?quH^3+OVT#zZwMvIs*RxmxuJ(%ZNwfK@}Hm5$HQlTcX~!dR-^NMM;;p%h(M;_|IG~CnV%rgFZLWfq9UO!m(Q27_wytNolRm6|3p=LkI_lF> z_m!-~0IP@))@h+cN)?a50?22McP%YZDr_u8*bxO)1e7JSSmsPNYpa3#ew~oc?!u*U zXb`qaC7rMoqS439fNwB_1U%_E3p|{uzSL2G!JXNf5T(o18SutJhG_|ZAU}W)-RTL< zt4Sj6B~kaHw0fcT7q(n8!;R$8+I7J6tJO=rlu;Lsc-bR#ccjnyJxdb(|2~f0VfyyW zp3gJP?&s2)CQWk9VVsNQGE`C0d;RJ9>>&&!zhe@Dfo#%`51zqobC$83DsX(73Z8sM z=*CcMM`3G1xZ41o%yvn8a%yMUO%GsB)H}c)-oWL8!cb4icLoB!lrLpoc8h%=ObJ38 z0@?@Ab7M>dV{&+!EEmp~0|iUTo66MHI#t|p2laTC3n|~&ZrjWtb_$^e20P8+CO+K6 zo2ED1#GBgZa1(cU*w#}mwX?c$(V6tt2bjDO$^HGaz?IMB<@&_8{p#ol4g>N>?P$%s7J_dmC7juP<8eAgq~i=lA)XBP7MJ}?JwP@ z2@GB4?2d~fj1DG^p-`w{NM-Ko3qCKAU2Ui`gx=(F_-z^YV}y#U{w3sUZ--~bk}b1i$#xmCc_)h zZ(;^Eyh1T2W!~_F-bbQVb@4c*8J)9H+3l!G4)UTeyRE0Qf|RNF!}<7p&&Rbb^RRY8 zP%?)vZkboOc|vMMa_#r}=;R{hN--PeT%HVwPl*HT>$%PF1aP2WTr+!LVr@DX0Jv%2 zb59n=def?QMtHhJChe!bpFy*MF| zB6!P{WOOxF3>K~Fu%T`>-A><}hu@=mgfWs0OQBi!rbz_%NRWJGAHXTfu+h?5Pj1;N z+&D-~;I)AYw%uaJK@HxlN0;3tee137=EN2RU*)zoC@WkWz@iBQ3UV8kST{=o8RVL5 zB7E{>8@m@ayJkC(pUJ)ed?OUR|3MZBr@zAz8UG#!zZzzFL8XnJSc3kT^W+UZ}j`J$q~zpILB`*H&4f_H6DMXNzX(0gSM-`g~ z1X|Tw%JMl>hl_&Xud3ElMq%4>U-t{Yx2=zP5nDF^R{d$aX~H$+zL(VgGY3WJ6(l-+ zn49euYr3hq;eb)=Na<2@o&T$q3phakug5)>=WE zHwn`l;mLsiRiU(Wt`m6bMmsH zu}u%ftmX26KnkfzTMp{+v|gjlIGCXDl<_T#W^qE*NTxs~y5w2Zx3}>J0I1`~u$rZz6 zD-~7j2=NeSiG@uG7DB0r`OHKRP8?_G5f+m$^&s3L%t75+^6t-xivM3RyeN1j)}tAl z<0!Y^qn=302a=>+2urp#;CexIds_glR%$b#fQ)UtRbTlY_^FtYmtioHPLDiArin=d zLJ^)tNfBfG5VBCst3vqv79`zD8>wEkI-t#WDMO1w8GIAtg~W<-&qiT&7JAJ-Q1cGC zWRGBVJP!nXtMidrYY~e&AZt`NKfdJCoRPBy%`%oag_zMb);eaY6pa=r3nOatSph4mF2WM@+X#}RaFTwTla*fJuuj`W_woPu6XP#$lBLkWnKVIZTbHAQ;2)#We7aU9SLEiw+l1_P6)px8Gpvw!S&y_XwOQx`B7!-XFYC$AV zi$u|ki9#kT?{@l-i_k4x6ZRetLYR;(AFm5rNZaC(U9iq;gAGurGH&DJQu3BX^VoY9 znPMx;q?q&8(pVo955I!;E+iEB_AjM_guknZ*bbup(^La)8sX_l%$6+csOKdz^0L~e zNRXXrZ*aUmK2-C8MHH(^4CjWs5J@Zvd#`fp{dgqOR4cM&e=|ZeKkEv*I_ygsh(BRv z03q%>GL+Xm4EB{e6>5JDyJ6adc7)llxC}R zK*?t$^9L^0M_0?{wf4Ej3HXnGiZ$G62I=_N!4V`PSBsHJRz3y=^7!}gf1TC_yi}dWvB}xQda0&5 z!`Oj%J%ZGpBU14A>n^KUFO{d$c(IMU@S)+~XHc*=3^vF-yJpcae>6&#W2tDjX0%u^ z%$YUP)RwZdA27ys4|mJ&xJ~3Fd68x9ix#AsQAH(FUFNvDepbDsxrcj3%unEdVlkl| zEW^~EC8NI>H7~36G81`87z;P#=f5_*7QU@%oU^E-`f`ZkUyzGT z7CEaxf!03LwTA<5E#v}g&C3_|e&X%8rwMc2OzG`=-@C#4u^@Ztx=pm}mhmw5(k0Cm zT(`nEXZea{h0Y(cU3cf@%qH~I{oq`d5BV?IWeqtkh1?zZvczQ}$1%G*mN9*H1VaeP z)~(NzO7i5{lQEg_lqWP#upn4j70o{9H;Ie990&xpojf6jB8L%0G$RSKg~beqS>dgq z*dE!t`Fk1nQe6)4c}JwW=B2Ci%30NME&2BBVobg>R~aLgk*n$X>pcxmd`Q9u z9o-!)1R?i%zYJKr>u>rTJ_f=3_Jr=63GS$^bt-AO)%C^M>CMIY3-Yb`0(X_?F}V?yx!7~zDO3Ioph!8kBV@va zW)+SiAP>-tTxMLADA#`7FG@sa!bfzk##%k+5bnb=^AY`X?JJ&68M#7G*K+8shK8Wq zl!ZVIza*3xw6-u2I-jb)qe$vB6lbEe(-4s(Se}8JACC{lkdtvRKCp^dVuK~Uof_Fx z4aUW9s&Bk8A#ZJRt6yW|8}XWDm}LcF+33V(Jdogl*3GovG!xvn{$BPVf>t zhtpmIuV3vx4m^!LMqwbn%ork(riezfC`+D&6}T)p$8K}z=B%Ad-_2bS1!&fKWY_qs zeSGbPTK?Ijr29h`;93Lxsuyy!f!LC^9+VMj$K{}ouP;u|Utf&Vco!DMvUn+Y<}cB( zhNEp?v3tk(uD-Z%i2;EWig;3kQX3$?}wA?oUly^xu(MHOYq~@UBW^dBob) z%ytz1PBy4H(v@d(*VooQU1x&JoQ+EKnHf(QRPVO3w%vD|Ri++mnX`60d^=gKh4MMN zSZlXnLg&)dtWAe1kB+KV1k*hS} z*=+@4G!v#sA|v_-yMJ>={rLs!(qX3^urv2py{yb&P}b+{ua-@iU=Q~)|KthGu0v!0 z$TJVP+V8+~xr`(aW4Q2WjoW7Aa&k7gtouc_jGqn&p!Qyn+x#MJ!W3q+A6+$C(B)~N z7Ux_-5yo%@W#dgJ2Lq9UjW>dDO=#~%t@=3o>**Dtg{Gb|FpWEE2GtxVnB&}ghUUE; z=4X3&h}rEoe7@$#>ae_1wS^zc-=`T_Z$FmbfAs-E@pqIoaH9WJd{;cUVekx zTcru5kV>R@!oZBlJL^`w0b}0fY_r2EFBI-TK=aE?-%7d)H`n8zk4VPu&6v1-pZupW zD+1a_@|PFE@Z}X$p+$T(wxUR68H>6=bHkS8mvFYnf63M?r8w39E@*<}d$SKSU!DBn zqpQ7U2U;z|o6=rV$c|_)nwgoYoMuw5TisuBV4R!>En4RJa(&$KU%_wW6_*g2(}hP54|8c?VL@}d(w9!Y{F%ma{sLw2 z%xF@1XTrFJVxU>+#XeN4rNKJ7IlF>hB>D57PC%3U;`yIWyo>)~^GcE8 z1OM$mTD|z2!^6D2?uWna<)Y7jX&dTD3%l0eTq194DDWM;`hjypRZLnF_=Pb^=={y( z=U1oST)YChVRVbfBB>TeE8iyWG5plFov+Si%R-%Y*_yj9BX3?#YM#0ck6gP2W!+P^ z>(wcnwNl`L?43^i>Dnb#69vHG^6^93<~$Q7AITl1XO7h9@*MSdM0G0`I|tVVQM6le z$@^qr9^msenNP0Szh?YQ%8=S?>$~ow?yQ{*z9Tb_ZQ1;?sT5HdhiiP*ln}C2 zlq&w`IaVfv%@#Cjs*PmhXlUx7X|?a~{nZ;|7$Mkb2PI+f}!r5(=b=ag2zPm}(xM|BzZNMzFwTbf^< z8x7NDqDiyKNWd;5n>+2nC3<{$zV*Qrxw;p5pCTRgSJonE%x~*Sr0utn$9485aQ4>l z_S(*%Un4Frtv!#R-w#sy-PVra<2riVOUaI^45irg&StBs3_IUHaLf1LIrar($j}Q{ zaU2RpE9WddG#9a>*_xB&;-63|G>nPZr_p$iAXQ1CS4Ctl43;UZ&>n7&5KSpvA`Co< z#eH^9^VkCx9hByP8?mb_W1$s|KQVB* z5-V2Wh+sk7j^z>gKa;m_syeCWS8vCr620CdB{|3B+zG#PZGBWNIuVwjT=Hz@l(yy4 zat2#)gwGcifL!mfgm+Yu2XhMAg>B6n46}sql#&Fda6}S*%g9Uu^teBl#MV&Gnba!3 zsa}mAs&kdH**AG1)i*^JCjo_HYqNcFw$TH|_L|fT3uhfv ze^{^4yD?P8RaJlQDoESSj|KE52juJRGoJq{^kZZxRP z@i?pR)>00sb^BA+&|f)vef*50XlhZlLWi2slpSeqYYQCLs|89v&sdK3m@v>Nx`x>X zB>Wrd&|1dmQ!fdf&jDTJE@YK5%_tK7YgNN=!N}FcYlr~{l~}7{ie;GG7*-QpZ@@|o z9usmQg@THHx@WUMvMdY^!6c(;A|ih|0c?>-4W>vI6L^U%7>zCZv{s9jOBRm_In#%6 z#v`hNzzp@xjLM34!a|3(i z_UM&yYD|sw4Nes_Nme8FUbAO@$3ldKZJ5$(@DP|f!cD!*nZmj~PjDWYfQ*bM@MvJy z09;Tgta{MaF}q{=%Jd!7Z-0QpZSgnzAaDkENj&hC4H%p~`t1!>v<1f5Lm+VmVsZBB z@s=4zb5FROJAmVDht2t5=$sFP&-oAtox5Ok?yz9n z;B-D5Qs+Zqbv^`I=L6z(J_usxgJ5>^V#B+o5Z`N7aU9}Lg)fe}3)0@E`b)w55R2LE#pxSm^_i{3BnhVB^_H;%3G&TBW} zR#r2DrXy3O+T1EzuBOg4Hn3;CD{5E!ux0Cc+=VSS-k*kAYUc$zBC*I0lrirtMU2It zie>l}-wEN@!gO;n4&_t5eY1m5?o96}Pv|tU_k0L-LK`0>AKT!nrjLb=Tb30XG6B2> z>QlHMtkyh~(UgO0v_HNv=EwTJ=s#X1jH%{VmWw8%PoJ&U<$JAKS2%IiYWwUOZXS2W zeFRWUPSvhlh+x=l0R~cZ{!L+~4|1le#d<8(ILCZF-jDI@#uQ4yDkn2jXx{)*HVizl zcbo26WEDth&Ca|3^$J%?y#c!GUk{Rj=pcVhmpxrd<$ErU1EEtELqPWQN))Kgg@^q?O5azq%77Fq=xfszxrr?kRpHOq57+0s|GE_ZrRjX0at379Z` zmommmnjoXl9zn;f|2R3}{O;oh>9+?++IJh5}FQpff2Frgkr z=6s_MJpoq>Qurk^#g(bG>p4m43m%1XERMjurboLI%EdkkE$UD;$yBk#;X;W4JzFLv z0y2}7=C>>^HLK87y3)ie%@es5dJu(2SG5A0;fCCU@%RAcWUENLk=4az|nHsHumS9w&ymxvzezkO?@mmTV2%CKy2W~VIM$@N)2?V zj9Tes5Ne5^!UQ;j2++@{5Y8TnL+2`wNJ^7WY2Z_Q2~9wk<7*q1N>+&48UnBm!smflEzb>sxRl zu-lNb6xcqTsnNb_VNc33b2UvqaDpNynn%Os$Ew>{zAjAo_cMF-+W5Oz!)D6UV zYl8D;i#@@4;*+%~T%tw631A}U)wI6vqy7Zn=(oQ-Z4B+D7L`>TN3hs%7gPz8vQP=% z3LG@tDP;18fzdqRg$(-dD0Z~o>fHU>NsIduKUM7A4&NxD|OehS7IYLu> zFGnIN!-(Cy?A*yd>!mwx8*5d`hn<2}Odpm?GFq^d_SUm>d#P9E%BYuBBr{PgPbCqv z{=pw^#@ds+XMxd{Rogm1VrHkZ6KSuU0@7%+ge!K^_wqxWY5=7LO*8vDT)PiMmhGZH zN>cV#lzWDA?Cy)kwwpz!!4hMwE4^09XNftoYER$GmCq7mttA)U{Y|#+g$ZWkg;B*g7~Ej}Q8+PKjM$u5E>yjs8qn7KzyA0L^&bbC6m| zuWSIWMGM$uY3u2KVsL(aCb*SowHetjcUG7yHMy+%A5dj$y_Jw8MqjQn#uJ`$lMSZX zDU&_@42RgkI9e7*rn7O%QjxEY$`199Jq24e8q*s@8l78VKBvq>~ zXo+ly7P3+Y>#aY|N+eo&RGDM+Us-lX?r83<-_+E$R<2`y$7TP_2%4?l_Bt8_Yiw@C z*z0d^6=GeF3Nk7J+TBH=mW491s`pYFR%h`y%F}sN-j{lJsqa_My7L<* z@8M5>{B88~>%+qzPe%Xok3Sy%G=_it;px{;f3yGb$EQyZ4}bjI>u+zaF8;xv{`O;5 zq__BAza9R_F8&eN$K_GBR z(Cq#L9o+4%f1RlOZR7hWx4-`GmdD~k>Crised%dq#+O>m%jUg4N+=Cl-dxb)~C-Q zN^OWz8=};PD7BC_M5zr?YI}%M!%JI3tXkmY3_e#CsP?ZErzZ6P;y@k028-Bf6p12J zH{zBB1fZ(@muUM`>G4>n8hqEe+b-ShMOoo z+c7yMIg9v`n=%O$|Bk!P6h02+w_Hh<%up+$SyeVUcP_=|a=62gQ5^z~a?d3rav_Q& zCOP}3@kfU;e4PJ{Gn!h|5RkZ>Lcf91B{$eJ55cF{`z6m|vrkwgvUnTA)kYtc$^`DP zU=vGR0ZlJK|zEuCiWsA`{h{v}z6BA4Uyem*57$TN5oO-lBDtw_SCROHzS zNqJVN&=LjCmE})g_Be*&mz6Dx8n_n}_;-0r39j5q;KHo;+fr~`O82&6L|6ZG7fsj{c11!!|HT#ve%>DU1RI>QIhQK$ zLlJzTzBx^)B$1P}NEBZtC3U=$xVD`cDd~DgsAGDkh%AGUu1m`O%=fSy1IrZP3_t`u zThg4CoVKY>8MuiN~uEj z0F2ZRckv&IrvIGZMJz-Y)qU>u{}Di4|v&Wr<~2FVMsOy`82NcuxlC%Nxp$nW#pb(~M*`mr z-`&8gFReAjS+z{m@K*UHBAH|O8UF1wMVRv$AJ9Y<`r zI5xd+&y4m=o>XL9i98&R=)k^#QZN!pVz6SwjGVFMqU^Fa>h+%I9}sd9ctH+vA}xjR zx+K;hn}tZ~#|r*}r^?wl`Lv^=-XqyYdxm6R2ORV>`>Yw@*TG-^U}WmXS&wq2I#NK^ zq1MeSa6fk(jkHh;mMI>gWmk3w9J;KFT+Z-T5wjSuZ*m?pNlN}yZ9;FC8Bab0bwk}Z zAr9PruGWZCgzV0K6A%4qzxndZ{_A8nTXpR1p_{K{YK>MdW0?B zRcOztV)ry$W}D|N`CxW5j&mlZ<)CJlc&B)JbxFR(u6D;M&xJzu^IW8qdjl2LF?4G$ z+DW+rHe@naWk1ApG!wZ~;?<+n76?plA0#a0-g>ly9`woUK~g3c8J#A6 zPiuWcm#>hQ37r!L|8X31ssEo$Ca=JgQe@^X;H3ii77YI_GfI&u?0!~}(e(oUcHSX8 zi!C6iQ$Nl;(HX2x%k&!zHkvYWv*6L~mE$}Sx&B+`{3+Oo)8oCMeP*V0)yHt)ykjMn zRm>)aO7I(%QbfEH2pIMPSh*@O7K0eueRiPigij3hs0Jx)SKmBA{j9^0x;(SA*?BU! zue!4R8(c+ioCG{OsdAh4@e|oxGqc!M?Oag21(nO)9Z3$7${rL*-FkERcPSJ$)CRba!k&7 z4gf>EM9{8W7Bpv|jAE7(+)2CHf>Gi?4aq;pg@&k+JQR#oRAy>g0!8C_UzL3h@0K<2^Tk_o~xO?&2-R+*!I?WRBc zK_6p!FGI#b^-iOC#@3DMR4+_CW>r#M3C#l^_;yUf_Gz~b0lz9*RgES)-P_5U6Se%a zRkV$dwB3n1_)hoXJKbkA>BsZp#h<4aPu6N{T4NkA4^Y#@Gz1|7xNmv>POmOWlrzX2 zrinyQ=(Tz_hTPs>jkaiYI;s>)mmN5`gA?zQcjE2EA+RV?nvL`c1~;A~RlO8g7b0U~ z&MJylu>3(vTaoc!3Kmi%jaE{5kG>Mh3Q*g@sus-9T!E8Ju5+;n4g|#S-BGa^4``mU19zM zHD$w^mAfIp$Q%o6m>U7k|3-J^a@lhE53DiX$)%RH-+NQc>PNtRqpmsLP2i_(^+r?W z05dpkGReo#9sPP^mxcDT)*DOTf5M~H<$3>@0esI&A6do==)aS1sO(^auW3f-Xk<8? zTz&KUsop7IJ-R$!_dvVa7JlH(?rDu{Bp&!2J8F62-c$E2XCM4Hlcf(oQYJk<7POv+HyQOvoR>FJ2u{AbG}R8J)3q==p3wB zvKA&=mL~#5s~XKco07IFO@2wG?>e{?&6}`sNPoS#xgtxN)08QeqZ`0CgzjO$P#BGw zFUhVpmRAP-(!7NS*%;Sp9HR(=f#ysuMJ9d4cET>3#+f@|QIT`CLT1>*r3ZQ-LcQvK zMz*7oVXHWmZqL-(tjKqnWJ(LtX+pDGzZeg^7wQKhb(7OfYR$opbc@MOpr=vURXJ-j z-L8A=#v-myms}-`N=8z8t2Lbt-t8oP7m}ej536iwjT>rbaA{Wp{YhW}>gaTui+c&% zP>@8>*gc%qMTEC|y41=N?G6gTbbZuJK?ZAijB$t1r7WiSUlJ}LK&DBTJHnbaVs^(8 zz30cN_?0IK9gBQ^%(BsU6Et-Ez^2E#$K$`^`8sZ0$9-(K_vT1w1sFZHeO%>stNnds zPnPayWIEc{u_CA7Qe|;fE+i8&x&Oqkl8?w6G0_v-%hJBo zVq%v?g$_1XGnlz!^RFfvgkqLq*p5DHBIon2thwFRwl=YcU(|D!bvX_vdZ}REds+Iw zPbO9B&sJ;y_sL}BWDiw~Pm^T(wclmY>Lc2nOaT^HJ>2JkY7wSDz}Yx{ZZH00@%k{)1Pm4s+jEO>DjmnN&CU8oAWw|hPOBuC=`yB-&7EERb%ESbLS(9gxNZYq% z_1R{+b;qHkJk#@fK~6j*Z4XUZ(mA`~DrvD@)W~^*sTuBFNWV^$e!qGA9Vb&IviF@x zFXqjw$@;a(qm^}EIwF>Ps;t$9VIUH#ap-sX3=)OPY|E}fzQNa^8K1wV-pY!b!^3wR zsQ$7%wRKT;7YtQ2pEJLw^mT~gq_=Kpkr8N{m1Rie2`lCVRiC*P&&LmeglAI zJy@yQ+F0p6U^?5>!Vj;5zpps5;L^0)*3l1v;9JuRA*#X8LLLv3s0V+QECULgjY%jD zHADhIIda?6E%Vj{z&5a*e{Ht2fvyoSpU1au$q<$~ptf-8TjpnDjsExNigVJ}2N zH^7;+PIiYI(6^hh0=0hl+4wm`)U$M{R!8JpQF<@xozuKKO8)eY@&qo1oA1UVDpHm? z`LlCT{*iXw-0d-+i-cx#xIZs%=f_dLS}Jj@5_$al$ERq zIZ8Oo)Xx#!MzcOU-B7HtKljhRi)y_bSy^vm(97L6=;ece?CNyT%Ll!D(8~w5)7}QX z{FC81A$mEeS*L|s_=76z$6*4R6F={a!g)A0xY-= z73B$(?Wmp;-CoZC{rRjm^Y(oou-~I1zhhTb!M9K{UL=YBcZ3}n4}%-nO&}n#FQKZ`qN4VOb7@h&*Qb*uAIVhw2v;PtR>vmmmV`Z(W{${Hw_=TTMD-Z-?Bq z)_*PL_G|U;6t@o~YPS%xfsDUfygp{p`T$u&fpiNKmi+}Kh{5jN5F#W)5v96>qhBxv z3B_#$VoKke#S3HvuZzp zy1~+{dBLh2uCl{QL?00ffTcN;{@{Nji9w=kc4=xZ0K$uFTs_ij>o;6>A7lR!9)VS^ z<)>2EHvS$E1K;pt%MVgfRjloux@>u^7cOTAW;zA($L6($aPKH7q|*A*wMFJzr(uIs znsc=%rsGJYj(|EgB~jJtc$$dmaY}W^T3o4|PYWe-iBVF=lFvspj}~09NEJCdrb|AG zM0TgOXpzZr8vkA{Su~t3!!M@M9h@$lck6yrhB9M zQzEXWJkzvV8HC`bR01?l*c>Tdi}sJ%G7&4ijW;)|KNiLp=%R>pIiI;kZe4nEm5?GG|y1Q+Ab4Ei;^|K^~FQgykxQmp%! z*>e*==+aRWDlu9eCUC<8a|t(Jv&lgKC2XdwpE7}E?W_td`R*ze(hj>-U$uKjcRlie zopwsBlu7Bqtg21=ul0;-^QtW?4Awii%*syVf4PAVBoaxLv}~Jk=S(d3`<7*<=y%Yh z-fq0MzKh2xQ#7WEj_2hw?aOIV-^wYHdus3@slM+POOfIApMz&h3rIdA@0z~yuKi*u z?T$%S*G;I7yf4)gLZSRQ8LsY<<@(UEcuY(&oP;S> zc3zebEgx$kX0y(nZ>OJSA{!}|r#yqyXHe4CwXKHPH7YmcP%RkGNkXSAktC%mGBw%Y zS|#$Q`_#~ErvdtzebiQ$bcVLYgxhr?&-s)Kn#ag=n)IZNA?v+N7F6#!1fT|hZ|r7F zE@-r{e?_d0rZH0@i!`NqW$M#!n-ktc6YWy)0`3S+v+R9SXcc_xH;B#q0@AHp6t-4w z3=xGpOulg&euz=SC)#T<)CPTNQ}OfKhT2+Y7TJDwj?+)@d3I+1umr}tNxahpJD9Xb zQ?{E3mF5M#vxRUZ2VAEmiym_NevM>f^&6kceLtG*+xT0<_G*5vzCs^Q!)cpLxRL`s zbcvS?L##c)s;T9(dOUVU_EEidcqFUG;Pv$fO`q{~1+Dhk21OIbP8;+4UPICnqOg{rkV;|Nja9|K$0X`2Q!*zWnmblYe{me)__w>0@mJ5sCl+M4>6uZK$HxZJmS^*EByuLC z{>slyJMVX+(dc)T+y4cjOU~XamO;uv3F7h~~K{mM=*2L)fJqIaO>9C_wn%Bho-}_yy@x3BBvWl(E-p(cc<=e8tVZ zahVi3O*ZC$Y+N=`YWW!oc^%bNMqE{oGW`sE2r$1j@VK`IA~B( zHg3zX;XNy$_dEn*Sl|OJHc%4U2A*ACOm0fcRg*G_baxyx)e>leLGeffu#$@u`ea$W z6g*Q-xWxK>q?o2$Nl>kU*6QKa_4anl95COmJBTS@q{rdfH}wb_ z@j=3T)tXHK0t@cT#3tbBJpjeZlNDilo5Dq>TrI|pB~bw$klK6G|!%Xz4f621`>Fvka3pBM-#1n zFj^C~I-=t13v2XB{ii9bpIDKveX+GmLME&p^i+@a|IzmOk3+kN_xJZ}7Y|sl>QBe0 z(;e4JtRCBreg+C0H^qzDCUWv)IsTnLT=rzX{QL3`Blv@FB>I8&2=A~kf0%fW$ZnUz zz`G7Co(bCrYm~P~V!w_^yQPAWn zI72Zw0j);S(`rai=q?6tGN7Ou0t1zRwjBe2D_W;!vp(R?EHCzc+29d(8_ia4y&yI= z*dh#`ZnPySHK+LRS2k+dB-~qd$xn)!KZIv$B=Qk{-Z8~Cw^>uXKfeaCL7PTmFoc`MDpF z|HjzzCA|1jlBhO4Gr6;vZNTs^H^B#D-0?!Tvy~!SQ$#u?}cxoK-*HdLO`;)yC^|Kp_YY0u*_ab zbSYCgj}V5TQ{HxTyI*M>l8&WWdF#&s%izz!{E%n~eI56WUk~-#e;DuN$M(-WbX)iT z$tb5whWh({ynh}3zb{X|{OaUSYyQ8JFHZ*l-v?=r$N#sv^ZrG=fAxudEZ^S*Mu7!{ z);xdiPaA%}+UQ%nelnWP*F1jiYrXt^G+8d__Fg}CE~QfJpd8jvEO3xN<0$K33R=6E z&vP~hV>^vQGf6leh81D!w4oz7MZ16 z_anM-ezjw}Mqi8!K!(wFzKK60xKIm`^Iu!?|LO)5_gM!aqL7S;jIcYFuk2vViuVP{ zaxjpaFDGkNSunw$+6pMt*19k*JqO$!K&woF$+j!Yg1G}g>ej0lHG5(@FU#JNF58xK z@uO4rDdiGyY4n$Hd93AI_FTFpfKJ^dTed9M;$Q8qQZ2jJcW5im(rF6lTb89g_p%Oa z=)ys)7kgbT*%*#e2_O|w$woYr3^FF~TFp#9W#p$P-%!b;pPpFM73%n=jr-)M%oa#_ zp`T|oXZcSpxIo@r;~^jYwaD4z9EC+!@W*NGVMoWb0UPiD*cEsLm@tt(VNGjVlOugH z%i0HoFw`VxD(CDDfN)Vvm`atv)*6Y2!~Xs?s3WZ899#JN>i_k7pzhBy{yC~HANAIm zC|BEbubEniR*PDlLH-{0=GvxBp@SpBIB3W z_hpui3V96EKBkdA1EZMd5Td?1Hkq#@#AnrW9>J5NQd=zl9v|8!N5lgjHP)-{3bYP2)OlokoYF@fWA_&I1L0+gw^_f%~o3 zTtfWZ9mVV(q?BFnLUut$^R6Oxk5vbHfC6?w^a?3n_YkG)f(qA#l&$N=jBYPk7vLc2 zU#^a2cRUwa%Ca!dpwAP^Pv_088Ix9kfeXIe z5gNxilky0y51#dR>2~0`ilV zL+GE&(AIWYqsaY&N44ljh`a+)P2*az7|{Jb#ZXnNbU@s3KOtC}xWHy@oUagLD!odt z8JVzlO1GEoE7%m%a*Tmm2^6?+UIVq{G`XiMSu&)Dj4D+uqy1GVeCu(jP=42PeM#D` zpZ%T8Q+;LBbL_aMePf>kH6Tw*bdo(9(Uw%o`_}k}x}NgRKK!oaq)qm;s_?PLc4oV3 zGMqK56NBaKv1*Nj<*YXHPr`CmNS3$nwzYF_AKt^cw_OeA-fhFVAI|-UIQO3JB9X}= z2{j>~)t_##A>$Jt(1=_gk~5;Vk$AnO*!F8iMiTF4M@|xPKT)&|f_o#3JrVar>gO#P zawdyJNpdKix!vF2k8yN*Zgk0yXN)R+VVncP-JFl+$e_+-*k2z zQ$-_|Db|`e-(ch-=jB^)A^_w@Yrh5(ASS#cYrc?JAUF{bO>D2HbCxNm@2!2=EThwe z{q^SN>Uq$tey|%$m{$qB>#-WOP7Hka`>`6$M3RX6Yle}JVU>H%Y^+9kSoBP)oEGd8 z|MW2XZxCKY5EB+U}ciWXwI+(RCVp**6K$t^e$TFTy~7ivz0P&m5cXb zb*i4cGf%(6RGCPmDIH0+q&dY}VL3(;QAHvTS22>DTwci|a&~!sO>D;H5s5{re*$`8 zPbTSX$_RTOB}MFEZIBJM)Fb5Cd<^kQSzPZjqFMQU2#mtAYMv4?KV{@nSw1M&N}b(3 zAAkA9_+;F6qYa>c%wDtggr=(XD+3WAr|KI`#H`Gz(p>x)gY=h_=a9@d;|VZ95o7Js zb}KBI^E;MVFMxkP?l~4uXcaGv6P8gj7-IW;eBw*hs#=iE7$cth+_dd0e$CfIcHNLC zzNX7{NNkwna|^e6Aj#$VUew>nez2F8(>P4ncP8Sd3wO^plc&(7^M=lbgemkJ^RY^O zsm6B=i1bk;NuV;mF|0?YkdrNx@}N#b6nD~>=AjBa2~py!EsuJ9(6k(vd7v&&dg${c zP^TvWdOZo(?a9{m1OgRUd>I(FtvTb?j0R1t=CQGoeb@)cwl)GXu&2+AJ#E++g61{w zroNvDNh2d|$|1hyRS>(BHLp%+Qq%^ssPDnSEV>*02D51Ew;sq1V|~Y0G^f6L)!P)B z3|#g#c2*zJyY;jyFxf5jZ%nb=zoC^Nbh%_%Z0ijvamzCD-SsOk9}6Nl)S(Iw`Z{=r zFTAT2T{2u2f5P(ZKH~{7ek+`Y%{1{%rK1Wt3120DZAkNs9@lqyc>3tUJACE()y6u1 zcl`*E~M0U2fwE+Oy{a`0UGEq!&HQ5j=5R!q!&}ovj7o zckQ?7%B^QW?r6^GG+~k)YJKkKtLwLa|DPk!?I!j=Xcm(qM@uh>9=J9x6@SHF+S9WL0ZGbL>O;^y?SL$KyZlP#>Uw1%?}2*&Sg7 zu!@4&f*4QAr-_Sikw_tncx&ZH>FuA-`x0w zhb%K;T=roBIHwNkAif696mx`41vxyoP+!h>ZZBsd|ZyU~lM<6RYpI|&pJ5tMLjS<{E)0H8kxrZq_M^gM#Xc-K3q}wM9-mE!7i|jF| z$yc6)Hl?vP%<zF#ptdA1Q@@9z>dfnj9l}#O z+7O;H_!aD|7{PAZ6Ec^AUgD#&H9wAI6ZqsZp?eF^(yqf zZ+CSq81zzNif$IK7|@5vfIiflUIY5je&hZ+P&L{-u3MsM#C*;bP2NU~W})S)&zs(9 zC}QnC*D&IalG0n3vThn)7c64q+5xL{dpnTiJQvG^DM?PRF303%!DYidT5If7Fihm~ z>h4PmxJ0Ha8?|wy|7~lm=sk|{Xo&C9*k^^pwC94`W@`vN^PS-vA4LIacP()ND}-#x z6ruQ-)I44B)YGXX@31b(yP*#CaD^JKP&?9ZxI(plYp6s0SnE*lt4^Qy`;WiG-qyn( ziqn1E;&l5gtX5Z~+V!S21q&3ZntHgKJ~h;6^UZdMk{H~OpP3tSh>~#I5GBzz*da<{ zU{6Do1hh|#Hw`t~oHkh49xt04ENq{bg>9(O);8{;Mq9&ULyb0UpG=K5(VkbcA!EVZ zr^PBdol~ZA9!VstcK|$7Y~GS#J(OV^%CHS(*oHD}Lm9R{^cu>rIc+Gz);f((z6{$n zz5^*qY?>SXo`mFZC&rtGiaUdP=Z<~gpPv!`9Mn6v4eDLn zU z(J;_GmA|M4gHM(_Bl7z1S|NBd@zUOGO z^v&eR6I$|o{N;amB1`VoKcEspn4>g$D5o(jKxdHTAsQ@8vOj07wb3wnofb1ZuP;vDERWtVm~`hcfg z>U%~r3vu+5HInUDXJFCu-6~{75tH(7?HAs<-e8~Bs|Ix)f0&Tdt4rSe-+s{66x{j50i_t=aylfUR_?}VTzdk zkDoW#_jsYww)`7HkSM^{Vi6T7%dk+-PGi7M;lpi*rsGz%E~L#c4JWQgnCcq}(%Ns) zs*C8>2g3}f-Nwa0I~!=P9X8s4EVv$ay^u_}kdz8d0?*Glv5tGQmmlo^R@{;t1`+zw z8st&MYQGqls@D-hV1o*!Hx{E%#mZ)K#I{ysrJS|XyGS}fWIyoTI$w)xZyh;R#(w2o zu%Jx#zf=@_&gP+2Rd0-{h_7t@L6NZ2>o;MV>JJlgEfPjfuiqRInh~m*VMZ0dW9Yrt zTia1zJIcvLjbIBUyn;cvHwK6G?74<~vi}&L=%*U`Y|qkD zdBGf$O@r0Yx9l^Xuu$9BOZV9Z;}`@8d^qcvI~->;4t;_Lv^JzYTZZ&7b}|j=4g#-{ zubj28E+kv|0_joGVN9-BB+`^+v9~HafdD%?l_Dt=(~8hSj>!)T-I-P%L6E~=iRRcx z)}diYG}3dU%2Rk_nG1cqNlI^-B$*)dg61?+Ot0ru6j(6SNCSd3XGIPyGxvj?tX;+Q zh`A(EiJZk_*g&$3eo94=DHfwX(%NgayT)ZKsMMU=`2pFtY%P}2=XNnPQo;dJmkuPwy7tcgVn`_9vd_QLxbY_UytZ zbG|*h_<$nki8C5^Pc`{szCF9rqibA^t$i^hZ8;=c`C|4f=H`)K%0WK8o^Ios;jt-6GqMwQN*ras98Q*g$o6KY2lb+$kDfN-+ue*;^)(=t5=t2r#F{x-~4=beQ|#A z=H~MBRkOpasn$ZBUXdQzcn-q^tD~dWvTV*XhD20_61g7vv75G&%|}X%ppVfqp=u`b zw2fq^WC_-aUkWa{!oVrKG`qWzn|kGRGmx{fW4tN0ky&?V1tQPPj;eTIV~;f<2w-Al zDPnTRDZKq|tQTgCcT@QeFJe%nLXW%Ich|XijSB@bzNPGw5*SeA>}@lmI)?tmu4#I2j_01wl9!ofB!_DisniOd{ll3E!Q!+u;PDT|`u| z43vaz$N#Iy!s5X6xb?%OHMORg!C793f@BPWOA^JGNjvA%9l@;2M{C+>+eX;^NceSe zbD$f4(?i~$n!XU3$X{vp)TO2WwR)073n8RAhsa3}BuVf$*bqs=rPjh51_~;wNsYl* zHmU40XTgYXc*>-r=`v{4)zhFdlhlWFz z8MFbE@FRf|PW70zqzSV&d}AMn1D>aiV#tacue+sD3^nnN2HA~XBA(4R;;KzYvh7fZ zaQP*b@?PY9P~q$=epI*)E1OFl+hLOs8-PE24-QgnH~I}yto2(DB*hBJ@_wOH@9Zmk z<2Sa$f!~-m@SB0(G|lLf=Qo{Q4=~Fg7iM{YX&GQz2AGxsre%O>=|isprp0LkOv}T_ z?neaE@}R*CrXb3Km6kc6>oPWu{B}-Zys@8Z=)-^z)BN25A*Q1ZCB%msz1W5l;#GSj zWqOAa;zJ4XfjwaxN{E~GNO;qL5HqNE?$`(Z`5E!gLA`U^px(6&c2MsI_B5z>&^|HV zG$6zb2r<}(C?tOyqL4nPD5OstA%>75&~f?Q^(%})LSLIRLcnd9F%>P^0XqV+L-*^F z^e2-Rd6PSF%Y4yW#+K)~ST0$7gB0m1yLQF$k;&0bc*+$KCRYprLMkK~5@Ww)+O)f? zBzlY{1E=RyGx{Z@pQS7n`D#o~NXlpiHKyQm##7!-wI*-!SB>6e%$$H!{6Br!LbK*H z{pzHJCVc*F^NohI&*~|gYuPZ*s{YD;l0P?xW&wy=p$i(fH5{<2vjcK{F}We9SC{*) zT2oPfw^Yz2p8)p@ufhtIMj^n#7U+pYCb?9aL=dcJIbkc~6S-U8XC^eIvF;7cO8MtT zKtcw5!e;HfpQTH+ss)*FNvM=Xii|ohGOgV5xYRVFq>zf`p3oI@AWIyEYU+~ELWz_r zgDOR{xQf3K@j<(eadukMjP;9C$gd(w+%i+_jVf|{NAu&n$jUM=b=}9D3>rnE$kg%H zrDv&9G+Ml7ipEs6y7X(C;H_Ps)Vu{bSJ01(T2a=I^awz2_{$gnK0X0i`fe%WcTl?* z$kkEKM>o^KSGI#P7waALQ2lqpBZghKmp%(n=ra+As}83?DySG3vlaFK5P){q4MuaN}bngjn(@~S3L_b0Q4Jon8h4uWbBR2~`* zf(qJ@=ib%^elzf!<~Hz~)^B~BJa>eB`RmQi)$<^=&&m43S0S(Gx!(%tBO{t%uNi_= zgkg4^*=(h?FbE;-e5- zqkU6W-n1weQWBatAK1;j=!50on4N`EslJVfBYL;oQxz>h;N(n-BKKl85=`F&%R1Uk z#eMe1_WF)+nH{K2!y|m7g9p||KzJ5NED{FW7{~}iARFBo2d}_o2Bjcm-EyX$o{mgB&Sc0U&P7OVr$$0v11>9~H` zZ=F!E5uh?;?_|4%psdS2b^!Z|UoZE-IO*0H@UJHhNh#{tA0tsmwk<$)kpMhz2Fv>AhU>e{tc}Jq06Ny zr3ZQ(gr9zQ{mRS7f(R}nvFu^6Ja~sM)a$7kE{i{vu)&>PHJpYGb?{B)!x|D{T%H

B)Dzv27l5E-|C9)PE7xm1hp={7S6v2xIdeWc!HcINL}3 zhuS{nKic-G_5*I8Xg}umiLS$LpX)mE>;3r-{zjhx&-$5weag=Yumx?z5N%cOxn0!N zZc{@|K})z)przfiRqL_#j%u?z970N$jc@E)w|u+Ym7!vJ=ah!fk-<0ndHH6C&=I!{ zp(AaB9YRM2_B4c!K>Nga(@?Q|Q19Ha5B&2p;-7lm^XAI>ovM* zq)&0*f%ag8t7n2v?W2BdVDYveNU5O-!y{2uZl#8@ux?&IWPsuv;G9Y&9yjBoHIWsLT@@W z!&?5>%+i7^$UC&cDp^vt@PY_$f5>wp5-f?$-Tji^m;Z*ZNGDMSWCs7s^lXZ5p9`M9 z`0oN|mh|_8l>gq9^cPPw>1OWGD>HNb$wU2UK>x%~&{g2?cZ76fHS{09_x&*yy#J5S zeSchHxmS%p@%{A=Jsxd7v3lIboyzON6M=kdep}iPm8RG!&M%>At|Eng1$97W`JhJuRs_jI9G+}H>NBEFLF~U39m8D^yb(-latMZ#pBAZ zT)U_B@L*G$)ZrYdvGM4+?o>a`g-%n9fQBUgn>q%3?)EZ}912r0_ zR~c@ws=czL9qlXTRwyrVSHAj=`yf2&;?xz?3$nLpJ$wbVMow>)#vo8s^x$qn6U?HJ9jX$@QH3p*(fNNCs}uGK-=8^Gn6Tb>gp6;AGA z>&3mv3A=8*&`~a~eCRz^i`(j$F*o$rii73qJegJZU-LCaS}I|dXw_uD_2SCOe^|M= zEW05Lv&a;|f3Mm0U4%`w7Is&|LgFl|#@@>U!9LLKgVUW9#!YN&v(`9I=kX?BW#!d z>+TTZqWe#;eRTkhe+nd+^9pFC-6(jN)w^4(B!v|3U&{)QN2chSuOMW2O;f1Z2ZwaS z3oUmNK}-R@`d51CHoo5Y3KH1vaZReMBC>l$zOGsH`;WI4$wR};IfTF28Sht*VpgI# z<2|%uT+9LY@}szx2fLM{T%yAYOMFPK~(-`i%odg4SenS-VE_ z8FT^SKyA>EpHI;neI)OxBqxYa{d2#p7Nk7;JBv$dzwKg(qvqwJ^-?WIOPTGW+Wt%I zL7!D}6zmaP+?hGibjo=-zr1j#3g=oF3W`<#d|Maq&T|r~rq=Xi*UO$Ydi~>Puu*>e zT=!p2(KF6uk@e37jC;ciQL0>rV7XI*XDG+am~6^w%J8HuVC*TFoP$u*^Q*5rJcX25 zQBfD8*3mnZQR^7Bj{PTk;{AbETL%fuPnG%48td#Cx|r9D{9fe%)e6aX=a(1$i?>;ie&{4_bjK~Ee8-pnGj=wH{zOVh9)_z{reqPmnmi6DuyHuuJZ_`~)8OZj);nFMIu}6SNFH z5}dt<=8OSMA&3iJgE2SN_%U>cXJp4x2;Ev~nZ zj5m#$V@CC^-uIDz{*L(PsNU7vsNVH;c2w_1_B5(@&^|KWG-i$&Gsl=VW{&Z}nA zIcB?kn3-b`;Ld0quIUF!#O^+I2XKr26tnSt+c@^NbG&zoEWTeh`O*iYZ0uFZ2+mJPsn}7O2rCOACBe_ix|M=}NF1gFJg97?D^J%HLKY%Wwb5XK(fMw`P<^D0Qh#kD4uQTf~gH=EhueW3IU|*IeU)G1pw< zi80sQm}_p#H8WXAurvK3n9VKMc-YC# zG8D5WJtavBMc1zT9f$BYiNOp`#HHP9p_MtfsclSH7e*FbQLKnF4r5Ca(zhv+WnC9t zgIQ_rbBSb;=UgabTCMG0$`BW+In23rb#z`+k3c(stSE}3j9{s%6cfjWD|Z-JOLJ%^>3Xf$~eo@;dNs%v_vrob1m&lb^v^!F+rW2T5}3#R;-e5UYApB^(idKQ}icv z`z9yMrt!QX@6_4!Z2IK3lkoP#Po`(nvs?X#s!uS8*6$qQFK)ZsKYg}~32IU8e5R}a zF%LrD179us-7D{U#Gy}LX*e@7DwXwSNYa*@<~-fd72!otI@X_coT?@YNy1qQ2lpOR zg$kx<*KO7vVkc{mqAMa1FFNGL2glJARy2uALV8_)wj=+jp;adO1n>s#;%9ud;>;@G zvdd>3#{ug7sdcBuW^9idw%cx1Z@(!<&3@QDxoGCS0>)wpOpCZFY~PzC!cx)&GjJ!I zWo23tXO0Kq;X_Uaa8Qqucg21)>N@k%?%FahE#SB4kWDk}LV9BmvJE+9QM3BnwQcLk z>ggEV<6R7Qyp!15WW+?hW0);X6D@d7+qR9)DT*4;e4_0kBz~N4(n6?|XIkS!xkiWfZIHiHL)AtC8i3}l!iE(^1bzkcE#Q)bk2Q+?krbKMz*2R$ zebf*f9FAR$XDxI@BfSz46VWSl;S&wfU&kR#zoLt|9xxzt4$VbF?;Gg_a}G~?<+Anr zv-aKiCng(CSCLn^`(f@OOYSc^a_sL3y|2{wveb7Fr22-!kc%w+%7GkDNVD2tgVM+t5@K_IDfDvFH_8unyNvvsTEWQQ=xtFgaB}KAhK{PVlDB_|Wip>j3Zi8BdGT0T zgJ!&fa-|u3^~LTd+b83`;~$m8EhLU~OeQu2tPS^(!E?D8PmCg-O+pa#ld%dBKrvmq z&o9m5nsg<-)!6WR)O3H&f43x43HZ;)uX1Wg~&~(4=H#Csycz&nM{MbIP zt*PjI9l@Sn=HW)01u3e2?p;z2QJ#!x-8jhay=@AdOg7&4;EH`TBAEj0$e=1xMX2G2 z)v9ir^{X(UJ_l~7woUI;%@pf+F>`Sc-~72*d731GKP%hPOr$CXM-7ztN@~&KxJei@ zycZ)r%Qi0)*(UFoWE$(}@xRsLc@vcAEjd-gN7m7T2ekCAK7wfN;8BAQVzQY9g<()3 zUZ&^FKqes6ug9}2XZx@N8_UZ9VW75Q0lC`4AigAL-{+auk z(zzFtc%>?5CH>iIZmalQEsR*JQ`yL~nCrSPRLz@#rHP1F){-hT)i$vojFiG2&wASy zy%FZ%kAZT`q$uxuyi`x|CfDN21pgw|#4Sc(bRv;*@=bb^ABQ?e6)Hs`$c?l2kjJw1 zs!BNT0Z!AB5xMH@BxFxYuR>Z9rAOA>mnSrdH1W~7SA2>{ga;Ejm|sM`M*dB=l>2R_ zZ-W>%l$Af}IM>Ous&$`ZEh_uKT4%(l#JwVSmuKQ(-q$uY#rhefQT$odp}9y*RQG`c z`FK;=ihJLd;5smyd^)_5==bG2{|28THO~prz&5>rOYg=j#~chdU&_H2Jb7l43F~ay z81!g4>v(jdX1V?hvSxr8RN{`Yr&JtE%s?FGs%LJ~8S-H;zMM#>`ynP+BjKenTCFXl>?e1gSUv&E~9{K1Hy zkOi`6aPuh_hCn$MNMA;?6J9M9J3RyXQ`B}b1Nsg=m1{@UK00{sddB=<{fXM=ANt*h zX)1$GD-i}q{_+oeKG}`dJ=Sw+&>M`thP>>B^o3t3u7%|0cDc)5V6?_Q z9T4zS@DBVlbX29XXKoH7IYnM_d$DgK$y_9M4Y^O(0Du!4Dp6l}p$uPuC8m*O)yX`@dUb*rld6N|N+KRW9hQ~?cW|5Flotm2 z2o73Csq%`Moi2fXIiKEOpwMk!px~x0R}GJslmkzB2^4a&1TFWc@miIeBd+m z=?Dpdg~{5QAoX~{Fm?R+!qmGc&U5=o?Y1dj&K<*riKXHk*7wvW%o}sssgPGwFH3a& z0rY#Q&T9e}ZmN?0HSCIwF0sTQ1;zqhsV~Il3$WDm+!P7*FY`_VXDfZ)5sREpL$^mK zL1G>EM_V_li31x|FU^)aO4gs-X^6muUkzqQSo3{FfQ@2rvOij$~wRvhNMmTw+}3D?3Cu-?q01vjcCg;eGDiUlMtOx;*~Hj&5_Ih??x= zqR@@Xntyiv~!MTWeHJ@=MhEL&uz#^2D971lx#0WAP5IpThnG1Rq4?tskrBj;<3*5FeEG zj7B~n>7i3;i0jJNNMQ^Yb$RA8oZT!-D0hJ2wlFJ$20RJ7B;Yl^ClKpyidofCgopt` z*I(Eb6HR;e8>YY#uD_NEgN$0yo+T)2cbb_i_o2ddo~dHBx=X#AO%<+| zkK}`Ue+md4DaF}NSE<&vu6lq~p?NBw3ZqBq@8t(9)tvadW9e#v+Z=M96-S%8lx7J4 z>q`7km?>AYG~J-!a8rllTo~YC=1Vo-5`)1p2-S;|F3*ido!cTh1nx{tpLDQ#t2g1X zIPa=o`IOEvThvXJ>;%YVtH={yCC4-O$9C?jtjNQo(}PuX`UByqWkgLyT=N*`%h{|3 zbhApz8$exx2UmG>%>ucb5aGz5h=`aHU$lbiba}IircL(CcHu2U<3Nj@ga^_ zkXpD;HAwunN?S zj+K}p{JMY!-z8*N@ZB2klpU#N%|L|uRNjgEVgQX9)kuXQ+;xag1jl=-8w*V|9e}ie z!kcE+@BVtW>O})MP8?50gkQCw8R;^Nf?_ea&|=;BvIb}dyutl=6}Kft;~4pU=+_op z*S;D6n?Z)(kXr;di8tR?ignT-6_Lr>Wd@WFg3K0i67N zR%uD$yyBT@&hL+;B$v3T21pnI2V;YZ1uQCzjmYz{K$jARbR&Q%%G0~D?81u*-pQb6 z+SAu3DNz0;5R^z`WS%Y+aC^ANV5$u4E3EPAU;@F}mTSx{(U~zm&;fYDv4H81l7%3|k_(sCswM|%&JCnY`ZHh)? zwO@9fyQ4|HV_HcR-|Bmjv;+nP?(f<=)EIDm?|!L*^YC^h>%)a<2JLM!ccjQ<8HosszLEX){cSg$|8L>^(Z35{x5hM0?>EI-gAq`|e&|?=pTa zU+;cgKZgxChsqXTn*lA#2%-^Tsd1I!(J@7b;J~ta$Ulp7YZ@1+E<=F~n;nO?QY%F^?@0bJc;#B&x`k=nc z3XB8$DB*s!@BRCXSNUfk&pbZdsd{%0ewexYwA=|z*@aK5QjU7ft~y*=Dw;Tjpo#cu zN9KT$FM6;Z;{oWdg2$p~sOTik9$VW13!lr=4j zPob*7jC;9v9g<-k(;eK!K&h8EyoZXKl+6WUCcqB$92o)*4E|S)gOT^F6@4sU>mX}v<-lMj2uBf zCen;GrH|(MOPFD{h#7{pk22{MD8M9QOVEtev%&J+9P=-aS`6^W_>=r>0lXwV94AOx zb$n80ERIn2N}iLN5@J~r@t}$ybBnmvRi~?R&m*ABOh8PzcO+Njl~e7D<`a7*r^8NR z_HHT6LA1{nv8F3&cjkqgYjV2~&$0%;!5KC2gBh^5m$R#mK?*i0jyjM_o#9+dR4>rJ zuQf+K@6>`juH`oL_>4J!fwW26+ODQ9F(Xb2{ZSo=YS;b%j)&dmCr<~HlMDHD1{PnW`@GoAT>~-s@<D)ciacON3PqY`kfDhNUDVdl=kIkq85ARG1j5uBB{u_qU}cu z25A@c@)BQNAGzVTq0pCS3X(vY27Vrk=dJA4r0sXFnZZzGR=-Jcr zwi1rwjU^%)yIS$4rIMpCu?#nWWr#^Qmo{@=_A`yrddd}(^+lSE*1K$zXRXV(XiH7m z>NM2eTS_f!4;Ey&P8`}~if>()=!~Y*>y(Af-C`F*2UGbU-q-E;W_=6@VPCF44M#$q=@xr z!EW&4qY71F#g2JKO&J06)5KJ6ClDr?QoYqjF9d1|YFXWB%3o3MlS@3m1tx`ABvEIl z;@e^5&+=s!()kO3lnvo~H!DL`;Sw#lPcWB0CNo}R`&Sd}*Ik?$T|Ynp`cM^lh=O!& zhntP!-WvB#C^9dyetZ}`f~6jPe*P$GGn_Xx&=W|FLw(W6Io!4$V>2$zW;nX z({3Tuay0_#v-D@pJ&fRYvoReKRBx|Pbza%7wuiZ3N4z+*v@F0mvD9#B^klrGZe2Zf zR8%bE{j8$iTMx64Hzmn5k2N~)F$aMUrhCtSptuGts2i3lYVHEp^}G#?Z2bS2S1GZv zW;g-i|G~WWFin0B9j~{dw;P>FpBe5CfM3XKqGr1Led5H&^?pc2$J6=wFk$D8PC0YR4V_REoPsXi4s z9xNloIQ>?M3VDAJyx{-Gyqco7s!d43L}gSm<%XCfGDtMxsGn4}t*<&|ND{_Kk$aEH z``L(Zko54FOI&<_4`<%TNL><(WafVP9lF~INN8pprbc;3UI&ShmPW9}UDG`L*uI)m zy@dx2`BoJpiq!ZT62b7Uxx2VCxZgH05I0qerz+!)8jMl6>rP1*(Ns=mdFCcXTtMf; zIfb@h)m%canw8bt$d-zKQ7_A1)GHJ7bTH`}=u=r|y2s|{h`1>>my-f$I$ev!(q=5E zTf6a@10H^-d(* zOHt;_g#WJN!AEs}dF%o`@syGas%`t7r%#4K z6BvliGi0>P0!5-t4R=uP@^?xh%3E{UCkYZl4D;VkUvtNFXEQS_Z*~^F0MHCLM^ESL z-NVIjO-m*^u|a8B(A3T``VQ~El6b+Epz`4%%yTsbtl)k2=Ix2-pVp-_51~4*ILcE}V64OOIiJ z9K3zzkC{|HOjyR<_-}4q04qio)Fi4z66K=#5PZ^h(xkIa z#s~^k^Dy`}lTjOxmFAObV`GWC%g5CY-))N(9Q$S&ItkVoD8~wDe6!8i z<`6MkfSjxfziyYU*2qD;E{t-!nbVj4c4vB~#}R8QXdj=OUHnm!sOSGj@P2HrHpvTH zd41VrmSN=S+}{8GrBxyuWACrVPIpzX*jB&@94NT3 zs*@jtWZP(H4bG^NYpIs;2}=TSCv!IbzzL>2z%WUhZjK&Yj=XOkD#H@+kemS?Sa{MZ zsvv;O7ZX#fRufm+!JoJeYAXuR7_T7gysBB&XZ2Byp?UIDHlB>7ED2a$EPg7+spcjn zDt4vg@eqPx$9cg)f4bjK2#%F;$(TTN*pA(9Z5!qt>Jr|_)xTID&crU7 zpCw6oQQ(A!Cjrk^{0AV@j*AL$+Ovsle82>vL6T(In^B2AOA0;5kh3jgArk9`nLdX> zcRq#5NraI-dtj=4U~DzDH^MEcX9}&}A)DX6hgb~K_mCu0jF|T75$J=3@PM4{=4w>g zBpZ_zJMLDWEzsR0S5XY_p5lZn`LaDLuWxSQ`;o zMvyN}H95<--E?RV#+!HrX>!eCyCIRLP^@?W*1wZYf%#tyu>@cCPwex%uxKHQCdF5q z2S2I=u?$>Rd>Q-&>)DJ*sJWIJ{4Xg4S-_JyM|HFG2|@C zzciej+`yW9{xhXH!fdO?R`^x(k76`+KYb-RIER{iVd0@&W~z_ulGu;C-T1_N*T5{6vy^1G72ejQn&Sh1m@I_wT zGpw5bLWIm0mBhGG{5$h7De>Td+#m}lFb8M>_){esuZ^Yxq*<{RdGOYJmAg?`L2}DL+ug^AW23U@Soi%-lz(QLG4j$+V8c(hNLm%I>3TG&S zalx6@vW}Ru93=x~v@!LN_j-(vRF_v6pV2lk7t9NgjhPz=d(`*?%_&z76ytxVbISr4 z#&g~Rm|sq8vQO7ZvK@(kY6g86hLRs0bqvi_Ce^Bs%}LCx6zX?&VVKsvj3xzCK*o^% z*^=3rSwYSg&=TKoCUOjz3LlIlN@_+4kKD$_>n2;#YB#T?xn1sQls8makkBJSTiA<1 zK;TnWP}t5@D+2+w3EP@M{{UfWgw3@G zG*wHcoJk8hnZS-Kpa!>LESS0bl&Ge~c@PWT1{|{QhKjpkdNF&k5BZDZD2q(CWk=!kc=6BDB8%g6CNH#ab9%$M$po)*>DU@htbko(b9>Z z%_M6Bv~H`zPFG>d%IuJ)Zofr(oo1t(++yIF+&k8#fb;s6o@omtsw&b8Q>N(Lh(p@o z1y=0RjZ8MxtH(G%S6r zLVA6*sCRy-Ls#*-N%GsblFaD##}oXa3g&^*F9HC<*?{Eg2qvT_bwVtORyO4V*o-t6 zLg~q&n8+^o^#wSH?4y-EI}o{Dik+j2&&Gt>wm{Bb=FIX#w(#sSugpiKSO|t5c@qZU zBv9AK)`No8J7pcq&-sYtO!P1KKxi8;lx$H2&@f`}-FF2W5e#2chd~ga3ny_}X`CXY z^1HU2H4F1V=oyKgo_qUP5S?DqC!JXVV8yn&w)PX;ZYnC89=8F-xLMI(oj1n51D)&m z^0r={Y&DQDEMxcsEagpGwIHBdr1G$ZaU^xSw}J~9Z!UIonXuEJIknqa779QS@Qr1oyiH;*TSRND49%6R2ABm^XuUsCD%x`%bg=P}sL8rH9;dY=lj^&Z1L2_hUZlyagU*lF!Z(@2LM9MOWAGzB^ zsVbE0FPl04imxU4&5h@uGccn^nV0vQz9+6;t1T2$mtRB0iPynLTqV>&1wt1A#)E*~1QyCf^m7D8ux z)@FTP{_BCZuuZ^ne|7zg|8NzWzq&zn$V#U?XB>CjTJ-NO1P(=sp)}b}y%vA>#qM*) zodr^P!c2?8^D10hWDrxiYCU9G!K!{Y9-&+I;}0h$sS=)KWj#qlBQ~qD+>}1-Pq;40 zoZo;7)_C}MU!yn?E7-N$qtEw^-C4Lf3IcAHTmgf*qj1p1X{0|hdx5aY;%OvCmRxnz zbm<&A6MI3rm80<0&0PpvLAk~Bu7v&9u-V*tF*V9>8cEO863yZ<=}#2fb?|1V5~%g0 z9MMI~9>{ZRC(X+BmYvH%bC!O4$b_BS(02ZH4~q}uF}{&QhbOKJj~QRSm1uC4Jqw`{ zl1O-TCfyRKVs(1e)*@anQdXkkzf{}lg;uHMQH13`Kgn`LM!JGc9Q$ZDU-6;Cu4Ro% zK=`UsNRekW4#o#01YzdV)|H*lnrR<6k=WE;OYTE5eg*))EZ4=T`;$sP`Wf`IeK>8Z z4Q!zr9P{Mve@iNt`My-Tie*13qx)c2&+5k440|($AS1Bp*S2oB60{(@e&UGhBS2bW z)-EM68`jpACyiCuRsqbeIFTv>UFyuPW_LOyp4*CfIekf+N0Fu){vk~`(lDU-w?!R9 zaJ{vMth!2HjEp4nDGG}2?lNCPo1}?po#dTHLXd>UUeoBGyQvMf+|Nol76Mx}*L7Qr ze05fuXt-lOY^AeGEhaip3cbd0K2u;T6GtoPG%F#}Tv0b25y@-aB2X)ZfidmznA6|z zbSV-d0i_no3(0=V_EQb@K3GH}`--8K7eDF8in zvoW!aUq@QeR4-SnI>d)#+fQZyqcwpPB)cs zE@U8Gr>X6@XVz8eNr93ch&>A+4E+OvqZi?W#QI*qI|te~yBCZ|#73r1CJrMk0=Z<& zBFYHjvK>YaD}OzZ>2Y_+SI*_?K$FbriGSUnJ|glBbOt!;c_YU&5PLxUk5kk|r}R!# z$kdlLN(B_h7XcS*?SV{=D+Yb;p{`-JiYz-W!W&4lkphEMw1GmM6e$dmsC~*}pp#Lw zF=6H1)hu@Z#&-oKde|*fG?Pz`by)(D2m{kkVeCBAdst|PkT3Q%#|AV};qiIzj zQdyu9o26lK)-MVs<$iREy-Qj}M&jZq+>757g1TSB zCCBgQs7&P82!NF&xA8`g%Y9uP!t{uex?W}ALq}N!?f+c;Q?&Zk=;|+iO*df%F-f+< z-CNcM#F@XkD!7TGaMU!}!hb>LKF0Qc+HaIWw)`&yEwQuyFa9V>e#BwlBHPEHY;3>E#{w4rC|s78>Pt%r6kcI zrb<*MvQk(#lkMhg0v7wOq09pYAWMUHvQ!mEkxX{O>8u7)nY@@5`;p++?x0L0t(b_L zQNQ9w>MnKrudu)VE8M=O?pjy;YQsP-Usjc24|^z%XccUpj!RhoIlz+nb&y*}GCe9Z z@}I*x66vM?V_6YS;lmo6Goo#W`duCWEt>EDt)ZA0>VT?W;n(c*zpFFZo&P#!SRO>T z*jKzdC=WY$Bb*MEodKTtc+4)Y%6)50PH|`_vrK|$;DDN8mN1{z76~x{D)K*R+iPTz zm~8FzXQ!Xdn@5W`VeCu3t@0n)hWOgl_-NUH+1ITbS>xIsT@InVt8#H)%W&a%tO55t z@j(`}^oCQ@mA1)Ej3n^igX-fr4XH?>%`?`XRZ`Lw(Fuf3IX%7HW8(FOtx7hOaBl_o z3UW9L+}Se(=*jl4a+D9fgyZsXo8v=_h!!|WL&ozE(jCg~yM7HeCNkEr>QYp|_ifW} za{T<;JwK!1H#-ST;~N5Ll1W7*Wi^#`1>-Lp&ytsxIPOP0Fzz0EKWrWv`;_`aO7{#w`Am&M+6$y}Dn&CE-8n=76f}S4`;tj62_!K!rH;b(XrKLSEF(Sh@ zIaYH_p_@csYV7Nb3)Ll&isYSBIg{;yfkkG{Eqz_yEQsiVpg}3L z80_Rxgy2!@0ccAuIXlyxd1>!E7_uVOQF82AkDKnM$J{n+{Rsz&W4w@nHH+Xey<+-G z%b~s{euaSj<`byOj6;ibYC!)NN&;Y6+YbBac2M0QhUfdbQ+q3HLzWO+rj;0mG^?w>1MLG z`yUn~K4`U)?bATQKfAcyxalYt_81QyZv{t#q**IDC6^rl)pq0JS3^s6mfB(cn^laK zaAFxCU(G}E2EKj^uqQZ?q+Z#ewU_TuvJagO?tcTPtZOXqXru_LP1XxbVmJzJwMSYB z3Jf*t+@kCN&9%tYJu-|>5N`b%a1CQbg4bp-EN}}<*=`+;jm^!88qxnsM!7&B83t`g zLF=5d-mSF<%=Y-lGrP;ZT4n2YyG;$k!fpasXCbjr;gJvpsgbFIn@BL92j|Px`|0`V ztpfFnM78T;tOKhdv9L5&NgRHoYe1lDXIas<8KCG;Kbls3qvk2TYS+(SG`mX7U0%f+ zLH_T*2Yxl_OA&eV>0u~P@sd+Enn%;0R}nNnJhM2}>XYYjT$#){L^N9(EwU#H#LSMm zm15{IXoc4;1qGzn-IyjG>{pKr$m$Nm=wfs&kS1paN7)j3A)3b)F?Eer?R;BfbT!1k z+)f>qq@uCFC}zTij8M7riT;p6;yFAA=Hb~%7smE_g>~ws{rA$@82Bt=b^QtJYFtd_ zYeah$$i`9)@0I6 z{)P|HszA)PxM&JJNu3#7xu`WZ)!%k+$NNuGv6zqXFt%Z6*0Vhc?rnmpn2!io2MBbV zxjs6N4q;^0vpX9&)bKOV?qW6$VS2yK^yZrSa4e@quO9~pfmH>pgAx~6HP}h^*H=c> zk&LB@N8Ura7H-ZNe!{pqwNEjvVbq3M*uu|$V>X~$w;FDF_jL=th7+g#7OnniedVwt z%Jyo7%{8>3&ggV@x4jN?%?_p;ziG ztk$6Tbn?ZAGD)?Q@`RG&knz5AIX^Y)(CLqK7tn&oLX&4>6A<&#see_1C;u7q6E8)k z`I|5O%E zfA`13K+iSk;b!bOd~%>S|2Vf#**fk>caCoctB;o7P!)~W5sbGat%vb)#LowGpMQY9 zq``qNz#B=d=3f&}gHH#v6QE%Hl?zJaMlaJn2jV)_sV3Qr8LP%+_!w5i4NN%Z`TOPKZ#|)t-nyLw z1N(FEf%@}0GccfIZSd^+#BJ+lpkxjA7j-1@F46QEP5@E6)_u_}{??z|6?wNiD3iKP z5jEwA$01u=Q+Te5iv}PhY!7r8KV39Nt#`=cMKqDF6*iquc&Naka4XvLGU_yyZJ2}8{_J=3bf91 zx+Cvk+SrG2_PEi;*e7nDC#B@4YW?{OZzM05IEUQi=iu%grig%@rX!uu8OnD2e1@!M zSI1`eL-ZD3`KY8zfQ;(OCp`{3h(MjgQiz$gS&W-7uA|dhiAuG>IxRil^-ydtqus3@0ru%E! zSGOIeXi87nE$^{tyRfk3gxdantRqvR^}SuNTWan05HDOdI*GuaOX%n0^n~Y!o$dSm z^t7a5Y{&cUVQxm@bx&qQ2lwav`|jl6pr9b9+vn+O>L^0z^sa`dPT$Go`{`m%=5UB6 zPB1#A=i5N>1tT-=zkHM8Z*qA80@^(N*pAE;3R28}$ExBDBCF>Df^iqKA%tO2o0d8( z&XC#-AV;h7%XfJOHxKE!e#a4ADMnH&81@4Yj|;rs$^t3PJ6;T6;@7}aQ(i+f6;GXaG4!=M@jnbKs(x0` zMX?+|99ER6uuc0az>h@X&ScYc*#Z`xmbK_%nAT+(l+}6UP0gMwg5V)5l(W)*_% zaBMeS{`7_rbm`*BODI$x2G_Foy-Rq)`A3@N7G87V)t#=cXWD#l%E?AVLXJ%YW!)3DBk?1-bV+@W`w~k=fv)=QhSnj=553mgD)* zjRp^;Lg1pvDlm17q;e zBlM)GH!Cy$h^e?W`_bVRlZKYBDrTk=hWWRl*V<8%irNp z4}@p?xHvexQOQv?=aaJ~{?)VyOLeGQC-qUYN54FCwrNu&DP0wH0tupX9Z18P0bGF>y_{8gEq=Lt>xJ;Q%wZzw%8S?;&Ij8r8#W!R>-2Xpy?Y96)#Y-5-)y zbfwP#G+gbJX(N4u9@2qI#3HgV?=`u3{oY~xXhhVpm@em~8yAhIu)TS#N((rzfj9rc z%=zNp>4V_3rZc&EnKiwMvlox51Y2az0Cuw*Ny1w2LqPt|ewwlpFi^suIL()WD7Vfk zMtV7)^HEqf6R^j5?e8TT4iEZUcQG4v(&2O$cDnESg`x#F#?FI?Bu|zI0Da3|YPbM5 z5IH={BzSZQkv?{d7YLQm^j8YgO1_`^fS!5)pE`D(3qYMd6Jb7p6{M1sVPl63J-U{ZfUBL7i}}d?v&hSp4nI6 z%A;tbeP`A_1qxuvQ<(;|kY$9(B^|w7x{MKDCO_=2qlnWf zl#$MTmkjK!Zstt!dyDK%=+1M_l?^Jo=DG}%lwg1DuoWBYe`t=j(Y4ELI_w{_F3au3 zHu)|@Dn_^aP^2y|s)f2u{o&K5C`yPoAP97Nd~gRK940WpIV}+8O3jm@9Vk5w?`v2K zP?H6~3}yj;DB}-{(r6QLDz|aIx?Ns}+bZ2_yyI3v!>^XBz3FIMcG8&j(Bu6!W7Y*_ zKBx{Y7Y*9*R>D1k>?Xc+wNLH1RmIqQKV-u+5>W<|C4aXclU0dzUL%9ytvMzL)jK-# z2p4h!^9aOce6Zg(@W&!@+1LBaWew&{J;j1C(7+|39au#{@}~iD5L5jG=2Z9Ei{Uju zx`}Arzer@`;3zK?u)Xu|2Igjhjep65`ok-D&&)`vnA!(*wu6PY;aw+6B1!JyF^wTa{<+>c*8jV01EGCsF3gaqO{3katwg1;} zZuOrLcC?)Fz4U*IQ{(^bxO7@z-uydfbnm6y`nXEB@F%Bji!NKOp;Z#JwF`sqfv=tL zcLt+N5cp3px#BIm`eHS zEmC)M=J7L%xHiEtjVLg&?smqJWX+aZhb$6&hw&!c49l!KHDm=9`b8}jiy~pw5-{e&ND8r=eJNw|bBFSvVF5Wg9IT2-HJeF{5qTVCLCGBp1&@aT5m^(t zz8p9^z-s{HwF4qSZ*OgMwJ?MQy68Ug(4X$zLe?cpKNg;QY~)U4IQuBo<4hOVFcAmw z#;5Y*OJA`XZO5Ro6KmH2>)Iyp3FZQ!;|+t5DP-i9?)q8>>sVuxj^!lVGdyNbX0wCF zE|<;F7uHVadq89Y3hRg=@(C5@7J2v`+`u#N;|W$+#wju!)o2LQxDV5?3-c0G@UJMD zIV#_S`V|OS_Ng992dLV`{-?Kqm-lcED%P{C&kkrwkDM^_(3(U1-jvho*^)SM%h5Ld z%RC9YYuK0k+!|z+#tb@k6)&(#*qM=~kqlCrk~=g8hid^ z)rg%`S~Vrix0HEHqxZvD8sXzcI%IDz{*iYjZSv~CByI9&=V*E{nn8A0%K&GYyfU}n zcVY)_1Jasp)bgQ+S@%v*nEfL2Qt3P$X8q|V=YtN`v(Y8GCtswkda}zTPIMbf`j<8XL`TX&%ns`zgg7gBence9z{Q7Qwv9YS+CY#B1W&D?_D8C-S@%xOU3D*D;jaWvg3xV z=j7cQKIw__L|-t^@AvnsiH9HeyLz9AuHLcwY}{mgC{AG4A%q(6{!-n?9>@z=j(=wE zn=*53&H?GDNO{FX&8-=L{=aV4>!JSf2wH@YNGI4GTsBaq-^$|F?yU@RRJQG zFnCczB{O^^8L#={yw{Kd2OwT#udz8%iy{C;eJ;2tOy$Un7wp)DS8vsgI2_^aL)NlPkDT02YMXZNAGq#WF%xE?wNTD%M=>nI3o7bT`ds67kmAJvY|lDK*^@i?iWj zK`mQ~0amYm*;Lo=G}PInK76a$Xe|c_$1RSXs-H(?mM9QzJB`eTQ=Rnykv)t0K-yDV z#nv7?f{Xbp-5WRCM$rCN!5ro@2be%|v$yX$54Og<8|V3934ineqIR@HnV zIU*48S#@*{>bL^%-%Y%Rce|$5C)SEFuP-VEuiv+b$Do>$BcPmOTUr;wPT7`Nic!I} zJ0qm_8Z3%weS}Zp-k%rId=HiQZN$Z>HXAH@d9nS^RK@&1OM6Y!|9V+({MX}aCbHEL zcP<9Snr?r-<^Qy4SPd3=tdEpzN47pz|2O9Uc>8}Z5pBXLF9j#|BSh^coFIiHe>|Ri z82_x4IM8v#g&*p<1)k`kbYYC0c>xi)HvP)D<+l~N6r}~L4GRd4vuyW$M21Au!kJ{Z zG$U()ZJTY9I7;5`EEYT!oFXF^k4kF#zdU#VamPkr5Ne(8tp&PQiihlr`em&L&JDCQ z?I3o+=$!VdRDVk=HtaI<*FtRvC!?jIm%!Ny8@L~XCCz7s)sesVYc5DSbA>TdsS3Cu z#U8^dZh4{sYN=`T=0HhwptpOjErAVeTB=;ni;9UEi3Vwl<#k*@m&t~y?_QQtEk6vd zH~@nGMtFQ9#;dbfOImaYX-kaGwHba3U3-dI4CI19rP>*4z7`=2!cE^7sX+Vx<=mm| z%n#GJexN)MYbDcFfVbaa+L^~zt%3+LK~Kwv0yBpkJVJ6koF9u7@E5s@ZhMJ;UmA(ri%Jf55U!Vl+ee>6rNUv(sRO z?e{HHVQM6=+(WJUv4yGDURVg0+f8el4Cu);fqW1l^QJ_J5u!T;5h%}$JR;pzop>IY!z`c2fy3)YsL`rv+jp6HPN zA|BXZj?8ZxG(Oz#ZV)`5XRj9rI$J#7o}QkLj|(p~-Cb>-caU7)Po&C~*ONWoT<>9> zuf*}AcXum3kIol=m+OlQy}>J#0^Cj+I{R;Sivv|Ub;o`8_;Nv|gAt$qD7{GyGW|b< zy;G26!4`(ww#{kVwr$(C?U}aS)i$Q5ZQHhO+wR-vocnMUaU*iq&Roc-dZ>rY{MMh5 ze3}?OELZb!rn%a9`%ChT9&dUcW&1~tE|lFuWP3Da51UfuC~QJ|9L0@SI8U}h;gISj zr7l{H4Q&N4m)KL`7Br6YYB@qmrko_GkTJnDTMc-BQcaB)tncWFNE@?19OJB`!AX3f z?Ho$hXw>HcSL6sBDv%0d9RfXxb;(s$+B+4$Do#LdOWf?-Lpbs}w80xwi?-~NXT2Zx zBUmDEK>&^F;uS&n7rz{eD7G$e>3cxdiKAN_#UI+1HaqyA4Mf|?%Oe&mQ=lO2N{Oh^=O=!auB|nF`mQl+EgpXVjOaJ;mHbw!Q{AtEy zS0Borpez&x`4!#qEU;02#JyVly=wU?&LZz^Lzs6k`NpM3Q)bZxAemU~4P@o|$SyV&&$e_=;%MTrvQT=&2sz`T2F& z;ngxi%d8}@cf#bHpzy?7YOY%%!0KW*vex+}DdgP+R~xm48rp_qvS*1K$bg~a^2cU8 z*iMkCfb(M54Tm8wa}jQ%_I3YNMT4PjX6JA90I346P)wtmRyLc)@hmm^!!9Z4SJXzB zSVqRzv3z}j7lqj*la+PNg2b+E%GXsUyjP3kREO$sr_ZO4*Yc?46E=;3lr`GMn4*U3 zH&^0K>EWK_ajKO{pu%Zoe}>K~xPGaJKqLk7Ah2UBX>tg9PaVzYwM%?GTOTXJh7g{L-HTPNIZX)rtTlN47F3Td5XuOw%*c!B%Xk)bVdDdu zk{PsYW~-BTH0{Y>b%dPoZ*$HSfFp#azPs3>Bh05jjW*!j#OpRsRBr#NeMsxL z-l!k~rEMENW-4(m?-hIKw^ygPP_s;$sYMPm{+N^Odym3|rj43Ba-%#x;PqI3)vwcX zWv^spDDSejxOJb*o*5@paGum{7=2H&89?CU<)BNaLIVU0CUMCC1oN?N@d(_jdQ8Db z5I<2Ri_-4F3{8g+XA-YCMWg-JliI}A_d~9QE?psm|HxW*lU2^bOd2|8Xq`|I`irWI z5K1Bn0L-n&S>`3BcX!>a`?tu7dNtQPB#XC+9%qr2)A9`m+7*uWZUkJ7zxMlGmzMK|1}%RNNQ=}yw=nS`r- zrVF~#;i{I}>4s}>7YY6}H6#Ssy*|bq)B(;!vAPP(c#lvaTy=C1oyIA>gVno4TOA*W ztjd{MhomiTo%daUfFv;l(>TG?1noI>_SU}Kl?nTKBxRQIp#W8AFCgR#lGfvfdad8w zZ~U?t+q|Byg4Gcov37MkCq?=$v2HILS{~aRqSYU`x?o@usiH!tpkyio8G5MI@**Ya zmm9_lEIT6$%FaAUeR6dZ_dj%_ImtvmJH~>vN)BqEI18|`kqodo|0b4@a}HOA4P{X1 z1PS%(jtFf zY~mzgEKM@~8TfD#;ArF z@$?Rg+#v~CQHp$tG1De7@ZqUsEhXi;Ow`^O2Gy*JJkDR8h{k7CbzfWSf#^G@7Jh)9-(cvk)2$(bxxzK!YVovVa@AFB8l@D#zRnwW({dU7ctw=9CeW#Jq*nxT{ z?h58IQ`a*I&8$Gy2cGk%kXB=_P$4AOTz-Q5uo*+niHd|Yg18>YyS)A25lh2F2A@~ z!ev!o{po;NxK`B7KQCySSya^3e{N`+{aqTO=9&3<^z42-+x@j;Q$Nn~7&Z4FTNnF( zvsV81dJteyU%f^Cv`mtSQ*&=x|KB8wKXExvxBoayQnIW57a0BDC;PjP{Zl4U z+{txitQ?|w6M%jjX+G$CwMvpf)Thzh1EYpf(}WmNiaH`b*Zl-!c}-*^;!Kp9MTml8 z!}zc_$#E!u+Wp#&N51v?XoQuo;Go%UQ)7|X4#IMwOM#vGl?-oAoJeLK-7=1;2E>tx z+=nzB;Z|xyN~L28W>oB0cVU4Z0QtKkeFuihI)EgF(9?`uJ088{2o;JV(1)<9oLGgG zK_HeCIKC|*;cx&Lr8qDpC$0MhFo6(P@2)?KD(Tiawm3w%Z%Zvpgf7i2&HLmvFOnpK z*kxcYVD3IJhmc;J7jZ02?C|!d4D@K8vUqMQ_5iwXj7dP#ljFMP;j;ymq*T^nqVREg zp^icwsPnGok)RTeVJygTPS8=KEwwV@kBSygWGAUnA!FT=PbR5oIe3c~@i^j@?~OI{ z9eM%^!$|S)+?IzIxQS0AR*hbrI9EluZADmEUH*BMY=E@irM_Y@6fNVy>0Je**(K#h zKlu=Br}F?!fwp+fg2-Z;y?J0FSBA=O3*ep8fg*+8p$|ebhMtB}UG8llN znpFZ)m-w+Iupo80VSv+9b9dgVxXiMCq_&l+?rufaKy*!rK8Yb)yg*w5_nIq^bifP6 z&Zvb)eaV?)M&hHD-6lD=;z-1O~i%Hbbaim9U;7sCnrmq7&4=o;1+I`1J^x_L{ zXT8x7j8xOUEBADggE;;v#&fME_p+FM?a7jMsh6ezjq~%WgCk(l(b=Ql;4Z|sUmvPh zto|B%N!L%ShR2}nqf7_eK(o3sh}uK0G=i&1cwjs%Gl=}!(hc7%{NyWxM)9-#Q+CDP zkgqZ~+wWGxMfOLHsM_MUL<&VH5wOA{h)YPl>~>wCxcZ>cZae$a)Vube-L7TI-O&+& zw!_77q;#UKzcDfCSmqiLRKCdgCLyUA57*|@o!Ml)u~<$LTMQllvS+`R7Be*w>>Ta> zD8BS1K?=vTj2f@qv>M$xpRJ&PI_ET%4pv7;&vQxp);s>p%saku>a+Ud9MV(7U96KD zL#x&Ft_A4CujC*YHDtU)teEIo7cdG}#6w&YLAn@ZW(91vXlG*5l#G;bC>pqsS&;q; zS$9MNd0DnG>lj|dpZigKD21#LfQsN^c;-`i8yR}@*{sxgk{dYj{8?dtIhO`h3@z}^ zzi}MqvnzN3yuT%|xfyRr`>~t!tfW^$6T1*%ApUGlAJWxmKQA0r&e#}27xw=R^WmB{ zuL(T%M6RHj!Et$C^p>nt|Ju1W{6c`^Y-W|zCIkANj5fq{Og^XfBM?q4c=F2u38TaC zbU_nJ+GVm_{jY(G_?UJiaM`4FreSz79A7#=*E^ji%|pqcwn1veVpeA7!}#GH{!Fum z`TmmfqjKs6s(&%VpCal-^t5LZ#fS!VttdXetFb(MrVU5KYFcwd-p`}7%t>ARFPvp~ zxRtK$UZ^t>eojG&qfrr5TXdj43HoOIuP6^M!h8)HP5R+CdIwglZC$X%v=ZO5+^BD2 znGSL!1aTJHz*^PFLK_x#RvI*7TRCtGnm#lTtP5I_$G3(6-nZKi7~g^X76#Nc>tO>) z4bO7^$hBc&VDeInh)t}AHy{O9letb!bQo(ZaLObPDRs%VEy#fI*?j4UH{3Es_L5t! z%gx2jkO#y=cTQ7@)rYStkMs#dGi)bikJz>Gx`^w`V#^s>`iNyfA29O!dprC4QnSZm z$N6Hna`0rUq-5D(+Juv^{@#zi$vFA(sqg4)V!RiKpwtAPVIrKnYvwdErmK)v$~l^a zXj-xmO#ztP*jRGBG!YYU>XPxC$Rg*{*;WoV6@zez$WxgGdBw*-v>+4SdrXekg)V17 zH}u;ixQb|n(=llugl}YyZ|h`)HaVY&NICCwa18`$^_yy#SA)9g?xUy5q7>i% zhYD-+_`kO!F_y06qHU-wPP=unqEF09@F*v%9Mq? z5(IOzerYg6*rPF|npL>9S))Ydf0BCRR(^foSZ{wtx2t8d<$lC`_UWye1Qqok*p7gYHWRh8ST5Z% z-_=EM(z*hxAE61K|EepcSw4CgWA|CDR(_3j(TwQ_wx3XW*PWki7D>Q=jq!J|J!HF^ z=;v4|nRb@*JN5Rv-#-bY-m`v0GZ;Grni4zOyJSD{JsJ)|syE(O+cQ0t;&jdjqFJN{ zPMo&??r8&-%<}^NbbLfvdNGf{PL-_J1O8Kr<>{49;TmIAaxIAO7b@8e6Fej-Tr^-@ zxVjd|NL(|AQ#x~2`;~Ly*%U;Z-OPPx$Yy*gD^y~;glF{d@z@vd6=dKEHGlBrY5P_~5{jtd1QSA`e~?om!00hf zo4ET=MfQnETKe3wo{xppO#QusrMUbI3vu$R6wTl5B={C7yG0eCdJG^oR;@PY)fn~# zG(9CkCb1wrUh2r9;%^BWndp$?40hr~&xgM3M!9YqxTf!+q zKZ($lt%u!i5?s`U>_wW`(^kLx4|vti2Ch*UsUtCSg>kgq ztS`ynqO5Geu0vGSnFpHyL5YJ7C&<-^+;$s9I*uyZYOcpaz&?oe#a2Rm!f<|;NnNZ2 z#%cuP_qZNYy(=@JUE_ijHdxt_?H7`oEUFl@Lr*6A%w zRCWsC&2Es^{BcDzexM3ksyr1Trz)mGfTLC$>&<~ely8_4ODl+=7~C)Aj={SxJWeY- zV=a^a`CtxIa$8e#s{@q$8Bk*m= z!pZ=pml4PE7@x3-qqbom~52L2Qw90<}C0=2Vm-J)s|=7VAkLelMugT z8L<17IvY(h6B+09)fq}~?HF9f!$I8#2wYzuzYFCEByZ!ocn6Fox~IrPT`?`aprpeh zeTBK8O|Lv?OtD0oE$2IkY#thYzRu}^>Uy6aaz6BX#QxI0n>Z41nk7eoaB+g}!sH;@ zgaq&5of;Ki6_Cq8Ky8C!)3-*34?_vZR{WyR3$LS5(%-q>m_JnB!!|(0(#U`z_Hlaw zniAN<3)}Ht2)Zm4NgAu-4u$G*il<|2j|=p^uY3WUt$Lv0Y4g(=<|$O}#qK7_jD5ZS zcwz^>Z$-BkfF0@0v%%KQplcy|+dVrZ9H#AX(=ydEFtu_?*~RrrFry4R9*6fS%@YWt zIEiO2BR@J(02}3$k2pbLdmH)Hlxw3(*->>!`W!KHmaT#n$NP!)#WTy7otES7D){VI z9O5R8R-!dbbRMcBJJahPc?-=ADQh%Lpr1tFdjoqmq@7669=^@vy=G!SYC5cus`k`ljHvL1xze z&y1XMBWY;1jwiyY4p(MDCRu8c-b17;rV1+sOkGjhd_~;fS(i3gAAnN0crw|Zidlk1EFe*a?jS4)M!vr0~QP=QY{%d4GpegPEtXgew@Uf%(wcfxD%)GfZ zOcefELQZ{*8<~{-7*6^CbE9%oQ?xkI42;8!ERt_n&78?8w{Sd1Tp1Yy;ZOp35vViT zu=(_Ohmco}*0%bl)?c4ep9^WO@PhXyb@7y=4%B!Y;|xvOJ=dFlms(2o_=&z>QDry# z3m9G2rP^URC$$#UIm`7Gi!N%dzFsO1`?0n1y_A35)8+^9qDhZzI!(*Zoc;qD>xlvC zJmmTEXc>~t2EK%$6f+66jGKPPbLpSVe%;0~f(XqK*NT;I>c197wKGN6rP70ijDQ{6 z|L6#aIQOIZKcmf3r56MVEPc7|T-(vNTqVlsfESLJ5`$Qrw>Yq4>s22Z+Gj5a> zueV;6Z*UG4%UFF^-lRs+rM!zhE2qNr7H}KH6l0bOIm|YX&=!_euDB_3S67J1H#08p zl&j&xM}Ox1Tp+*B+PR}Db7LuuTA~;v`0m4@JXq%aA6E4$m%(OCqlOB{;(a9yNNk9-8tDf-?b3deD>jrpld9Etnz+!Q7=;1Z-i zx2|i9#c`4wjgyKs0Pjup_0u$%l+tUGSJ>(AaC!F4+J_(9L-ufSIe7mAIMe@p5vNAX5kb+x!V**L zzGl$=e=Xkh;Y&wmO;k}^ZVs{v_F_C2=JsfSzRw{w^b1m)u`wpa-zzGpgl+%7Br$7} zF&GRJ9GB-d)E4fOx%T|3rSVyoUE2>vyq0y8FQ;d+qV88dS7#&Vlu`~vPSvHZ7qR=1 zV_PLzQ#Js|6$omvxY#&r5#9)~VSy_z#S9L1dvYW!n{WKncVT6UuQ#!C?FR8k0_zZU} z5{6E~Ow~5V%zCO~F&(|SOjE!JW_I?og#4g*QIwytdzFz~Aq?Q={fN6Z_}CXfL=0$+ zS>0cJ?>Usai-tf}k^l4mVu%+@8ngc&2=O*L^qPG?miHi$J7?n4B+6I{?dTZV=8L!k zkV99#d10Nj%p_rV)sk$6S`gxr0&nV-H||LCQB^&UEU+Nh<;!GMb-b;Tca@D9H|+lx zKp$-vxr;Jt)ONcLmB2E#NI7*!-WiXCy_4EGEIidW9fK(SL zjURfNR;2Zz_AI-kQ`%u@yr1b|84}8;4Bq?kv1F_cYOpX+qIm8s3<|AVbdWjkM z@lHfcu*@trbu#J%G)E;<8^8P;b(?={g6_$4Y$?3fxZ;3(7CTL*wj-X|>}%CE-F)xN z$qhl;i`J+t|2z7yGW5fAo6%5;{I&0r#W|z@o|vgomlx*#z3I=5?dJbl{M?@&9+r?0 z@cTYn&m70>p4m4DFdjUc`Z!-M%pHj`AdbKz2C4+nFxWBmoT ze!by26D}N%w{FShd17HqeDjM3mCnc}*Qj+K@kI60`!z$}L8^PW^^(>VP6fc94dZTD zXp_rQQ<1ku2?s&QH-^E!otL1u1TCkfRacp6 z{jd37T?YjqkwP(jYoVK}{C?%XbnrbI6$&X4l@WA$#$3Uo?i9NscXXxRzvce`!h3uy zha9*K1mk|?&G~NH3mq+leB^P{-ePHO z+tRWagD!@8RX;Ppb!wGcKk(G|m}Zc~VF{-2s@ktWNGwVlw|}ZVag9sqKYQ`szT-Nd zcVpLw?!dcYV!fxN&#I``rJg%}b^Fgt*zs7|4L2=0CuYaIyD5^E!H{6|kV}rHJo-QL zwO|4XVL<;TqjEBjOTXM{+B!SsHh#e= zuO&w7I`qkTFyIPVn!l^XA2eB)c)Ro;FE@CVy7%0Uf#6({ zwasngd7@zgo6ZP%sfMEvb%`_;+WAtCbEOczw45TmvV4Pb*_$?vHW9|Uo}I=MG9{>7 zRN!H-Lo@Aiq*^VDSeKPCx`Up0BIB%-!NDEv4DVfEKjXD^1ZU9V3^~zUmUF*XBN#q}5Dbb$A!SReciiEu5Kg zDb3{Rt1%7)OhFeJPwG}$JJbiQoeK;+P~t{8@XOBcs-ur8lxxVaW0v|!RZ?S~V^5=7 zl^Qn^W2}rhz=tN>G92Wqv2i^NYEha?^51(htsN?juX!HrS-MRkVhL%PL-Kb)c+hCW z=Q>NgOh!q+NwMp?4Y#A^!`4VBy?Q%VK5d{+@s)3)F+L~&NhQgB!EQafiAsY!?c0zp@Ge%oP=@G+c@{-f2q*gq43b+9Siw+fCJQ8}t@0 z1xrS?eZD3+BWB9JRMj&9zU>K6}o&QvOYhLUE6N`Jko+QDedGEblMk ztA2`Zn`P)c;Mjaf%_vp&_F#qNk(rLF*BO_*S=elON}j6K%0~LOA-Ts)YhAW%HiR0M zeZb($Y4KuJKwq2~1mt$C%%y3(S+jKdl!9wm95znalb3NbJUWldwyt7oND564WHw)~ zGoGAaDbcz3n_P+nur0AwMr_s8xsOK4WNdECQbU%bSH31<@&cSCkrE5~oMVN(HAhEQ zvh@h$s7v6X+W;L*cdLry4yT{HhSuXWd;Q}vvqn3k226M z?+)a0SfRm0&k|O~2XACkt3~T$k$TI_o>u{()4FU>|UD5pO?Oafva@rb|0f@Tmu?SA_Sxl|FfU{MYmKKWlD++UZH*UG)!!yz}FYn)Bm0;|haOhzXR0Aiv(K1+)zPaFOI7T3AN}PJ2=p6fv~{H^cfB>hr|fu3SwEvmBmc8~7vRFQ#$USB<&;di&8S#QcR=+*hYaZ*K-nB6v&gha~_eB8PnM zZI)NwW|6NYQjUze3L`Nq#!t%o8mJU3ct)Ba6`A*Q)Bw!Y%C5#%L3|;V*|Z}bNUe*} zP}vJX@}^-(&A?9E?q%_zVKGLN-hT90qD9Z#>Q{h?{pFxFa+pM?##Y|2VCcETJFnWJ z$vpTb8{Q&9zT@~;hv3L8X<1igmY%@mK4KbUHw!rJfKeC7M|1ib*3;mi#aJBF0MsNvgDz0)& zmN`;COtB3SO|Fp7#@SzD3Ua+dju)W}f$ITMF6(yVWS5B{TJCFuVDzvs<|kQoB-AW` zyhb#^a1tKiJ7h2uZ1^BR_0hb_78UE7gH25^I>?^!a;d$@gD+VcqQ7wKry98o(ox=Bsx zIONhcxI`1C>L*m{S@Y-+lw`uRl@n@Ulto!4?E>92B8GhUOv9rcmxnl|8;%V04!tDi zMopB2GP4G&_Fy7uTVrJ#J9VVD_V#1>1-8z&xWA!^$B+%Zi+FLYC3 zsw-j3q;1?;2}=dms3LbFL4$<>`j_t%QGl?`yt;r`LSh7d#V#m3k64Yk`T!!Ei=7zM z_Zr|OM&(@=V*9s{?4>haMI!};7{s8f^=p^|Gr;R{@+HwntUe|Vkp>Dp2c*MT!~r|M z2>By7<~A&wD+Bmhg?Y$c^)8UOAql3t2bFwDkQ!p-(FX;9mrCUbEqs!#2%iOzIfAmcExKalHK-WzoFT}GoW0r|U*)Qgxb<9ZE6H63O$ zZ>dm&3G>@vTs*j>6`Y+Sb(Sd|#Z(*0Mua=g79!Tf^T{nPW712^NeRjnN0xrS8t2{c zeM8wF+6Rg)$j{bOHCW`}$~bkq!{3#Cm0k`$qsk>i9*Va9CZC*VA?}Age3lnI&`M~x z{h_Agu;uXnUCiciMb7m)xUxcr-vp}~ee?lLGHn-~R)~uv_XO4$X?I9Bt|VQboHime z+UUAFTpXtiKU_|*KNsh8I^6PLqWi`~|B03AA1xy=RQzKWXl^O_F?z!O&)<3eUmv30 ze`DKPWB;3V>+|y_D(GHwjQWZFa{F)c^?zb|9?wx7ew-r9|MzMCH`^Z{kiXi4Xx|Zf zqk$IfMUv?rC8Vj$afv87Zr;MIcaM2_8y@;BpF6rbgcGdfq^I(>=4H0FIydIp@jL$r zj<3CyWSS3JrOW-rz3&Cb>vm-6MjFAfhrp|Ik}7P?O~;$+*2IEOOkv?D4R#k&0pPt7|UYcWwJg-X!N9;<_K{#19|fC|#BK<4zFooPT_e z(zWfq*oVj+m;Z{*cgC;R{nHB5+L6r1>Qx(q=m_Z0As>vaj=_5eqZymjGdR*`YkCK3 zHv>O3BG6gB{06xS14~%CMeG%;vSK-ZK#&1z(ZP6PBIWJBA)|v2!eTTUfNs0VMk0lx z88HiI)i%C07>RYXQK0ln-gz z9kF9E#P9sV=lwD)Wx7j;?(8Wx43219JJo7Ud~M80o7zOyuxp!UncQx@#tnv;;j-F0AQ$E0!YZ#Jg9F%GJhsH{; zUYeKU`~9N-ntyWo`$u)SfBv=m^V7qE0OMx2`8a_8+w}Wo^YwLg6*0i)^YOi0BjETr z*B@!}3F7;7MX`SFyzZ!-LrmC#1$_s)ov6sRvs2%U^v;>t*S4doNaiFQU`W^`-nMGICH!)fwA2M_p+N$t$ooMvS0JgF1*p#i!FAAngIuU|?MTL@GnWEK4GR z!bPdbv;d_A+|`O0qe>2+GKlRzS1olpWmj~Ct|{g%#3q&&iEz);_=k^O#mJl6oRH+P z6sA548>_I5C>JK1Hj9C}#%cEzJ`%pRdKP>Vk#3ilaSDb6X=Dgib&|g$v*Aa9D4T*QwW_E(DiEI;|>*LWW_aNuf>VUuc*?!&s=~!MKpN83Pwi zO$O8m=T-x@rL|A7YrfVA^>+Mas9=-kpvN~G#*5xyUJ-k*zVf$=M#}QeJ!@K-Y#zt6 z8tD&fg(80VVkmh^#@D6%D~FHcgH^5yyRz^?0(NPCCj_0Z!NEm=s_*?b0!#)2gDxTm z(@EQZaujx^Q(d@;5f6uWps@ci44VV7{1V1_&|S?4DP~doKqMis>8O%rqbWUoYOlV6 z92pN^lR6T~*jFF~JWECl84R9aAlG!M(%#jc&CVlX!Xe>xF5B*{&KT*5=aJn3_1wJp zL~e>8a&9M{${3ir(gE1gMoUt_ygq}5s1O2n6~MnbX$ zf<6dM_?`dy`j>^hTHwI=RQJ~}EpU5Q+{*DcNN=+TVASL(m|t^{hJVdWhb8> za9}Wikm4t^Xp1?nY0CTMLpRGmzx=2U@I_NYVuwvYXU@S!PAnx~e*r(JG@2<=8w{-U zey4YG0GE2ItIHV!w)q+BAH=vjhuSwFG0y>utbg4+{}Kkd zg?v%SdZ?n~8`bBZBFshCuOh4CDD|7zKBZa|NBZ@P-6JQdl%xw)8T1yC){}s-L?x8A zHky+{?CZKOjoFH3fqw}zO+<^mxQdC*ikRPC6j;n;s-?3_t4mBITbd6;LMB>{&!kt+aWK(Cn^?gpes*bW~l(- zcf{sVuM@7IYh~gEXz2+wS5tk+_-Bo*J9!YC zgwlz9K#aij&xuQ!*#9*DU`LrtY~{&&z(lQfuu3mine$k(Gj@fr0%;kA(8kj^-^Jj+ zLj^Z<{T}++8T#z;pmxV%3+-j;#!40VQHTr)A|J!0cTPG!j+1HlTXh8vlLC8(-?Qu^ zyT|oP7>>)!tdMrMB^_J2Kq;m>daWql2+`)XLX1mhjk~(dxDBHj+P3LZ8R&5xmH0Sx zXECS}-C~&-OVv+~sHE04EK!Mxos^_RK22hN?m_3{*YkOGrf;+ zj%va!vOwn>Uh-Z~5FUb2{A-U>*8$y6>C}E93uD;~D<|us_q?mC>x3U+Eq2t!=!Pf% z6+nXy+di|5uVY`-L1fW8$LTV-d={Z~;j$UT$;gfw0&5K-r?EiK28om2s^24O2p_@O zyut;uY1Gq(>|I81QM8&{=4o%|+O~xS-WXPod-<}HmT&v+xKluBX(lTNiuq~tjyH-u z6wVyZQsBd6O?CXI(K|`y_XSgtf*ZQC7`AwN7RaV zo7<-M#7@$c&dw}}?z>cnE5w@NLRFIKIjs5f)mMlee(-L@UBH&kl7UaV-*f=R_CKseZ;mFV0 zf?1liu&Ow1o_A^R(M%X?{)OnH_SQ11$`I$!Rq6e6d|{os zAI7&|TO?M^{ihX>cvsZ1xG(=ezd!BEZpa&qJJvL# zaK$gbz$?E<``^&wBj?A5G;2WT`S73opA45DM%m*h2&~E=*Yjt<@_%OhB>i6-+y-j2 z&DbmDkRFU`IP{kTi~eYxRwamyD2u4)xqgjs6v)yuQLQ(GCF#BkBJd@Nshl*u;i#G{ za7`#-^_Ksk`dG`61CwmvK5nE@FncHZepPRR&@br;Mw4uGxe>SMtd1`XIrwOLT{+hCK07;Vb&ZLVAkV+<{9xT%Ee6^ z?29EO4fbj3LItTw48OllyFY2Pt{zUgrIlb~8Y`@JA!-ecE03Wx;ddlF@(OE#72imB zfu_4!wA{tQD7f1F_9oXm;?D;KFaHMh20{WXxz#%cVTf_P!aN;7X=rnZG{@?wW~@I_ z^j8^;V~pRuxP#uQSPKBIaqnrG-0PxN#zwxhvqC zK-(VgII_`r&^iycaatV5Au6~H*yx_ONp%ea+r8e=-=3#9^Gd)}nd0ncX8@1F3oPO}Sj?Fk37 z3UYCrZiT|PI7M&?a26T;jvX$lUH0ItFdc@}fZB$uaG^GNm(xzr%te3nquQp+F=urD z0Yq}s?&~Z(bu*$}H-{H}fJ8dG3R`2>I%?8H;Ef#pc zP2i_L3+E8O`ahmbU?)Oy2w;jeM+~!yi@}c4m!2;{Z0}S;*3d|Ynrx`5iu*{*hvfC>-B6^?a#ZWiABUz_kCYTgpmwGc&P`r;CyKGi0 zx(9Smj0TkbIr-OP%y96QX~ekFc}28$K0Vu>MVI-h#|6pp+_UV_uy%cqN0+QJWIR5| zBYuAiblRJzQhWSrAa(xySG1tnjl;m zSlo%BsU%0_;-bejOo+m$7L`kP*U=WJNVGUI8QE`aBX+)q*A93d~<@f9nh1m13uI%t3JIjpBh0sWusTsr+vDM9CdusJ79XW%B{%;jBB_z7m+V_cMb+ z3F$sZ1>+jrQPS*zwiXhEqzo8@4dp)*-V&OzL_pA?r_o2;fe)5r=dcx}*_8QiGiqlK z<0?;+I#@tO7++n}1nlpeC|36J`}K?WAQSH`v?OlB^4$?z#V9K)4EV6r*n{KW(sr*Sj*QZcAWLH3&sA#2{NhZ1%q4`L2pW!B~BH8;m0I5f9 zop{Iam7w-f9S%j1$*@r@irJ`>3w3(bVKfX?8xTd$F<)`ok5mEjU~S%C&7YN5!UlP` z*FS0;jEAu~eLSv*GFz9ZGiijwCP7JyomcsYOkJ0J^u)yWG23W*!?tP9M`*P9R6+0U z$7pS<9c>T` zyqIMsp-*&`x_``|m+i$ou!5}{jpU))V|}8Rr!0QFZAHdzgR68T4cd2Y1&dSpOu~6X z2u~GT%HOt{ji5B-2e~1NvdA!77t)eiD`Q#J+|gwWws7M(F~*C&n>Sf}x8zKuAkjJ8 zCQAf&E%)CUfV|`0Pw%z_oxZG0pg#J!?GYQZ#*)DA0l3uiRc-aRX_l1Hxe4fz%;PPj zd35PDM;Kotf`!skaGK^(96nB zsKJ1YYPGDT(EGU(fR6TkG{zvMcSeXhJxy0JI)ghs3d;RzK7}$;PQ4wK4ITV$!QEi3 z63G(`N>woyGqA$eJf2~eMwS{3dI)IndyhAjnHplyaYp?KSF6c*%|XzMp0}h?K-+-f zOp%!UlJ?YZTZXU`bjO~B#1GG5fPg%zK+M})L7sHXVsCbzTZS7xy7qS%0?FojTSuktH<-6S~5s_dwk8)5s{XHfCJ4U4zneLvET@V z`+*|IfSpVKN0|G>b6uO#0Cz$H%=q>SMQ`GEuLpVcDLjcHm~+udzI_qN@7dEM+$4w- zbRHV2>Jga|bkxp}oO3mH!Nf~M>@swSTdRQW=%KL(K%@YRxp8y?1P!pr1qe5F)`mcf z)TXQJ{{f6ZbH8aCR38&w!K-#s>^$=Y%r?|k#gSig}Y9UM*CSwzW=!m)YGmW3U5w$;HOp^>BJ&B<;(5KNOyk;W?8vpWo6 zynrtW=sL}2a8Y>~TgDKCDXssPe)N^d0btz*^;k9Dtar^w%8@fd)A$kIzre=+NM3w% zNocih(8vE)^DAd^w;nub_U+TfvuBH^gKAJtHhjGq@-rR{p+`gLe>X$uKRdfL!aI6f z8oR5k-&vF5tRk%4^OYQmH8$~D(r_1hU;!_%JAXdMA0Y(|)@JM5-ro$P2YrInJj70o z9vB;sHy+rS=X!VSQg;UJpAYFrb4pgEb?=-qW+w_axG|lWCAWnvA4tuGbkKo8yaIcq ziN~?10ARy4nwxq8Nh)p9)li*^Fn7%g%E#iGyH;3;U{@e2x8S)(dv|JSTRm+|?Jrrj zun!=~nW>3cF)z)cQzjB^vZJ{=`(E5Z_yB11%blQGp4mCNC~OhF5!zko0Mm{(QkHMG zO^vSVOEji{G#@M(Vh(*vOsN%HnnVwM=kC$0zm#b5SWNG} z&^nSV(P?KX^Tog#3B1P@iMf)wtHR~ak*SqDx~+_3(pHip@solJW1)q? zpuuR!U@Ty;b5Q9G4X4!&+l@?jpf&rX@a0=Ngw%rU>i%}nAVyRgHdGQOR7vZaRpL4M z-)A3w_i29q=d-gPp3eX6@4q|yuz(N0J^%ChC;P|m&d<-ze)!w#uisz2`a3`W^n+-& zH~815vme;2zb6toKmYT;j5O#V%%-l?`3)W!ZC;ArJ6rk@f`cfg)5sMRscCA@v^X{u zSaEBz+}crujQ?uxUe-El)RTiuwf2NOuC-rstr;x*z13zy(e_r`D7Ul{mF1ft7CGu% z5xM1cZEU0-n;r{Z=5+@*-Rud-LZ;v_Z_AkYc>1xlj#_F-gm23NavwDB=`+d7*SCxl~xYk7CdwSJ|0^A9ti9tBP zN>(9A8Lr6b7r)<}VpvvB{;1fxDQKmqxBRU zi-Ow_O8f7Yi>8Jov8GZar#nUN7`stq$7rPnvRJ%b|MZjHtmnISUMcasus@61%i)?7hhFhI%PP>~nIRRmU; zmCu!h+czrRx0l95Yh1Zf5K2_DoR@ScUN_Vw-}E;g2sv!eRIU{T!q9031D*s#kGW+g zXTmT1J7oLNYJmfiZ%xTm?Zq>gs6;>!8PDGfi|DY*U%7>yqTux(K}u${v;fN{}ZxSV-66j;9^nt@F)=L7r#5&TeqDYvB^sU8tpv-O^pwR>$@wT3X!5b9b|{gz2( zP(uYN;&y;>Rx@|Q{y265I1s-1MIx8HS0g{)VkFc5tJyOwqW=$b*cRks+J z<`dU+wY2}whTqb{#KllS7z*$sMCEGlw49NasV8h4mp|buoe}@*r5@oNw)4lcLp$DTE(&11*hNVmN(F30V zt8gZ(oQw5QqB=@cqr-8OsNgzERHN5Xq9R9$YHItA64gbSl*A-ua5py_mBa_>07w4JPTD($*sr2fRIs}O;fXnwAF;xUx zsjn;9lqtaq7ihJo1yN0Dt?>m}>*r0l0iNBkI!7y2c(}F@yI;zDL9XT2VgKm)F_uf{ z?BALtE7(YuJ5by}kr!CNcbc$1hNba4SfXHY)Ia+*AjbH>2SJ|U%9VEz3;y-xOBWEL zpZ)3@hK+oIcPUn{c}|u*fPSiv{WHPt5JJzv0d#SX>^odT&5On`g}%i1m#o@yG`BCg zkzwDyJ2gnH)!n?ho6qNSQ`i{FDU5k>{1HchYgwCgpWX;-CeXfEjpNf&u`~T!=o6h; zsm}!zv`Kz?kRU$;;uWM?1YGt|;8f)DZYw@U#OBfVy*7K^o(R>{%D-Y8a~eh7$mX4NK>p`cjEa8PaR<%ZHN=yieJj%3x zJRWin4QloZplUQ*Yu;2|s- z@v$~)1L`%o(QEub<8i6Lk6jokgk&l@x%vPV75QM2fxh}Ja~X%;{2UL_`yHZjlXo$U zlV`6kULF_$xN=j6Rsl4A_w?nEc`l_qLRx^d96|*eZ2LQj1wyE6yngr>Y&+bN*7o;VH^__@8>#BxcukiWM=SQ6d=SJ951F@XF6jNU8EGkN?j4Fdu_Bca(myCC zS!01+xV;wyD1IwoLFu;6wLZ2_!H&$Z1(76W_6NOVn5S?MclE67V6sUOVlP}Q1lZ-3 zFT1v7B8@tvM|fPx>q?h1m~G=rBLkIT5T0g7zP@}JvMpOX;Td1G!r?i(l+cy1W!ohh z4^=$(aRAKRVY)OtAJ+uO#o&(QE~Mu`bYogE1H=t}xZMLa3))S3sN zHI?788DY0f5I)xV+J^|^!T?T~qZroK&#DFnz1%5PpW~@SVfV7ysjMDzEqA?vcqe!P zy^tgqmY!Q$4tOXqtsvb1x-WQe8ez=r9+k7c#ANy9n6C)R1UNHSK^u{6l0~S1diOAs zIWU&aXF|yhs{~-|S9Hs22RS}&{(78jTP#C{q<0uY7dp_=gKd(V@tuYAf8ci9D|vPv zGy&W*z{4!5X?C3N@gaM{WX_@fFt!{kKsin~R>!HfjdaLM*iUFP+Ev`$JC5p~pzrMccJ_vlt*Wc~6<@0-W{u8ut3%~~xF{TWMW-sQZCK`74#NqXE+@-Sg&zBC zx6{C`V5;t=v+Pep2vaLM6&A{{4?U~z40N$SDrhpeSI3OPt;M9o-u$(F!Y^INSDjko zK^#?rqL6cpoobrI5IyxRxfex&O#TY0wzOO-KpFJ;4ZUS;PMj0_I;}_EaE7HD#)UT0 zW}@p62g;Oe`A6L{pPpy4(Tqw+G53BB&8vIxd_gW-^nhNraaHDJFFdU4!U`LW!p^NBk3ve& zS$iFJvPEKr4>{c)$qE|y(Ex~EWSMNlC~#>fps@eI5VP#wHPB*XQT5iBLAAnMSb#tp z>BrzG7<~U&<4qL+%b}Hl8{5b`S0MnC%|{J4VX+|RiY^N*0ec-Py4VfI^1(Pwu|m>7 z6?7|tRctG7ojI$i#}F~Klw2^XSTOcxoPj{>(P%}vZmVCJd$BO*wLu7%4p-VJe_k=1 zfc)x7uYADkCU03qesp10KfK zDrPeXRI`Od1#G>6-qVtAM>mi8y-JzyZ8^aUf6ujHYDJ`_TtOigFv|W6+vKyuC^`_O zC}QSu$4k=bXK%j$@#kvNVFhlJ^&-g4AE6N4GgygVUzpO|e^`*OWMztt-6BS-8nd%N zq#ze-w9z8LXa5Fg>@ZN`*G#aAXT9EyS+_eVi3cu2H|leD)~>M;^Sz%AJ$wfQg;Y1^ z_nr|95Xm7a0#fCu$mg_>f|0B5UWB6^0LBvnc$Lp%L0yBf=`%-@z)gS+)yR4u4_xYbp zxIFyzqcMT~*!va_wg6FRgR8Uuqs6?$;Zr}F+%HgI!DA5*cO};PSup_RsqI8L{Z>os z0q@nh~S`vCIxh(_^<%BSYJ9qgVJF?|UOYI>&``R)iH=GL1 zCCfr)H!L^hHL=W?eZ$pvFT!izs77d@hJ*{PILvMGk{&>=;DD_?_8eNHZp>zGS#Kyt z0b$h|4r$hABn_iDId4aFYE0JYU>D8ADQRjJ=o2&C?X*P>FU~YK zc`ft!t$k_Od$dim#x!6BS|#2PrAo-wRSq_1i{XVEdV9ExS-PRLlH_l&L34QB{Moqf zW3L1-1H2L;4g8x!zy|ix90*0hsdLjl**ZpjiPl&_6S~H@W-!mmZ#yE25mN;%)kX^* zh+}xB%ohtY35%OQ!VGn954j@k_1is(=xlIroTngo;N|3TDA){kDqF{}Ekp+cWujVA zCVu1F`@vO#$g7}k0fTkQgJ5?F6s>fH`Kn^jC)<5v0#R&}DSN9aT!VSEef)aM&9${H-d^%h#XAYhr48H4pwMRXHrhC8NN;pb8q!%JkGH9Uho%C9Z zBjcfcaz%nDq5Z4gYf+o$?vJ8(&xpO>ztRr^0=7FINW$%3-2l(4OAMeiCP|DZv?q5G z27h~D&bSk?ad`bVbaR^%sdjhH-?)VdUa~gEv10l?*!q}cVXv>c@z@()<@1t41k|o% zYUZ~*k)*@EPP%{6=|do1`+ymS0kZ~Y{qnT&7V7w7fMye(CFIPROlx0)c9B+q( zQg5K4GYqnEU|;EnJ3!|s1h)qj+i38>McS5*mM-%z_xhG`kuw3PJ(cY@IM_y%i-cF0 zLVi|y@J#}q3)c!RJeF+ZM%lr8s3rqod}C^}+@;H~6PKTX-eL;NpDa+0E85a0th=N@$A}{9tVb}{WD2e#R(bi zy(a}FKo;Wl9(|<{x>_+t_*Sb+RMaHrD_4}tx?(3ac5i~U=J`F8XZ4l=8X=*GK#FL6 zC)M_nCeMpC=+Wra# zI6bGX9z@DkIQt8$={fmU;lR3c!FgIowCx1azCLbiQKQ1WAZHgvx$)lXX`U&Rp^0+U z8;?$#UsMvSfCJHoZT?YjJ?gDTy|sJ&SJqoS$>zW=I;p5@Wb-fflKqmIa{Ndxqb1FF zy)*SOQWbQNLTXw!sudqW_5dgR13GElj6bxL7^+CyzowwG?>!q14ews5f#SnYC7-|K z^Ni^2UI)eZ_OHnma8u9WfjYUrSVi3aHOW99tC-urj(g$1#$HIO(0iZVFjlcQmF7rY z-{A_rsnU6Jo!_(w1pgoU*Lc<`SMt9F<4b~V-q!=iP(!Y{}=*YxM zWr-w7zoJ>Fa#=dfNQJY6N@o3Q+5*EXQt7OJrB^t6?)-)WW7`d4b@ zv`L-sRaqOUq&A6Ur0UwF%4+@Vpuvt;Uh7}UmDnZ?Ot)&=Xl1rZH0yX3w{cLuagqsD z-5#cydsW^hkquXbORf60M|BF%H~l+gT}}z6wpY424X;oYx&3=1rONu(M8$1tf1_3C z`qzDQy~#5;wQ60mZr7=87pY>GNHkLGZm_CdBI&f+cdp7^vXYWq3onu4)RMIQ>#;P7 z`-L1DOXQ1!qFZ391-t+J05nd*=v;eLT1StC(tKzWYo=e5k^nPdOC#+IS5olc0Uage zKFkWV^UE5GiqeN%D%{Lb7z5@F6RcIy%jB;qtA$>Wq6( zyj8_iDN_S22fJ@1h7BOT*|4pelucDimDXk|_I3{#YfP?@O|8T1&a{01(*Oy2HqG;* z3nyb-94&T6it|~HtbiU4xe1R;ZYHMRo}OY{hvx;->VywVnYLOx-Gqc;1YChSWL>z= zL{)V(X%fLima-9M4d+o5bbEzIQuo{DQo9%5hVUdh!`h}{VG%2vDQ)C^&u6w`Rh7yH zmb+o@)E3`jw#ssnRbhBYdr_v8Tw&P(@S3rtP&&=}R|V$cno-0OmnMJCbHB|naXx{n zDM(0YJA8InAMQlKe_NIXCrm=)`f+T-@pFS~Zi3hLpb)glXWwg?Pd)6~2OX$%2lFd= z=kQUf!67q$#S5l(s)m^56p~wHiWuJAgIz}k2++C@KadAV4ef(g{xWvmHZUd4zCXk= zA@tp?&qPTfxz|aOZd_@d!)%QHoQQTkdP6drPG?$Pnni693@iv^4=?Gel3P}9*l3J{ z2dHH1){H97R88*a&VtiXSBMpiycP+1gk(4DzfJGJSV2p%XMm=K6l=xvF`^E3!y%FM zmeRphd}ztPL)@oSIKhjnOY?-|Eg)v%cPu9=+psM~9VN&SoPiYK7~~49-zF*OnpK*l zjkw9AxMfuh!6j?Ke+r(_*a8Oy0tWON+`o{O4Wb@q0c+9U;PNygiMBNr4xDjoUfYno zOm4T0;Pvha$R7H5k(GLqvs+d?QG7k8RkqacG@OEPL&~vfE5aG}=acE3T$Sx_b_N@Vgj}(L-6F10VBn~5wBDJC9zU|u98VsQP55v=#Dyng(!pgnUAWK%Pt~k48gjP#lR}_FT zabDq6i2O7ny`MTjbd*UsLusoe; z%_|Zt)j#9vdW{xn!)9;TPI+EkCuPJ@C@koTtkzT?TR>7N#op2 zNrduixpABnE zpt$h%Scfk?6{?H0<<9sSp+&us&3c2$#i_x`TWV6)2({-SSc%{$#Iz&Kplih|T57Ib z@wojE49y*BcHy!@u=p7-b|6%V+(cJl&!SQIJZIZd)}TDWlJ!H3P^jcs;P6F6*tVBD z%%`p{4uv*WoK@;KFrz68EwbSgR$~~ttPkkk3fUF`E8EbDW;Lt0s(FS8!f~bEEDY3- z07Y2O??zS+X|d^>=RmCu3~Qp0kP0XE7*JxLBaB#*Cw!uP+kek;K6{tCZIhgCrUp~lJvtTNIyn$^pwQt2`SGL?BHps&J)v|r=<8!Xlq*H z^2DU&M`>$Xro)Grov+-6MhtEtVe*wdJIUc>2O#b zu&W65WGJkCpsSe75rM%P0=bF}MF;|G6yPel6N2zpBj8pkosU6Z^+FC}N=6*^Y7f*Z zCQd{kuSO)Kn7kE$yBYzrit&{ZsH-#}s|fLZn5#4ps~C6jaE_(`SjDCcDLw{x)nfox zJpyReM8GOf?vLuBaj>d;P1-n6)jkYW5_64#R82v{NC&8L_yLo2q=y5iIyfrmsAnDZ ztfQWl+SWw4r^kVMngH`O8L2%T;we3`CkfuE3*3}IH3IE40NRv5Gz9B32-uWBa|F`q zQ9%g*OWPF^0ht~NjOmepm?nZSrSgkp0HzjRDMimr17GSwD9Yk?nuL3o3Z!&g5xU!hLM1oZYo^SWQ>cc zFh!{lMIG3oC18)hN%CpWSS!o zM8mK`$yA5nhXx^qk}2K`dT1i-&;gJ`)8KceKnWDpp-5~A%yziKxwAr5PZ-eFi-;7Vd$WHiBk_PO&=aiH4Ox4G5}Bq z-Y3l(`3r;pByPR~L4OVa{D~_xlfi!CsD2ZHei9rgj|1}a@Bly4!F_O0BV?xl`;5Z& zMBXtB>Jx$NiEhd+picr^PlRF*%qI=1C&BGvY(U8{JrR=qETLqGp6GxgfX@_oo@np+ zNZisWG*4{%18~nYSf1##*Sz?AA$cATj_2T@n@J~89D?Tn_?@_#H3@ph0_()s*nMDk zCJoxe^EwS`CxN%z8)oOhvOHk}r{{T=70dJ)_PiJMEtnP!H2?qH(kN+GH?%NhF;6nx zh2KsKg7|FcgYv1*J)2Ss2{7FjNCWwOY-SEodpooFJ;o_{2X4hO#&4OL4y}iL!$_$h ze8R!l?UC^*B}ZAIW2tVBX@|?BFb7WmXyQLG{x^-JX{MHn3RauwKp)0nLq%R_aVw@` z9)|mHGCJc3<)PAoHHKpC6#vPj;6r#Pzhy#isRj8PGx5B!V^l)?m#V4AX_b?|xjko{ z3_DTNkJhT2WO&pXdz-fVt(cb;Tk(&GpQ4_Vxm5(n98WhB!xHa*%S0CBx8y~mYUu?L zp@p+i>@b(2YbA`Elc_$Ou#Z&JTV!(kWV@SZ@LK2b`*0GWdr{KLCKxT{tuvI&5dp*n z2sOgBAZm+)oRbcJG|dM}))mKc;s9LY0G}H;X|;-MGDo$FyC5IL;NDIGDlviOa%I+; zL$8EY@WO4ucd(Ez#Z@KO6;n~Z2JmeleS=hk`@rXHs-_iyLn;1~Icm}rY3=|{=4C+z zTMV@GE6NLfSVJokj0k=kwf4M%=k=@h8?Je2YZJQ}sOYARs0?rC2FCW5itC8QG}b3N7(ifuQ&wbEY=dRu5d zk82>MVetK~5lT{!n8(~QRYzhqwa*WlhSZ?grSdW(_M@ds-NJ^g#+ZU2jW zI2q?9A5MOT*WtK7%B%La_9MULkw>n-RzI8!<-0;U39Dy>M?+&f_C3SL&`H6i0QFdC zB?)?DC8Y>WPmcN^$c4`qO=%`0H|x!kgvky&-iG}aJJN_7A@78fT|oPl)zC>BJ79cy z7`B@ccdbxp?6ZIkLNO6~E#lr3kAskTS8aeX0Iv;rW>u-hjb#yywwB&e;5bW1s+;w? zbM@VenEeYnYr)gNhHM1W!#jStA zNOfz^T7;#^`#P@H6%|cEL*nJ$E`J2QG${ymS^(;#;M31G@bq3mrxB%7k-DPUjp~@y zqq>}>V(i@Cy)gDey@@#lUE)SQs+>oa^C4BvHm{|Wm0o|4S-*{j7AzSlXtlQaF`lu~ z9XTdp&!{5Ia^l!p@MQ{zsE4AqU0nZcWKCx#oUn>BEp$=hJne5+eaPr@2+Z@T8iwPktxx#!Ymd{3dq2-SVRYSo* zo)}yPA=_59bJ~#-qlcG6Q86pax-xAqFl>hwTB3tSq&UZG6ZnEajV)UZN}w^1h?7s# zBv6#TVk>O2sHyizgy=+3Q@=7*2Tv9?tuPs%tz=QN6(%D@CyJU6k@om7qNW|)qmcM3 z6cRnY+sV&0d@c1{FdM58Ijf2t#NMF9^dJjN3S2u=JeTc5)~B2`6@)9ju)pY5w|=iQ z>#&RS^84Bn@vNKRph+^i=IGjjcre$tUVzCdT1>8MszssG@VF-VG&`d;806A6#(I9E z-Zqn{uVv1rMq?k0@@=eU;*@lcf~xwel1uiUr{Gugy4rc#!EWf!cYx!%Ov*eXCZf)a zP)gYEN>*E1KPUQxnd=uOJToa`S@#TW#JWib4^EdvXM|F#c)M}Y3a25a zi5RR52k2)pep{v#T~8l0^PC;2O|zw9UNM?OJq7b?(ztPv1IWHu_jZz=7jG<6hHFyS zD@IkQ4$w%?@iB~uN0fZ5c-Iz?Na{A#_gD3dY-zUP!ki;$9?(N*A|gTTS|r#eAT^niQ3ioc0ilPco9yU{L17(7qU{ zo^TfFr=I{Z9FA-w%D_Ys%|9mSd1?e$L(oJ3_K`kyByQrrW0N66fy2W(UV_14UjrN- z0jt|3tb^!!-j7M1F!itZ1DbnEMq1Dl!t=_N7lq-IZcR8IZf=5!Y=6#&sTww{Rl#jp z#~JY?CH@|9&kvPZpL!1TQ^3tV+1R{tMNX}i&T>Gz)v8=~% zG)p3ZxGM;W8wJWEG!UxXh%Ku-S!^?~?I#eWs)vj?yvOpgm{KAZ zH;3*10$#IX6isyANB5eP!`3#$+JZM!zX(F8{u6U1sBSo(^vje~!)~tCg1m=70a+N_ zW{9VmIpN{Z;=qEjT$xk^ueQS*S+nQl4ZAb@geAt#5j2~%30$iNHoo3qs@eJ_GB+%{ zNwA0#$QMDguWO6uOe@L@zd%&6d)RWuHLcd{P=SKvrpSA6fB&4fmN)!yOep_3GBbiV1b(b#Vf{q=gV7m)%T+(e zvy}O%@+s_hbktziA^jeSp>+A)(F0+lu4%S0sfqXt?Cjy?T2m~g3Uw$E${$QT6)y`$ z{^YXKXKb}%S^e)QO1JsxV^iQV!~bMI{rk|Q#$*$G>)4{}RyT!PJBJuE-9svRK%mVF z@+t^FkE%OXKBPs_A)<_A=}l}R1$BR6p75er~wQMC#yolyLqZGABV_8qT5=?=@l2$d(nu1ogO6l(4 zd1#mEwLW6=Q9Yo`Jo``e52=z>Tp|=|$hO?uC}&u;#;#epvV6yFY7g!%K-3R?MSpaD z?s0OrV;AHvJDZUmjE(ifJWLzSwlP!GGJ%z>AkFYBmn2j?o89uPK3|akhgA}monUKP z^ESB1`mAbGs-7zHl$^n1MhRrWTC3s@KyW;|@ovl4b8%oWv-SqAdUWzN3 zcQxYic^`#ZBWL!*(%GfXNAn6IM|;9S6F&$$oJtRH3ak2feZ%gOc+}}@yt!A!BWLT1 z!PwyTg6$T{Ra-LB2zJZ#wC7wIuVZdJ4L|CY`>jy4=4!?D0ROL-@{O#m-{8k_p&KD1 z{rWkj*41mRy0;~0U5#0Q-T*~?+|0Dt|)JNa_{ zlN5|(MWbp~jZc$ZMhNez7tS-36kvoMnk(?t4kH+)XlzzgkuRQn@#NWat%0ELgeo>0 zm>K=oU@e3gm#A zaho*`$1J`QSWYGbW_jJ^FaD=}7E_an6o$gHGw8xOxdY4UmfoQC_5!k8bYsuHoIe{c zD=0;m%q^Etpx*TK?AiIE({@p?fn0r-}-aFuZMI8Rht^7t97?We{qF)m@5VqIgw zC!X#9%P$VL{cqh4P83Vs{hp_p4sa>UP)7TgTgLJIro(%OxDhgepb46Ic6XGEJDjTR z=Zsv*+}XcL=GfM7ZW0CA0&%a%X&c z;{e&WP@G~7&Zuf{M8eKET(}zwfuY8wCxypf7UaU9?SdyR2niLL=g~Q{JYk&Y-ys5& zjV9P)TCOHz7}1r)h%`_^+T|C>cW_%)@M31OoUEf{$u<;fqeC4qe_xm(4=huC*7$%9 zpy!xvs=Cwk)NByAG3Mfgc7EUIkItOck()q@4&@>T5A%UT`-_=33Za&o`~Q%1#2>T%CV7YkPYRkpY%TjGLt#^U%Vb=NVo&0Y&k;0R}Q7g5ANX z7(~Gv8MHPW*Bz!aEjQ3cF$8@3XtxBd{21jDt4X%H-1R&DRyp729&zAiu?Ca>K^7y7BXiSs}&c#-d)LpXK4{l zfF{;`&O4(w@{Zgw9ClcGqy4`EpR?bHC;AV!w&0v{N1c&c?U54`$rViV+Y-}}X~mwC zIr*y;vU-kEuK;{yca)7bBWRiOQN4Qyvx2~EuueQbl)AHR6oqMT{G}hp&qVNLxP#U| zZs3gJXh7`-C+S~d*a8BzUIB~Vewr8TmKAOz*n$S$A`fk1y}~YaHx-uxmsn_NT9L?QjQ=7NV09Iyg4n+MG z%kjpbArcKOdp0?&PSVEZ%sT6I+#1o{Aa)=Umi{=&58}{eKE7=btK^V%K5mM_sH>lK zo!hnax5UWR_wD`GNOZ&C{Ds*mkd{q5FByK9y!n-&fnZCl|z; zQL|6Sy&jtHW9^_^G9^w=o%PD(()A(;wK&w#3r8uRS@Dj|1{j6oHe{-PR zwaksr#{fZ}m~mj*1W15DgYzteyS?VOfI6LG;6r2Rh7~2LSl$HcGe9HZw7>AJ0T7q( zQ2$Ok6M2HQAI3L%=g{UW;BScYwIb}6RXb8jg~@}!;;B-6Ew&-w{>sLsfd&b^v&z7@|IP% zoZUUSla(Igx#pg8TxseFpyodLO$c=O{o9vspOcF`H!Pr0Y}FLF+0=s2l7Gi41qbU5 z*AiO8^FI%7!pOMD665L;#^xFl4giQ28l{Km8fQgogf;3T)`T5`62>+yr7YOMY9Hqc z!x4N`y^31?M~O zR&>Ft?f^p)pNkvq+&{DrPzjbztrb-$o3d((YgXezHjrDLm$qmZcYH9N6RWQ>k)!P$ zw{nMsQN1fTl}&V|meDNK^#IW|6Oap+yULkO0l|i*xU$}*W-}a|bL6$r3mKwhtG_WN zaj3Zq*5-&_Oxr#kl3N6?yQ1|bNiWkkn#NcUAikPtwZYll3{+VJGZ^4eAa%$dP+Dy= zB+2RXQ83G>(utj_WJ%YsOq2T7f2d7nLX~rCwm}7GkZ@NhBGLwaZlr?q*c&Pn4rL@y zx+AobjW8iz0Yin!+I-z>=o=4u+lV+WWpv5vaqXB3@@(A}N1e%|j1Dcs=U+e|V> ztPo(=_`$8vdeK`^q?rp}n_M;h(A;X3b=9yLesx#znk6LWo~>x1upW0?UHQD5X$`l8 zyES$^>q|AugZHJ`fUpSLZRwWa>9}gw*}9!A>UZtrgq-1>r;1ck*5?OLpXuhSk7ZhF zBOqF9ZeOPKd0dNY$0E?@znHKF#kahmRj~_cu-DiV&;yyc<&^}h1i7UZYSgZ1ts?(& z_T9z1f4sSP{px(=48n`q-nK{^e#^br^e1S1>Gmehwp-5t@Fv?2&FQv{R=0~CVIRQ( zaC_)Jji{ma6dL?h^O=X!Eln}lB+(~H>M;a|O%cfgPhZyYC7U6-R@Sj^ca=MSKH;UC zRH$bu_lhM_gGmGV!^752-9;$Li_TW9S+~ce+18~TG(N}V8hWQ*6;#zc`%59S8{PLr z)AP`iot_GE@%=S<$<~9Q#;G6?Q7wu{pFnva5-F zrx%WZTWIr89fns4n}$>8*)d{TM9g7LEiK7 zIe8^igAs6b?=@kXre&%rO{p2<5`|easDxr|aVf|vGzv%3=1$nh`u8U@a`JJd^gp6r zsgnh{+?EB;c&%j;v0_z)t7XZ|13{X6lXHo9>Fm}kNNI8E+S%2| zbdX{uw`C=_+(VdxO%j5iOG-!tH)cpHAbV@nJz|%s_eDqe2s)VcHLyT0S43hO#>`}l zpm&}tcTv>(jh$p1nL!QWh?XO?Ue1cHnJ!Dz-Uv&##;P+&! zsgXBQurNr)J4SP&ckT>7eqZsL1-Vhu5u}%R4BoWUA^U*sjPF6o^-Sp-%vlsZEDE`_ zHR&Fr7vH|SoYaYL-(A{1=#O%>*L-MVm{C4V^Ct?bt(^K*PeyHza^qzSww~_ExQWtb zkm{?Ci=RWTyziLba1jL1=0dl0G=6@djM#^CotIuM}?w$$}jP=*mB(9 z4d1@Iyw_Ph?El1T|E$(`51!TV!!@gJdB!3NgK@z<-f?~SV#-utPMXLffYVItpMAo14;dnSO#&M$u}Sd_+z#;g zEw+t^ZsLrUn^jt_u)g)H>#oK01uHk?t7|7=d;Y?Z9RZrH(7t-ANWoVufJ~c&k0=GN zWtEPLcxs=}v-{4@1$pg#OGze+f@M*wfL!4~IFbXZn~Anmi<>`!@@CYuCpZC5-o3hb z`TEsjoBw7b@8-3fH;T=99Xc5kz{C>atx_Pk%77z}449khjUK|cHg1S~tI(Veb};M0 zl(G(MFBLU0MAVjvC<+|wx?*hRTbiEy{8B_~IO%jXB2z@wHK6#=#{l_x7Uh)FCL}xT zk9GS@ceJlN-^a8^SeUhh7&O50TgghV&8oYWSe?NjJZTiG7WnGVWpHB$9W?eG3sx5% z8rpYkH8is3Yv=h5GaQt_Xj5-6*9a`HMjRL%YJ_C`<;OM^nKcwKGCaDUN33XjJz&&m zdrdlNqL`4=A^G&B_iFm0*LOp$egBB6tgKhjTtYlj1v4H#rM%@#^DP&j4L69(m{w#t zSbpRpA0PFeNfYkv3XtYI2iE zy@$FiYpcIm^4c++use{|h-7L?bVxxmgC+%sx`$P?X!GDAHmR((Ko1TO_36{6PZ6x} zfBcXCL1aZJ$yvs?{TqXXk-{IIJzJ3fxOn{ygg=(GLa<|etG=N_H~>kfumAJiNArlOw_1EIq+Qt%r_R_g0YHf4*oj^VIDng)0i8_}*s z#p<6i4k>D%?Cd4DyC)GEB#0J8yrjGe0d5l$%K@Pum_gD#STmNB7mm%?Amy@>SbR6^ zNEXJrVU-nSgl-G!;kgPd*-=~YBDz~qH0>1J+Hri$m&P0X{oT*h!$d@nvVn! zn>K5Nnvwb}st%bJnzi|$1#q^JV|k!qr(1I>zdi6&#%thgHYE)U<{!rXftx_~L62m- z9^qidYs&F_xe9_Hd*j+=a%RK(0+r=;kQA6eamtAt4|B%rfzM^U4m_RlNXyVdZa(b3VX1yj`)WAhe6N`j%G|U}WIPVXDFqp=` znArzx(nAMZqZMZnUL)9H5cN{9H~j8|Eq#wtVJTBQ&61!vIeXy}?rZ?jrKnl8q8U5) zEex#Ouq~?sfX+4;(&oHvsK^bBYQ`#HB4C~{yJgM7+j4EQv6tAzn2PEwEDOEfARQA};Z8P{@p;(-N>BndtH)Aa+lwkv zv9(q~Di35Ap{wp4rh8aubn9{%k46sm3B?~hqqjV_vJJSnTbR_6Dq~FqSGiI$<3S(> zeR zk9=X-epp7=BteC>^(_I|UD^?W&<&I-T$RnqP88*pUy#BQm_aRk^PUzNAl%xDnGFzH zc1LH+uin{mcocPukVQR^lNc&id)b4?9$1Hm<4wsUyr@l`v(KPHX}?7v*J#xp*x1|* zaOQ2Z1Pis>xciK;6$(Hez%K&`Ezn?@H2_oy{8-Of8?O*APMolG6A~n#orj9GxxB5N zV{;{3u!UpWw(Xo4GqG*kwr$(yiEUdao;aCs@=k2$PTddp7u>4;30<{U_kMaUOs}>X z)&|M@$sP(Fzd_dYt;U$ILKbj-5;r^ul*WMT4c2pLLXM_2)z%gvB~51;B;RiFQ-M;7 zsts^O?zVL38j=EAsj98U(HhSfMecx9aUsp>fa6<2L;!ZjvcjWYpFJOOZ0Bbis}e?% z49BwlmM4zT=H5tjlpR8f3<+8>wu&*#oBvj2GT8w)Kdca{IX$JNP@m{Vh^>;gjpeB< zGh!YxbgG36C(hWeEvr`(O?K7_tgKPJcdqL*fT~&|c3<`7C4;0BovFSgjmU$RNA|3r+OLCYAnv?wQEQEu#hwJXli4UgsJ*_8|X1K(lL<|5PnsO zUkx9v1w7V~SO0m#FHb!D7me5S=huxzeEH>W531Dy#1s77^g<-5SmC*OaG^W)2R{EB1!TdJ#G3{05e zPNz`-B_JIA8JC%(anE)lExSbvnD~D4;NA=vA4<|RvHoxEBPl#a9%j~}{g2owk7GbwGho7GZZG}fx z$m=TO^6#Pb)DiLMvzc(KoG$9lHq~2_v~yF<1*h&*nOTXV=-y8FxD4ZzWO(_8cX`(% zSUIW<0x%doY(BTT3)^Q7vb@W^Khu4nkww50!~dfzarvnSl`zYH-F2sc2M{=rB_x`- z_{Wo+pEMuMY59Yc_J3+WFX+7OI-!$TOsXatKm3>IRm2ZC{H8J0@Gf_E$0Bka z{oxOUDsPaj)+Hk_PP+dl9k4eNE{$x&cRlKxgA1iyeiPJ7A75e1u{J-4C@48=Us6}9R)qWi4(!F{8|#x_>!gs zod)gFYE!tVqRy>298+Y+eoRld-oeZ7+50}Zq`$ap>59oKd<(!b`B`7~($Be`ZN|E7 z0n;5pTQY9O={znTHRN$9iK+yaFgXq@ui6_*W(dp|F zh<2~3qL2X^bgTYX5(N>P%2q~ea_Ge@wOM>r&&yB%iWaM5Je4`>(djt(%)?uSEumIjuL#`(ON=9ft0 zvwv1d!h4#%KOSAkzJi_xzWEZjXYK>{1|cf+v1`IP(_Oo%fU?vQap|LR)NT@64U7Vv z6Yge~Xis35FuAK2mfz2eeXzw#LPEPk{wa0bw{i6*&vyO+-uoh8CLcrhxj5eF#%GMj#6 zw!zuv5R>5T!e!WE^C~ocG_B^DL4oa7DtCS3H~t*cuube$`42@#kwZP5isf1s677D6j;)XV%>WfihUe#}t7Rro zmn9r`9B3=Y+tfHM3zO3JGf6V%%_N*)kywuwZ)&21cGwGnQm=DZMbkyBGm>hGrfGy`ewaG8Zp;C}b!CXS9OVKR0T%rby1(VJJv7tki6p zd#b)tK6TN#N@=wjjho#>hC#z(wPFX+a-6J*A&*zbIfuTgx~z7mDCw)qPDc3CVn}V2 zz~akpib+TDj-dQDGAHpBs*Jf+vPUj=sh8$f#IDrdgyqeA-sP-`yjx*#yDR6K5%x`C#HD=SJDDD7cK|ZUa%2*B9>+ z^YbTGPv#$+xM?kq`J3u(>rUmWyQH9BExv@EzbV5g4r=txcAAQ{j(;(OPa=#OmQ9A| zt5qE;U50jTsa#H)5v`ZsizUy!wX5d=u+HS&QwLMV7};ui(`8yq7@BleS^% z8EAsLLK_-6oxW~sV~vwkxC;Cwu{G1&UEqzMTvcMO@>~)}D~)hUb{D(Q?DFSkjaQ0? zX=DEzL7J@=3|&BQD}l`eJ$D9NkI&a2(e+!o+10=Y*{&p&Zte6lzj?oQL@Nc5*%5I^74wD;YnJm zrf&jIF6aKc^@z!q`LWzIJGq&%xQSgouYAFMwhk{aNg;SGj>M{UW|F`{KC#lSUL|so zaN+Qdli0$44o3a@0CFv16jn;@&**|AjMoLOsKQpCD{1$1lkOTycq}ovjad0C8~nm- zk8u#2kpQKCZv>BhIvhO>kIue;Q{B!bMf6gos6@WGRRqsJU*E68(=Cp3HFD?{A%7l4 z=O1#qjjmzleGk9yrp=r-ikZ?dle(tn1HgV!;BlxISTNnu>9m?ehwtcvd-`gsTgg>M zr>blhps}P^N-PvGe=9O;!0Lvdm<&riRxyCZ&Cgp{>5WJY+cjt)%U`!8bIXvcM)nOD_{*kfp!`>;Z1 z#vVy(M-9r=>a?7<%sE*O~ioC4IHb0(+ZVUDmN5-U7J0#>62H&5=3S7Tg! zgg?ghI{hj1dn{_FS&>7FMuYvoMcYPeL6^QlwAVT`q}maL2mk$$i8xg4E9r}VgzK(f z21J0>HV_vj|A}}&+GTm3Q4FsCw6NY9T5K^z)n9kHH(tTBO~8m;*3>e(i;(j4D49X? zRXe%eVvEF5GXQ$lXCOfkp!_*}vI|0r!AxT}Z3I4GxYlw2TDU(HpbElxTeLz z{>+-lxoe^o#eTaB2X>zw^)uct@{xOUPO>(D7nXag{OCEl^?A|jq?4}eUtXrRo}7U)VsFDGe1TP(0vr=3QU#a)NInO!gkEB1`XYuxv*DOZR( zJM@mb7HPaa(prP(a%1n$t#No0hSJXbq6uZwapR>-B*MR%oR^2Y7JdKh2q&a@m?AuP z-(k=dQVt$`2!`nWH>x|=)BHLTc~LM3y&XzIiFM`x*28(!$waK@+m1DPn0i?L8XFyA zI={ETa1@A?l{cEkdJHh8$aq(1j$oBMoBd4weIreBy;wL=gYUOKN=uy0M{3lUE6W-3 zzZe3}Jb9CyY=+f5RRH4Zo{i)9|Jv{sgl(QobtSjLLalaP9{DUXL-G6OS)1bT{?UQf zC=+XSTgpaKC981eGOMuGu`Zg<=H~8ixD7?q=X}4C+eli~i}NA_XPtJ<8)Jh<;hPlo zqWMVop+E$5Dq&`*p^s|IxM)$Jr(Uxy_}cFlx))$2^`6Rx=gu#i-_H#d*|^OKkjCvr zo_REe;))uoD^e(-i(1Kl9$dJ#iPzoLTt`vLR+~#Je)E6Zgd+k(zcY#_?_?FGp;h#m z5O9{AE&C7Lmdq8B;tLjNr0IQGhN+DNMdJ|XPU_Z8RpL@H3b*6t zHoFe^5u!sU?rBLCE(;6HO5Tw)@Nb})4gTbjoi1jj3cH3Z($Zx?I261keDXSry;53B zgKQ1cmD=;%n2X{SBSG(j@|^AyER?FW!NU|Mp!=6F8<=E+P}(BbZk$RTrok&n5Z@>H8rTiH_Pe0&QsFeYNJeBM{?Yco zNx$W%6I;=H(s$SdJ7m@77tl~$3;`SIf8?yG-Gn>Q!#hJbACTf?Hhj@c&`;@F9DPyG zevW_hAhE_7#yE1B1reQG`MHy&-bwSG@zg_BYvna zOy>dcrh_Sr$XEV__tn7T>ZqLIiG(w)-qhj=500i^Q6+FS^5ZX_^u8>XYQyYLdXqPg zTyHWpEQl}vp`^aXqxORUa^8rJbxM-vI5C?p!I;l)olf%lYqH<@%0w7TC`aQPnCZRC zw^wdrTtmnr?K5NxtC-Vg&@kpQci&KRM6gx~rHZJ`)R)S}Z}Ow(K8=~KY+Y^!_Z~Jo zk=gAhX83>u+OOV`|0}eBWaF-07&(Cgp*U?_8h0Q2o&;p-Lp^h}vLY>?FrQo64s&GP zr5vXUa=`uMaAh#Zsm2@2H-1P?&zjcIna1`~xpP#yFbM9fh9ytgV$@$9VR8!NxXldV6eQv@&ky;|4BTSl+MK;`WDJ}$j?gu=xkl0_A`&ez;3eVI|k@^T59xJ4%{rK~`C`}>XD%@;B_ z9~di}t?{q|l6_^cjMgG5L#K7I{|;Y1u;ZG1xH50R3c|T$)dlqhY>IE;fObL?YLQt) zE2g{VL7($~<4&7VnOes+XG#rk>MNRs8At*lFAW&!?u7c%!7;uDw8k9K9*e5*oPQxI zDYKr&D^;9sii@y@KL?YX-MaW<>nnMf z43Roo>Hj&?nUH=V>gnNc>Ivw4OPMeFn457qDXslr* z_XW`ZGfQ|fukkEFHG?-0hs&oA#^g6_^l+_7#fRg79m6zOvYX+No2VwtXAp2Hy5B&wN-H31P@BAWtTO zCn-jA=*-y5^>}6+BtOebhyP?ZyHT}|-oU(oBwcdHR2s}^qc#lut?rL8g2qluV=Rh16*xdhAMV)pHgnv=hU~$oEFLhKTz-Zu-sk&lSDiv{ zr+I&8S8X`10b*!hgF+hUBqGlDH%0#`Lx*8}2f8xVe@gQC#^T`H$ya9S?@kUoJM^s! z>lzIVP(1yTp0C(f1X$ATC~ERxA3cw?>6q8KR2=K?UPE5C_ntQcSpml}Fa(y)gY^~) zBn}}iz^a|wJM33ZKn`iZQurpf8p^WpgCpLDc6xuC z9VYP#N1_=kvrUEAZtQ;k7xpAV80pwR@4fF8hVNFObHm4vQQ3}0ax2FYXV`(;E~aBa zE7UWFDppW2BKu~~rHiUWVZT2Qb*KM0qAs>J+MJez8wfdX%;H1D|6I<;DO3Cq+qXyQA&u40FbYU?Ek&?^*OYXy@-f-Yw?~uy@s%4ojGus= zS(fD}o}1!hW5`<83NQRRgvx~FX=OGy$ZqD{1V*aN3W?Q_s==7pEy8T$qAj(>AVcC5 z`ixESiN4KZ=_vJ^)FLdBau><|>S?Q|YFoakZey$rH0Lv1?AzIE*ZJ-!VfLR(CZE;h zXlI-UZ8~f6ct8yMm8G?;Ak1z)36b3V8E{*xEal-cfR4U?GpcXoo{>Z10J`j7t-1KL_U{A$=XQdaP) zsV2|pRsL!Wj|V`Sa%FW$Q&66wI(}=$qnzQoJ4N+-4(;t2(dy(37uzpv8WHC;4U?2` zd|%+lJN`8N9-~ZBzI;CD*0+eAWY_m@8+U;29ta5V+kJ|=z!M=x7Zs`|bGnbS@0B;< zib~OFrDkTe?e9BAh-I5Vq$6~K;lqo@vhPO^&x;}1sYXut71ZBF-p12f#VmhRm@39O zp*QEENKz8ulyu97Fp0NoGJ2*LScg67hpRz0P(`}_>>ew4(A&omSx0aJ33B^6rm53 zI23jH6@5(SVuN3nz1#z>U+!RP7s&^7^;Bpgj5qPD9d}Ycus}~QN~LD><0W5u3W)gw zUQulK?NuTJUiH89rSdGi&n~Ik31Xf~M-t_=)ahCs9fGD~JX{Nt)e17gzWT$?hOpY< z#<4HXh*>7JgkdeKf zuem@#EpJ#*J(WfRO}qdd3_~C8yLzKsoQM1K8%#Cb-Q@ltn9Nns=hHJQ4DR>QX#tg9 zs&K8Bi<1kDP)8T`52SYb_;ioB-|dlETR%|poWjYE=j+k`shIJJz|-btmVRV>wTB5r zqfqy21)8WFl9%sMDaM#VXhHkr|4d%C#jpatRyrCK=PWuOoqkhY+tT6jdtXB0M%He$ z(cB=KrT&vz+d<#|9mWA0jrZ7k{%!1QvNRd?sjt|-RviV5%4TUy3p`7Y8ZeJ{^R_dq z97icTcSDySS1*D5F|X_RfYg4@YLQ)4)%kKTfIhcSrGumK&@pCr+1!*1&xD}Vd1AH8 zu<7+dPc=R ze`!zGz>e@_VCm}3V2GIo=Z3ym*}#@GOJZaL1Xum#`@8dFWRFE<@yCqe941}4MAVyB z&-A2TrJX$aX1=U=Mz4p%i*9y>Khi?qvwm9HtpliIkZ>M@I6vL6|C)_|9{{@Xv$R@h zO4p4g=CFG+v=_K^I*9po1ihV09CACmmjMMBeNs#x;mg=!sSsND!%AQOl}z&)kYt-f zOVyGSwVQ8bqzk`dQA*!CEj`9UnI~}hhu1eSp!YHPC;^~}(^~Lnu@7?C<6wpRJKM#xs_1Q>)ktyr zyWd*?Ygvg&zhNX+V}nvVIk}0S^84)Rua`oB`ui`jkq(UhP0B$2s$&kgK}3fDbyg|# z%wqjJf0Fc$lNlEnE$497OqAfAi{{WI^;B3lgsljJc|xSt@ava6s7Pl?gh0@gKKR3n zGfNDxM4#Q_XP{m`fyn9w%Q#FN8u27dEHTgJ%ro)&B#i%i5<~!4#2j5|H%w+rw8~=MlZN^tAjt{yDpo4qMs*#t@ z*uGI=-r0Fw~^)RyC`T)#vujViqE=L=j{;c~0iZ8pGmA74P~^ngy>UW33kg3~M! zqCw$`^<>*NM);28XzMnctI-N)!3k!$Lu-{CVT%p@mQovao}QXt(~Aa&P1%Sx&4rhD zj0eIy@)%RXD8YBDs{W7=`LW58d>1iHo^8&-XwIXr2&PA)g`@WK7|M`BPSPz(*kP=+ z#r!or`PV7if{&yJR(6SxfR&HKv27k@kNHnczR|lEw^V4TOJXdaz5|+r$|%kTM%{W9 zZx=hOZ0tkKp9^dc2ulRy@3z`Iw9~+)Pi-YIoLo8Y#3*VBZvAD3ZB)Z z=wbZWp?s+~h^dS~{7Ak64odA7;3aXq1D_mJT!#aR+sEx!Cvf zH*SXu=*+|ACfaA%0yP{}6d<{IQd>5$nb(9JH!+fW-A^V~&K7sJH0BqiS;k{JP772| zwLfZDjx;{V!uRzn^Wt1;3-c}+6-~Q21B&~KTSv}c(f$kP5tbJA9M%Z#qmC(3lUtwv~KT(_J`k?yBM@P zbHVfF(8M{^T6w_?LJ92UMi%FYbq5#422^L?y#?s&hws3HaCX-I+^jv zn}L6PMvItaGnUu{mKao1>+ojlX<+2-xPtDP(_l9zk zF4Vqsj?&OtC|2OOy^Fd(zL~)dCjTL`9Lyaz4H%d_y)Z~9rd*aF>WiN45 zpe5(jS$lw7S#`!6mJ@OWoeHaWDqvN--FF$9@SUcFauk(r_AcmW@FggxI~MszoNQ_q zDNN1Do5p16P=HytD17!@5K@fxy&FZC4xsIe1<(8^t_p2ePt{eel#QYOUEvVC44gm7 zNxCu|a1Jp=BT#@ixMa&!0+ixAF;y9^GKkt#y0-&h3K)=>X<9F|IW!nZRVwDZ{z#N9 z0t_fuy4!~nwu8ea z1S0Ie*w}I@8O%k4$~ROx+q>WWyxw(y47z5AehCUacO0*b&J*Yi;j4H^ZjqHQR8kbT zCSxXsp4jF^)YFN*6S~~bqlRLPnUQ(a8O)opG_yLD{p^fxX5+IosX5k~^;jwQ4+lwX zW|y0zvs2cym6t?h>|Q@M?QESwA(pLMp|E?UH3FrTk%#maky?X>+TW#DTJ#>Gie}8k z_}KH!f{eI2KAAX#GV!boWOOUY<>qoQ{Zuo6&R!jkxxUbni=1F&m zroW!{bL`+n6{c%t?LUr|{R?I`UelJwH&PV~;I<%DOh#5-{B3As*E|g8MS9Iokx_b# zY)gv(%NPYMdurfg4z2Z9ws$jkfAcEYR>Oe-`vw6hrKA4z2$urjDkoyparoBPKuZ>T|)v?UDC}$p!%eoO-zFJ;~kKOYK`y z<+bziI?X!h{3^334v{(pkQpG;#(pHHH#hr~#J9#ypClqMdsh{$F^X~ZZ`}>CCVV8y zt$&_r19k{feIr1xh`hB*`Ds?RIy6q)|5#MIIMccm>U2JFUo8k@3bLt`n(w>$TaX zYNBH+?On0y#x?nN%4X?Mz@>bSW6|5wJyOzQKS$rn%zXtHsyj@!n03`JB&2yRKih~us)ujKcIbwFX?(YQ)GrA)z6 zF<)6DIM6A-RAfUH1i;-^QtKR}Y~3fZ#((8)t~g%dff25xV%@qxo_gj+_1gv1f&d=A5c6r}l;OE|niY*eM`LX3(%2C1c~ZCo-#`RWl8K8;r2WYiW8i*`hJf-FzF$ z#Wd{5ikk4e*QXO@aja@ytYr*n{It*M1yRil-wnH{1g0-4wddD+5UEE6bQt7b?_VSl z-h}AA6AvNiz6`+#InTVaI@fW(knJAgc#7RR`6~sGW-kVFf}uaum194y6iu(i)}H+dC+8yUj$q=+?Z#!)$w4>BtevH;pyDK((w1D{>+XaDBcvp;UYV{*E0dgjWC z^#`w*VZn#H{B6+dVc3H3#T0dV{b8nkjG&tDPPuZC%1q`-YEusuBqlbN>N6jK=Q9f$kCuOm`)=nwfeu5M3J zrtxvaenQjjSKO@AYW6N28W5DTu)wLw0%CsL%~=YmvZ99m)e;*;$?xKHz??l@+N90y zs%Y5idM-m#W=QJ&flf}Fg5VmDS(EO%NWBzdpV?wf%J>4Uq|xM3yD2TvUo_3b&qn2k z>+GDp85bD54;J)UrmzB7@YQ4&N`+XC(Cust4kQ5(*wkmqTNjpCXeGcBMb^dXgH(W1 zIw%{w*q|JWLJ{sf=j8H(@yYC6?PrB}NZxsm_vV6@nyU>uDLafJLhiB4Q+@emTqRKx zdv@s;i!iZe|8c3ivLGBLSAA;8Zri^`yA?e^$x060roWA4Z5;3RdhEUJA}}v=l(thm z@i&97^FPApIt364Z-%!CRGR*xcX;p1`H6A+mEU@G1neo0#wVWojHkO2tzM-LlN;{n zr?IB2egRH0R_Ou_XRuGgLJ$m)cGu$^m48DMa%K}MMQMw>=|EEl+Ld+c2Hata|8R=_ zS%k{v-ZQl-EgxdAP=RNBoTyWteG(xzDaC;{9)5TLg=DJ36ppH;5C|5lF`BvAcpNG~ z#?~6(Y)VgBC-qJR8}Q_stU8`i+-kb9vbDGPg?(39j5u;Ydd8=9I(Yo_#CTJ_I`SVx zSS#27^lVUoBt1vOCy0Tghw9+^fOtgW3QR2Q&E+ZPA93;DiWA$0^~BU}s?{@Z$o)^v zQ(p8BEQc`}Z&ur)O_SYnJd{wcorDvpRy_f^8{XRVxfeP#whQPcSMFchD8Dwo${ z1C8mV9(P6@A1^W8vpt|XS#TX4pGc6dfX(*bBw9f%bs z+Kh9bpF|FxpIv@H^YPsBduL~_6zyVbyw=j@&0g9r7RPs(i*O@jbimk5VD)Te7IRB? zscrm+HeOpAtF~EQyD^l8JNUhOU3r|U1;#ce6zX}@Qj860g4LQYN=s~os2NRKExVSc zzt{P@U$b#h-32zPaxJa|^Y&;A(=*FVnggotyC2iPsNXz2@;M%LA0@gE zvrM8rj*r?r2;(y280U2xlFG8v3w)72eP;W%vx*>v$uQSqN`;fsGnMo*uk`d+l~UJy zDsgSb9C19lYba~2$7E1JIn{QW&PycoTb#w)MsF~;r^n^A6qua*t6e|kndk;Jp56<< z3A`HO<>=^d$zE~IQcDFp!~iR8O;d}tK{=&zI>nb#zL@?=GS8Ao_7@pkhHrc+o`t2; zSkMkYaerVu4ds$k8S`49aQdgo{h#4kvZ=fhhj~8N^+T?e{%*C(?Z|`~5A|9*m*CCR z7JJNg)e#TWN9_)S?Zf8zz`!Ly2BEfk##<YP0FyJ?rVNL$TVrKb<(FKWfm0wa)ouxNg@S#aj(Oe` zxJ5Zc(_Uv}pgDyKBV_H;fVKMPyeA(c1oUnXJwa97vKR1OYxt2YD;tJ{+8C6aq z9r#^6sR$bmeJBnT+lC_#RD`lDPC5Jf-H&gdSylx<{&{Qp8-h~7{4vz~xk4bImhArS z*yj^&0}wr}>>ZpauJLtT$a5S4?+H8)f3GS7QV z&)SaD! zm9g}vNCv6q#>N-m(_c*vG=6kFa_U1*{bUXDleEN<{5?s3Rctd#KJfkS?&b(1^Zi%H z3Qiwg=Z6P<>u-Gn!qGMlD@fES4RBWHAEeuRKd5^rKHad(7+i7tqNGO0V#BUM1HJxo zPL6udo!4^hB$J45C~>5wlZ!t+^4S%k?-D9ow{`C*er9jd$An*B>%w<#uh1 zUCYMZy;gq>xeJ8;+e~$PR%xc;T2XL^rfi?m(GcAg?PGao z(Xq3E7JzSXoMv$Qqb*Fmj;hQyFX%$bcS5?&k@G3}X3KjkFU`JGr|SZ)OFCt3U;?yq z|2)LyTa%k$L|m8uhX<~cBEw0y_y$UY*WTFU@s+-dzW&hT>~mdS?G4! z<=zhDqh@Y!VXcuOe~l*e6$L2pa3?S9-`*1Z*7ZgTF@ZncUuh&y`vL15mq>g|h* yk%Mu!uE*VE`JP7jNNwyn9*cbcHTs%V^RrdDtjT%$h6Drq{>C + echo "Applying CRDs..."; + mkdir -p /etc/crd; + base64 -d /etc/config/crd-manifest.tgz.b64 | tar -xzv -C /etc/crd; + kubectl replace -Rf /etc/crd || kubectl create -Rf /etc/crd; + echo "Done!" + volumeMounts: + - name: crd-manifest + readOnly: true + mountPath: /etc/config + restartPolicy: OnFailure + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} + {{- if .Values.nodeSelector }} + {{- toYaml .Values.nodeSelector | nindent 8 }} + {{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} + {{- if .Values.tolerations }} + {{- toYaml .Values.tolerations | nindent 8 }} + {{- end }} + volumes: + - name: crd-manifest + configMap: + name: {{ .Chart.Name }}-manifest +--- +apiVersion: batch/v1 +kind: Job +metadata: + name: {{ .Chart.Name }}-delete + namespace: {{ .Release.Namespace }} + labels: + app: {{ .Chart.Name }} + annotations: + "helm.sh/hook": pre-delete + "helm.sh/hook-delete-policy": before-hook-creation, hook-succeeded, hook-failed +spec: + template: + metadata: + name: {{ .Chart.Name }}-delete + labels: + app: {{ .Chart.Name }} + spec: + serviceAccountName: {{ .Chart.Name }}-manager + securityContext: + runAsNonRoot: false + runAsUser: 0 + containers: + - name: delete-crds + image: {{ template "system_default_registry" . }}{{ .Values.image.repository }}:{{ .Values.image.tag }} + imagePullPolicy: IfNotPresent + command: + - /bin/sh + - -c + - > + echo "Deleting CRDs..."; + mkdir -p /etc/crd; + base64 -d /etc/config/crd-manifest.tgz.b64 | tar -xzv -C /etc/crd; + kubectl delete --ignore-not-found=true -Rf /etc/crd; + volumeMounts: + - name: crd-manifest + readOnly: true + mountPath: /etc/config + restartPolicy: OnFailure + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} + {{- if .Values.nodeSelector }} + {{- toYaml .Values.nodeSelector | nindent 8 }} + {{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} + {{- if .Values.tolerations }} + {{- toYaml .Values.tolerations | nindent 8 }} + {{- end }} + volumes: + - name: crd-manifest + configMap: + name: {{ .Chart.Name }}-manifest diff --git a/charts/rancher-monitoring-crd/103.2.2+up57.0.3/templates/manifest.yaml b/charts/rancher-monitoring-crd/103.2.2+up57.0.3/templates/manifest.yaml new file mode 100644 index 00000000000..8dc9dfb4471 --- /dev/null +++ b/charts/rancher-monitoring-crd/103.2.2+up57.0.3/templates/manifest.yaml @@ -0,0 +1,8 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ .Chart.Name }}-manifest + namespace: {{ .Release.Namespace }} +data: + crd-manifest.tgz.b64: + {{- .Files.Get "files/crd-manifest.tgz" | b64enc | indent 4 }} diff --git a/charts/rancher-monitoring-crd/103.2.2+up57.0.3/templates/rbac.yaml b/charts/rancher-monitoring-crd/103.2.2+up57.0.3/templates/rbac.yaml new file mode 100644 index 00000000000..a4d498b0fa4 --- /dev/null +++ b/charts/rancher-monitoring-crd/103.2.2+up57.0.3/templates/rbac.yaml @@ -0,0 +1,76 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ .Chart.Name }}-manager + labels: + app: {{ .Chart.Name }}-manager +rules: +- apiGroups: + - apiextensions.k8s.io + resources: + - customresourcedefinitions + verbs: ['create', 'get', 'patch', 'delete', 'update', 'list'] +{{- if .Values.global.cattle.psp.enabled }} +- apiGroups: ['policy'] + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - {{ .Chart.Name }}-manager +{{- end }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ .Chart.Name }}-manager + labels: + app: {{ .Chart.Name }}-manager +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ .Chart.Name }}-manager +subjects: +- kind: ServiceAccount + name: {{ .Chart.Name }}-manager + namespace: {{ .Release.Namespace }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ .Chart.Name }}-manager + namespace: {{ .Release.Namespace }} + labels: + app: {{ .Chart.Name }}-manager +--- +{{- if .Values.global.cattle.psp.enabled }} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ .Chart.Name }}-manager + namespace: {{ .Release.Namespace }} + labels: + app: {{ .Chart.Name }}-manager +spec: + privileged: false + allowPrivilegeEscalation: false + hostNetwork: false + hostIPC: false + hostPID: false + runAsUser: + rule: 'RunAsAny' + seLinux: + rule: 'RunAsAny' + supplementalGroups: + rule: 'MustRunAs' + ranges: + - min: 1 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + - min: 1 + max: 65535 + readOnlyRootFilesystem: false + volumes: + - 'configMap' + - 'secret' +{{- end }} diff --git a/charts/rancher-monitoring-crd/103.2.2+up57.0.3/templates/validate-psp-install.yaml b/charts/rancher-monitoring-crd/103.2.2+up57.0.3/templates/validate-psp-install.yaml new file mode 100644 index 00000000000..a30c59d3b7c --- /dev/null +++ b/charts/rancher-monitoring-crd/103.2.2+up57.0.3/templates/validate-psp-install.yaml @@ -0,0 +1,7 @@ +#{{- if gt (len (lookup "rbac.authorization.k8s.io/v1" "ClusterRole" "" "")) 0 -}} +#{{- if .Values.global.cattle.psp.enabled }} +#{{- if not (.Capabilities.APIVersions.Has "policy/v1beta1/PodSecurityPolicy") }} +#{{- fail "The target cluster does not have the PodSecurityPolicy API resource. Please disable PSPs in this chart before proceeding." -}} +#{{- end }} +#{{- end }} +#{{- end }} diff --git a/charts/rancher-monitoring-crd/103.2.2+up57.0.3/values.yaml b/charts/rancher-monitoring-crd/103.2.2+up57.0.3/values.yaml new file mode 100644 index 00000000000..99e63600c42 --- /dev/null +++ b/charts/rancher-monitoring-crd/103.2.2+up57.0.3/values.yaml @@ -0,0 +1,17 @@ +# Default values for rancher-monitoring-crd. +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. + +global: + cattle: + psp: + enabled: false + systemDefaultRegistry: "" + +image: + repository: rancher/shell + tag: v0.2.1 + +nodeSelector: {} + +tolerations: [] diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/.editorconfig b/charts/rancher-monitoring/103.2.2+up57.0.3/.editorconfig new file mode 100644 index 00000000000..f5ee2f46103 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/.editorconfig @@ -0,0 +1,5 @@ +root = true + +[files/dashboards/*.json] +indent_size = 2 +indent_style = space \ No newline at end of file diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/.helmignore b/charts/rancher-monitoring/103.2.2+up57.0.3/.helmignore new file mode 100644 index 00000000000..9bdbec92b40 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/.helmignore @@ -0,0 +1,29 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*~ +# Various IDEs +.project +.idea/ +*.tmproj +# helm/charts +OWNERS +hack/ +ci/ +kube-prometheus-*.tgz + +unittests/ +files/dashboards/ diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/CHANGELOG.md b/charts/rancher-monitoring/103.2.2+up57.0.3/CHANGELOG.md new file mode 100644 index 00000000000..8178169b918 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/CHANGELOG.md @@ -0,0 +1,47 @@ +# Changelog +All notable changes from the upstream Prometheus Operator chart will be added to this file. + +## [Package Version 00] - 2020-07-19 +### Added +- Added [Prometheus Adapter](https://github.com/helm/charts/tree/master/stable/prometheus-adapter) as a dependency to the upstream Prometheus Operator chart to allow users to expose custom metrics from the default Prometheus instance deployed by this chart +- Remove `prometheus-operator/cleanup-crds.yaml` and `prometheus-operator/crds.yaml` from the Prometheus Operator upstream chart in favor of just using the CRD directory to install the CRDs. +- Added support for `rkeControllerManager`, `rkeScheduler`, `rkeProxy`, and `rkeEtcd` PushProx exporters for monitoring k8s components within RKE clusters +- Added support for a `k3sServer` PushProx exporter that monitors k3s server components (`kubeControllerManager`, `kubeScheduler`, and `kubeProxy`) within k3s clusters +- Added support for `kubeAdmControllerManager`, `kubeAdmScheduler`, `kubeAdmProxy`, and `kubeAdmEtcd` PushProx exporters for monitoring k8s components within kubeAdm clusters +- Added support for `rke2ControllerManager`, `rke2Scheduler`, `rke2Proxy`, and `rke2Etcd` PushProx exporters for monitoring k8s components within rke2 clusters +- Exposed `prometheus.prometheusSpec.ignoreNamespaceSelectors` on values.yaml and set it to `false` by default. This value instructs the default Prometheus server deployed with this chart to ignore the `namespaceSelector` field within any created ServiceMonitor or PodMonitor CRs that it selects. This prevents ServiceMonitors and PodMonitors from configuring the Prometheus scrape configuration to monitor resources outside the namespace that they are deployed in; if a user needs to have one ServiceMonitor / PodMonitor monitor resources within several namespaces (such as the resources that are used to monitor Istio in a default installation), they should not enable this option since it would require them to create one ServiceMonitor / PodMonitor CR per namespace that they would like to monitor. Relevant fields were also updated in the default README.md. +- Added `grafana.sidecar.dashboards.searchNamespace` to `values.yaml` with a default value of `cattle-dashboards`. The namespace provided should contain all ConfigMaps with the label `grafana_dashboard` and will be searched by the Grafana Dashboards sidecar for updates. The namespace specified is also created along with this deployment. All default dashboard ConfigMaps have been relocated from the deployment namespace to the namespace specified +- Added `monitoring-admin`, `monitoring-edit`, and `monitoring-view` default `ClusterRoles` to allow admins to assign roles to users to interact with Prometheus Operator CRs. These can be enabled by setting `.Values.global.rbac.userRoles.create` (default: `true`). In a typical RBAC setup, you might want to use a `ClusterRoleBinding` to bind these roles to a Subject to allow them to set up or view `ServiceMonitors` / `PodMonitors` / `PrometheusRules` and view `Prometheus` or `Alertmanager` CRs across the cluster. If `.Values.global.rbac.userRoles.aggregateRolesForRBAC` is enabled, these ClusterRoles will aggregate into the respective default ClusterRoles provided by Kubernetes +- Added `monitoring-config-admin`, `monitoring-config-edit` and `monitoring-config-view` default `Roles` to allow admins to assign roles to users to be able to edit / view `Secrets` and `ConfigMaps` within the `cattle-monitoring-system` namespace. These can be enabled by setting `.Values.global.rbac.userRoles.create` (default: `true`). In a typical RBAC setup, you might want to use a `RoleBinding` to bind these roles to a Subject within the `cattle-monitoring-system` namespace to allow them to modify Secrets / ConfigMaps tied to the deployment, such as your Alertmanager Config Secret. +- Added `monitoring-dashboard-admin`, `monitoring-dashboard-edit` and `monitoring-dashboard-view` default `Roles` to allow admins to assign roles to users to be able to edit / view `ConfigMaps` within the `cattle-dashboards` namespace. These can be enabled by setting `.Values.global.rbac.userRoles.create` (default: `true`) and deploying Grafana as part of this chart. In a typical RBAC setup, you might want to use a `RoleBinding` to bind these roles to a Subject within the `cattle-dashboards` namespace to allow them to create / modify ConfigMaps that contain the JSON used to persist Grafana Dashboards on the cluster. +- Added default resource limits for `Prometheus Operator`, `Prometheus`, `AlertManager`, `Grafana`, `kube-state-metrics`, `node-exporter` +- Added a default template `rancher_defaults.tmpl` to AlertManager that Rancher will offer to users in order to help configure the way alerts are rendered on a notifier. Also updated the default template deployed with this chart to reference that template and added an example of a Slack config using this template as a comment in the `values.yaml`. +- Added support for private registries via introducing a new field for `global.cattle.systemDefaultRegistry` that, if supplied, will automatically be prepended onto every image used by the chart. +- Added a default `nginx` proxy container deployed with Grafana whose config is set in the `ConfigMap` located in `charts/grafana/templates/nginx-config.yaml`. The purpose of this container is to make it possible to view Grafana's UI through a proxy that has a subpath (e.g. Rancher's proxy). This proxy container is set to listen on port `8080` (with a `portName` of `nginx-http` instead of the default `service`), which is also where the Grafana service will now point to, and will forward all requests to the Grafana container listening on the default port `3000`. +- Added a default `nginx` proxy container deployed with Prometheus whose config is set in the `ConfigMap` located in `templates/prometheus/nginx-config.yaml`. The purpose of this container is to make it possible to view Prometheus's UI through a proxy that has a subpath (e.g. Rancher's proxy). This proxy container is set to listen on port `8081` (with a `portName` of `nginx-http` instead of the default `web`), which is also where the Prometheus service will now point to, and will forward all requests to the Prometheus container listening on the default port `9090`. +- Added support for passing CIS Scans in a hardened cluster by introducing a Job that patches the default service account within the `cattle-monitoring-system` and `cattle-dashboards` namespaces on install or upgrade and adding a default allow all `NetworkPolicy` to the `cattle-monitoring-system` and `cattle-dashboards` namespaces. +### Modified +- Updated the chart name from `prometheus-operator` to `rancher-monitoring` and added the `io.rancher.certified: rancher` annotation to `Chart.yaml` +- Modified the default `node-exporter` port from `9100` to `9796` +- Modified the default `nameOverride` to `rancher-monitoring`. This change is necessary as the Prometheus Adapter's default URL (`http://{{ .Values.nameOverride }}-prometheus.{{ .Values.namespaceOverride }}.svc`) is based off of the value used here; if modified, the default Adapter URL must also be modified +- Modified the default `namespaceOverride` to `cattle-monitoring-system`. This change is necessary as the Prometheus Adapter's default URL (`http://{{ .Values.nameOverride }}-prometheus.{{ .Values.namespaceOverride }}.svc`) is based off of the value used here; if modified, the default Adapter URL must also be modified +- Configured some default values for `grafana.service` values and exposed them in the default README.md +- The default namespaces the following ServiceMonitors were changed from the deployment namespace to allow them to continue to monitor metrics when `prometheus.prometheusSpec.ignoreNamespaceSelectors` is enabled: + - `core-dns`: `kube-system` + - `api-server`: `default` + - `kube-controller-manager`: `kube-system` + - `kubelet`: `{{ .Values.kubelet.namespace }}` +- Disabled the following deployments by default (can be enabled if required): + - `AlertManager` + - `kube-controller-manager` metrics exporter + - `kube-etcd` metrics exporter + - `kube-scheduler` metrics exporter + - `kube-proxy` metrics exporter +- Updated default Grafana `deploymentStrategy` to `Recreate` to prevent deployments from being stuck on upgrade if a PV is attached to Grafana +- Modified the default `SelectorNilUsesHelmValues` to default to `false`. As a result, we look for all CRs with any labels in all namespaces by default rather than just the ones tagged with the label `release: rancher-monitoring`. +- Modified the default images used by the `rancher-monitoring` chart to point to Rancher mirrors of the original images from upstream. +- Modified the behavior of the chart to create the Alertmanager Config Secret via a pre-install hook instead of using the normal Helm lifecycle to manage the secret. The benefit of this approach is that all changes to the Config Secret done on a live cluster will never get overridden on a `helm upgrade` since the secret only gets created on a `helm install`. If you would like the secret to be cleaned up on an `helm uninstall`, enable `alertmanager.cleanupOnUninstall`; however, this is disabled by default to prevent the loss of alerting configuration on an uninstall. This secret will never be modified on a `helm upgrade`. +- Modified the default `securityContext` for `Pod` templates across the chart to `{"runAsNonRoot": "true", "runAsUser": "1000"}` and replaced `grafana.rbac.pspUseAppArmor` in favor of `grafana.rbac.pspAnnotations={}` in order to make it possible to deploy this chart on a hardened cluster which does not support Seccomp or AppArmor annotations in PSPs. Users can always choose to specify the annotations they want to use for the PSP directly as part of the values provided. +- Modified `.Values.prometheus.prometheusSpec.containers` to take in a string representing a template that should be rendered by Helm (via `tpl`) instead of allowing a user to provide YAML directly. +- Modified the default Grafana configuration to auto assign users who access Grafana to the Viewer role and enable anonymous access to Grafana dashboards by default. This default works well for a Rancher user who is accessing Grafana via the `kubectl proxy` on the Rancher Dashboard UI since anonymous users who enter via the proxy are authenticated by the k8s API Server, but you can / should modify this behavior if you plan on exposing Grafana in a way that does not require authentication (e.g. as a `NodePort` service). +- Modified the default Grafana configuration to add a default dashboard for Rancher on the Grafana home page. \ No newline at end of file diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/CONTRIBUTING.md b/charts/rancher-monitoring/103.2.2+up57.0.3/CONTRIBUTING.md new file mode 100644 index 00000000000..f6ce2a32352 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/CONTRIBUTING.md @@ -0,0 +1,12 @@ +# Contributing Guidelines + +## How to contribute to this chart + +1. Fork this repository, develop and test your Chart. +1. Bump the chart version for every change. +1. Ensure PR title has the prefix `[kube-prometheus-stack]` +1. When making changes to rules or dashboards, see the README.md section on how to sync data from upstream repositories +1. Check the `hack/minikube` folder has scripts to set up minikube and components of this chart that will allow all components to be scraped. You can use this configuration when validating your changes. +1. Check for changes of RBAC rules. +1. Check for changes in CRD specs. +1. PR must pass the linter (`helm lint`) diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/Chart.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/Chart.yaml new file mode 100644 index 00000000000..d71222b02d4 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/Chart.yaml @@ -0,0 +1,148 @@ +annotations: + artifacthub.io/license: Apache-2.0 + artifacthub.io/links: | + - name: Chart Source + url: https://github.com/prometheus-community/helm-charts + - name: Upstream Project + url: https://github.com/prometheus-operator/kube-prometheus + artifacthub.io/operator: "true" + catalog.cattle.io/auto-install: rancher-monitoring-crd=match + catalog.cattle.io/certified: rancher + catalog.cattle.io/deploys-on-os: windows + catalog.cattle.io/display-name: Monitoring + catalog.cattle.io/kube-version: '>= 1.26.0-0 < 1.31.0-0' + catalog.cattle.io/namespace: cattle-monitoring-system + catalog.cattle.io/permits-os: linux,windows + catalog.cattle.io/provides-gvr: monitoring.coreos.com.prometheus/v1 + catalog.cattle.io/rancher-version: '>= 2.8.0-0 < 2.19.0-0' + catalog.cattle.io/release-name: rancher-monitoring + catalog.cattle.io/requests-cpu: 4500m + catalog.cattle.io/requests-memory: 4000Mi + catalog.cattle.io/type: cluster-tool + catalog.cattle.io/ui-component: monitoring + catalog.cattle.io/upstream-version: 57.0.3 +apiVersion: v2 +appVersion: v0.72.0 +dependencies: +- condition: grafana.enabled + name: grafana + repository: file://./charts/grafana + version: 7.3.11 +- condition: hardenedKubelet.enabled + name: hardenedKubelet + repository: file://./charts/hardenedKubelet + version: 0.2.1 +- condition: hardenedNodeExporter.enabled + name: hardenedNodeExporter + repository: file://./charts/hardenedNodeExporter + version: 0.2.1 +- condition: k3sServer.enabled + name: k3sServer + repository: file://./charts/k3sServer + version: 0.2.1 +- condition: kubeStateMetrics.enabled + name: kube-state-metrics + repository: file://./charts/kube-state-metrics + version: 5.16.4 +- condition: kubeAdmControllerManager.enabled + name: kubeAdmControllerManager + repository: file://./charts/kubeAdmControllerManager + version: 0.2.1 +- condition: kubeAdmEtcd.enabled + name: kubeAdmEtcd + repository: file://./charts/kubeAdmEtcd + version: 0.2.1 +- condition: kubeAdmProxy.enabled + name: kubeAdmProxy + repository: file://./charts/kubeAdmProxy + version: 0.2.1 +- condition: kubeAdmScheduler.enabled + name: kubeAdmScheduler + repository: file://./charts/kubeAdmScheduler + version: 0.2.1 +- condition: prometheus-adapter.enabled + name: prometheus-adapter + repository: file://./charts/prometheus-adapter + version: 4.2.0 +- condition: nodeExporter.enabled + name: prometheus-node-exporter + repository: file://./charts/prometheus-node-exporter + version: 4.30.3 +- condition: rke2ControllerManager.enabled + name: rke2ControllerManager + repository: file://./charts/rke2ControllerManager + version: 0.2.1 +- condition: rke2Etcd.enabled + name: rke2Etcd + repository: file://./charts/rke2Etcd + version: 0.2.1 +- condition: rke2IngressNginx.enabled + name: rke2IngressNginx + repository: file://./charts/rke2IngressNginx + version: 0.2.1 +- condition: rke2Proxy.enabled + name: rke2Proxy + repository: file://./charts/rke2Proxy + version: 0.2.1 +- condition: rke2Scheduler.enabled + name: rke2Scheduler + repository: file://./charts/rke2Scheduler + version: 0.2.1 +- condition: rkeControllerManager.enabled + name: rkeControllerManager + repository: file://./charts/rkeControllerManager + version: 0.2.1 +- condition: rkeEtcd.enabled + name: rkeEtcd + repository: file://./charts/rkeEtcd + version: 0.2.1 +- condition: rkeIngressNginx.enabled + name: rkeIngressNginx + repository: file://./charts/rkeIngressNginx + version: 0.2.1 +- condition: rkeProxy.enabled + name: rkeProxy + repository: file://./charts/rkeProxy + version: 0.2.1 +- condition: rkeScheduler.enabled + name: rkeScheduler + repository: file://./charts/rkeScheduler + version: 0.2.1 +- condition: windowsExporter.enabled + name: windowsExporter + repository: file://./charts/windowsExporter + version: 0.3.1 +description: kube-prometheus-stack collects Kubernetes manifests, Grafana dashboards, + and Prometheus rules combined with documentation and scripts to provide easy to + operate end-to-end Kubernetes cluster monitoring with Prometheus using the Prometheus + Operator. +home: https://github.com/prometheus-operator/kube-prometheus +icon: file://assets/logos/rancher-monitoring.png +keywords: +- operator +- prometheus +- kube-prometheus +kubeVersion: '>=1.19.0-0' +maintainers: +- email: andrew@quadcorps.co.uk + name: andrewgkew +- email: gianrubio@gmail.com + name: gianrubio +- email: github.gkarthiks@gmail.com + name: gkarthiks +- email: kube-prometheus-stack@sisti.pt + name: GMartinez-Sisti +- email: github@jkroepke.de + name: jkroepke +- email: scott@r6by.com + name: scottrigby +- email: miroslav.hadzhiev@gmail.com + name: Xtigyro +- email: quentin.bisson@gmail.com + name: QuentinBisson +name: rancher-monitoring +sources: +- https://github.com/prometheus-community/helm-charts +- https://github.com/prometheus-operator/kube-prometheus +type: application +version: 103.2.2+up57.0.3 diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/README.md b/charts/rancher-monitoring/103.2.2+up57.0.3/README.md new file mode 100644 index 00000000000..9baf58bb161 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/README.md @@ -0,0 +1,1080 @@ +# kube-prometheus-stack + +Installs the [kube-prometheus stack](https://github.com/prometheus-operator/kube-prometheus), a collection of Kubernetes manifests, [Grafana](http://grafana.com/) dashboards, and [Prometheus rules](https://prometheus.io/docs/prometheus/latest/configuration/recording_rules/) combined with documentation and scripts to provide easy to operate end-to-end Kubernetes cluster monitoring with [Prometheus](https://prometheus.io/) using the [Prometheus Operator](https://github.com/prometheus-operator/prometheus-operator). + +See the [kube-prometheus](https://github.com/prometheus-operator/kube-prometheus) README for details about components, dashboards, and alerts. + +_Note: This chart was formerly named `prometheus-operator` chart, now renamed to more clearly reflect that it installs the `kube-prometheus` project stack, within which Prometheus Operator is only one component._ + +## Prerequisites + +- Kubernetes 1.19+ +- Helm 3+ + +## Get Helm Repository Info + +```console +helm repo add prometheus-community https://prometheus-community.github.io/helm-charts +helm repo update +``` + +_See [`helm repo`](https://helm.sh/docs/helm/helm_repo/) for command documentation._ + +## Install Helm Chart + +```console +helm install [RELEASE_NAME] prometheus-community/kube-prometheus-stack +``` + +_See [configuration](#configuration) below._ + +_See [helm install](https://helm.sh/docs/helm/helm_install/) for command documentation._ + +## Dependencies + +By default this chart installs additional, dependent charts: + +- [prometheus-community/kube-state-metrics](https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-state-metrics) +- [prometheus-community/prometheus-node-exporter](https://github.com/prometheus-community/helm-charts/tree/main/charts/prometheus-node-exporter) +- [grafana/grafana](https://github.com/grafana/helm-charts/tree/main/charts/grafana) + +To disable dependencies during installation, see [multiple releases](#multiple-releases) below. + +_See [helm dependency](https://helm.sh/docs/helm/helm_dependency/) for command documentation._ + +## Uninstall Helm Chart + +```console +helm uninstall [RELEASE_NAME] +``` + +This removes all the Kubernetes components associated with the chart and deletes the release. + +_See [helm uninstall](https://helm.sh/docs/helm/helm_uninstall/) for command documentation._ + +CRDs created by this chart are not removed by default and should be manually cleaned up: + +```console +kubectl delete crd alertmanagerconfigs.monitoring.coreos.com +kubectl delete crd alertmanagers.monitoring.coreos.com +kubectl delete crd podmonitors.monitoring.coreos.com +kubectl delete crd probes.monitoring.coreos.com +kubectl delete crd prometheusagents.monitoring.coreos.com +kubectl delete crd prometheuses.monitoring.coreos.com +kubectl delete crd prometheusrules.monitoring.coreos.com +kubectl delete crd scrapeconfigs.monitoring.coreos.com +kubectl delete crd servicemonitors.monitoring.coreos.com +kubectl delete crd thanosrulers.monitoring.coreos.com +``` + +## Upgrading Chart + +```console +helm upgrade [RELEASE_NAME] prometheus-community/kube-prometheus-stack +``` + +With Helm v3, CRDs created by this chart are not updated by default and should be manually updated. +Consult also the [Helm Documentation on CRDs](https://helm.sh/docs/chart_best_practices/custom_resource_definitions). + +_See [helm upgrade](https://helm.sh/docs/helm/helm_upgrade/) for command documentation._ + +### Upgrading an existing Release to a new major version + +A major chart version change (like v1.2.3 -> v2.0.0) indicates that there is an incompatible breaking change needing manual actions. + +### From 56.x to 57.x + +This version upgrades Prometheus-Operator to v0.72.0 + +Run these commands to update the CRDs before applying the upgrade. + +```console +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.72.0/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagerconfigs.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.72.0/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagers.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.72.0/example/prometheus-operator-crd/monitoring.coreos.com_podmonitors.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.72.0/example/prometheus-operator-crd/monitoring.coreos.com_probes.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.72.0/example/prometheus-operator-crd/monitoring.coreos.com_prometheusagents.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.72.0/example/prometheus-operator-crd/monitoring.coreos.com_prometheuses.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.72.0/example/prometheus-operator-crd/monitoring.coreos.com_prometheusrules.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.72.0/example/prometheus-operator-crd/monitoring.coreos.com_scrapeconfigs.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.72.0/example/prometheus-operator-crd/monitoring.coreos.com_servicemonitors.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.72.0/example/prometheus-operator-crd/monitoring.coreos.com_thanosrulers.yaml +``` + +### From 55.x to 56.x + +This version upgrades Prometheus-Operator to v0.71.0, Prometheus to 2.49.1 + +Run these commands to update the CRDs before applying the upgrade. + +```console +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.71.0/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagerconfigs.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.71.0/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagers.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.71.0/example/prometheus-operator-crd/monitoring.coreos.com_podmonitors.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.71.0/example/prometheus-operator-crd/monitoring.coreos.com_probes.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.71.0/example/prometheus-operator-crd/monitoring.coreos.com_prometheusagents.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.71.0/example/prometheus-operator-crd/monitoring.coreos.com_prometheuses.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.71.0/example/prometheus-operator-crd/monitoring.coreos.com_prometheusrules.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.71.0/example/prometheus-operator-crd/monitoring.coreos.com_scrapeconfigs.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.71.0/example/prometheus-operator-crd/monitoring.coreos.com_servicemonitors.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.71.0/example/prometheus-operator-crd/monitoring.coreos.com_thanosrulers.yaml +``` + +### From 54.x to 55.x + +This version upgrades Prometheus-Operator to v0.70.0 + +Run these commands to update the CRDs before applying the upgrade. + +```console +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.70.0/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagerconfigs.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.70.0/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagers.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.70.0/example/prometheus-operator-crd/monitoring.coreos.com_podmonitors.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.70.0/example/prometheus-operator-crd/monitoring.coreos.com_probes.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.70.0/example/prometheus-operator-crd/monitoring.coreos.com_prometheusagents.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.70.0/example/prometheus-operator-crd/monitoring.coreos.com_prometheuses.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.70.0/example/prometheus-operator-crd/monitoring.coreos.com_prometheusrules.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.70.0/example/prometheus-operator-crd/monitoring.coreos.com_scrapeconfigs.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.70.0/example/prometheus-operator-crd/monitoring.coreos.com_servicemonitors.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.70.0/example/prometheus-operator-crd/monitoring.coreos.com_thanosrulers.yaml +``` + +### From 53.x to 54.x + +Grafana Helm Chart has bumped to version 7 + +Please note Grafana Helm Chart [changelog](https://github.com/grafana/helm-charts/tree/main/charts/grafana#to-700). + +### From 52.x to 53.x + +This version upgrades Prometheus-Operator to v0.69.1, Prometheus to 2.47.2 + +Run these commands to update the CRDs before applying the upgrade. + +```console +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.69.1/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagerconfigs.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.69.1/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagers.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.69.1/example/prometheus-operator-crd/monitoring.coreos.com_podmonitors.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.69.1/example/prometheus-operator-crd/monitoring.coreos.com_probes.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.69.1/example/prometheus-operator-crd/monitoring.coreos.com_prometheusagents.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.69.1/example/prometheus-operator-crd/monitoring.coreos.com_prometheuses.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.69.1/example/prometheus-operator-crd/monitoring.coreos.com_prometheusrules.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.69.1/example/prometheus-operator-crd/monitoring.coreos.com_scrapeconfigs.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.69.1/example/prometheus-operator-crd/monitoring.coreos.com_servicemonitors.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.69.1/example/prometheus-operator-crd/monitoring.coreos.com_thanosrulers.yaml +``` + +### From 51.x to 52.x + +This includes the ability to select between using existing secrets or create new secret objects for various thanos config. The defaults have not changed but if you were setting: + +- `thanosRuler.thanosRulerSpec.alertmanagersConfig` or +- `thanosRuler.thanosRulerSpec.objectStorageConfig` or +- `thanosRuler.thanosRulerSpec.queryConfig` or +- `prometheus.prometheusSpec.thanos.objectStorageConfig` + +you will have to need to set `existingSecret` or `secret` based on your requirement + +For instance, the `thanosRuler.thanosRulerSpec.alertmanagersConfig` used to be configured as follow: + +```yaml +thanosRuler: + thanosRulerSpec: + alertmanagersConfig: + alertmanagers: + - api_version: v2 + http_config: + basic_auth: + username: some_user + password: some_pass + static_configs: + - alertmanager.thanos.io + scheme: http + timeout: 10s +``` + +But it now moved to: + +```yaml +thanosRuler: + thanosRulerSpec: + alertmanagersConfig: + secret: + alertmanagers: + - api_version: v2 + http_config: + basic_auth: + username: some_user + password: some_pass + static_configs: + - alertmanager.thanos.io + scheme: http + timeout: 10s +``` + +or the `thanosRuler.thanosRulerSpec.objectStorageConfig` used to be configured as follow: + +```yaml +thanosRuler: + thanosRulerSpec: + objectStorageConfig: + name: existing-secret-not-created-by-this-chart + key: object-storage-configs.yaml +``` + +But it now moved to: + +```yaml +thanosRuler: + thanosRulerSpec: + objectStorageConfig: + existingSecret: + name: existing-secret-not-created-by-this-chart + key: object-storage-configs.yaml +``` + +### From 50.x to 51.x + +This version upgrades Prometheus-Operator to v0.68.0, Prometheus to 2.47.0 and Thanos to v0.32.2 + +Run these commands to update the CRDs before applying the upgrade. + +```console +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.68.0/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagerconfigs.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.68.0/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagers.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.68.0/example/prometheus-operator-crd/monitoring.coreos.com_podmonitors.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.68.0/example/prometheus-operator-crd/monitoring.coreos.com_probes.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.68.0/example/prometheus-operator-crd/monitoring.coreos.com_prometheusagents.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.68.0/example/prometheus-operator-crd/monitoring.coreos.com_prometheuses.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.68.0/example/prometheus-operator-crd/monitoring.coreos.com_prometheusrules.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.68.0/example/prometheus-operator-crd/monitoring.coreos.com_scrapeconfigs.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.68.0/example/prometheus-operator-crd/monitoring.coreos.com_servicemonitors.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.68.0/example/prometheus-operator-crd/monitoring.coreos.com_thanosrulers.yaml +``` + +### From 49.x to 50.x + +This version requires Kubernetes 1.19+. + +We do not expect any breaking changes in this version. + +### From 48.x to 49.x + +This version upgrades Prometheus-Operator to v0.67.1, 0, Alertmanager to v0.26.0, Prometheus to 2.46.0 and Thanos to v0.32.0 + +Run these commands to update the CRDs before applying the upgrade. + +```console +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.67.1/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagerconfigs.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.67.1/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagers.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.67.1/example/prometheus-operator-crd/monitoring.coreos.com_podmonitors.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.67.1/example/prometheus-operator-crd/monitoring.coreos.com_probes.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.67.1/example/prometheus-operator-crd/monitoring.coreos.com_prometheusagents.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.67.1/example/prometheus-operator-crd/monitoring.coreos.com_prometheuses.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.67.1/example/prometheus-operator-crd/monitoring.coreos.com_prometheusrules.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.67.1/example/prometheus-operator-crd/monitoring.coreos.com_scrapeconfigs.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.67.1/example/prometheus-operator-crd/monitoring.coreos.com_servicemonitors.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.67.1/example/prometheus-operator-crd/monitoring.coreos.com_thanosrulers.yaml +``` + +### From 47.x to 48.x + +This version moved all CRDs into a dedicated sub-chart. No new CRDs are introduced in this version. +See [#3548](https://github.com/prometheus-community/helm-charts/issues/3548) for more context. + +We do not expect any breaking changes in this version. + +### From 46.x to 47.x + +This version upgrades Prometheus-Operator to v0.66.0 with new CRDs (PrometheusAgent and ScrapeConfig). + +Run these commands to update the CRDs before applying the upgrade. + +```console +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.66.0/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagerconfigs.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.66.0/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagers.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.66.0/example/prometheus-operator-crd/monitoring.coreos.com_podmonitors.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.66.0/example/prometheus-operator-crd/monitoring.coreos.com_probes.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.66.0/example/prometheus-operator-crd/monitoring.coreos.com_prometheusagents.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.66.0/example/prometheus-operator-crd/monitoring.coreos.com_prometheuses.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.66.0/example/prometheus-operator-crd/monitoring.coreos.com_prometheusrules.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.66.0/example/prometheus-operator-crd/monitoring.coreos.com_scrapeconfigs.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.66.0/example/prometheus-operator-crd/monitoring.coreos.com_servicemonitors.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.66.0/example/prometheus-operator-crd/monitoring.coreos.com_thanosrulers.yaml +``` + +### From 45.x to 46.x + +This version upgrades Prometheus-Operator to v0.65.1 with new CRDs (PrometheusAgent and ScrapeConfig), Prometheus to v2.44.0 and Thanos to v0.31.0. + +Run these commands to update the CRDs before applying the upgrade. + +```console +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.65.1/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagerconfigs.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.65.1/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagers.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.65.1/example/prometheus-operator-crd/monitoring.coreos.com_podmonitors.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.65.1/example/prometheus-operator-crd/monitoring.coreos.com_probes.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.65.1/example/prometheus-operator-crd/monitoring.coreos.com_prometheusagents.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.65.1/example/prometheus-operator-crd/monitoring.coreos.com_prometheuses.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.65.1/example/prometheus-operator-crd/monitoring.coreos.com_prometheusrules.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.65.1/example/prometheus-operator-crd/monitoring.coreos.com_scrapeconfigs.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.65.1/example/prometheus-operator-crd/monitoring.coreos.com_servicemonitors.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.65.1/example/prometheus-operator-crd/monitoring.coreos.com_thanosrulers.yaml +``` + +### From 44.x to 45.x + +This version upgrades Prometheus-Operator to v0.63.0, Prometheus to v2.42.0 and Thanos to v0.30.2. + +Run these commands to update the CRDs before applying the upgrade. + +```console +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.63.0/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagerconfigs.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.63.0/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagers.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.63.0/example/prometheus-operator-crd/monitoring.coreos.com_podmonitors.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.63.0/example/prometheus-operator-crd/monitoring.coreos.com_probes.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.63.0/example/prometheus-operator-crd/monitoring.coreos.com_prometheuses.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.63.0/example/prometheus-operator-crd/monitoring.coreos.com_prometheusrules.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.63.0/example/prometheus-operator-crd/monitoring.coreos.com_servicemonitors.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.63.0/example/prometheus-operator-crd/monitoring.coreos.com_thanosrulers.yaml +``` + +### From 43.x to 44.x + +This version upgrades Prometheus-Operator to v0.62.0, Prometheus to v2.41.0 and Thanos to v0.30.1. + +Run these commands to update the CRDs before applying the upgrade. + +```console +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.62.0/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagerconfigs.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.62.0/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagers.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.62.0/example/prometheus-operator-crd/monitoring.coreos.com_podmonitors.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.62.0/example/prometheus-operator-crd/monitoring.coreos.com_probes.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.62.0/example/prometheus-operator-crd/monitoring.coreos.com_prometheuses.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.62.0/example/prometheus-operator-crd/monitoring.coreos.com_prometheusrules.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.62.0/example/prometheus-operator-crd/monitoring.coreos.com_servicemonitors.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.62.0/example/prometheus-operator-crd/monitoring.coreos.com_thanosrulers.yaml +``` + +If you have explicitly set `prometheusOperator.admissionWebhooks.failurePolicy`, this value is now always used even when `.prometheusOperator.admissionWebhooks.patch.enabled` is `true` (the default). + +The values for `prometheusOperator.image.tag` & `prometheusOperator.prometheusConfigReloader.image.tag` are now empty by default and the Chart.yaml `appVersion` field is used instead. + +### From 42.x to 43.x + +This version upgrades Prometheus-Operator to v0.61.1, Prometheus to v2.40.5 and Thanos to v0.29.0. + +Run these commands to update the CRDs before applying the upgrade. + +```console +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.61.1/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagerconfigs.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.61.1/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagers.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.61.1/example/prometheus-operator-crd/monitoring.coreos.com_podmonitors.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.61.1/example/prometheus-operator-crd/monitoring.coreos.com_probes.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.61.1/example/prometheus-operator-crd/monitoring.coreos.com_prometheuses.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.61.1/example/prometheus-operator-crd/monitoring.coreos.com_prometheusrules.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.61.1/example/prometheus-operator-crd/monitoring.coreos.com_servicemonitors.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.61.1/example/prometheus-operator-crd/monitoring.coreos.com_thanosrulers.yaml +``` + +### From 41.x to 42.x + +This includes the overridability of container registry for all containers at the global level using `global.imageRegistry` or per container image. The defaults have not changed but if you were using a custom image, you will have to override the registry of said custom container image before you upgrade. + +For instance, the prometheus-config-reloader used to be configured as follow: + +```yaml + image: + repository: quay.io/prometheus-operator/prometheus-config-reloader + tag: v0.60.1 + sha: "" +``` + +But it now moved to: + +```yaml + image: + registry: quay.io + repository: prometheus-operator/prometheus-config-reloader + tag: v0.60.1 + sha: "" +``` + +### From 40.x to 41.x + +This version upgrades Prometheus-Operator to v0.60.1, Prometheus to v2.39.1 and Thanos to v0.28.1. +This version also upgrades the Helm charts of kube-state-metrics to 4.20.2, prometheus-node-exporter to 4.3.0 and Grafana to 6.40.4. + +Run these commands to update the CRDs before applying the upgrade. + +```console +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.60.1/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagerconfigs.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.60.1/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagers.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.60.1/example/prometheus-operator-crd/monitoring.coreos.com_podmonitors.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.60.1/example/prometheus-operator-crd/monitoring.coreos.com_probes.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.60.1/example/prometheus-operator-crd/monitoring.coreos.com_prometheuses.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.60.1/example/prometheus-operator-crd/monitoring.coreos.com_prometheusrules.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.60.1/example/prometheus-operator-crd/monitoring.coreos.com_servicemonitors.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.60.1/example/prometheus-operator-crd/monitoring.coreos.com_thanosrulers.yaml +``` + +This version splits kubeScheduler recording and altering rules in separate config values. +Instead of `defaultRules.rules.kubeScheduler` the 2 new variables `defaultRules.rules.kubeSchedulerAlerting` and `defaultRules.rules.kubeSchedulerRecording` are used. + +### From 39.x to 40.x + +This version upgrades Prometheus-Operator to v0.59.1, Prometheus to v2.38.0, kube-state-metrics to v2.6.0 and Thanos to v0.28.0. +This version also upgrades the Helm charts of kube-state-metrics to 4.18.0 and prometheus-node-exporter to 4.2.0. + +Run these commands to update the CRDs before applying the upgrade. + +```console +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.59.1/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagerconfigs.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.59.1/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagers.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.59.1/example/prometheus-operator-crd/monitoring.coreos.com_podmonitors.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.59.1/example/prometheus-operator-crd/monitoring.coreos.com_probes.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.59.1/example/prometheus-operator-crd/monitoring.coreos.com_prometheuses.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.59.1/example/prometheus-operator-crd/monitoring.coreos.com_prometheusrules.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.59.1/example/prometheus-operator-crd/monitoring.coreos.com_servicemonitors.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.59.1/example/prometheus-operator-crd/monitoring.coreos.com_thanosrulers.yaml +``` + +Starting from prometheus-node-exporter version 4.0.0, the `node exporter` chart is using the [Kubernetes recommended labels](https://kubernetes.io/docs/concepts/overview/working-with-objects/common-labels/). Therefore you have to delete the daemonset before you upgrade. + +```console +kubectl delete daemonset -l app=prometheus-node-exporter +helm upgrade -i kube-prometheus-stack prometheus-community/kube-prometheus-stack +``` + +If you use your own custom [ServiceMonitor](https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#servicemonitor) or [PodMonitor](https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#podmonitor), please ensure to upgrade their `selector` fields accordingly to the new labels. + +### From 38.x to 39.x + +This upgraded prometheus-operator to v0.58.0 and prometheus to v2.37.0 + +Run these commands to update the CRDs before applying the upgrade. + +```console +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.58.0/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagerconfigs.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.58.0/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagers.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.58.0/example/prometheus-operator-crd/monitoring.coreos.com_podmonitors.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.58.0/example/prometheus-operator-crd/monitoring.coreos.com_probes.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.58.0/example/prometheus-operator-crd/monitoring.coreos.com_prometheuses.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.58.0/example/prometheus-operator-crd/monitoring.coreos.com_prometheusrules.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.58.0/example/prometheus-operator-crd/monitoring.coreos.com_servicemonitors.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.58.0/example/prometheus-operator-crd/monitoring.coreos.com_thanosrulers.yaml +``` + +### From 37.x to 38.x + +Reverted one of the default metrics relabelings for cAdvisor added in 36.x, due to it breaking container_network_* and various other statistics. If you do not want this change, you will need to override the `kubelet.cAdvisorMetricRelabelings`. + +### From 36.x to 37.x + +This includes some default metric relabelings for cAdvisor and apiserver metrics to reduce cardinality. If you do not want these defaults, you will need to override the `kubeApiServer.metricRelabelings` and or `kubelet.cAdvisorMetricRelabelings`. + +### From 35.x to 36.x + +This upgraded prometheus-operator to v0.57.0 and prometheus to v2.36.1 + +Run these commands to update the CRDs before applying the upgrade. + +```console +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.57.0/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagerconfigs.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.57.0/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagers.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.57.0/example/prometheus-operator-crd/monitoring.coreos.com_podmonitors.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.57.0/example/prometheus-operator-crd/monitoring.coreos.com_probes.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.57.0/example/prometheus-operator-crd/monitoring.coreos.com_prometheuses.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.57.0/example/prometheus-operator-crd/monitoring.coreos.com_prometheusrules.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.57.0/example/prometheus-operator-crd/monitoring.coreos.com_servicemonitors.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.57.0/example/prometheus-operator-crd/monitoring.coreos.com_thanosrulers.yaml +``` + +### From 34.x to 35.x + +This upgraded prometheus-operator to v0.56.0 and prometheus to v2.35.0 + +Run these commands to update the CRDs before applying the upgrade. + +```console +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.56.0/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagerconfigs.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.56.0/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagers.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.56.0/example/prometheus-operator-crd/monitoring.coreos.com_podmonitors.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.56.0/example/prometheus-operator-crd/monitoring.coreos.com_probes.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.56.0/example/prometheus-operator-crd/monitoring.coreos.com_prometheuses.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.56.0/example/prometheus-operator-crd/monitoring.coreos.com_prometheusrules.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.56.0/example/prometheus-operator-crd/monitoring.coreos.com_servicemonitors.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.56.0/example/prometheus-operator-crd/monitoring.coreos.com_thanosrulers.yaml +``` + +### From 33.x to 34.x + +This upgrades to prometheus-operator to v0.55.0 and prometheus to v2.33.5. + +Run these commands to update the CRDs before applying the upgrade. + +```console +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.55.0/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagerconfigs.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.55.0/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagers.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.55.0/example/prometheus-operator-crd/monitoring.coreos.com_podmonitors.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.55.0/example/prometheus-operator-crd/monitoring.coreos.com_probes.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.55.0/example/prometheus-operator-crd/monitoring.coreos.com_prometheuses.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.55.0/example/prometheus-operator-crd/monitoring.coreos.com_prometheusrules.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.55.0/example/prometheus-operator-crd/monitoring.coreos.com_servicemonitors.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.55.0/example/prometheus-operator-crd/monitoring.coreos.com_thanosrulers.yaml +``` + +### From 32.x to 33.x + +This upgrades the prometheus-node-exporter Chart to v3.0.0. Please review the changes to this subchart if you make customizations to hostMountPropagation. + +### From 31.x to 32.x + +This upgrades to prometheus-operator to v0.54.0 and prometheus to v2.33.1. It also changes the default for `grafana.serviceMonitor.enabled` to `true. + +Run these commands to update the CRDs before applying the upgrade. + +```console +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.54.0/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagerconfigs.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.54.0/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagers.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.54.0/example/prometheus-operator-crd/monitoring.coreos.com_podmonitors.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.54.0/example/prometheus-operator-crd/monitoring.coreos.com_probes.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.54.0/example/prometheus-operator-crd/monitoring.coreos.com_prometheuses.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.54.0/example/prometheus-operator-crd/monitoring.coreos.com_prometheusrules.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.54.0/example/prometheus-operator-crd/monitoring.coreos.com_servicemonitors.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.54.0/example/prometheus-operator-crd/monitoring.coreos.com_thanosrulers.yaml +``` + +### From 30.x to 31.x + +This version removes the built-in grafana ServiceMonitor and instead relies on the ServiceMonitor of the sub-chart. +`grafana.serviceMonitor.enabled` must be set instead of `grafana.serviceMonitor.selfMonitor` and the old ServiceMonitor may +need to be manually cleaned up after deploying the new release. + +### From 29.x to 30.x + +This version updates kube-state-metrics to 4.3.0 and uses the new option `kube-state-metrics.releaseLabel=true` which adds the "release" label to kube-state-metrics labels, making scraping of the metrics by kube-prometheus-stack work out of the box again, independent of the used kube-prometheus-stack release name. If you already set the "release" label via `kube-state-metrics.customLabels` you might have to remove that and use it via the new option. + +### From 28.x to 29.x + +This version makes scraping port for kube-controller-manager and kube-scheduler dynamic to reflect changes to default serving ports +for those components in Kubernetes versions v1.22 and v1.23 respectively. + +If you deploy on clusters using version v1.22+, kube-controller-manager will be scraped over HTTPS on port 10257. + +If you deploy on clusters running version v1.23+, kube-scheduler will be scraped over HTTPS on port 10259. + +### From 27.x to 28.x + +This version disables PodSecurityPolicies by default because they are deprecated in Kubernetes 1.21 and will be removed in Kubernetes 1.25. + +If you are using PodSecurityPolicies you can enable the previous behaviour by setting `kube-state-metrics.podSecurityPolicy.enabled`, `prometheus-node-exporter.rbac.pspEnabled`, `grafana.rbac.pspEnabled` and `global.rbac.pspEnabled` to `true`. + +### From 26.x to 27.x + +This version splits prometheus-node-exporter chart recording and altering rules in separate config values. +Instead of `defaultRules.rules.node` the 2 new variables `defaultRules.rules.nodeExporterAlerting` and `defaultRules.rules.nodeExporterRecording` are used. + +Also the following defaultRules.rules has been removed as they had no effect: `kubeApiserverError`, `kubePrometheusNodeAlerting`, `kubernetesAbsent`, `time`. + +The ability to set a rubookUrl via `defaultRules.rules.rubookUrl` was reintroduced. + +### From 25.x to 26.x + +This version enables the prometheus-node-exporter subchart servicemonitor by default again, by setting `prometheus-node-exporter.prometheus.monitor.enabled` to `true`. + +### From 24.x to 25.x + +This version upgrade to prometheus-operator v0.53.1. It removes support for setting a runbookUrl, since the upstream format for runbooks changed. + +```console +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.53.1/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagerconfigs.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.53.1/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagers.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.53.1/example/prometheus-operator-crd/monitoring.coreos.com_podmonitors.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.53.1/example/prometheus-operator-crd/monitoring.coreos.com_probes.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.53.1/example/prometheus-operator-crd/monitoring.coreos.com_prometheuses.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.53.1/example/prometheus-operator-crd/monitoring.coreos.com_prometheusrules.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.53.1/example/prometheus-operator-crd/monitoring.coreos.com_servicemonitors.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.53.1/example/prometheus-operator-crd/monitoring.coreos.com_thanosrulers.yaml +``` + +### From 23.x to 24.x + +The custom `ServiceMonitor` for the _kube-state-metrics_ & _prometheus-node-exporter_ charts have been removed in favour of the built-in sub-chart `ServiceMonitor`; for both sub-charts this means that `ServiceMonitor` customisations happen via the values passed to the chart. If you haven't directly customised this behaviour then there are no changes required to upgrade, but if you have please read the following. + +For _kube-state-metrics_ the `ServiceMonitor` customisation is now set via `kube-state-metrics.prometheus.monitor` and the `kubeStateMetrics.serviceMonitor.selfMonitor.enabled` value has moved to `kube-state-metrics.selfMonitor.enabled`. + +For _prometheus-node-exporter_ the `ServiceMonitor` customisation is now set via `prometheus-node-exporter.prometheus.monitor` and the `nodeExporter.jobLabel` values has moved to `prometheus-node-exporter.prometheus.monitor.jobLabel`. + +### From 22.x to 23.x + +Port names have been renamed for Istio's +[explicit protocol selection](https://istio.io/latest/docs/ops/configuration/traffic-management/protocol-selection/#explicit-protocol-selection). + +| | old value | new value | +|-|-----------|-----------| +| `alertmanager.alertmanagerSpec.portName` | `web` | `http-web` | +| `grafana.service.portName` | `service` | `http-web` | +| `prometheus-node-exporter.service.portName` | `metrics` (hardcoded) | `http-metrics` | +| `prometheus.prometheusSpec.portName` | `web` | `http-web` | + +### From 21.x to 22.x + +Due to the upgrade of the `kube-state-metrics` chart, removal of its deployment/stateful needs to done manually prior to upgrading: + +```console +kubectl delete deployments.apps -l app.kubernetes.io/instance=prometheus-operator,app.kubernetes.io/name=kube-state-metrics --cascade=orphan +``` + +or if you use autosharding: + +```console +kubectl delete statefulsets.apps -l app.kubernetes.io/instance=prometheus-operator,app.kubernetes.io/name=kube-state-metrics --cascade=orphan +``` + +### From 20.x to 21.x + +The config reloader values have been refactored. All the values have been moved to the key `prometheusConfigReloader` and the limits and requests can now be set separately. + +### From 19.x to 20.x + +Version 20 upgrades prometheus-operator from 0.50.x to 0.52.x. Helm does not automatically upgrade or install new CRDs on a chart upgrade, so you have to install the CRDs manually before updating: + +```console +kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.52.0/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagerconfigs.yaml +kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.52.0/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagers.yaml +kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.52.0/example/prometheus-operator-crd/monitoring.coreos.com_podmonitors.yaml +kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.52.0/example/prometheus-operator-crd/monitoring.coreos.com_probes.yaml +kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.52.0/example/prometheus-operator-crd/monitoring.coreos.com_prometheuses.yaml +kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.52.0/example/prometheus-operator-crd/monitoring.coreos.com_prometheusrules.yaml +kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.52.0/example/prometheus-operator-crd/monitoring.coreos.com_servicemonitors.yaml +kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.52.0/example/prometheus-operator-crd/monitoring.coreos.com_thanosrulers.yaml +``` + +### From 18.x to 19.x + +`kubeStateMetrics.serviceMonitor.namespaceOverride` was removed. +Please use `kube-state-metrics.namespaceOverride` instead. + +### From 17.x to 18.x + +Version 18 upgrades prometheus-operator from 0.49.x to 0.50.x. Helm does not automatically upgrade or install new CRDs on a chart upgrade, so you have to install the CRDs manually before updating: + +```console +kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.50.0/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagerconfigs.yaml +kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.50.0/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagers.yaml +kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.50.0/example/prometheus-operator-crd/monitoring.coreos.com_podmonitors.yaml +kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.50.0/example/prometheus-operator-crd/monitoring.coreos.com_probes.yaml +kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.50.0/example/prometheus-operator-crd/monitoring.coreos.com_prometheuses.yaml +kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.50.0/example/prometheus-operator-crd/monitoring.coreos.com_prometheusrules.yaml +kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.50.0/example/prometheus-operator-crd/monitoring.coreos.com_servicemonitors.yaml +kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.50.0/example/prometheus-operator-crd/monitoring.coreos.com_thanosrulers.yaml +``` + +### From 16.x to 17.x + +Version 17 upgrades prometheus-operator from 0.48.x to 0.49.x. Helm does not automatically upgrade or install new CRDs on a chart upgrade, so you have to install the CRDs manually before updating: + +```console +kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.49.0/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagerconfigs.yaml +kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.49.0/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagers.yaml +kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.49.0/example/prometheus-operator-crd/monitoring.coreos.com_podmonitors.yaml +kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.49.0/example/prometheus-operator-crd/monitoring.coreos.com_probes.yaml +kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.49.0/example/prometheus-operator-crd/monitoring.coreos.com_prometheuses.yaml +kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.49.0/example/prometheus-operator-crd/monitoring.coreos.com_prometheusrules.yaml +kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.49.0/example/prometheus-operator-crd/monitoring.coreos.com_servicemonitors.yaml +kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.49.0/example/prometheus-operator-crd/monitoring.coreos.com_thanosrulers.yaml +``` + +### From 15.x to 16.x + +Version 16 upgrades kube-state-metrics to v2.0.0. This includes changed command-line arguments and removed metrics, see this [blog post](https://kubernetes.io/blog/2021/04/13/kube-state-metrics-v-2-0/). This version also removes Grafana dashboards that supported Kubernetes 1.14 or earlier. + +### From 14.x to 15.x + +Version 15 upgrades prometheus-operator from 0.46.x to 0.47.x. Helm does not automatically upgrade or install new CRDs on a chart upgrade, so you have to install the CRDs manually before updating: + +```console +kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.47.0/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagerconfigs.yaml +kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.47.0/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagers.yaml +kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.47.0/example/prometheus-operator-crd/monitoring.coreos.com_podmonitors.yaml +kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.47.0/example/prometheus-operator-crd/monitoring.coreos.com_probes.yaml +kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.47.0/example/prometheus-operator-crd/monitoring.coreos.com_prometheuses.yaml +kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.47.0/example/prometheus-operator-crd/monitoring.coreos.com_servicemonitors.yaml +kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.47.0/example/prometheus-operator-crd/monitoring.coreos.com_thanosrulers.yaml +``` + +### From 13.x to 14.x + +Version 14 upgrades prometheus-operator from 0.45.x to 0.46.x. Helm does not automatically upgrade or install new CRDs on a chart upgrade, so you have to install the CRDs manually before updating: + +```console +kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.46.0/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagerconfigs.yaml +kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.46.0/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagers.yaml +kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.46.0/example/prometheus-operator-crd/monitoring.coreos.com_podmonitors.yaml +kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.46.0/example/prometheus-operator-crd/monitoring.coreos.com_probes.yaml +kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.46.0/example/prometheus-operator-crd/monitoring.coreos.com_prometheuses.yaml +kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.46.0/example/prometheus-operator-crd/monitoring.coreos.com_servicemonitors.yaml +kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.46.0/example/prometheus-operator-crd/monitoring.coreos.com_thanosrulers.yaml +``` + +### From 12.x to 13.x + +Version 13 upgrades prometheus-operator from 0.44.x to 0.45.x. Helm does not automatically upgrade or install new CRDs on a chart upgrade, so you have to install the CRD manually before updating: + +```console +kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.45.0/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagerconfigs.yaml +kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.45.0/example/prometheus-operator-crd/monitoring.coreos.com_prometheuses.yaml +kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.45.0/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagers.yaml +``` + +### From 11.x to 12.x + +Version 12 upgrades prometheus-operator from 0.43.x to 0.44.x. Helm does not automatically upgrade or install new CRDs on a chart upgrade, so you have to install the CRD manually before updating: + +```console +kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/release-0.44/example/prometheus-operator-crd/monitoring.coreos.com_prometheuses.yaml +``` + +The chart was migrated to support only helm v3 and later. + +### From 10.x to 11.x + +Version 11 upgrades prometheus-operator from 0.42.x to 0.43.x. Starting with 0.43.x an additional `AlertmanagerConfigs` CRD is introduced. Helm does not automatically upgrade or install new CRDs on a chart upgrade, so you have to install the CRD manually before updating: + +```console +kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/release-0.43/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagerconfigs.yaml +``` + +Version 11 removes the deprecated tlsProxy via ghostunnel in favor of native TLS support the prometheus-operator gained with v0.39.0. + +### From 9.x to 10.x + +Version 10 upgrades prometheus-operator from 0.38.x to 0.42.x. Starting with 0.40.x an additional `Probes` CRD is introduced. Helm does not automatically upgrade or install new CRDs on a chart upgrade, so you have to install the CRD manually before updating: + +```console +kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/release-0.42/example/prometheus-operator-crd/monitoring.coreos.com_probes.yaml +``` + +### From 8.x to 9.x + +Version 9 of the helm chart removes the existing `additionalScrapeConfigsExternal` in favour of `additionalScrapeConfigsSecret`. This change lets users specify the secret name and secret key to use for the additional scrape configuration of prometheus. This is useful for users that have prometheus-operator as a subchart and also have a template that creates the additional scrape configuration. + +### From 7.x to 8.x + +Due to new template functions being used in the rules in version 8.x.x of the chart, an upgrade to Prometheus Operator and Prometheus is necessary in order to support them. First, upgrade to the latest version of 7.x.x + +```console +helm upgrade [RELEASE_NAME] prometheus-community/kube-prometheus-stack --version 7.5.0 +``` + +Then upgrade to 8.x.x + +```console +helm upgrade [RELEASE_NAME] prometheus-community/kube-prometheus-stack --version [8.x.x] +``` + +Minimal recommended Prometheus version for this chart release is `2.12.x` + +### From 6.x to 7.x + +Due to a change in grafana subchart, version 7.x.x now requires Helm >= 2.12.0. + +### From 5.x to 6.x + +Due to a change in deployment labels of kube-state-metrics, the upgrade requires `helm upgrade --force` in order to re-create the deployment. If this is not done an error will occur indicating that the deployment cannot be modified: + +```console +invalid: spec.selector: Invalid value: v1.LabelSelector{MatchLabels:map[string]string{"app.kubernetes.io/name":"kube-state-metrics"}, MatchExpressions:[]v1.LabelSelectorRequirement(nil)}: field is immutable +``` + +If this error has already been encountered, a `helm history` command can be used to determine which release has worked, then `helm rollback` to the release, then `helm upgrade --force` to this new one + +## Configuration + +See [Customizing the Chart Before Installing](https://helm.sh/docs/intro/using_helm/#customizing-the-chart-before-installing). To see all configurable options with detailed comments: + +```console +helm show values prometheus-community/kube-prometheus-stack +``` + +You may also run `helm show values` on this chart's [dependencies](#dependencies) for additional options. + +### Rancher Monitoring Configuration + +The following table shows values exposed by Rancher Monitoring's additions to the chart: + +| Parameter | Description | Default | +| ----- | ----------- | ------ | +| `nameOverride` | Provide a name that should be used instead of the chart name when naming all resources deployed by this chart |`"rancher-monitoring"`| +| `namespaceOverride` | Override the deployment namespace | `"cattle-monitoring-system"` | +| `global.rbac.userRoles.create` | Create default user ClusterRoles to allow users to interact with Prometheus CRs, ConfigMaps, and Secrets | `true` | +| `global.rbac.userRoles.aggregateToDefaultRoles` | Aggregate default user ClusterRoles into default k8s ClusterRoles | `true` | +| `prometheus-adapter.enabled` | Whether to install [prometheus-adapter](https://github.com/prometheus-community/helm-charts/tree/main/charts/prometheus-adapter) within the cluster | `true` | +| `prometheus-adapter.prometheus.url` | A URL pointing to the Prometheus deployment within your cluster. The default value is set based on the assumption that you plan to deploy the default Prometheus instance from this chart where `.Values.namespaceOverride=cattle-monitoring-system` and `.Values.nameOverride=rancher-monitoring` | `http://rancher-monitoring-prometheus.cattle-monitoring-system.svc` | +| `prometheus-adapter.prometheus.port` | The port on the Prometheus deployment that Prometheus Adapter can make requests to | `9090` | +| `prometheus.prometheusSpec.ignoreNamespaceSelectors` | Ignore NamespaceSelector settings from the PodMonitor and ServiceMonitor configs. If true, PodMonitors and ServiceMonitors can only discover Pods and Services within the namespace they are deployed into | `false` | + +The following values are enabled for different distributions via [rancher-pushprox](https://github.com/rancher/dev-charts/tree/master/packages/rancher-pushprox). See the rancher-pushprox `README.md` for more information on what all values can be configured for the PushProxy chart. + +| Parameter | Description | Default | +| ----- | ----------- | ------ | +| `rkeControllerManager.enabled` | Create a PushProx installation for monitoring kube-controller-manager metrics in RKE clusters | `false` | +| `rkeScheduler.enabled` | Create a PushProx installation for monitoring kube-scheduler metrics in RKE clusters | `false` | +| `rkeProxy.enabled` | Create a PushProx installation for monitoring kube-proxy metrics in RKE clusters | `false` | +| `rkeIngressNginx.enabled` | Create a PushProx installation for monitoring ingress-nginx metrics in RKE clusters | `false` | +| `rkeEtcd.enabled` | Create a PushProx installation for monitoring etcd metrics in RKE clusters | `false` | +| `rke2IngressNginx.enabled` | Create a PushProx installation for monitoring ingress-nginx metrics in RKE2 clusters | `false` | +| `k3sServer.enabled` | Create a PushProx installation for monitoring k3s-server metrics (accounts for kube-controller-manager, kube-scheduler, and kube-proxy metrics) in k3s clusters | `false` | +| `kubeAdmControllerManager.enabled` | Create a PushProx installation for monitoring kube-controller-manager metrics in kubeAdm clusters | `false` | +| `kubeAdmScheduler.enabled` | Create a PushProx installation for monitoring kube-scheduler metrics in kubeAdm clusters | `false` | +| `kubeAdmProxy.enabled` | Create a PushProx installation for monitoring kube-proxy metrics in kubeAdm clusters | `false` | +| `kubeAdmEtcd.enabled` | Create a PushProx installation for monitoring etcd metrics in kubeAdm clusters | `false` | + + +### Multiple releases + +The same chart can be used to run multiple Prometheus instances in the same cluster if required. To achieve this, it is necessary to run only one instance of prometheus-operator and a pair of alertmanager pods for an HA configuration, while all other components need to be disabled. To disable a dependency during installation, set `kubeStateMetrics.enabled`, `nodeExporter.enabled` and `grafana.enabled` to `false`. + +## Work-Arounds for Known Issues + +### Running on private GKE clusters + +When Google configure the control plane for private clusters, they automatically configure VPC peering between your Kubernetes cluster’s network and a separate Google managed project. In order to restrict what Google are able to access within your cluster, the firewall rules configured restrict access to your Kubernetes pods. This means that in order to use the webhook component with a GKE private cluster, you must configure an additional firewall rule to allow the GKE control plane access to your webhook pod. + +You can read more information on how to add firewall rules for the GKE control plane nodes in the [GKE docs](https://cloud.google.com/kubernetes-engine/docs/how-to/private-clusters#add_firewall_rules) + +Alternatively, you can disable the hooks by setting `prometheusOperator.admissionWebhooks.enabled=false`. + +## PrometheusRules Admission Webhooks + +With Prometheus Operator version 0.30+, the core Prometheus Operator pod exposes an endpoint that will integrate with the `validatingwebhookconfiguration` Kubernetes feature to prevent malformed rules from being added to the cluster. + +### How the Chart Configures the Hooks + +A validating and mutating webhook configuration requires the endpoint to which the request is sent to use TLS. It is possible to set up custom certificates to do this, but in most cases, a self-signed certificate is enough. The setup of this component requires some more complex orchestration when using helm. The steps are created to be idempotent and to allow turning the feature on and off without running into helm quirks. + +1. A pre-install hook provisions a certificate into the same namespace using a format compatible with provisioning using end user certificates. If the certificate already exists, the hook exits. +2. The prometheus operator pod is configured to use a TLS proxy container, which will load that certificate. +3. Validating and Mutating webhook configurations are created in the cluster, with their failure mode set to Ignore. This allows rules to be created by the same chart at the same time, even though the webhook has not yet been fully set up - it does not have the correct CA field set. +4. A post-install hook reads the CA from the secret created by step 1 and patches the Validating and Mutating webhook configurations. This process will allow a custom CA provisioned by some other process to also be patched into the webhook configurations. The chosen failure policy is also patched into the webhook configurations + +### Alternatives + +It should be possible to use [jetstack/cert-manager](https://github.com/jetstack/cert-manager) if a more complete solution is required, but it has not been tested. + +You can enable automatic self-signed TLS certificate provisioning via cert-manager by setting the `prometheusOperator.admissionWebhooks.certManager.enabled` value to true. + +### Limitations + +Because the operator can only run as a single pod, there is potential for this component failure to cause rule deployment failure. Because this risk is outweighed by the benefit of having validation, the feature is enabled by default. + +## Developing Prometheus Rules and Grafana Dashboards + +This chart Grafana Dashboards and Prometheus Rules are just a copy from [prometheus-operator/prometheus-operator](https://github.com/prometheus-operator/prometheus-operator) and other sources, synced (with alterations) by scripts in [hack](hack) folder. In order to introduce any changes you need to first [add them to the original repository](https://github.com/prometheus-operator/kube-prometheus/blob/main/docs/customizations/developing-prometheus-rules-and-grafana-dashboards.md) and then sync there by scripts. + +## Further Information + +For more in-depth documentation of configuration options meanings, please see + +- [Prometheus Operator](https://github.com/prometheus-operator/prometheus-operator) +- [Prometheus](https://prometheus.io/docs/introduction/overview/) +- [Grafana](https://github.com/grafana/helm-charts/tree/main/charts/grafana#grafana-helm-chart) + +## prometheus.io/scrape + +The prometheus operator does not support annotation-based discovery of services, using the `PodMonitor` or `ServiceMonitor` CRD in its place as they provide far more configuration options. +For information on how to use PodMonitors/ServiceMonitors, please see the documentation on the `prometheus-operator/prometheus-operator` documentation here: + +- [ServiceMonitors](https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/user-guides/getting-started.md#include-servicemonitors) +- [PodMonitors](https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/user-guides/getting-started.md#include-podmonitors) +- [Running Exporters](https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/user-guides/running-exporters.md) + +By default, Prometheus discovers PodMonitors and ServiceMonitors within its namespace, that are labeled with the same release tag as the prometheus-operator release. +Sometimes, you may need to discover custom PodMonitors/ServiceMonitors, for example used to scrape data from third-party applications. +An easy way of doing this, without compromising the default PodMonitors/ServiceMonitors discovery, is allowing Prometheus to discover all PodMonitors/ServiceMonitors within its namespace, without applying label filtering. +To do so, you can set `prometheus.prometheusSpec.podMonitorSelectorNilUsesHelmValues` and `prometheus.prometheusSpec.serviceMonitorSelectorNilUsesHelmValues` to `false`. + +## Migrating from stable/prometheus-operator chart + +## Zero downtime + +Since `kube-prometheus-stack` is fully compatible with the `stable/prometheus-operator` chart, a migration without downtime can be achieved. +However, the old name prefix needs to be kept. If you want the new name please follow the step by step guide below (with downtime). + +You can override the name to achieve this: + +```console +helm upgrade prometheus-operator prometheus-community/kube-prometheus-stack -n monitoring --reuse-values --set nameOverride=prometheus-operator +``` + +**Note**: It is recommended to run this first with `--dry-run --debug`. + +## Redeploy with new name (downtime) + +If the **prometheus-operator** values are compatible with the new **kube-prometheus-stack** chart, please follow the below steps for migration: + +> The guide presumes that chart is deployed in `monitoring` namespace and the deployments are running there. If in other namespace, please replace the `monitoring` to the deployed namespace. + +1. Patch the PersistenceVolume created/used by the prometheus-operator chart to `Retain` claim policy: + + ```console + kubectl patch pv/ -p '{"spec":{"persistentVolumeReclaimPolicy":"Retain"}}' + ``` + + **Note:** To execute the above command, the user must have a cluster wide permission. Please refer [Kubernetes RBAC](https://kubernetes.io/docs/reference/access-authn-authz/rbac/) + +2. Uninstall the **prometheus-operator** release and delete the existing PersistentVolumeClaim, and verify PV become Released. + + ```console + helm uninstall prometheus-operator -n monitoring + kubectl delete pvc/ -n monitoring + ``` + + Additionally, you have to manually remove the remaining `prometheus-operator-kubelet` service. + + ```console + kubectl delete service/prometheus-operator-kubelet -n kube-system + ``` + + You can choose to remove all your existing CRDs (ServiceMonitors, Podmonitors, etc.) if you want to. + +3. Remove current `spec.claimRef` values to change the PV's status from Released to Available. + + ```console + kubectl patch pv/ --type json -p='[{"op": "remove", "path": "/spec/claimRef"}]' -n monitoring + ``` + +**Note:** To execute the above command, the user must have a cluster wide permission. Please refer to [Kubernetes RBAC](https://kubernetes.io/docs/reference/access-authn-authz/rbac/) + +After these steps, proceed to a fresh **kube-prometheus-stack** installation and make sure the current release of **kube-prometheus-stack** matching the `volumeClaimTemplate` values in the `values.yaml`. + +The binding is done via matching a specific amount of storage requested and with certain access modes. + +For example, if you had storage specified as this with **prometheus-operator**: + +```yaml +volumeClaimTemplate: + spec: + storageClassName: gp2 + accessModes: ["ReadWriteOnce"] + resources: + requests: + storage: 50Gi +``` + +You have to specify matching `volumeClaimTemplate` with 50Gi storage and `ReadWriteOnce` access mode. + +Additionally, you should check the current AZ of your legacy installation's PV, and configure the fresh release to use the same AZ as the old one. If the pods are in a different AZ than the PV, the release will fail to bind the existing one, hence creating a new PV. + +This can be achieved either by specifying the labels through `values.yaml`, e.g. setting `prometheus.prometheusSpec.nodeSelector` to: + +```yaml +nodeSelector: + failure-domain.beta.kubernetes.io/zone: east-west-1a +``` + +or passing these values as `--set` overrides during installation. + +The new release should now re-attach your previously released PV with its content. + +## Migrating from coreos/prometheus-operator chart + +The multiple charts have been combined into a single chart that installs prometheus operator, prometheus, alertmanager, grafana as well as the multitude of exporters necessary to monitor a cluster. + +There is no simple and direct migration path between the charts as the changes are extensive and intended to make the chart easier to support. + +The capabilities of the old chart are all available in the new chart, including the ability to run multiple prometheus instances on a single cluster - you will need to disable the parts of the chart you do not wish to deploy. + +You can check out the tickets for this change [here](https://github.com/prometheus-operator/prometheus-operator/issues/592) and [here](https://github.com/helm/charts/pull/6765). + +### High-level overview of Changes + +#### Added dependencies + +The chart has added 3 [dependencies](#dependencies). + +- Node-Exporter, Kube-State-Metrics: These components are loaded as dependencies into the chart, and are relatively simple components +- Grafana: The Grafana chart is more feature-rich than this chart - it contains a sidecar that is able to load data sources and dashboards from configmaps deployed into the same cluster. For more information check out the [documentation for the chart](https://github.com/grafana/helm-charts/blob/main/charts/grafana/README.md) + +#### Kubelet Service + +Because the kubelet service has a new name in the chart, make sure to clean up the old kubelet service in the `kube-system` namespace to prevent counting container metrics twice. + +#### Persistent Volumes + +If you would like to keep the data of the current persistent volumes, it should be possible to attach existing volumes to new PVCs and PVs that are created using the conventions in the new chart. For example, in order to use an existing Azure disk for a helm release called `prometheus-migration` the following resources can be created: + +```yaml +apiVersion: v1 +kind: PersistentVolume +metadata: + name: pvc-prometheus-migration-prometheus-0 +spec: + accessModes: + - ReadWriteOnce + azureDisk: + cachingMode: None + diskName: pvc-prometheus-migration-prometheus-0 + diskURI: /subscriptions/f5125d82-2622-4c50-8d25-3f7ba3e9ac4b/resourceGroups/sample-migration-resource-group/providers/Microsoft.Compute/disks/pvc-prometheus-migration-prometheus-0 + fsType: "" + kind: Managed + readOnly: false + capacity: + storage: 1Gi + persistentVolumeReclaimPolicy: Delete + storageClassName: prometheus + volumeMode: Filesystem +``` + +```yaml +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + labels: + app.kubernetes.io/name: prometheus + prometheus: prometheus-migration-prometheus + name: prometheus-prometheus-migration-prometheus-db-prometheus-prometheus-migration-prometheus-0 + namespace: monitoring +spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 1Gi + storageClassName: prometheus + volumeMode: Filesystem + volumeName: pvc-prometheus-migration-prometheus-0 +``` + +The PVC will take ownership of the PV and when you create a release using a persistent volume claim template it will use the existing PVCs as they match the naming convention used by the chart. For other cloud providers similar approaches can be used. + +#### KubeProxy + +The metrics bind address of kube-proxy is default to `127.0.0.1:10249` that prometheus instances **cannot** access to. You should expose metrics by changing `metricsBindAddress` field value to `0.0.0.0:10249` if you want to collect them. + +Depending on the cluster, the relevant part `config.conf` will be in ConfigMap `kube-system/kube-proxy` or `kube-system/kube-proxy-config`. For example: + +```console +kubectl -n kube-system edit cm kube-proxy +``` + +```yaml +apiVersion: v1 +data: + config.conf: |- + apiVersion: kubeproxy.config.k8s.io/v1alpha1 + kind: KubeProxyConfiguration + # ... + # metricsBindAddress: 127.0.0.1:10249 + metricsBindAddress: 0.0.0.0:10249 + # ... + kubeconfig.conf: |- + # ... +kind: ConfigMap +metadata: + labels: + app: kube-proxy + name: kube-proxy + namespace: kube-system +``` diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/app-README.md b/charts/rancher-monitoring/103.2.2+up57.0.3/app-README.md new file mode 100644 index 00000000000..39208543848 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/app-README.md @@ -0,0 +1,46 @@ +# Rancher Monitoring and Alerting + + This chart is based on the upstream [kube-prometheus-stack](https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack) chart. The chart deploys [Prometheus Operator](https://github.com/prometheus-operator/prometheus-operator) and its CRDs along with [Grafana](https://github.com/grafana/helm-charts/tree/main/charts/grafana), [Prometheus Adapter](https://github.com/prometheus-community/helm-charts/tree/main/charts/prometheus-adapter) and additional charts / Kubernetes manifests to gather metrics. It allows users to monitor their Kubernetes clusters, view metrics in Grafana dashboards, and set up alerts and notifications. + +For more information on how to use the feature, refer to our [docs](https://rancher.com/docs/rancher/v2.x/en/monitoring-alerting/v2.5/). + +The chart installs the following components: + +- [Prometheus Operator](https://github.com/coreos/prometheus-operator) - The operator provides easy monitoring definitions for Kubernetes services, manages [Prometheus](https://prometheus.io/) and [AlertManager](https://prometheus.io/docs/alerting/latest/alertmanager/) instances, and adds default scrape targets for some Kubernetes components. +- [kube-prometheus](https://github.com/prometheus-operator/kube-prometheus/) - A collection of community-curated Kubernetes manifests, Grafana Dashboards, and PrometheusRules that deploy a default end-to-end cluster monitoring configuration. +- [Grafana](https://github.com/grafana/helm-charts/tree/main/charts/grafana) - Grafana allows a user to create / view dashboards based on the cluster metrics collected by Prometheus. +- [node-exporter](https://github.com/prometheus-community/helm-charts/tree/main/charts/prometheus-node-exporter) / [kube-state-metrics](https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-state-metrics) / [rancher-pushprox](https://github.com/rancher/charts/tree/dev-v2.7/packages/rancher-monitoring/rancher-pushprox/charts) - These charts monitor various Kubernetes components across different Kubernetes cluster types. +- [Prometheus Adapter](https://github.com/prometheus-community/helm-charts/tree/main/charts/prometheus-adapter) - The adapter allows a user to expose custom metrics, resource metrics, and external metrics on the default [Prometheus](https://prometheus.io/) instance to the Kubernetes API Server. + +For more information, review the Helm README of this chart. + +## Upgrading to Kubernetes v1.25+ + +Starting in Kubernetes v1.25, [Pod Security Policies](https://kubernetes.io/docs/concepts/security/pod-security-policy/) have been removed from the Kubernetes API. + +As a result, **before upgrading to Kubernetes v1.25** (or on a fresh install in a Kubernetes v1.25+ cluster), users are expected to perform an in-place upgrade of this chart with `global.cattle.psp.enabled` set to `false` if it has been previously set to `true`. +​ +> **Note:** +> In this chart release, any previous field that was associated with any PSP resources have been removed in favor of a single global field: `global.cattle.psp.enabled`. + +> **Note:** +> If you upgrade your cluster to Kubernetes v1.25+ before removing PSPs via a `helm upgrade` (even if you manually clean up resources), **it will leave the Helm release in a broken state within the cluster such that further Helm operations will not work (`helm uninstall`, `helm upgrade`, etc.).** +> +> If your charts get stuck in this state, please consult the Rancher docs on how to clean up your Helm release secrets. + +Upon setting `global.cattle.psp.enabled` to false, the chart will remove any PSP resources deployed on its behalf from the cluster. This is the default setting for this chart. +​ +As a replacement for PSPs, [Pod Security Admission](https://kubernetes.io/docs/concepts/security/pod-security-admission/) should be used. Please consult the Rancher docs for more details on how to configure your chart release namespaces to work with the new Pod Security Admission and apply Pod Security Standards. + +## Upgrading from 100.0.0+up16.6.0 to 100.1.0+up19.0.3 + +### Noticeable changes: +Grafana: +- `sidecar.dashboards.searchNamespace`, `sidecar.datasources.searchNamespace` and `sidecar.notifiers.searchNamespace` support a list of namespaces now. + +Kube-state-metrics +- the type of `collectors` is changed from Dictionary to List. +- `kubeStateMetrics.serviceMonitor.namespaceOverride` was replaced by `kube-state-metrics.namespaceOverride`. + +### Known issues: +- Occasionally, the upgrade fails with errors related to the webhook `prometheusrulemutate.monitoring.coreos.com`. This is a known issue in the upstream, and the workaround is to trigger the upgrade one more time. [32416](https://github.com/rancher/rancher/issues/32416#issuecomment-828881726) diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/.helmignore b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/.helmignore new file mode 100644 index 00000000000..8cade1318fb --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/.helmignore @@ -0,0 +1,23 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*~ +# Various IDEs +.vscode +.project +.idea/ +*.tmproj +OWNERS diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/Chart.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/Chart.yaml new file mode 100644 index 00000000000..ff6bcb26aac --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/Chart.yaml @@ -0,0 +1,39 @@ +annotations: + artifacthub.io/license: Apache-2.0 + artifacthub.io/links: | + - name: Chart Source + url: https://github.com/grafana/helm-charts + - name: Upstream Project + url: https://github.com/grafana/grafana + catalog.cattle.io/hidden: "true" + catalog.cattle.io/kube-version: '>= 1.26.0-0 < 1.31.0-0' + catalog.cattle.io/os: linux + catalog.rancher.io/certified: rancher + catalog.rancher.io/namespace: cattle-monitoring-system + catalog.rancher.io/release-name: rancher-grafana +apiVersion: v2 +appVersion: 10.4.1 +description: The leading tool for querying and visualizing time series and metrics. +home: https://grafana.com +icon: https://artifacthub.io/image/b4fed1a7-6c8f-4945-b99d-096efa3e4116 +keywords: +- monitoring +- metric +kubeVersion: '>=1.26.0-0' +maintainers: +- email: zanhsieh@gmail.com + name: zanhsieh +- email: rluckie@cisco.com + name: rtluckie +- email: maor.friedman@redhat.com + name: maorfr +- email: miroslav.hadzhiev@gmail.com + name: Xtigyro +- email: mail@torstenwalter.de + name: torstenwalter +name: grafana +sources: +- https://github.com/grafana/grafana +- https://github.com/grafana/helm-charts +type: application +version: 7.3.11 diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/README.md b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/README.md new file mode 100644 index 00000000000..0ff07f297db --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/README.md @@ -0,0 +1,770 @@ +# Grafana Helm Chart + +* Installs the web dashboarding system [Grafana](http://grafana.org/) + +## Get Repo Info + +```console +helm repo add grafana https://grafana.github.io/helm-charts +helm repo update +``` + +_See [helm repo](https://helm.sh/docs/helm/helm_repo/) for command documentation._ + +## Installing the Chart + +To install the chart with the release name `my-release`: + +```console +helm install my-release grafana/grafana +``` + +## Uninstalling the Chart + +To uninstall/delete the my-release deployment: + +```console +helm delete my-release +``` + +The command removes all the Kubernetes components associated with the chart and deletes the release. + +## Upgrading an existing Release to a new major version + +A major chart version change (like v1.2.3 -> v2.0.0) indicates that there is an +incompatible breaking change needing manual actions. + +### To 4.0.0 (And 3.12.1) + +This version requires Helm >= 2.12.0. + +### To 5.0.0 + +You have to add --force to your helm upgrade command as the labels of the chart have changed. + +### To 6.0.0 + +This version requires Helm >= 3.1.0. + +### To 7.0.0 + +For consistency with other Helm charts, the `global.image.registry` parameter was renamed +to `global.imageRegistry`. If you were not previously setting `global.image.registry`, no action +is required on upgrade. If you were previously setting `global.image.registry`, you will +need to instead set `global.imageRegistry`. + +## Configuration + +| Parameter | Description | Default | +|-------------------------------------------|-----------------------------------------------|---------------------------------------------------------| +| `replicas` | Number of nodes | `1` | +| `podDisruptionBudget.minAvailable` | Pod disruption minimum available | `nil` | +| `podDisruptionBudget.maxUnavailable` | Pod disruption maximum unavailable | `nil` | +| `podDisruptionBudget.apiVersion` | Pod disruption apiVersion | `nil` | +| `deploymentStrategy` | Deployment strategy | `{ "type": "RollingUpdate" }` | +| `livenessProbe` | Liveness Probe settings | `{ "httpGet": { "path": "/api/health", "port": 3000 } "initialDelaySeconds": 60, "timeoutSeconds": 30, "failureThreshold": 10 }` | +| `readinessProbe` | Readiness Probe settings | `{ "httpGet": { "path": "/api/health", "port": 3000 } }`| +| `securityContext` | Deployment securityContext | `{"runAsUser": 472, "runAsGroup": 472, "fsGroup": 472}` | +| `priorityClassName` | Name of Priority Class to assign pods | `nil` | +| `image.registry` | Image registry | `docker.io` | +| `image.repository` | Image repository | `grafana/grafana` | +| `image.tag` | Overrides the Grafana image tag whose default is the chart appVersion (`Must be >= 5.0.0`) | `` | +| `image.sha` | Image sha (optional) | `` | +| `image.pullPolicy` | Image pull policy | `IfNotPresent` | +| `image.pullSecrets` | Image pull secrets (can be templated) | `[]` | +| `service.enabled` | Enable grafana service | `true` | +| `service.type` | Kubernetes service type | `ClusterIP` | +| `service.port` | Kubernetes port where service is exposed | `80` | +| `service.portName` | Name of the port on the service | `service` | +| `service.appProtocol` | Adds the appProtocol field to the service | `` | +| `service.targetPort` | Internal service is port | `3000` | +| `service.nodePort` | Kubernetes service nodePort | `nil` | +| `service.annotations` | Service annotations (can be templated) | `{}` | +| `service.labels` | Custom labels | `{}` | +| `service.clusterIP` | internal cluster service IP | `nil` | +| `service.loadBalancerIP` | IP address to assign to load balancer (if supported) | `nil` | +| `service.loadBalancerSourceRanges` | list of IP CIDRs allowed access to lb (if supported) | `[]` | +| `service.externalIPs` | service external IP addresses | `[]` | +| `service.externalTrafficPolicy` | change the default externalTrafficPolicy | `nil` | +| `headlessService` | Create a headless service | `false` | +| `extraExposePorts` | Additional service ports for sidecar containers| `[]` | +| `hostAliases` | adds rules to the pod's /etc/hosts | `[]` | +| `ingress.enabled` | Enables Ingress | `false` | +| `ingress.annotations` | Ingress annotations (values are templated) | `{}` | +| `ingress.labels` | Custom labels | `{}` | +| `ingress.path` | Ingress accepted path | `/` | +| `ingress.pathType` | Ingress type of path | `Prefix` | +| `ingress.hosts` | Ingress accepted hostnames | `["chart-example.local"]` | +| `ingress.extraPaths` | Ingress extra paths to prepend to every host configuration. Useful when configuring [custom actions with AWS ALB Ingress Controller](https://kubernetes-sigs.github.io/aws-load-balancer-controller/v2.6/guide/ingress/annotations/#actions). Requires `ingress.hosts` to have one or more host entries. | `[]` | +| `ingress.tls` | Ingress TLS configuration | `[]` | +| `ingress.ingressClassName` | Ingress Class Name. MAY be required for Kubernetes versions >= 1.18 | `""` | +| `resources` | CPU/Memory resource requests/limits | `{}` | +| `nodeSelector` | Node labels for pod assignment | `{}` | +| `tolerations` | Toleration labels for pod assignment | `[]` | +| `affinity` | Affinity settings for pod assignment | `{}` | +| `extraInitContainers` | Init containers to add to the grafana pod | `{}` | +| `extraContainers` | Sidecar containers to add to the grafana pod | `""` | +| `extraContainerVolumes` | Volumes that can be mounted in sidecar containers | `[]` | +| `extraLabels` | Custom labels for all manifests | `{}` | +| `schedulerName` | Name of the k8s scheduler (other than default) | `nil` | +| `persistence.enabled` | Use persistent volume to store data | `false` | +| `persistence.type` | Type of persistence (`pvc` or `statefulset`) | `pvc` | +| `persistence.size` | Size of persistent volume claim | `10Gi` | +| `persistence.existingClaim` | Use an existing PVC to persist data (can be templated) | `nil` | +| `persistence.storageClassName` | Type of persistent volume claim | `nil` | +| `persistence.accessModes` | Persistence access modes | `[ReadWriteOnce]` | +| `persistence.annotations` | PersistentVolumeClaim annotations | `{}` | +| `persistence.finalizers` | PersistentVolumeClaim finalizers | `[ "kubernetes.io/pvc-protection" ]` | +| `persistence.extraPvcLabels` | Extra labels to apply to a PVC. | `{}` | +| `persistence.subPath` | Mount a sub dir of the persistent volume (can be templated) | `nil` | +| `persistence.inMemory.enabled` | If persistence is not enabled, whether to mount the local storage in-memory to improve performance | `false` | +| `persistence.inMemory.sizeLimit` | SizeLimit for the in-memory local storage | `nil` | +| `initChownData.enabled` | If false, don't reset data ownership at startup | true | +| `initChownData.image.registry` | init-chown-data container image registry | `docker.io` | +| `initChownData.image.repository` | init-chown-data container image repository | `busybox` | +| `initChownData.image.tag` | init-chown-data container image tag | `1.31.1` | +| `initChownData.image.sha` | init-chown-data container image sha (optional)| `""` | +| `initChownData.image.pullPolicy` | init-chown-data container image pull policy | `IfNotPresent` | +| `initChownData.resources` | init-chown-data pod resource requests & limits | `{}` | +| `schedulerName` | Alternate scheduler name | `nil` | +| `env` | Extra environment variables passed to pods | `{}` | +| `envValueFrom` | Environment variables from alternate sources. See the API docs on [EnvVarSource](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.17/#envvarsource-v1-core) for format details. Can be templated | `{}` | +| `envFromSecret` | Name of a Kubernetes secret (must be manually created in the same namespace) containing values to be added to the environment. Can be templated | `""` | +| `envFromSecrets` | List of Kubernetes secrets (must be manually created in the same namespace) containing values to be added to the environment. Can be templated | `[]` | +| `envFromConfigMaps` | List of Kubernetes ConfigMaps (must be manually created in the same namespace) containing values to be added to the environment. Can be templated | `[]` | +| `envRenderSecret` | Sensible environment variables passed to pods and stored as secret. (passed through [tpl](https://helm.sh/docs/howto/charts_tips_and_tricks/#using-the-tpl-function)) | `{}` | +| `enableServiceLinks` | Inject Kubernetes services as environment variables. | `true` | +| `extraSecretMounts` | Additional grafana server secret mounts | `[]` | +| `extraVolumeMounts` | Additional grafana server volume mounts | `[]` | +| `extraVolumes` | Additional Grafana server volumes | `[]` | +| `automountServiceAccountToken` | Mounted the service account token on the grafana pod. Mandatory, if sidecars are enabled | `true` | +| `createConfigmap` | Enable creating the grafana configmap | `true` | +| `extraConfigmapMounts` | Additional grafana server configMap volume mounts (values are templated) | `[]` | +| `extraEmptyDirMounts` | Additional grafana server emptyDir volume mounts | `[]` | +| `plugins` | Plugins to be loaded along with Grafana | `[]` | +| `datasources` | Configure grafana datasources (passed through tpl) | `{}` | +| `alerting` | Configure grafana alerting (passed through tpl) | `{}` | +| `notifiers` | Configure grafana notifiers | `{}` | +| `dashboardProviders` | Configure grafana dashboard providers | `{}` | +| `dashboards` | Dashboards to import | `{}` | +| `dashboardsConfigMaps` | ConfigMaps reference that contains dashboards | `{}` | +| `grafana.ini` | Grafana's primary configuration | `{}` | +| `global.imageRegistry` | Global image pull registry for all images. | `null` | +| `global.imagePullSecrets` | Global image pull secrets (can be templated). Allows either an array of {name: pullSecret} maps (k8s-style), or an array of strings (more common helm-style). | `[]` | +| `ldap.enabled` | Enable LDAP authentication | `false` | +| `ldap.existingSecret` | The name of an existing secret containing the `ldap.toml` file, this must have the key `ldap-toml`. | `""` | +| `ldap.config` | Grafana's LDAP configuration | `""` | +| `annotations` | Deployment annotations | `{}` | +| `labels` | Deployment labels | `{}` | +| `podAnnotations` | Pod annotations | `{}` | +| `podLabels` | Pod labels | `{}` | +| `podPortName` | Name of the grafana port on the pod | `grafana` | +| `lifecycleHooks` | Lifecycle hooks for podStart and preStop [Example](https://kubernetes.io/docs/tasks/configure-pod-container/attach-handler-lifecycle-event/#define-poststart-and-prestop-handlers) | `{}` | +| `sidecar.image.registry` | Sidecar image registry | `quay.io` | +| `sidecar.image.repository` | Sidecar image repository | `kiwigrid/k8s-sidecar` | +| `sidecar.image.tag` | Sidecar image tag | `1.26.0` | +| `sidecar.image.sha` | Sidecar image sha (optional) | `""` | +| `sidecar.imagePullPolicy` | Sidecar image pull policy | `IfNotPresent` | +| `sidecar.resources` | Sidecar resources | `{}` | +| `sidecar.securityContext` | Sidecar securityContext | `{}` | +| `sidecar.enableUniqueFilenames` | Sets the kiwigrid/k8s-sidecar UNIQUE_FILENAMES environment variable. If set to `true` the sidecar will create unique filenames where duplicate data keys exist between ConfigMaps and/or Secrets within the same or multiple Namespaces. | `false` | +| `sidecar.alerts.enabled` | Enables the cluster wide search for alerts and adds/updates/deletes them in grafana |`false` | +| `sidecar.alerts.label` | Label that config maps with alerts should have to be added | `grafana_alert` | +| `sidecar.alerts.labelValue` | Label value that config maps with alerts should have to be added | `""` | +| `sidecar.alerts.searchNamespace` | Namespaces list. If specified, the sidecar will search for alerts config-maps inside these namespaces. Otherwise the namespace in which the sidecar is running will be used. It's also possible to specify ALL to search in all namespaces. | `nil` | +| `sidecar.alerts.watchMethod` | Method to use to detect ConfigMap changes. With WATCH the sidecar will do a WATCH requests, with SLEEP it will list all ConfigMaps, then sleep for 60 seconds. | `WATCH` | +| `sidecar.alerts.resource` | Should the sidecar looks into secrets, configmaps or both. | `both` | +| `sidecar.alerts.reloadURL` | Full url of datasource configuration reload API endpoint, to invoke after a config-map change | `"http://localhost:3000/api/admin/provisioning/alerting/reload"` | +| `sidecar.alerts.skipReload` | Enabling this omits defining the REQ_URL and REQ_METHOD environment variables | `false` | +| `sidecar.alerts.initAlerts` | Set to true to deploy the alerts sidecar as an initContainer. This is needed if skipReload is true, to load any alerts defined at startup time. | `false` | +| `sidecar.alerts.extraMounts` | Additional alerts sidecar volume mounts. | `[]` | +| `sidecar.dashboards.enabled` | Enables the cluster wide search for dashboards and adds/updates/deletes them in grafana | `false` | +| `sidecar.dashboards.SCProvider` | Enables creation of sidecar provider | `true` | +| `sidecar.dashboards.provider.name` | Unique name of the grafana provider | `sidecarProvider` | +| `sidecar.dashboards.provider.orgid` | Id of the organisation, to which the dashboards should be added | `1` | +| `sidecar.dashboards.provider.folder` | Logical folder in which grafana groups dashboards | `""` | +| `sidecar.dashboards.provider.disableDelete` | Activate to avoid the deletion of imported dashboards | `false` | +| `sidecar.dashboards.provider.allowUiUpdates` | Allow updating provisioned dashboards from the UI | `false` | +| `sidecar.dashboards.provider.type` | Provider type | `file` | +| `sidecar.dashboards.provider.foldersFromFilesStructure` | Allow Grafana to replicate dashboard structure from filesystem. | `false` | +| `sidecar.dashboards.watchMethod` | Method to use to detect ConfigMap changes. With WATCH the sidecar will do a WATCH requests, with SLEEP it will list all ConfigMaps, then sleep for 60 seconds. | `WATCH` | +| `sidecar.skipTlsVerify` | Set to true to skip tls verification for kube api calls | `nil` | +| `sidecar.dashboards.label` | Label that config maps with dashboards should have to be added | `grafana_dashboard` | +| `sidecar.dashboards.labelValue` | Label value that config maps with dashboards should have to be added | `""` | +| `sidecar.dashboards.folder` | Folder in the pod that should hold the collected dashboards (unless `sidecar.dashboards.defaultFolderName` is set). This path will be mounted. | `/tmp/dashboards` | +| `sidecar.dashboards.folderAnnotation` | The annotation the sidecar will look for in configmaps to override the destination folder for files | `nil` | +| `sidecar.dashboards.defaultFolderName` | The default folder name, it will create a subfolder under the `sidecar.dashboards.folder` and put dashboards in there instead | `nil` | +| `sidecar.dashboards.searchNamespace` | Namespaces list. If specified, the sidecar will search for dashboards config-maps inside these namespaces. Otherwise the namespace in which the sidecar is running will be used. It's also possible to specify ALL to search in all namespaces. | `nil` | +| `sidecar.dashboards.script` | Absolute path to shell script to execute after a configmap got reloaded. | `nil` | +| `sidecar.dashboards.reloadURL` | Full url of dashboards configuration reload API endpoint, to invoke after a config-map change | `"http://localhost:3000/api/admin/provisioning/dashboards/reload"` | +| `sidecar.dashboards.skipReload` | Enabling this omits defining the REQ_USERNAME, REQ_PASSWORD, REQ_URL and REQ_METHOD environment variables | `false` | +| `sidecar.dashboards.resource` | Should the sidecar looks into secrets, configmaps or both. | `both` | +| `sidecar.dashboards.extraMounts` | Additional dashboard sidecar volume mounts. | `[]` | +| `sidecar.datasources.enabled` | Enables the cluster wide search for datasources and adds/updates/deletes them in grafana |`false` | +| `sidecar.datasources.label` | Label that config maps with datasources should have to be added | `grafana_datasource` | +| `sidecar.datasources.labelValue` | Label value that config maps with datasources should have to be added | `""` | +| `sidecar.datasources.searchNamespace` | Namespaces list. If specified, the sidecar will search for datasources config-maps inside these namespaces. Otherwise the namespace in which the sidecar is running will be used. It's also possible to specify ALL to search in all namespaces. | `nil` | +| `sidecar.datasources.watchMethod` | Method to use to detect ConfigMap changes. With WATCH the sidecar will do a WATCH requests, with SLEEP it will list all ConfigMaps, then sleep for 60 seconds. | `WATCH` | +| `sidecar.datasources.resource` | Should the sidecar looks into secrets, configmaps or both. | `both` | +| `sidecar.datasources.reloadURL` | Full url of datasource configuration reload API endpoint, to invoke after a config-map change | `"http://localhost:3000/api/admin/provisioning/datasources/reload"` | +| `sidecar.datasources.skipReload` | Enabling this omits defining the REQ_URL and REQ_METHOD environment variables | `false` | +| `sidecar.datasources.initDatasources` | Set to true to deploy the datasource sidecar as an initContainer in addition to a container. This is needed if skipReload is true, to load any datasources defined at startup time. | `false` | +| `sidecar.notifiers.enabled` | Enables the cluster wide search for notifiers and adds/updates/deletes them in grafana | `false` | +| `sidecar.notifiers.label` | Label that config maps with notifiers should have to be added | `grafana_notifier` | +| `sidecar.notifiers.labelValue` | Label value that config maps with notifiers should have to be added | `""` | +| `sidecar.notifiers.searchNamespace` | Namespaces list. If specified, the sidecar will search for notifiers config-maps (or secrets) inside these namespaces. Otherwise the namespace in which the sidecar is running will be used. It's also possible to specify ALL to search in all namespaces. | `nil` | +| `sidecar.notifiers.watchMethod` | Method to use to detect ConfigMap changes. With WATCH the sidecar will do a WATCH requests, with SLEEP it will list all ConfigMaps, then sleep for 60 seconds. | `WATCH` | +| `sidecar.notifiers.resource` | Should the sidecar looks into secrets, configmaps or both. | `both` | +| `sidecar.notifiers.reloadURL` | Full url of notifier configuration reload API endpoint, to invoke after a config-map change | `"http://localhost:3000/api/admin/provisioning/notifications/reload"` | +| `sidecar.notifiers.skipReload` | Enabling this omits defining the REQ_URL and REQ_METHOD environment variables | `false` | +| `sidecar.notifiers.initNotifiers` | Set to true to deploy the notifier sidecar as an initContainer in addition to a container. This is needed if skipReload is true, to load any notifiers defined at startup time. | `false` | +| `smtp.existingSecret` | The name of an existing secret containing the SMTP credentials. | `""` | +| `smtp.userKey` | The key in the existing SMTP secret containing the username. | `"user"` | +| `smtp.passwordKey` | The key in the existing SMTP secret containing the password. | `"password"` | +| `admin.existingSecret` | The name of an existing secret containing the admin credentials (can be templated). | `""` | +| `admin.userKey` | The key in the existing admin secret containing the username. | `"admin-user"` | +| `admin.passwordKey` | The key in the existing admin secret containing the password. | `"admin-password"` | +| `serviceAccount.automountServiceAccountToken` | Automount the service account token on all pods where is service account is used | `false` | +| `serviceAccount.annotations` | ServiceAccount annotations | | +| `serviceAccount.create` | Create service account | `true` | +| `serviceAccount.labels` | ServiceAccount labels | `{}` | +| `serviceAccount.name` | Service account name to use, when empty will be set to created account if `serviceAccount.create` is set else to `default` | `` | +| `serviceAccount.nameTest` | Service account name to use for test, when empty will be set to created account if `serviceAccount.create` is set else to `default` | `nil` | +| `rbac.create` | Create and use RBAC resources | `true` | +| `rbac.namespaced` | Creates Role and Rolebinding instead of the default ClusterRole and ClusteRoleBindings for the grafana instance | `false` | +| `rbac.useExistingRole` | Set to a rolename to use existing role - skipping role creating - but still doing serviceaccount and rolebinding to the rolename set here. | `nil` | +| `rbac.pspEnabled` | Create PodSecurityPolicy (with `rbac.create`, grant roles permissions as well) | `false` | +| `rbac.pspUseAppArmor` | Enforce AppArmor in created PodSecurityPolicy (requires `rbac.pspEnabled`) | `false` | +| `rbac.extraRoleRules` | Additional rules to add to the Role | [] | +| `rbac.extraClusterRoleRules` | Additional rules to add to the ClusterRole | [] | +| `command` | Define command to be executed by grafana container at startup | `nil` | +| `args` | Define additional args if command is used | `nil` | +| `testFramework.enabled` | Whether to create test-related resources | `true` | +| `testFramework.image.registry` | `test-framework` image registry. | `docker.io` | +| `testFramework.image.repository` | `test-framework` image repository. | `bats/bats` | +| `testFramework.image.tag` | `test-framework` image tag. | `v1.4.1` | +| `testFramework.imagePullPolicy` | `test-framework` image pull policy. | `IfNotPresent` | +| `testFramework.securityContext` | `test-framework` securityContext | `{}` | +| `downloadDashboards.env` | Environment variables to be passed to the `download-dashboards` container | `{}` | +| `downloadDashboards.envFromSecret` | Name of a Kubernetes secret (must be manually created in the same namespace) containing values to be added to the environment. Can be templated | `""` | +| `downloadDashboards.resources` | Resources of `download-dashboards` container | `{}` | +| `downloadDashboardsImage.registry` | Curl docker image registry | `docker.io` | +| `downloadDashboardsImage.repository` | Curl docker image repository | `curlimages/curl` | +| `downloadDashboardsImage.tag` | Curl docker image tag | `7.73.0` | +| `downloadDashboardsImage.sha` | Curl docker image sha (optional) | `""` | +| `downloadDashboardsImage.pullPolicy` | Curl docker image pull policy | `IfNotPresent` | +| `namespaceOverride` | Override the deployment namespace | `""` (`Release.Namespace`) | +| `serviceMonitor.enabled` | Use servicemonitor from prometheus operator | `false` | +| `serviceMonitor.namespace` | Namespace this servicemonitor is installed in | | +| `serviceMonitor.interval` | How frequently Prometheus should scrape | `1m` | +| `serviceMonitor.path` | Path to scrape | `/metrics` | +| `serviceMonitor.scheme` | Scheme to use for metrics scraping | `http` | +| `serviceMonitor.tlsConfig` | TLS configuration block for the endpoint | `{}` | +| `serviceMonitor.labels` | Labels for the servicemonitor passed to Prometheus Operator | `{}` | +| `serviceMonitor.scrapeTimeout` | Timeout after which the scrape is ended | `30s` | +| `serviceMonitor.relabelings` | RelabelConfigs to apply to samples before scraping. | `[]` | +| `serviceMonitor.metricRelabelings` | MetricRelabelConfigs to apply to samples before ingestion. | `[]` | +| `revisionHistoryLimit` | Number of old ReplicaSets to retain | `10` | +| `imageRenderer.enabled` | Enable the image-renderer deployment & service | `false` | +| `imageRenderer.image.registry` | image-renderer Image registry | `docker.io` | +| `imageRenderer.image.repository` | image-renderer Image repository | `grafana/grafana-image-renderer` | +| `imageRenderer.image.tag` | image-renderer Image tag | `latest` | +| `imageRenderer.image.sha` | image-renderer Image sha (optional) | `""` | +| `imageRenderer.image.pullPolicy` | image-renderer ImagePullPolicy | `Always` | +| `imageRenderer.env` | extra env-vars for image-renderer | `{}` | +| `imageRenderer.envValueFrom` | Environment variables for image-renderer from alternate sources. See the API docs on [EnvVarSource](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.17/#envvarsource-v1-core) for format details. Can be templated | `{}` | +| `imageRenderer.serviceAccountName` | image-renderer deployment serviceAccountName | `""` | +| `imageRenderer.securityContext` | image-renderer deployment securityContext | `{}` | +| `imageRenderer.podAnnotations ` | image-renderer image-renderer pod annotation | `{}` | +| `imageRenderer.hostAliases` | image-renderer deployment Host Aliases | `[]` | +| `imageRenderer.priorityClassName` | image-renderer deployment priority class | `''` | +| `imageRenderer.service.enabled` | Enable the image-renderer service | `true` | +| `imageRenderer.service.portName` | image-renderer service port name | `http` | +| `imageRenderer.service.port` | image-renderer port used by deployment | `8081` | +| `imageRenderer.service.targetPort` | image-renderer service port used by service | `8081` | +| `imageRenderer.appProtocol` | Adds the appProtocol field to the service | `` | +| `imageRenderer.grafanaSubPath` | Grafana sub path to use for image renderer callback url | `''` | +| `imageRenderer.podPortName` | name of the image-renderer port on the pod | `http` | +| `imageRenderer.revisionHistoryLimit` | number of image-renderer replica sets to keep | `10` | +| `imageRenderer.networkPolicy.limitIngress` | Enable a NetworkPolicy to limit inbound traffic from only the created grafana pods | `true` | +| `imageRenderer.networkPolicy.limitEgress` | Enable a NetworkPolicy to limit outbound traffic to only the created grafana pods | `false` | +| `imageRenderer.resources` | Set resource limits for image-renderer pods | `{}` | +| `imageRenderer.nodeSelector` | Node labels for pod assignment | `{}` | +| `imageRenderer.tolerations` | Toleration labels for pod assignment | `[]` | +| `imageRenderer.affinity` | Affinity settings for pod assignment | `{}` | +| `networkPolicy.enabled` | Enable creation of NetworkPolicy resources. | `false` | +| `networkPolicy.allowExternal` | Don't require client label for connections | `true` | +| `networkPolicy.explicitNamespacesSelector` | A Kubernetes LabelSelector to explicitly select namespaces from which traffic could be allowed | `{}` | +| `networkPolicy.ingress` | Enable the creation of an ingress network policy | `true` | +| `networkPolicy.egress.enabled` | Enable the creation of an egress network policy | `false` | +| `networkPolicy.egress.ports` | An array of ports to allow for the egress | `[]` | +| `enableKubeBackwardCompatibility` | Enable backward compatibility of kubernetes where pod's defintion version below 1.13 doesn't have the enableServiceLinks option | `false` | + +### Example ingress with path + +With grafana 6.3 and above + +```yaml +grafana.ini: + server: + domain: monitoring.example.com + root_url: "%(protocol)s://%(domain)s/grafana" + serve_from_sub_path: true +ingress: + enabled: true + hosts: + - "monitoring.example.com" + path: "/grafana" +``` + +### Example of extraVolumeMounts and extraVolumes + +Configure additional volumes with `extraVolumes` and volume mounts with `extraVolumeMounts`. + +Example for `extraVolumeMounts` and corresponding `extraVolumes`: + +```yaml +extraVolumeMounts: + - name: plugins + mountPath: /var/lib/grafana/plugins + subPath: configs/grafana/plugins + readOnly: false + - name: dashboards + mountPath: /var/lib/grafana/dashboards + hostPath: /usr/shared/grafana/dashboards + readOnly: false + +extraVolumes: + - name: plugins + existingClaim: existing-grafana-claim + - name: dashboards + hostPath: /usr/shared/grafana/dashboards +``` + +Volumes default to `emptyDir`. Set to `persistentVolumeClaim`, +`hostPath`, `csi`, or `configMap` for other types. For a +`persistentVolumeClaim`, specify an existing claim name with +`existingClaim`. + +## Import dashboards + +There are a few methods to import dashboards to Grafana. Below are some examples and explanations as to how to use each method: + +```yaml +dashboards: + default: + some-dashboard: + json: | + { + "annotations": + + ... + # Complete json file here + ... + + "title": "Some Dashboard", + "uid": "abcd1234", + "version": 1 + } + custom-dashboard: + # This is a path to a file inside the dashboards directory inside the chart directory + file: dashboards/custom-dashboard.json + prometheus-stats: + # Ref: https://grafana.com/dashboards/2 + gnetId: 2 + revision: 2 + datasource: Prometheus + loki-dashboard-quick-search: + gnetId: 12019 + revision: 2 + datasource: + - name: DS_PROMETHEUS + value: Prometheus + - name: DS_LOKI + value: Loki + local-dashboard: + url: https://raw.githubusercontent.com/user/repository/master/dashboards/dashboard.json +``` + +## BASE64 dashboards + +Dashboards could be stored on a server that does not return JSON directly and instead of it returns a Base64 encoded file (e.g. Gerrit) +A new parameter has been added to the url use case so if you specify a b64content value equals to true after the url entry a Base64 decoding is applied before save the file to disk. +If this entry is not set or is equals to false not decoding is applied to the file before saving it to disk. + +### Gerrit use case + +Gerrit API for download files has the following schema: where {project-name} and +{file-id} usually has '/' in their values and so they MUST be replaced by %2F so if project-name is user/repo, branch-id is master and file-id is equals to dir1/dir2/dashboard +the url value is + +## Sidecar for dashboards + +If the parameter `sidecar.dashboards.enabled` is set, a sidecar container is deployed in the grafana +pod. This container watches all configmaps (or secrets) in the cluster and filters out the ones with +a label as defined in `sidecar.dashboards.label`. The files defined in those configmaps are written +to a folder and accessed by grafana. Changes to the configmaps are monitored and the imported +dashboards are deleted/updated. + +A recommendation is to use one configmap per dashboard, as a reduction of multiple dashboards inside +one configmap is currently not properly mirrored in grafana. + +Example dashboard config: + +```yaml +apiVersion: v1 +kind: ConfigMap +metadata: + name: sample-grafana-dashboard + labels: + grafana_dashboard: "1" +data: + k8s-dashboard.json: |- + [...] +``` + +## Sidecar for datasources + +If the parameter `sidecar.datasources.enabled` is set, an init container is deployed in the grafana +pod. This container lists all secrets (or configmaps, though not recommended) in the cluster and +filters out the ones with a label as defined in `sidecar.datasources.label`. The files defined in +those secrets are written to a folder and accessed by grafana on startup. Using these yaml files, +the data sources in grafana can be imported. + +Should you aim for reloading datasources in Grafana each time the config is changed, set `sidecar.datasources.skipReload: false` and adjust `sidecar.datasources.reloadURL` to `http://..svc.cluster.local/api/admin/provisioning/datasources/reload`. + +Secrets are recommended over configmaps for this usecase because datasources usually contain private +data like usernames and passwords. Secrets are the more appropriate cluster resource to manage those. + +Example values to add a postgres datasource as a kubernetes secret: +```yaml +apiVersion: v1 +kind: Secret +metadata: + name: grafana-datasources + labels: + grafana_datasource: 'true' # default value for: sidecar.datasources.label +stringData: + pg-db.yaml: |- + apiVersion: 1 + datasources: + - name: My pg db datasource + type: postgres + url: my-postgresql-db:5432 + user: db-readonly-user + secureJsonData: + password: 'SUperSEcretPa$$word' + jsonData: + database: my_datase + sslmode: 'disable' # disable/require/verify-ca/verify-full + maxOpenConns: 0 # Grafana v5.4+ + maxIdleConns: 2 # Grafana v5.4+ + connMaxLifetime: 14400 # Grafana v5.4+ + postgresVersion: 1000 # 903=9.3, 904=9.4, 905=9.5, 906=9.6, 1000=10 + timescaledb: false + # allow users to edit datasources from the UI. + editable: false +``` + +Example values to add a datasource adapted from [Grafana](http://docs.grafana.org/administration/provisioning/#example-datasource-config-file): + +```yaml +datasources: + datasources.yaml: + apiVersion: 1 + datasources: + # name of the datasource. Required + - name: Graphite + # datasource type. Required + type: graphite + # access mode. proxy or direct (Server or Browser in the UI). Required + access: proxy + # org id. will default to orgId 1 if not specified + orgId: 1 + # url + url: http://localhost:8080 + # database password, if used + password: + # database user, if used + user: + # database name, if used + database: + # enable/disable basic auth + basicAuth: + # basic auth username + basicAuthUser: + # basic auth password + basicAuthPassword: + # enable/disable with credentials headers + withCredentials: + # mark as default datasource. Max one per org + isDefault: + # fields that will be converted to json and stored in json_data + jsonData: + graphiteVersion: "1.1" + tlsAuth: true + tlsAuthWithCACert: true + # json object of data that will be encrypted. + secureJsonData: + tlsCACert: "..." + tlsClientCert: "..." + tlsClientKey: "..." + version: 1 + # allow users to edit datasources from the UI. + editable: false +``` + +## Sidecar for notifiers + +If the parameter `sidecar.notifiers.enabled` is set, an init container is deployed in the grafana +pod. This container lists all secrets (or configmaps, though not recommended) in the cluster and +filters out the ones with a label as defined in `sidecar.notifiers.label`. The files defined in +those secrets are written to a folder and accessed by grafana on startup. Using these yaml files, +the notification channels in grafana can be imported. The secrets must be created before +`helm install` so that the notifiers init container can list the secrets. + +Secrets are recommended over configmaps for this usecase because alert notification channels usually contain +private data like SMTP usernames and passwords. Secrets are the more appropriate cluster resource to manage those. + +Example datasource config adapted from [Grafana](https://grafana.com/docs/grafana/latest/administration/provisioning/#alert-notification-channels): + +```yaml +notifiers: + - name: notification-channel-1 + type: slack + uid: notifier1 + # either + org_id: 2 + # or + org_name: Main Org. + is_default: true + send_reminder: true + frequency: 1h + disable_resolve_message: false + # See `Supported Settings` section for settings supporter for each + # alert notification type. + settings: + recipient: 'XXX' + token: 'xoxb' + uploadImage: true + url: https://slack.com + +delete_notifiers: + - name: notification-channel-1 + uid: notifier1 + org_id: 2 + - name: notification-channel-2 + # default org_id: 1 +``` + +## Sidecar for alerting resources + +If the parameter `sidecar.alerts.enabled` is set, a sidecar container is deployed in the grafana +pod. This container watches all configmaps (or secrets) in the cluster (namespace defined by `sidecar.alerts.searchNamespace`) and filters out the ones with +a label as defined in `sidecar.alerts.label` (default is `grafana_alert`). The files defined in those configmaps are written +to a folder and accessed by grafana. Changes to the configmaps are monitored and the imported alerting resources are updated, however, deletions are a little more complicated (see below). + +This sidecar can be used to provision alert rules, contact points, notification policies, notification templates and mute timings as shown in [Grafana Documentation](https://grafana.com/docs/grafana/next/alerting/set-up/provision-alerting-resources/file-provisioning/). + +To fetch the alert config which will be provisioned, use the alert provisioning API ([Grafana Documentation](https://grafana.com/docs/grafana/next/developers/http_api/alerting_provisioning/)). +You can use either JSON or YAML format. + +Example config for an alert rule: + +```yaml +apiVersion: v1 +kind: ConfigMap +metadata: + name: sample-grafana-alert + labels: + grafana_alert: "1" +data: + k8s-alert.yml: |- + apiVersion: 1 + groups: + - orgId: 1 + name: k8s-alert + [...] +``` + +To delete provisioned alert rules is a two step process, you need to delete the configmap which defined the alert rule +and then create a configuration which deletes the alert rule. + +Example deletion configuration: +```yaml +apiVersion: v1 +kind: ConfigMap +metadata: + name: delete-sample-grafana-alert + namespace: monitoring + labels: + grafana_alert: "1" +data: + delete-k8s-alert.yml: |- + apiVersion: 1 + deleteRules: + - orgId: 1 + uid: 16624780-6564-45dc-825c-8bded4ad92d3 +``` + +## Statically provision alerting resources +If you don't need to change alerting resources (alert rules, contact points, notification policies and notification templates) regularly you could use the `alerting` config option instead of the sidecar option above. +This will grab the alerting config and apply it statically at build time for the helm file. + +There are two methods to statically provision alerting configuration in Grafana. Below are some examples and explanations as to how to use each method: + +```yaml +alerting: + team1-alert-rules.yaml: + file: alerting/team1/rules.yaml + team2-alert-rules.yaml: + file: alerting/team2/rules.yaml + team3-alert-rules.yaml: + file: alerting/team3/rules.yaml + notification-policies.yaml: + file: alerting/shared/notification-policies.yaml + notification-templates.yaml: + file: alerting/shared/notification-templates.yaml + contactpoints.yaml: + apiVersion: 1 + contactPoints: + - orgId: 1 + name: Slack channel + receivers: + - uid: default-receiver + type: slack + settings: + # Webhook URL to be filled in + url: "" + # We need to escape double curly braces for the tpl function. + text: '{{ `{{ template "default.message" . }}` }}' + title: '{{ `{{ template "default.title" . }}` }}' +``` + +The two possibilities for static alerting resource provisioning are: + +* Inlining the file contents as shown for contact points in the above example. +* Importing a file using a relative path starting from the chart root directory as shown for the alert rules in the above example. + +### Important notes on file provisioning + +* The format of the files is defined in the [Grafana documentation](https://grafana.com/docs/grafana/next/alerting/set-up/provision-alerting-resources/file-provisioning/) on file provisioning. +* The chart supports importing YAML and JSON files. +* The filename must be unique, otherwise one volume mount will overwrite the other. +* In case of inlining, double curly braces that arise from the Grafana configuration format and are not intended as templates for the chart must be escaped. +* The number of total files under `alerting:` is not limited. Each file will end up as a volume mount in the corresponding provisioning folder of the deployed Grafana instance. +* The file size for each import is limited by what the function `.Files.Get` can handle, which suffices for most cases. + +## How to serve Grafana with a path prefix (/grafana) + +In order to serve Grafana with a prefix (e.g., ), add the following to your values.yaml. + +```yaml +ingress: + enabled: true + annotations: + kubernetes.io/ingress.class: "nginx" + nginx.ingress.kubernetes.io/rewrite-target: /$1 + nginx.ingress.kubernetes.io/use-regex: "true" + + path: /grafana/?(.*) + hosts: + - k8s.example.dev + +grafana.ini: + server: + root_url: http://localhost:3000/grafana # this host can be localhost +``` + +## How to securely reference secrets in grafana.ini + +This example uses Grafana [file providers](https://grafana.com/docs/grafana/latest/administration/configuration/#file-provider) for secret values and the `extraSecretMounts` configuration flag (Additional grafana server secret mounts) to mount the secrets. + +In grafana.ini: + +```yaml +grafana.ini: + [auth.generic_oauth] + enabled = true + client_id = $__file{/etc/secrets/auth_generic_oauth/client_id} + client_secret = $__file{/etc/secrets/auth_generic_oauth/client_secret} +``` + +Existing secret, or created along with helm: + +```yaml +--- +apiVersion: v1 +kind: Secret +metadata: + name: auth-generic-oauth-secret +type: Opaque +stringData: + client_id: + client_secret: +``` + +Include in the `extraSecretMounts` configuration flag: + +```yaml +- extraSecretMounts: + - name: auth-generic-oauth-secret-mount + secretName: auth-generic-oauth-secret + defaultMode: 0440 + mountPath: /etc/secrets/auth_generic_oauth + readOnly: true +``` + +### extraSecretMounts using a Container Storage Interface (CSI) provider + +This example uses a CSI driver e.g. retrieving secrets using [Azure Key Vault Provider](https://github.com/Azure/secrets-store-csi-driver-provider-azure) + +```yaml +- extraSecretMounts: + - name: secrets-store-inline + mountPath: /run/secrets + readOnly: true + csi: + driver: secrets-store.csi.k8s.io + readOnly: true + volumeAttributes: + secretProviderClass: "my-provider" + nodePublishSecretRef: + name: akv-creds +``` + +## Image Renderer Plug-In + +This chart supports enabling [remote image rendering](https://github.com/grafana/grafana-image-renderer/blob/master/README.md#run-in-docker) + +```yaml +imageRenderer: + enabled: true +``` + +### Image Renderer NetworkPolicy + +By default the image-renderer pods will have a network policy which only allows ingress traffic from the created grafana instance + +### High Availability for unified alerting + +If you want to run Grafana in a high availability cluster you need to enable +the headless service by setting `headlessService: true` in your `values.yaml` +file. + +As next step you have to setup the `grafana.ini` in your `values.yaml` in a way +that it will make use of the headless service to obtain all the IPs of the +cluster. You should replace ``{{ Name }}`` with the name of your helm deployment. + +```yaml +grafana.ini: + ... + unified_alerting: + enabled: true + ha_peers: {{ Name }}-headless:9094 + ha_listen_address: ${POD_IP}:9094 + ha_advertise_address: ${POD_IP}:9094 + + alerting: + enabled: false +``` diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/dashboards/custom-dashboard.json b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/dashboards/custom-dashboard.json new file mode 100644 index 00000000000..9e26dfeeb6e --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/dashboards/custom-dashboard.json @@ -0,0 +1 @@ +{} \ No newline at end of file diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/NOTES.txt b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/NOTES.txt new file mode 100644 index 00000000000..d86419fe232 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/NOTES.txt @@ -0,0 +1,55 @@ +1. Get your '{{ .Values.adminUser }}' user password by running: + + kubectl get secret --namespace {{ include "grafana.namespace" . }} {{ .Values.admin.existingSecret | default (include "grafana.fullname" .) }} -o jsonpath="{.data.{{ .Values.admin.passwordKey | default "admin-password" }}}" | base64 --decode ; echo + + +2. The Grafana server can be accessed via port {{ .Values.service.port }} on the following DNS name from within your cluster: + + {{ include "grafana.fullname" . }}.{{ include "grafana.namespace" . }}.svc.cluster.local +{{ if .Values.ingress.enabled }} + If you bind grafana to 80, please update values in values.yaml and reinstall: + ``` + securityContext: + runAsUser: 0 + runAsGroup: 0 + fsGroup: 0 + + command: + - "setcap" + - "'cap_net_bind_service=+ep'" + - "/usr/sbin/grafana-server &&" + - "sh" + - "/run.sh" + ``` + Details refer to https://grafana.com/docs/installation/configuration/#http-port. + Or grafana would always crash. + + From outside the cluster, the server URL(s) are: + {{- range .Values.ingress.hosts }} + http://{{ . }} + {{- end }} +{{- else }} + Get the Grafana URL to visit by running these commands in the same shell: + {{- if contains "NodePort" .Values.service.type }} + export NODE_PORT=$(kubectl get --namespace {{ include "grafana.namespace" . }} -o jsonpath="{.spec.ports[0].nodePort}" services {{ include "grafana.fullname" . }}) + export NODE_IP=$(kubectl get nodes --namespace {{ include "grafana.namespace" . }} -o jsonpath="{.items[0].status.addresses[0].address}") + echo http://$NODE_IP:$NODE_PORT + {{- else if contains "LoadBalancer" .Values.service.type }} + NOTE: It may take a few minutes for the LoadBalancer IP to be available. + You can watch the status of by running 'kubectl get svc --namespace {{ include "grafana.namespace" . }} -w {{ include "grafana.fullname" . }}' + export SERVICE_IP=$(kubectl get svc --namespace {{ include "grafana.namespace" . }} {{ include "grafana.fullname" . }} -o jsonpath='{.status.loadBalancer.ingress[0].ip}') + http://$SERVICE_IP:{{ .Values.service.port -}} + {{- else if contains "ClusterIP" .Values.service.type }} + export POD_NAME=$(kubectl get pods --namespace {{ include "grafana.namespace" . }} -l "app.kubernetes.io/name={{ include "grafana.name" . }},app.kubernetes.io/instance={{ .Release.Name }}" -o jsonpath="{.items[0].metadata.name}") + kubectl --namespace {{ include "grafana.namespace" . }} port-forward $POD_NAME 3000 + {{- end }} +{{- end }} + +3. Login with the password from step 1 and the username: {{ .Values.adminUser }} + +{{- if not .Values.persistence.enabled }} +################################################################################# +###### WARNING: Persistence is disabled!!! You will lose your data when ##### +###### the Grafana pod is terminated. ##### +################################################################################# +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/_config.tpl b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/_config.tpl new file mode 100644 index 00000000000..19df19cd2ab --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/_config.tpl @@ -0,0 +1,171 @@ +{{/* + Generate config map data + */}} +{{- define "grafana.configData" -}} +{{ include "grafana.assertNoLeakedSecrets" . }} +{{- $files := .Files }} +{{- $root := . -}} +{{- with .Values.plugins }} +plugins: {{ join "," . }} +{{- end }} +grafana.ini: | +{{- range $elem, $elemVal := index .Values "grafana.ini" }} + {{- if not (kindIs "map" $elemVal) }} + {{- if kindIs "invalid" $elemVal }} + {{ $elem }} = + {{- else if kindIs "string" $elemVal }} + {{ $elem }} = {{ tpl $elemVal $ }} + {{- else }} + {{ $elem }} = {{ $elemVal }} + {{- end }} + {{- end }} +{{- end }} +{{- range $key, $value := index .Values "grafana.ini" }} + {{- if kindIs "map" $value }} + [{{ $key }}] + {{- range $elem, $elemVal := $value }} + {{- if kindIs "invalid" $elemVal }} + {{ $elem }} = + {{- else if kindIs "string" $elemVal }} + {{ $elem }} = {{ tpl $elemVal $ }} + {{- else }} + {{ $elem }} = {{ $elemVal }} + {{- end }} + {{- end }} + {{- end }} +{{- end }} + +{{- range $key, $value := .Values.datasources }} +{{- if not (hasKey $value "secret") }} +{{ $key }}: | + {{- tpl (toYaml $value | nindent 2) $root }} +{{- end }} +{{- end }} + +{{- range $key, $value := .Values.notifiers }} +{{- if not (hasKey $value "secret") }} +{{ $key }}: | + {{- toYaml $value | nindent 2 }} +{{- end }} +{{- end }} + +{{- range $key, $value := .Values.alerting }} +{{- if (hasKey $value "file") }} +{{ $key }}: +{{- toYaml ( $files.Get $value.file ) | nindent 2 }} +{{- else if (or (hasKey $value "secret") (hasKey $value "secretFile"))}} +{{/* will be stored inside secret generated by "configSecret.yaml"*/}} +{{- else }} +{{ $key }}: | + {{- tpl (toYaml $value | nindent 2) $root }} +{{- end }} +{{- end }} + +{{- range $key, $value := .Values.dashboardProviders }} +{{ $key }}: | + {{- toYaml $value | nindent 2 }} +{{- end }} + +{{- if .Values.dashboards }} +download_dashboards.sh: | + #!/usr/bin/env sh + set -euf + {{- if .Values.dashboardProviders }} + {{- range $key, $value := .Values.dashboardProviders }} + {{- range $value.providers }} + mkdir -p {{ .options.path }} + {{- end }} + {{- end }} + {{- end }} +{{ $dashboardProviders := .Values.dashboardProviders }} +{{- range $provider, $dashboards := .Values.dashboards }} + {{- range $key, $value := $dashboards }} + {{- if (or (hasKey $value "gnetId") (hasKey $value "url")) }} + curl -skf \ + --connect-timeout 60 \ + --max-time 60 \ + {{- if not $value.b64content }} + {{- if not $value.acceptHeader }} + -H "Accept: application/json" \ + {{- else }} + -H "Accept: {{ $value.acceptHeader }}" \ + {{- end }} + {{- if $value.token }} + -H "Authorization: token {{ $value.token }}" \ + {{- end }} + {{- if $value.bearerToken }} + -H "Authorization: Bearer {{ $value.bearerToken }}" \ + {{- end }} + {{- if $value.basic }} + -H "Authorization: Basic {{ $value.basic }}" \ + {{- end }} + {{- if $value.gitlabToken }} + -H "PRIVATE-TOKEN: {{ $value.gitlabToken }}" \ + {{- end }} + -H "Content-Type: application/json;charset=UTF-8" \ + {{- end }} + {{- $dpPath := "" -}} + {{- range $kd := (index $dashboardProviders "dashboardproviders.yaml").providers }} + {{- if eq $kd.name $provider }} + {{- $dpPath = $kd.options.path }} + {{- end }} + {{- end }} + {{- if $value.url }} + "{{ $value.url }}" \ + {{- else }} + "https://grafana.com/api/dashboards/{{ $value.gnetId }}/revisions/{{- if $value.revision -}}{{ $value.revision }}{{- else -}}1{{- end -}}/download" \ + {{- end }} + {{- if $value.datasource }} + {{- if kindIs "string" $value.datasource }} + | sed '/-- .* --/! s/"datasource":.*,/"datasource": "{{ $value.datasource }}",/g' \ + {{- end }} + {{- if kindIs "slice" $value.datasource }} + {{- range $value.datasource }} + | sed '/-- .* --/! s/${{"{"}}{{ .name }}}/{{ .value }}/g' \ + {{- end }} + {{- end }} + {{- end }} + {{- if $value.b64content }} + | base64 -d \ + {{- end }} + > "{{- if $dpPath -}}{{ $dpPath }}{{- else -}}/var/lib/grafana/dashboards/{{ $provider }}{{- end -}}/{{ $key }}.json" + {{ end }} + {{- end }} +{{- end }} +{{- end }} +{{- end -}} + +{{/* + Generate dashboard json config map data + */}} +{{- define "grafana.configDashboardProviderData" -}} +provider.yaml: |- + apiVersion: 1 + providers: + - name: '{{ .Values.sidecar.dashboards.provider.name }}' + orgId: {{ .Values.sidecar.dashboards.provider.orgid }} + {{- if not .Values.sidecar.dashboards.provider.foldersFromFilesStructure }} + folder: '{{ .Values.sidecar.dashboards.provider.folder }}' + {{- end }} + type: {{ .Values.sidecar.dashboards.provider.type }} + disableDeletion: {{ .Values.sidecar.dashboards.provider.disableDelete }} + allowUiUpdates: {{ .Values.sidecar.dashboards.provider.allowUiUpdates }} + updateIntervalSeconds: {{ .Values.sidecar.dashboards.provider.updateIntervalSeconds | default 30 }} + options: + foldersFromFilesStructure: {{ .Values.sidecar.dashboards.provider.foldersFromFilesStructure }} + path: {{ .Values.sidecar.dashboards.folder }}{{- with .Values.sidecar.dashboards.defaultFolderName }}/{{ . }}{{- end }} +{{- end -}} + +{{- define "grafana.secretsData" -}} +{{- if and (not .Values.env.GF_SECURITY_DISABLE_INITIAL_ADMIN_CREATION) (not .Values.admin.existingSecret) (not .Values.env.GF_SECURITY_ADMIN_PASSWORD__FILE) (not .Values.env.GF_SECURITY_ADMIN_PASSWORD) }} +admin-user: {{ .Values.adminUser | b64enc | quote }} +{{- if .Values.adminPassword }} +admin-password: {{ .Values.adminPassword | b64enc | quote }} +{{- else }} +admin-password: {{ include "grafana.password" . }} +{{- end }} +{{- end }} +{{- if not .Values.ldap.existingSecret }} +ldap-toml: {{ tpl .Values.ldap.config $ | b64enc | quote }} +{{- end }} +{{- end -}} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/_helpers.tpl b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/_helpers.tpl new file mode 100644 index 00000000000..68d2d815d82 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/_helpers.tpl @@ -0,0 +1,305 @@ +# Rancher +{{- define "system_default_registry" -}} +{{- if .Values.global.cattle.systemDefaultRegistry -}} +{{- printf "%s/" .Values.global.cattle.systemDefaultRegistry -}} +{{- end -}} +{{- end -}} + +# Windows Support + +{{/* +Windows cluster will add default taint for linux nodes, +add below linux tolerations to workloads could be scheduled to those linux nodes +*/}} + +{{- define "linux-node-tolerations" -}} +- key: "cattle.io/os" + value: "linux" + effect: "NoSchedule" + operator: "Equal" +{{- end -}} + +{{- define "linux-node-selector" -}} +{{- if semverCompare "<1.14-0" .Capabilities.KubeVersion.GitVersion -}} +beta.kubernetes.io/os: linux +{{- else -}} +kubernetes.io/os: linux +{{- end -}} +{{- end -}} + +{{/* vim: set filetype=mustache: */}} +{{/* +Expand the name of the chart. +*/}} +{{- define "grafana.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "grafana.fullname" -}} +{{- if .Values.fullnameOverride }} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- $name := default .Chart.Name .Values.nameOverride }} +{{- if contains $name .Release.Name }} +{{- .Release.Name | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }} +{{- end }} +{{- end }} +{{- end }} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "grafana.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Create the name of the service account +*/}} +{{- define "grafana.serviceAccountName" -}} +{{- if .Values.serviceAccount.create }} +{{- default (include "grafana.fullname" .) .Values.serviceAccount.name }} +{{- else }} +{{- default "default" .Values.serviceAccount.name }} +{{- end }} +{{- end }} + +{{- define "grafana.serviceAccountNameTest" -}} +{{- if .Values.serviceAccount.create }} +{{- default (print (include "grafana.fullname" .) "-test") .Values.serviceAccount.nameTest }} +{{- else }} +{{- default "default" .Values.serviceAccount.nameTest }} +{{- end }} +{{- end }} + +{{/* +Allow the release namespace to be overridden for multi-namespace deployments in combined charts +*/}} +{{- define "grafana.namespace" -}} +{{- if .Values.namespaceOverride }} +{{- .Values.namespaceOverride }} +{{- else }} +{{- .Release.Namespace }} +{{- end }} +{{- end }} + +{{/* +Common labels +*/}} +{{- define "grafana.labels" -}} +helm.sh/chart: {{ include "grafana.chart" . }} +{{ include "grafana.selectorLabels" . }} +{{- if or .Chart.AppVersion .Values.image.tag }} +app.kubernetes.io/version: {{ mustRegexReplaceAllLiteral "@sha.*" .Values.image.tag "" | default .Chart.AppVersion | quote }} +{{- end }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- with .Values.extraLabels }} +{{ toYaml . }} +{{- end }} +{{- end }} + +{{/* +Selector labels +*/}} +{{- define "grafana.selectorLabels" -}} +app.kubernetes.io/name: {{ include "grafana.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end }} + +{{/* +Common labels +*/}} +{{- define "grafana.imageRenderer.labels" -}} +helm.sh/chart: {{ include "grafana.chart" . }} +{{ include "grafana.imageRenderer.selectorLabels" . }} +{{- if or .Chart.AppVersion .Values.image.tag }} +app.kubernetes.io/version: {{ mustRegexReplaceAllLiteral "@sha.*" .Values.image.tag "" | default .Chart.AppVersion | quote }} +{{- end }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- end }} + +{{/* +Selector labels ImageRenderer +*/}} +{{- define "grafana.imageRenderer.selectorLabels" -}} +app.kubernetes.io/name: {{ include "grafana.name" . }}-image-renderer +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end }} + +{{/* +Looks if there's an existing secret and reuse its password. If not it generates +new password and use it. +*/}} +{{- define "grafana.password" -}} +{{- $secret := (lookup "v1" "Secret" (include "grafana.namespace" .) (include "grafana.fullname" .) ) }} +{{- if $secret }} +{{- index $secret "data" "admin-password" }} +{{- else }} +{{- (randAlphaNum 40) | b64enc | quote }} +{{- end }} +{{- end }} + +{{/* +Return the appropriate apiVersion for rbac. +*/}} +{{- define "grafana.rbac.apiVersion" -}} +{{- if $.Capabilities.APIVersions.Has "rbac.authorization.k8s.io/v1" }} +{{- print "rbac.authorization.k8s.io/v1" }} +{{- else }} +{{- print "rbac.authorization.k8s.io/v1beta1" }} +{{- end }} +{{- end }} + +{{/* +Return the appropriate apiVersion for ingress. +*/}} +{{- define "grafana.ingress.apiVersion" -}} +{{- if and ($.Capabilities.APIVersions.Has "networking.k8s.io/v1") (semverCompare ">= 1.19-0" .Capabilities.KubeVersion.Version) }} +{{- print "networking.k8s.io/v1" }} +{{- else if $.Capabilities.APIVersions.Has "networking.k8s.io/v1beta1" }} +{{- print "networking.k8s.io/v1beta1" }} +{{- else }} +{{- print "extensions/v1beta1" }} +{{- end }} +{{- end }} + +{{/* +Return the appropriate apiVersion for Horizontal Pod Autoscaler. +*/}} +{{- define "grafana.hpa.apiVersion" -}} +{{- if .Capabilities.APIVersions.Has "autoscaling/v2" }} +{{- print "autoscaling/v2" }} +{{- else }} +{{- print "autoscaling/v2beta2" }} +{{- end }} +{{- end }} + +{{/* +Return the appropriate apiVersion for podDisruptionBudget. +*/}} +{{- define "grafana.podDisruptionBudget.apiVersion" -}} +{{- if $.Values.podDisruptionBudget.apiVersion }} +{{- print $.Values.podDisruptionBudget.apiVersion }} +{{- else if $.Capabilities.APIVersions.Has "policy/v1/PodDisruptionBudget" }} +{{- print "policy/v1" }} +{{- else }} +{{- print "policy/v1beta1" }} +{{- end }} +{{- end }} + +{{/* +Return if ingress is stable. +*/}} +{{- define "grafana.ingress.isStable" -}} +{{- eq (include "grafana.ingress.apiVersion" .) "networking.k8s.io/v1" }} +{{- end }} + +{{/* +Return if ingress supports ingressClassName. +*/}} +{{- define "grafana.ingress.supportsIngressClassName" -}} +{{- or (eq (include "grafana.ingress.isStable" .) "true") (and (eq (include "grafana.ingress.apiVersion" .) "networking.k8s.io/v1beta1") (semverCompare ">= 1.18-0" .Capabilities.KubeVersion.Version)) }} +{{- end }} + +{{/* +Return if ingress supports pathType. +*/}} +{{- define "grafana.ingress.supportsPathType" -}} +{{- or (eq (include "grafana.ingress.isStable" .) "true") (and (eq (include "grafana.ingress.apiVersion" .) "networking.k8s.io/v1beta1") (semverCompare ">= 1.18-0" .Capabilities.KubeVersion.Version)) }} +{{- end }} + +{{/* +Formats imagePullSecrets. Input is (dict "root" . "imagePullSecrets" .{specific imagePullSecrets}) +*/}} +{{- define "grafana.imagePullSecrets" -}} +{{- $root := .root }} +{{- range (concat .root.Values.global.imagePullSecrets .imagePullSecrets) }} +{{- if eq (typeOf .) "map[string]interface {}" }} +- {{ toYaml (dict "name" (tpl .name $root)) | trim }} +{{- else }} +- name: {{ tpl . $root }} +{{- end }} +{{- end }} +{{- end }} + + +{{/* + Checks whether or not the configSecret secret has to be created + */}} +{{- define "grafana.shouldCreateConfigSecret" -}} +{{- $secretFound := false -}} +{{- range $key, $value := .Values.datasources }} + {{- if hasKey $value "secret" }} + {{- $secretFound = true}} + {{- end }} +{{- end }} +{{- range $key, $value := .Values.notifiers }} + {{- if hasKey $value "secret" }} + {{- $secretFound = true}} + {{- end }} +{{- end }} +{{- range $key, $value := .Values.alerting }} + {{- if (or (hasKey $value "secret") (hasKey $value "secretFile")) }} + {{- $secretFound = true}} + {{- end }} +{{- end }} +{{- $secretFound}} +{{- end -}} + +{{/* + Checks whether the user is attempting to store secrets in plaintext + in the grafana.ini configmap +*/}} +{{/* grafana.assertNoLeakedSecrets checks for sensitive keys in values */}} +{{- define "grafana.assertNoLeakedSecrets" -}} + {{- $sensitiveKeysYaml := ` +sensitiveKeys: +- path: ["database", "password"] +- path: ["smtp", "password"] +- path: ["security", "secret_key"] +- path: ["security", "admin_password"] +- path: ["auth.basic", "password"] +- path: ["auth.ldap", "bind_password"] +- path: ["auth.google", "client_secret"] +- path: ["auth.github", "client_secret"] +- path: ["auth.gitlab", "client_secret"] +- path: ["auth.generic_oauth", "client_secret"] +- path: ["auth.okta", "client_secret"] +- path: ["auth.azuread", "client_secret"] +- path: ["auth.grafana_com", "client_secret"] +- path: ["auth.grafananet", "client_secret"] +- path: ["azure", "user_identity_client_secret"] +- path: ["unified_alerting", "ha_redis_password"] +- path: ["metrics", "basic_auth_password"] +- path: ["external_image_storage.s3", "secret_key"] +- path: ["external_image_storage.webdav", "password"] +- path: ["external_image_storage.azure_blob", "account_key"] +` | fromYaml -}} + {{- if $.Values.assertNoLeakedSecrets -}} + {{- $grafanaIni := index .Values "grafana.ini" -}} + {{- range $_, $secret := $sensitiveKeysYaml.sensitiveKeys -}} + {{- $currentMap := $grafanaIni -}} + {{- $shouldContinue := true -}} + {{- range $index, $elem := $secret.path -}} + {{- if and $shouldContinue (hasKey $currentMap $elem) -}} + {{- if eq (len $secret.path) (add1 $index) -}} + {{- if not (regexMatch "\\$(?:__(?:env|file|vault))?{[^}]+}" (index $currentMap $elem)) -}} + {{- fail (printf "Sensitive key '%s' should not be defined explicitly in values. Use variable expansion instead. You can disable this client-side validation by changing the value of assertNoLeakedSecrets." (join "." $secret.path)) -}} + {{- end -}} + {{- else -}} + {{- $currentMap = index $currentMap $elem -}} + {{- end -}} + {{- else -}} + {{- $shouldContinue = false -}} + {{- end -}} + {{- end -}} + {{- end -}} + {{- end -}} +{{- end -}} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/_pod.tpl b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/_pod.tpl new file mode 100644 index 00000000000..2ebf7d5f103 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/_pod.tpl @@ -0,0 +1,1296 @@ +{{- define "grafana.pod" -}} +{{- $sts := list "sts" "StatefulSet" "statefulset" -}} +{{- $root := . -}} +{{- with .Values.schedulerName }} +schedulerName: "{{ . }}" +{{- end }} +serviceAccountName: {{ include "grafana.serviceAccountName" . }} +automountServiceAccountToken: {{ .Values.automountServiceAccountToken }} +{{- with .Values.securityContext }} +securityContext: + {{- toYaml . | nindent 2 }} +{{- end }} +{{- with .Values.hostAliases }} +hostAliases: + {{- toYaml . | nindent 2 }} +{{- end }} +{{- if .Values.dnsPolicy }} +dnsPolicy: {{ .Values.dnsPolicy }} +{{- end }} +{{- with .Values.dnsConfig }} +dnsConfig: + {{- toYaml . | nindent 2 }} +{{- end }} +{{- with .Values.priorityClassName }} +priorityClassName: {{ . }} +{{- end }} +{{- if ( or .Values.persistence.enabled .Values.dashboards .Values.extraInitContainers (and .Values.sidecar.alerts.enabled .Values.sidecar.alerts.initAlerts) (and .Values.sidecar.datasources.enabled .Values.sidecar.datasources.initDatasources) (and .Values.sidecar.notifiers.enabled .Values.sidecar.notifiers.initNotifiers)) }} +initContainers: +{{- end }} +{{- if ( and .Values.persistence.enabled .Values.initChownData.enabled ) }} + - name: init-chown-data + {{- $registry := include "system_default_registry" . | default .Values.initChownData.image.registry -}} + {{- if .Values.initChownData.image.sha }} + image: "{{ $registry }}{{ .Values.initChownData.image.repository }}:{{ .Values.initChownData.image.tag }}@sha256:{{ .Values.initChownData.image.sha }}" + {{- else }} + image: "{{ $registry }}{{ .Values.initChownData.image.repository }}:{{ .Values.initChownData.image.tag }}" + {{- end }} + imagePullPolicy: {{ .Values.initChownData.image.pullPolicy }} + {{- with .Values.initChownData.securityContext }} + securityContext: + {{- toYaml . | nindent 6 }} + {{- end }} + command: + - chown + - -R + - {{ .Values.securityContext.runAsUser }}:{{ .Values.securityContext.runAsGroup }} + - /var/lib/grafana + {{- with .Values.initChownData.resources }} + resources: + {{- toYaml . | nindent 6 }} + {{- end }} + volumeMounts: + - name: storage + mountPath: "/var/lib/grafana" + {{- with .Values.persistence.subPath }} + subPath: {{ tpl . $root }} + {{- end }} +{{- end }} +{{- if .Values.dashboards }} + - name: download-dashboards + {{- $registry := include "system_default_registry" . | default .Values.downloadDashboardsImage.registry -}} + {{- if .Values.downloadDashboardsImage.sha }} + image: "{{ $registry }}{{ .Values.downloadDashboardsImage.repository }}:{{ .Values.downloadDashboardsImage.tag }}@sha256:{{ .Values.downloadDashboardsImage.sha }}" + {{- else }} + image: "{{ $registry }}{{ .Values.downloadDashboardsImage.repository }}:{{ .Values.downloadDashboardsImage.tag }}" + {{- end }} + imagePullPolicy: {{ .Values.downloadDashboardsImage.pullPolicy }} + command: ["/bin/sh"] + args: [ "-c", "mkdir -p /var/lib/grafana/dashboards/default && /bin/sh -x /etc/grafana/download_dashboards.sh" ] + {{- with .Values.downloadDashboards.resources }} + resources: + {{- toYaml . | nindent 6 }} + {{- end }} + env: + {{- range $key, $value := .Values.downloadDashboards.env }} + - name: "{{ $key }}" + value: "{{ $value }}" + {{- end }} + {{- range $key, $value := .Values.downloadDashboards.envValueFrom }} + - name: {{ $key | quote }} + valueFrom: + {{- tpl (toYaml $value) $ | nindent 10 }} + {{- end }} + {{- with .Values.downloadDashboards.securityContext }} + securityContext: + {{- toYaml . | nindent 6 }} + {{- end }} + {{- with .Values.downloadDashboards.envFromSecret }} + envFrom: + - secretRef: + name: {{ tpl . $root }} + {{- end }} + volumeMounts: + - name: config + mountPath: "/etc/grafana/download_dashboards.sh" + subPath: download_dashboards.sh + - name: storage + mountPath: "/var/lib/grafana" + {{- with .Values.persistence.subPath }} + subPath: {{ tpl . $root }} + {{- end }} + {{- range .Values.extraSecretMounts }} + - name: {{ .name }} + mountPath: {{ .mountPath }} + readOnly: {{ .readOnly }} + {{- end }} +{{- end }} +{{- if and .Values.sidecar.alerts.enabled .Values.sidecar.alerts.initAlerts }} + - name: {{ include "grafana.name" . }}-init-sc-alerts + {{- $registry := include "system_default_registry" . | default .Values.sidecar.image.registry -}} + {{- if .Values.sidecar.image.sha }} + image: "{{ $registry }}{{ .Values.sidecar.image.repository }}:{{ .Values.sidecar.image.tag }}@sha256:{{ .Values.sidecar.image.sha }}" + {{- else }} + image: "{{ $registry }}{{ .Values.sidecar.image.repository }}:{{ .Values.sidecar.image.tag }}" + {{- end }} + imagePullPolicy: {{ .Values.sidecar.imagePullPolicy }} + env: + {{- range $key, $value := .Values.sidecar.alerts.env }} + - name: "{{ $key }}" + value: "{{ $value }}" + {{- end }} + {{- if .Values.sidecar.alerts.ignoreAlreadyProcessed }} + - name: IGNORE_ALREADY_PROCESSED + value: "true" + {{- end }} + - name: METHOD + value: "LIST" + - name: LABEL + value: "{{ .Values.sidecar.alerts.label }}" + {{- with .Values.sidecar.alerts.labelValue }} + - name: LABEL_VALUE + value: {{ quote . }} + {{- end }} + {{- if or .Values.sidecar.logLevel .Values.sidecar.alerts.logLevel }} + - name: LOG_LEVEL + value: {{ default .Values.sidecar.logLevel .Values.sidecar.alerts.logLevel }} + {{- end }} + - name: FOLDER + value: "/etc/grafana/provisioning/alerting" + - name: RESOURCE + value: {{ quote .Values.sidecar.alerts.resource }} + {{- with .Values.sidecar.enableUniqueFilenames }} + - name: UNIQUE_FILENAMES + value: "{{ . }}" + {{- end }} + {{- with .Values.sidecar.alerts.searchNamespace }} + - name: NAMESPACE + value: {{ . | join "," | quote }} + {{- end }} + {{- with .Values.sidecar.alerts.skipTlsVerify }} + - name: SKIP_TLS_VERIFY + value: {{ quote . }} + {{- end }} + {{- with .Values.sidecar.alerts.script }} + - name: SCRIPT + value: {{ quote . }} + {{- end }} + {{- with .Values.sidecar.livenessProbe }} + livenessProbe: + {{- toYaml . | nindent 6 }} + {{- end }} + {{- with .Values.sidecar.readinessProbe }} + readinessProbe: + {{- toYaml . | nindent 6 }} + {{- end }} + {{- with .Values.sidecar.resources }} + resources: + {{- toYaml . | nindent 6 }} + {{- end }} + {{- with .Values.sidecar.securityContext }} + securityContext: + {{- toYaml . | nindent 6 }} + {{- end }} + volumeMounts: + - name: sc-alerts-volume + mountPath: "/etc/grafana/provisioning/alerting" + {{- with .Values.sidecar.alerts.extraMounts }} + {{- toYaml . | trim | nindent 6 }} + {{- end }} +{{- end }} +{{- if and .Values.sidecar.datasources.enabled .Values.sidecar.datasources.initDatasources }} + - name: {{ include "grafana.name" . }}-init-sc-datasources + {{- $registry := include "system_default_registry" . | default .Values.sidecar.image.registry -}} + {{- if .Values.sidecar.image.sha }} + image: "{{ $registry }}{{ .Values.sidecar.image.repository }}:{{ .Values.sidecar.image.tag }}@sha256:{{ .Values.sidecar.image.sha }}" + {{- else }} + image: "{{ $registry }}{{ .Values.sidecar.image.repository }}:{{ .Values.sidecar.image.tag }}" + {{- end }} + imagePullPolicy: {{ .Values.sidecar.imagePullPolicy }} + env: + {{- range $key, $value := .Values.sidecar.datasources.env }} + - name: "{{ $key }}" + value: "{{ $value }}" + {{- end }} + {{- if .Values.sidecar.datasources.ignoreAlreadyProcessed }} + - name: IGNORE_ALREADY_PROCESSED + value: "true" + {{- end }} + - name: METHOD + value: "LIST" + - name: LABEL + value: "{{ .Values.sidecar.datasources.label }}" + {{- with .Values.sidecar.datasources.labelValue }} + - name: LABEL_VALUE + value: {{ quote . }} + {{- end }} + {{- if or .Values.sidecar.logLevel .Values.sidecar.datasources.logLevel }} + - name: LOG_LEVEL + value: {{ default .Values.sidecar.logLevel .Values.sidecar.datasources.logLevel }} + {{- end }} + - name: FOLDER + value: "/etc/grafana/provisioning/datasources" + - name: RESOURCE + value: {{ quote .Values.sidecar.datasources.resource }} + {{- with .Values.sidecar.enableUniqueFilenames }} + - name: UNIQUE_FILENAMES + value: "{{ . }}" + {{- end }} + {{- if .Values.sidecar.datasources.searchNamespace }} + - name: NAMESPACE + value: "{{ tpl (.Values.sidecar.datasources.searchNamespace | join ",") . }}" + {{- end }} + {{- with .Values.sidecar.skipTlsVerify }} + - name: SKIP_TLS_VERIFY + value: "{{ . }}" + {{- end }} + {{- with .Values.sidecar.resources }} + resources: + {{- toYaml . | nindent 6 }} + {{- end }} + {{- with .Values.sidecar.securityContext }} + securityContext: + {{- toYaml . | nindent 6 }} + {{- end }} + volumeMounts: + - name: sc-datasources-volume + mountPath: "/etc/grafana/provisioning/datasources" +{{- end }} +{{- if and .Values.sidecar.notifiers.enabled .Values.sidecar.notifiers.initNotifiers }} + - name: {{ include "grafana.name" . }}-init-sc-notifiers + {{- $registry := include "system_default_registry" . | default .Values.sidecar.image.registry -}} + {{- if .Values.sidecar.image.sha }} + image: "{{ $registry }}{{ .Values.sidecar.image.repository }}:{{ .Values.sidecar.image.tag }}@sha256:{{ .Values.sidecar.image.sha }}" + {{- else }} + image: "{{ $registry }}{{ .Values.sidecar.image.repository }}:{{ .Values.sidecar.image.tag }}" + {{- end }} + imagePullPolicy: {{ .Values.sidecar.imagePullPolicy }} + env: + {{- range $key, $value := .Values.sidecar.notifiers.env }} + - name: "{{ $key }}" + value: "{{ $value }}" + {{- end }} + {{- if .Values.sidecar.notifiers.ignoreAlreadyProcessed }} + - name: IGNORE_ALREADY_PROCESSED + value: "true" + {{- end }} + - name: METHOD + value: LIST + - name: LABEL + value: "{{ .Values.sidecar.notifiers.label }}" + {{- with .Values.sidecar.notifiers.labelValue }} + - name: LABEL_VALUE + value: {{ quote . }} + {{- end }} + {{- if or .Values.sidecar.logLevel .Values.sidecar.notifiers.logLevel }} + - name: LOG_LEVEL + value: {{ default .Values.sidecar.logLevel .Values.sidecar.notifiers.logLevel }} + {{- end }} + - name: FOLDER + value: "/etc/grafana/provisioning/notifiers" + - name: RESOURCE + value: {{ quote .Values.sidecar.notifiers.resource }} + {{- with .Values.sidecar.enableUniqueFilenames }} + - name: UNIQUE_FILENAMES + value: "{{ . }}" + {{- end }} + {{- with .Values.sidecar.notifiers.searchNamespace }} + - name: NAMESPACE + value: "{{ tpl (. | join ",") $root }}" + {{- end }} + {{- with .Values.sidecar.skipTlsVerify }} + - name: SKIP_TLS_VERIFY + value: "{{ . }}" + {{- end }} + {{- with .Values.sidecar.livenessProbe }} + livenessProbe: + {{- toYaml . | nindent 6 }} + {{- end }} + {{- with .Values.sidecar.readinessProbe }} + readinessProbe: + {{- toYaml . | nindent 6 }} + {{- end }} + {{- with .Values.sidecar.resources }} + resources: + {{- toYaml . | nindent 6 }} + {{- end }} + {{- with .Values.sidecar.securityContext }} + securityContext: + {{- toYaml . | nindent 6 }} + {{- end }} + volumeMounts: + - name: sc-notifiers-volume + mountPath: "/etc/grafana/provisioning/notifiers" +{{- end}} +{{- with .Values.extraInitContainers }} + {{- tpl (toYaml .) $root | nindent 2 }} +{{- end }} +{{- if or .Values.image.pullSecrets .Values.global.imagePullSecrets }} +imagePullSecrets: + {{- include "grafana.imagePullSecrets" (dict "root" $root "imagePullSecrets" .Values.image.pullSecrets) | nindent 2 }} +{{- end }} +{{- if not .Values.enableKubeBackwardCompatibility }} +enableServiceLinks: {{ .Values.enableServiceLinks }} +{{- end }} +containers: +{{- if and .Values.sidecar.alerts.enabled (not .Values.sidecar.alerts.initAlerts) }} + - name: {{ include "grafana.name" . }}-sc-alerts + {{- $registry := include "system_default_registry" . | default .Values.sidecar.image.registry -}} + {{- if .Values.sidecar.image.sha }} + image: "{{ $registry }}{{ .Values.sidecar.image.repository }}:{{ .Values.sidecar.image.tag }}@sha256:{{ .Values.sidecar.image.sha }}" + {{- else }} + image: "{{ $registry }}{{ .Values.sidecar.image.repository }}:{{ .Values.sidecar.image.tag }}" + {{- end }} + imagePullPolicy: {{ .Values.sidecar.imagePullPolicy }} + env: + {{- range $key, $value := .Values.sidecar.alerts.env }} + - name: "{{ $key }}" + value: "{{ $value }}" + {{- end }} + {{- if .Values.sidecar.alerts.ignoreAlreadyProcessed }} + - name: IGNORE_ALREADY_PROCESSED + value: "true" + {{- end }} + - name: METHOD + value: {{ .Values.sidecar.alerts.watchMethod }} + - name: LABEL + value: "{{ .Values.sidecar.alerts.label }}" + {{- with .Values.sidecar.alerts.labelValue }} + - name: LABEL_VALUE + value: {{ quote . }} + {{- end }} + {{- if or .Values.sidecar.logLevel .Values.sidecar.alerts.logLevel }} + - name: LOG_LEVEL + value: {{ default .Values.sidecar.logLevel .Values.sidecar.alerts.logLevel }} + {{- end }} + - name: FOLDER + value: "/etc/grafana/provisioning/alerting" + - name: RESOURCE + value: {{ quote .Values.sidecar.alerts.resource }} + {{- with .Values.sidecar.enableUniqueFilenames }} + - name: UNIQUE_FILENAMES + value: "{{ . }}" + {{- end }} + {{- with .Values.sidecar.alerts.searchNamespace }} + - name: NAMESPACE + value: {{ . | join "," | quote }} + {{- end }} + {{- with .Values.sidecar.alerts.skipTlsVerify }} + - name: SKIP_TLS_VERIFY + value: {{ quote . }} + {{- end }} + {{- with .Values.sidecar.alerts.script }} + - name: SCRIPT + value: {{ quote . }} + {{- end }} + {{- if and (not .Values.env.GF_SECURITY_ADMIN_USER) (not .Values.env.GF_SECURITY_DISABLE_INITIAL_ADMIN_CREATION) }} + - name: REQ_USERNAME + valueFrom: + secretKeyRef: + name: {{ (tpl .Values.admin.existingSecret .) | default (include "grafana.fullname" .) }} + key: {{ .Values.admin.userKey | default "admin-user" }} + {{- end }} + {{- if and (not .Values.env.GF_SECURITY_ADMIN_PASSWORD) (not .Values.env.GF_SECURITY_ADMIN_PASSWORD__FILE) (not .Values.env.GF_SECURITY_DISABLE_INITIAL_ADMIN_CREATION) }} + - name: REQ_PASSWORD + valueFrom: + secretKeyRef: + name: {{ (tpl .Values.admin.existingSecret .) | default (include "grafana.fullname" .) }} + key: {{ .Values.admin.passwordKey | default "admin-password" }} + {{- end }} + {{- if not .Values.sidecar.alerts.skipReload }} + - name: REQ_URL + value: {{ .Values.sidecar.alerts.reloadURL }} + - name: REQ_METHOD + value: POST + {{- end }} + {{- if .Values.sidecar.alerts.watchServerTimeout }} + {{- if ne .Values.sidecar.alerts.watchMethod "WATCH" }} + {{- fail (printf "Cannot use .Values.sidecar.alerts.watchServerTimeout with .Values.sidecar.alerts.watchMethod %s" .Values.sidecar.alerts.watchMethod) }} + {{- end }} + - name: WATCH_SERVER_TIMEOUT + value: "{{ .Values.sidecar.alerts.watchServerTimeout }}" + {{- end }} + {{- if .Values.sidecar.alerts.watchClientTimeout }} + {{- if ne .Values.sidecar.alerts.watchMethod "WATCH" }} + {{- fail (printf "Cannot use .Values.sidecar.alerts.watchClientTimeout with .Values.sidecar.alerts.watchMethod %s" .Values.sidecar.alerts.watchMethod) }} + {{- end }} + - name: WATCH_CLIENT_TIMEOUT + value: "{{ .Values.sidecar.alerts.watchClientTimeout }}" + {{- end }} + {{- with .Values.sidecar.livenessProbe }} + livenessProbe: + {{- toYaml . | nindent 6 }} + {{- end }} + {{- with .Values.sidecar.readinessProbe }} + readinessProbe: + {{- toYaml . | nindent 6 }} + {{- end }} + {{- with .Values.sidecar.resources }} + resources: + {{- toYaml . | nindent 6 }} + {{- end }} + {{- with .Values.sidecar.securityContext }} + securityContext: + {{- toYaml . | nindent 6 }} + {{- end }} + volumeMounts: + - name: sc-alerts-volume + mountPath: "/etc/grafana/provisioning/alerting" + {{- with .Values.sidecar.alerts.extraMounts }} + {{- toYaml . | trim | nindent 6 }} + {{- end }} +{{- end}} +{{- if .Values.sidecar.dashboards.enabled }} + - name: {{ include "grafana.name" . }}-sc-dashboard + {{- $registry := include "system_default_registry" . | default .Values.sidecar.image.registry -}} + {{- if .Values.sidecar.image.sha }} + image: "{{ $registry }}{{ .Values.sidecar.image.repository }}:{{ .Values.sidecar.image.tag }}@sha256:{{ .Values.sidecar.image.sha }}" + {{- else }} + image: "{{ $registry }}{{ .Values.sidecar.image.repository }}:{{ .Values.sidecar.image.tag }}" + {{- end }} + imagePullPolicy: {{ .Values.sidecar.imagePullPolicy }} + env: + {{- range $key, $value := .Values.sidecar.dashboards.env }} + - name: "{{ $key }}" + value: "{{ $value }}" + {{- end }} + {{- range $key, $value := .Values.sidecar.datasources.envValueFrom }} + - name: {{ $key | quote }} + valueFrom: + {{- tpl (toYaml $value) $ | nindent 10 }} + {{- end }} + {{- if .Values.sidecar.dashboards.ignoreAlreadyProcessed }} + - name: IGNORE_ALREADY_PROCESSED + value: "true" + {{- end }} + - name: METHOD + value: {{ .Values.sidecar.dashboards.watchMethod }} + - name: LABEL + value: "{{ .Values.sidecar.dashboards.label }}" + {{- with .Values.sidecar.dashboards.labelValue }} + - name: LABEL_VALUE + value: {{ quote . }} + {{- end }} + {{- if or .Values.sidecar.logLevel .Values.sidecar.dashboards.logLevel }} + - name: LOG_LEVEL + value: {{ default .Values.sidecar.logLevel .Values.sidecar.dashboards.logLevel }} + {{- end }} + - name: FOLDER + value: "{{ .Values.sidecar.dashboards.folder }}{{- with .Values.sidecar.dashboards.defaultFolderName }}/{{ . }}{{- end }}" + - name: RESOURCE + value: {{ quote .Values.sidecar.dashboards.resource }} + {{- with .Values.sidecar.enableUniqueFilenames }} + - name: UNIQUE_FILENAMES + value: "{{ . }}" + {{- end }} + {{- with .Values.sidecar.dashboards.searchNamespace }} + - name: NAMESPACE + value: "{{ tpl (. | join ",") $root }}" + {{- end }} + {{- with .Values.sidecar.skipTlsVerify }} + - name: SKIP_TLS_VERIFY + value: "{{ . }}" + {{- end }} + {{- with .Values.sidecar.dashboards.folderAnnotation }} + - name: FOLDER_ANNOTATION + value: "{{ . }}" + {{- end }} + {{- with .Values.sidecar.dashboards.script }} + - name: SCRIPT + value: "{{ . }}" + {{- end }} + {{- if not .Values.sidecar.dashboards.skipReload }} + {{- if and (not .Values.env.GF_SECURITY_ADMIN_USER) (not .Values.env.GF_SECURITY_DISABLE_INITIAL_ADMIN_CREATION) }} + - name: REQ_USERNAME + valueFrom: + secretKeyRef: + name: {{ (tpl .Values.admin.existingSecret .) | default (include "grafana.fullname" .) }} + key: {{ .Values.admin.userKey | default "admin-user" }} + {{- end }} + {{- if and (not .Values.env.GF_SECURITY_ADMIN_PASSWORD) (not .Values.env.GF_SECURITY_ADMIN_PASSWORD__FILE) (not .Values.env.GF_SECURITY_DISABLE_INITIAL_ADMIN_CREATION) }} + - name: REQ_PASSWORD + valueFrom: + secretKeyRef: + name: {{ (tpl .Values.admin.existingSecret .) | default (include "grafana.fullname" .) }} + key: {{ .Values.admin.passwordKey | default "admin-password" }} + {{- end }} + - name: REQ_URL + value: {{ .Values.sidecar.dashboards.reloadURL }} + - name: REQ_METHOD + value: POST + {{- end }} + {{- if .Values.sidecar.dashboards.watchServerTimeout }} + {{- if ne .Values.sidecar.dashboards.watchMethod "WATCH" }} + {{- fail (printf "Cannot use .Values.sidecar.dashboards.watchServerTimeout with .Values.sidecar.dashboards.watchMethod %s" .Values.sidecar.dashboards.watchMethod) }} + {{- end }} + - name: WATCH_SERVER_TIMEOUT + value: "{{ .Values.sidecar.dashboards.watchServerTimeout }}" + {{- end }} + {{- if .Values.sidecar.dashboards.watchClientTimeout }} + {{- if ne .Values.sidecar.dashboards.watchMethod "WATCH" }} + {{- fail (printf "Cannot use .Values.sidecar.dashboards.watchClientTimeout with .Values.sidecar.dashboards.watchMethod %s" .Values.sidecar.dashboards.watchMethod) }} + {{- end }} + - name: WATCH_CLIENT_TIMEOUT + value: {{ .Values.sidecar.dashboards.watchClientTimeout | quote }} + {{- end }} + {{- with .Values.sidecar.livenessProbe }} + livenessProbe: + {{- toYaml . | nindent 6 }} + {{- end }} + {{- with .Values.sidecar.readinessProbe }} + readinessProbe: + {{- toYaml . | nindent 6 }} + {{- end }} + {{- with .Values.sidecar.resources }} + resources: + {{- toYaml . | nindent 6 }} + {{- end }} + {{- with .Values.sidecar.securityContext }} + securityContext: + {{- toYaml . | nindent 6 }} + {{- end }} + volumeMounts: + - name: sc-dashboard-volume + mountPath: {{ .Values.sidecar.dashboards.folder | quote }} + {{- with .Values.sidecar.dashboards.extraMounts }} + {{- toYaml . | trim | nindent 6 }} + {{- end }} +{{- end}} +{{- if and .Values.sidecar.datasources.enabled (not .Values.sidecar.datasources.initDatasources) }} + - name: {{ include "grafana.name" . }}-sc-datasources + {{- $registry := include "system_default_registry" . | default .Values.sidecar.image.registry -}} + {{- if .Values.sidecar.image.sha }} + image: "{{ $registry }}{{ .Values.sidecar.image.repository }}:{{ .Values.sidecar.image.tag }}@sha256:{{ .Values.sidecar.image.sha }}" + {{- else }} + image: "{{ $registry }}{{ .Values.sidecar.image.repository }}:{{ .Values.sidecar.image.tag }}" + {{- end }} + imagePullPolicy: {{ .Values.sidecar.imagePullPolicy }} + env: + {{- range $key, $value := .Values.sidecar.datasources.env }} + - name: "{{ $key }}" + value: "{{ $value }}" + {{- end }} + {{- if .Values.sidecar.datasources.ignoreAlreadyProcessed }} + - name: IGNORE_ALREADY_PROCESSED + value: "true" + {{- end }} + - name: METHOD + value: {{ .Values.sidecar.datasources.watchMethod }} + - name: LABEL + value: "{{ .Values.sidecar.datasources.label }}" + {{- with .Values.sidecar.datasources.labelValue }} + - name: LABEL_VALUE + value: {{ quote . }} + {{- end }} + {{- if or .Values.sidecar.logLevel .Values.sidecar.datasources.logLevel }} + - name: LOG_LEVEL + value: {{ default .Values.sidecar.logLevel .Values.sidecar.datasources.logLevel }} + {{- end }} + - name: FOLDER + value: "/etc/grafana/provisioning/datasources" + - name: RESOURCE + value: {{ quote .Values.sidecar.datasources.resource }} + {{- with .Values.sidecar.enableUniqueFilenames }} + - name: UNIQUE_FILENAMES + value: "{{ . }}" + {{- end }} + {{- with .Values.sidecar.datasources.searchNamespace }} + - name: NAMESPACE + value: "{{ tpl (. | join ",") $root }}" + {{- end }} + {{- if .Values.sidecar.skipTlsVerify }} + - name: SKIP_TLS_VERIFY + value: "{{ .Values.sidecar.skipTlsVerify }}" + {{- end }} + {{- if .Values.sidecar.datasources.script }} + - name: SCRIPT + value: "{{ .Values.sidecar.datasources.script }}" + {{- end }} + {{- if and (not .Values.env.GF_SECURITY_ADMIN_USER) (not .Values.env.GF_SECURITY_DISABLE_INITIAL_ADMIN_CREATION) }} + - name: REQ_USERNAME + valueFrom: + secretKeyRef: + name: {{ (tpl .Values.admin.existingSecret .) | default (include "grafana.fullname" .) }} + key: {{ .Values.admin.userKey | default "admin-user" }} + {{- end }} + {{- if and (not .Values.env.GF_SECURITY_ADMIN_PASSWORD) (not .Values.env.GF_SECURITY_ADMIN_PASSWORD__FILE) (not .Values.env.GF_SECURITY_DISABLE_INITIAL_ADMIN_CREATION) }} + - name: REQ_PASSWORD + valueFrom: + secretKeyRef: + name: {{ (tpl .Values.admin.existingSecret .) | default (include "grafana.fullname" .) }} + key: {{ .Values.admin.passwordKey | default "admin-password" }} + {{- end }} + {{- if not .Values.sidecar.datasources.skipReload }} + - name: REQ_URL + value: {{ .Values.sidecar.datasources.reloadURL }} + - name: REQ_METHOD + value: POST + {{- end }} + {{- if .Values.sidecar.datasources.watchServerTimeout }} + {{- if ne .Values.sidecar.datasources.watchMethod "WATCH" }} + {{- fail (printf "Cannot use .Values.sidecar.datasources.watchServerTimeout with .Values.sidecar.datasources.watchMethod %s" .Values.sidecar.datasources.watchMethod) }} + {{- end }} + - name: WATCH_SERVER_TIMEOUT + value: "{{ .Values.sidecar.datasources.watchServerTimeout }}" + {{- end }} + {{- if .Values.sidecar.datasources.watchClientTimeout }} + {{- if ne .Values.sidecar.datasources.watchMethod "WATCH" }} + {{- fail (printf "Cannot use .Values.sidecar.datasources.watchClientTimeout with .Values.sidecar.datasources.watchMethod %s" .Values.sidecar.datasources.watchMethod) }} + {{- end }} + - name: WATCH_CLIENT_TIMEOUT + value: "{{ .Values.sidecar.datasources.watchClientTimeout }}" + {{- end }} + {{- with .Values.sidecar.livenessProbe }} + livenessProbe: + {{- toYaml . | nindent 6 }} + {{- end }} + {{- with .Values.sidecar.readinessProbe }} + readinessProbe: + {{- toYaml . | nindent 6 }} + {{- end }} + {{- with .Values.sidecar.resources }} + resources: + {{- toYaml . | nindent 6 }} + {{- end }} + {{- with .Values.sidecar.securityContext }} + securityContext: + {{- toYaml . | nindent 6 }} + {{- end }} + volumeMounts: + - name: sc-datasources-volume + mountPath: "/etc/grafana/provisioning/datasources" +{{- end}} +{{- if .Values.sidecar.notifiers.enabled }} + - name: {{ include "grafana.name" . }}-sc-notifiers + {{- $registry := include "system_default_registry" . | default .Values.sidecar.image.registry -}} + {{- if .Values.sidecar.image.sha }} + image: "{{ $registry }}{{ .Values.sidecar.image.repository }}:{{ .Values.sidecar.image.tag }}@sha256:{{ .Values.sidecar.image.sha }}" + {{- else }} + image: "{{ $registry }}{{ .Values.sidecar.image.repository }}:{{ .Values.sidecar.image.tag }}" + {{- end }} + imagePullPolicy: {{ .Values.sidecar.imagePullPolicy }} + env: + {{- range $key, $value := .Values.sidecar.notifiers.env }} + - name: "{{ $key }}" + value: "{{ $value }}" + {{- end }} + {{- if .Values.sidecar.notifiers.ignoreAlreadyProcessed }} + - name: IGNORE_ALREADY_PROCESSED + value: "true" + {{- end }} + - name: METHOD + value: {{ .Values.sidecar.notifiers.watchMethod }} + - name: LABEL + value: "{{ .Values.sidecar.notifiers.label }}" + {{- with .Values.sidecar.notifiers.labelValue }} + - name: LABEL_VALUE + value: {{ quote . }} + {{- end }} + {{- if or .Values.sidecar.logLevel .Values.sidecar.notifiers.logLevel }} + - name: LOG_LEVEL + value: {{ default .Values.sidecar.logLevel .Values.sidecar.notifiers.logLevel }} + {{- end }} + - name: FOLDER + value: "/etc/grafana/provisioning/notifiers" + - name: RESOURCE + value: {{ quote .Values.sidecar.notifiers.resource }} + {{- if .Values.sidecar.enableUniqueFilenames }} + - name: UNIQUE_FILENAMES + value: "{{ .Values.sidecar.enableUniqueFilenames }}" + {{- end }} + {{- with .Values.sidecar.notifiers.searchNamespace }} + - name: NAMESPACE + value: "{{ tpl (. | join ",") $root }}" + {{- end }} + {{- with .Values.sidecar.skipTlsVerify }} + - name: SKIP_TLS_VERIFY + value: "{{ . }}" + {{- end }} + {{- if .Values.sidecar.notifiers.script }} + - name: SCRIPT + value: "{{ .Values.sidecar.notifiers.script }}" + {{- end }} + {{- if and (not .Values.env.GF_SECURITY_ADMIN_USER) (not .Values.env.GF_SECURITY_DISABLE_INITIAL_ADMIN_CREATION) }} + - name: REQ_USERNAME + valueFrom: + secretKeyRef: + name: {{ (tpl .Values.admin.existingSecret .) | default (include "grafana.fullname" .) }} + key: {{ .Values.admin.userKey | default "admin-user" }} + {{- end }} + {{- if and (not .Values.env.GF_SECURITY_ADMIN_PASSWORD) (not .Values.env.GF_SECURITY_ADMIN_PASSWORD__FILE) (not .Values.env.GF_SECURITY_DISABLE_INITIAL_ADMIN_CREATION) }} + - name: REQ_PASSWORD + valueFrom: + secretKeyRef: + name: {{ (tpl .Values.admin.existingSecret .) | default (include "grafana.fullname" .) }} + key: {{ .Values.admin.passwordKey | default "admin-password" }} + {{- end }} + {{- if not .Values.sidecar.notifiers.skipReload }} + - name: REQ_URL + value: {{ .Values.sidecar.notifiers.reloadURL }} + - name: REQ_METHOD + value: POST + {{- end }} + {{- if .Values.sidecar.notifiers.watchServerTimeout }} + {{- if ne .Values.sidecar.notifiers.watchMethod "WATCH" }} + {{- fail (printf "Cannot use .Values.sidecar.notifiers.watchServerTimeout with .Values.sidecar.notifiers.watchMethod %s" .Values.sidecar.notifiers.watchMethod) }} + {{- end }} + - name: WATCH_SERVER_TIMEOUT + value: "{{ .Values.sidecar.notifiers.watchServerTimeout }}" + {{- end }} + {{- if .Values.sidecar.notifiers.watchClientTimeout }} + {{- if ne .Values.sidecar.notifiers.watchMethod "WATCH" }} + {{- fail (printf "Cannot use .Values.sidecar.notifiers.watchClientTimeout with .Values.sidecar.notifiers.watchMethod %s" .Values.sidecar.notifiers.watchMethod) }} + {{- end }} + - name: WATCH_CLIENT_TIMEOUT + value: "{{ .Values.sidecar.notifiers.watchClientTimeout }}" + {{- end }} + {{- with .Values.sidecar.livenessProbe }} + livenessProbe: + {{- toYaml . | nindent 6 }} + {{- end }} + {{- with .Values.sidecar.readinessProbe }} + readinessProbe: + {{- toYaml . | nindent 6 }} + {{- end }} + {{- with .Values.sidecar.resources }} + resources: + {{- toYaml . | nindent 6 }} + {{- end }} + {{- with .Values.sidecar.securityContext }} + securityContext: + {{- toYaml . | nindent 6 }} + {{- end }} + volumeMounts: + - name: sc-notifiers-volume + mountPath: "/etc/grafana/provisioning/notifiers" +{{- end}} +{{- if .Values.sidecar.plugins.enabled }} + - name: {{ include "grafana.name" . }}-sc-plugins + {{- $registry := include "system_default_registry" . | default .Values.sidecar.image.registry -}} + {{- if .Values.sidecar.image.sha }} + image: "{{ $registry }}{{ .Values.sidecar.image.repository }}:{{ .Values.sidecar.image.tag }}@sha256:{{ .Values.sidecar.image.sha }}" + {{- else }} + image: "{{ $registry }}{{ .Values.sidecar.image.repository }}:{{ .Values.sidecar.image.tag }}" + {{- end }} + imagePullPolicy: {{ .Values.sidecar.imagePullPolicy }} + env: + {{- range $key, $value := .Values.sidecar.plugins.env }} + - name: "{{ $key }}" + value: "{{ $value }}" + {{- end }} + {{- if .Values.sidecar.plugins.ignoreAlreadyProcessed }} + - name: IGNORE_ALREADY_PROCESSED + value: "true" + {{- end }} + - name: METHOD + value: {{ .Values.sidecar.plugins.watchMethod }} + - name: LABEL + value: "{{ .Values.sidecar.plugins.label }}" + {{- if .Values.sidecar.plugins.labelValue }} + - name: LABEL_VALUE + value: {{ quote .Values.sidecar.plugins.labelValue }} + {{- end }} + {{- if or .Values.sidecar.logLevel .Values.sidecar.plugins.logLevel }} + - name: LOG_LEVEL + value: {{ default .Values.sidecar.logLevel .Values.sidecar.plugins.logLevel }} + {{- end }} + - name: FOLDER + value: "/etc/grafana/provisioning/plugins" + - name: RESOURCE + value: {{ quote .Values.sidecar.plugins.resource }} + {{- with .Values.sidecar.enableUniqueFilenames }} + - name: UNIQUE_FILENAMES + value: "{{ . }}" + {{- end }} + {{- with .Values.sidecar.plugins.searchNamespace }} + - name: NAMESPACE + value: "{{ tpl (. | join ",") $root }}" + {{- end }} + {{- with .Values.sidecar.plugins.script }} + - name: SCRIPT + value: "{{ . }}" + {{- end }} + {{- with .Values.sidecar.skipTlsVerify }} + - name: SKIP_TLS_VERIFY + value: "{{ . }}" + {{- end }} + {{- if and (not .Values.env.GF_SECURITY_ADMIN_USER) (not .Values.env.GF_SECURITY_DISABLE_INITIAL_ADMIN_CREATION) }} + - name: REQ_USERNAME + valueFrom: + secretKeyRef: + name: {{ (tpl .Values.admin.existingSecret .) | default (include "grafana.fullname" .) }} + key: {{ .Values.admin.userKey | default "admin-user" }} + {{- end }} + {{- if and (not .Values.env.GF_SECURITY_ADMIN_PASSWORD) (not .Values.env.GF_SECURITY_ADMIN_PASSWORD__FILE) (not .Values.env.GF_SECURITY_DISABLE_INITIAL_ADMIN_CREATION) }} + - name: REQ_PASSWORD + valueFrom: + secretKeyRef: + name: {{ (tpl .Values.admin.existingSecret .) | default (include "grafana.fullname" .) }} + key: {{ .Values.admin.passwordKey | default "admin-password" }} + {{- end }} + {{- if not .Values.sidecar.plugins.skipReload }} + - name: REQ_URL + value: {{ .Values.sidecar.plugins.reloadURL }} + - name: REQ_METHOD + value: POST + {{- end }} + {{- if .Values.sidecar.plugins.watchServerTimeout }} + {{- if ne .Values.sidecar.plugins.watchMethod "WATCH" }} + {{- fail (printf "Cannot use .Values.sidecar.plugins.watchServerTimeout with .Values.sidecar.plugins.watchMethod %s" .Values.sidecar.plugins.watchMethod) }} + {{- end }} + - name: WATCH_SERVER_TIMEOUT + value: "{{ .Values.sidecar.plugins.watchServerTimeout }}" + {{- end }} + {{- if .Values.sidecar.plugins.watchClientTimeout }} + {{- if ne .Values.sidecar.plugins.watchMethod "WATCH" }} + {{- fail (printf "Cannot use .Values.sidecar.plugins.watchClientTimeout with .Values.sidecar.plugins.watchMethod %s" .Values.sidecar.plugins.watchMethod) }} + {{- end }} + - name: WATCH_CLIENT_TIMEOUT + value: "{{ .Values.sidecar.plugins.watchClientTimeout }}" + {{- end }} + {{- with .Values.sidecar.livenessProbe }} + livenessProbe: + {{- toYaml . | nindent 6 }} + {{- end }} + {{- with .Values.sidecar.readinessProbe }} + readinessProbe: + {{- toYaml . | nindent 6 }} + {{- end }} + {{- with .Values.sidecar.resources }} + resources: + {{- toYaml . | nindent 6 }} + {{- end }} + {{- with .Values.sidecar.securityContext }} + securityContext: + {{- toYaml . | nindent 6 }} + {{- end }} + volumeMounts: + - name: sc-plugins-volume + mountPath: "/etc/grafana/provisioning/plugins" +{{- end}} + - name: {{ .Chart.Name }} + {{- $registry := include "system_default_registry" . | default .Values.image.registry -}} + {{- if .Values.image.sha }} + image: "{{ $registry }}{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}@sha256:{{ .Values.image.sha }}" + {{- else }} + image: "{{ $registry }}{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}" + {{- end }} + imagePullPolicy: {{ .Values.image.pullPolicy }} + {{- if .Values.command }} + command: + {{- range .Values.command }} + - {{ . | quote }} + {{- end }} + {{- end }} + {{- if .Values.args }} + args: + {{- range .Values.args }} + - {{ . | quote }} + {{- end }} + {{- end }} + {{- with .Values.containerSecurityContext }} + securityContext: + {{- toYaml . | nindent 6 }} + {{- end }} + volumeMounts: + - name: config + mountPath: "/etc/grafana/grafana.ini" + subPath: grafana.ini + {{- if .Values.ldap.enabled }} + - name: ldap + mountPath: "/etc/grafana/ldap.toml" + subPath: ldap.toml + {{- end }} + {{- range .Values.extraConfigmapMounts }} + - name: {{ tpl .name $root }} + mountPath: {{ tpl .mountPath $root }} + subPath: {{ tpl (.subPath | default "") $root }} + readOnly: {{ .readOnly }} + {{- end }} + - name: storage + mountPath: "/var/lib/grafana" + {{- with .Values.persistence.subPath }} + subPath: {{ tpl . $root }} + {{- end }} + {{- with .Values.dashboards }} + {{- range $provider, $dashboards := . }} + {{- range $key, $value := $dashboards }} + {{- if (or (hasKey $value "json") (hasKey $value "file")) }} + - name: dashboards-{{ $provider }} + mountPath: "/var/lib/grafana/dashboards/{{ $provider }}/{{ $key }}.json" + subPath: "{{ $key }}.json" + {{- end }} + {{- end }} + {{- end }} + {{- end }} + {{- with .Values.dashboardsConfigMaps }} + {{- range (keys . | sortAlpha) }} + - name: dashboards-{{ . }} + mountPath: "/var/lib/grafana/dashboards/{{ . }}" + {{- end }} + {{- end }} + {{- with .Values.datasources }} + {{- $datasources := . }} + {{- range (keys . | sortAlpha) }} + {{- if (or (hasKey (index $datasources .) "secret")) }} {{/*check if current datasource should be handeled as secret */}} + - name: config-secret + mountPath: "/etc/grafana/provisioning/datasources/{{ . }}" + subPath: {{ . | quote }} + {{- else }} + - name: config + mountPath: "/etc/grafana/provisioning/datasources/{{ . }}" + subPath: {{ . | quote }} + {{- end }} + {{- end }} + {{- end }} + {{- with .Values.notifiers }} + {{- $notifiers := . }} + {{- range (keys . | sortAlpha) }} + {{- if (or (hasKey (index $notifiers .) "secret")) }} {{/*check if current notifier should be handeled as secret */}} + - name: config-secret + mountPath: "/etc/grafana/provisioning/notifiers/{{ . }}" + subPath: {{ . | quote }} + {{- else }} + - name: config + mountPath: "/etc/grafana/provisioning/notifiers/{{ . }}" + subPath: {{ . | quote }} + {{- end }} + {{- end }} + {{- end }} + {{- with .Values.alerting }} + {{- $alertingmap := .}} + {{- range (keys . | sortAlpha) }} + {{- if (or (hasKey (index $.Values.alerting .) "secret") (hasKey (index $.Values.alerting .) "secretFile")) }} {{/*check if current alerting entry should be handeled as secret */}} + - name: config-secret + mountPath: "/etc/grafana/provisioning/alerting/{{ . }}" + subPath: {{ . | quote }} + {{- else }} + - name: config + mountPath: "/etc/grafana/provisioning/alerting/{{ . }}" + subPath: {{ . | quote }} + {{- end }} + {{- end }} + {{- end }} + {{- with .Values.dashboardProviders }} + {{- range (keys . | sortAlpha) }} + - name: config + mountPath: "/etc/grafana/provisioning/dashboards/{{ . }}" + subPath: {{ . | quote }} + {{- end }} + {{- end }} + {{- with .Values.sidecar.alerts.enabled }} + - name: sc-alerts-volume + mountPath: "/etc/grafana/provisioning/alerting" + {{- end}} + {{- if .Values.sidecar.dashboards.enabled }} + - name: sc-dashboard-volume + mountPath: {{ .Values.sidecar.dashboards.folder | quote }} + {{- if .Values.sidecar.dashboards.SCProvider }} + - name: sc-dashboard-provider + mountPath: "/etc/grafana/provisioning/dashboards/sc-dashboardproviders.yaml" + subPath: provider.yaml + {{- end}} + {{- end}} + {{- if .Values.sidecar.datasources.enabled }} + - name: sc-datasources-volume + mountPath: "/etc/grafana/provisioning/datasources" + {{- end}} + {{- if .Values.sidecar.plugins.enabled }} + - name: sc-plugins-volume + mountPath: "/etc/grafana/provisioning/plugins" + {{- end}} + {{- if .Values.sidecar.notifiers.enabled }} + - name: sc-notifiers-volume + mountPath: "/etc/grafana/provisioning/notifiers" + {{- end}} + {{- range .Values.extraSecretMounts }} + - name: {{ .name }} + mountPath: {{ .mountPath }} + readOnly: {{ .readOnly }} + subPath: {{ .subPath | default "" }} + {{- end }} + {{- range .Values.extraVolumeMounts }} + - name: {{ .name }} + mountPath: {{ .mountPath }} + subPath: {{ .subPath | default "" }} + readOnly: {{ .readOnly }} + {{- end }} + {{- range .Values.extraEmptyDirMounts }} + - name: {{ .name }} + mountPath: {{ .mountPath }} + {{- end }} + ports: + - name: {{ .Values.podPortName }} + containerPort: {{ .Values.service.targetPort }} + protocol: TCP + - name: {{ .Values.gossipPortName }}-tcp + containerPort: 9094 + protocol: TCP + - name: {{ .Values.gossipPortName }}-udp + containerPort: 9094 + protocol: UDP + env: + - name: POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + {{- if and (not .Values.env.GF_SECURITY_ADMIN_USER) (not .Values.env.GF_SECURITY_DISABLE_INITIAL_ADMIN_CREATION) }} + - name: GF_SECURITY_ADMIN_USER + valueFrom: + secretKeyRef: + name: {{ (tpl .Values.admin.existingSecret .) | default (include "grafana.fullname" .) }} + key: {{ .Values.admin.userKey | default "admin-user" }} + {{- end }} + {{- if and (not .Values.env.GF_SECURITY_ADMIN_PASSWORD) (not .Values.env.GF_SECURITY_ADMIN_PASSWORD__FILE) (not .Values.env.GF_SECURITY_DISABLE_INITIAL_ADMIN_CREATION) }} + - name: GF_SECURITY_ADMIN_PASSWORD + valueFrom: + secretKeyRef: + name: {{ (tpl .Values.admin.existingSecret .) | default (include "grafana.fullname" .) }} + key: {{ .Values.admin.passwordKey | default "admin-password" }} + {{- end }} + {{- if .Values.plugins }} + - name: GF_INSTALL_PLUGINS + valueFrom: + configMapKeyRef: + name: {{ include "grafana.fullname" . }} + key: plugins + {{- end }} + {{- if .Values.smtp.existingSecret }} + - name: GF_SMTP_USER + valueFrom: + secretKeyRef: + name: {{ .Values.smtp.existingSecret }} + key: {{ .Values.smtp.userKey | default "user" }} + - name: GF_SMTP_PASSWORD + valueFrom: + secretKeyRef: + name: {{ .Values.smtp.existingSecret }} + key: {{ .Values.smtp.passwordKey | default "password" }} + {{- end }} + {{- if .Values.imageRenderer.enabled }} + - name: GF_RENDERING_SERVER_URL + value: http://{{ include "grafana.fullname" . }}-image-renderer.{{ include "grafana.namespace" . }}:{{ .Values.imageRenderer.service.port }}/render + - name: GF_RENDERING_CALLBACK_URL + value: {{ .Values.imageRenderer.grafanaProtocol }}://{{ include "grafana.fullname" . }}.{{ include "grafana.namespace" . }}:{{ .Values.service.port }}/{{ .Values.imageRenderer.grafanaSubPath }} + {{- end }} + - name: GF_PATHS_DATA + value: {{ (get .Values "grafana.ini").paths.data }} + - name: GF_PATHS_LOGS + value: {{ (get .Values "grafana.ini").paths.logs }} + - name: GF_PATHS_PLUGINS + value: {{ (get .Values "grafana.ini").paths.plugins }} + - name: GF_PATHS_PROVISIONING + value: {{ (get .Values "grafana.ini").paths.provisioning }} + {{- range $key, $value := .Values.envValueFrom }} + - name: {{ $key | quote }} + valueFrom: + {{- tpl (toYaml $value) $ | nindent 10 }} + {{- end }} + {{- range $key, $value := .Values.env }} + - name: "{{ tpl $key $ }}" + value: "{{ tpl (print $value) $ }}" + {{- end }} + {{- if or .Values.envFromSecret (or .Values.envRenderSecret .Values.envFromSecrets) .Values.envFromConfigMaps }} + envFrom: + {{- if .Values.envFromSecret }} + - secretRef: + name: {{ tpl .Values.envFromSecret . }} + {{- end }} + {{- if .Values.envRenderSecret }} + - secretRef: + name: {{ include "grafana.fullname" . }}-env + {{- end }} + {{- range .Values.envFromSecrets }} + - secretRef: + name: {{ tpl .name $ }} + optional: {{ .optional | default false }} + {{- if .prefix }} + prefix: {{ tpl .prefix $ }} + {{- end }} + {{- end }} + {{- range .Values.envFromConfigMaps }} + - configMapRef: + name: {{ tpl .name $ }} + optional: {{ .optional | default false }} + {{- if .prefix }} + prefix: {{ tpl .prefix $ }} + {{- end }} + {{- end }} + {{- end }} + {{- with .Values.livenessProbe }} + livenessProbe: + {{- toYaml . | nindent 6 }} + {{- end }} + {{- with .Values.readinessProbe }} + readinessProbe: + {{- toYaml . | nindent 6 }} + {{- end }} + {{- with .Values.lifecycleHooks }} + lifecycle: + {{- tpl (toYaml .) $root | nindent 6 }} + {{- end }} + {{- with .Values.resources }} + resources: + {{- toYaml . | nindent 6 }} + {{- end }} +{{- with .Values.extraContainers }} + {{- tpl . $ | nindent 2 }} +{{- end }} +nodeSelector: {{ include "linux-node-selector" . | nindent 2 }} +{{- with .Values.nodeSelector }} + {{- toYaml . | nindent 2 }} +{{- end }} +{{- with .Values.affinity }} +affinity: + {{- tpl (toYaml .) $root | nindent 2 }} +{{- end }} +{{- with .Values.topologySpreadConstraints }} +topologySpreadConstraints: + {{- toYaml . | nindent 2 }} +{{- end }} +tolerations: {{ include "linux-node-tolerations" . | nindent 2 }} +{{- with .Values.tolerations }} + {{- toYaml . | nindent 2 }} +{{- end }} +volumes: + - name: config + configMap: + name: {{ include "grafana.fullname" . }} + {{- $createConfigSecret := eq (include "grafana.shouldCreateConfigSecret" .) "true" -}} + {{- if and .Values.createConfigmap $createConfigSecret }} + - name: config-secret + secret: + secretName: {{ include "grafana.fullname" . }}-config-secret + {{- end }} + {{- range .Values.extraConfigmapMounts }} + - name: {{ tpl .name $root }} + configMap: + name: {{ tpl .configMap $root }} + {{- with .items }} + items: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + {{- if .Values.dashboards }} + {{- range (keys .Values.dashboards | sortAlpha) }} + - name: dashboards-{{ . }} + configMap: + name: {{ include "grafana.fullname" $ }}-dashboards-{{ . }} + {{- end }} + {{- end }} + {{- if .Values.dashboardsConfigMaps }} + {{- range $provider, $name := .Values.dashboardsConfigMaps }} + - name: dashboards-{{ $provider }} + configMap: + name: {{ tpl $name $root }} + {{- end }} + {{- end }} + {{- if .Values.ldap.enabled }} + - name: ldap + secret: + {{- if .Values.ldap.existingSecret }} + secretName: {{ .Values.ldap.existingSecret }} + {{- else }} + secretName: {{ include "grafana.fullname" . }} + {{- end }} + items: + - key: ldap-toml + path: ldap.toml + {{- end }} + {{- if and .Values.persistence.enabled (eq .Values.persistence.type "pvc") }} + - name: storage + persistentVolumeClaim: + claimName: {{ tpl (.Values.persistence.existingClaim | default (include "grafana.fullname" .)) . }} + {{- else if and .Values.persistence.enabled (has .Values.persistence.type $sts) }} + {{/* nothing */}} + {{- else }} + - name: storage + {{- if .Values.persistence.inMemory.enabled }} + emptyDir: + medium: Memory + {{- with .Values.persistence.inMemory.sizeLimit }} + sizeLimit: {{ . }} + {{- end }} + {{- else }} + emptyDir: {} + {{- end }} + {{- end }} + {{- if .Values.sidecar.alerts.enabled }} + - name: sc-alerts-volume + emptyDir: + {{- with .Values.sidecar.alerts.sizeLimit }} + sizeLimit: {{ . }} + {{- else }} + {} + {{- end }} + {{- end }} + {{- if .Values.sidecar.dashboards.enabled }} + - name: sc-dashboard-volume + emptyDir: + {{- with .Values.sidecar.dashboards.sizeLimit }} + sizeLimit: {{ . }} + {{- else }} + {} + {{- end }} + {{- if .Values.sidecar.dashboards.SCProvider }} + - name: sc-dashboard-provider + configMap: + name: {{ include "grafana.fullname" . }}-config-dashboards + {{- end }} + {{- end }} + {{- if .Values.sidecar.datasources.enabled }} + - name: sc-datasources-volume + emptyDir: + {{- with .Values.sidecar.datasources.sizeLimit }} + sizeLimit: {{ . }} + {{- else }} + {} + {{- end }} + {{- end }} + {{- if .Values.sidecar.plugins.enabled }} + - name: sc-plugins-volume + emptyDir: + {{- with .Values.sidecar.plugins.sizeLimit }} + sizeLimit: {{ . }} + {{- else }} + {} + {{- end }} + {{- end }} + {{- if .Values.sidecar.notifiers.enabled }} + - name: sc-notifiers-volume + emptyDir: + {{- with .Values.sidecar.notifiers.sizeLimit }} + sizeLimit: {{ . }} + {{- else }} + {} + {{- end }} + {{- end }} + {{- range .Values.extraSecretMounts }} + {{- if .secretName }} + - name: {{ .name }} + secret: + secretName: {{ .secretName }} + defaultMode: {{ .defaultMode }} + {{- with .items }} + items: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- else if .projected }} + - name: {{ .name }} + projected: + {{- toYaml .projected | nindent 6 }} + {{- else if .csi }} + - name: {{ .name }} + csi: + {{- toYaml .csi | nindent 6 }} + {{- end }} + {{- end }} + {{- range .Values.extraVolumes }} + - name: {{ .name }} + {{- if .existingClaim }} + persistentVolumeClaim: + claimName: {{ .existingClaim }} + {{- else if .hostPath }} + hostPath: + {{ toYaml .hostPath | nindent 6 }} + {{- else if .csi }} + csi: + {{- toYaml .csi | nindent 6 }} + {{- else if .configMap }} + configMap: + {{- toYaml .configMap | nindent 6 }} + {{- else if .emptyDir }} + emptyDir: + {{- toYaml .emptyDir | nindent 6 }} + {{- else }} + emptyDir: {} + {{- end }} + {{- end }} + {{- range .Values.extraEmptyDirMounts }} + - name: {{ .name }} + emptyDir: {} + {{- end }} + {{- with .Values.extraContainerVolumes }} + {{- tpl (toYaml .) $root | nindent 2 }} + {{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/clusterrole.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/clusterrole.yaml new file mode 100644 index 00000000000..3af4b62b630 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/clusterrole.yaml @@ -0,0 +1,25 @@ +{{- if and .Values.rbac.create (or (not .Values.rbac.namespaced) .Values.rbac.extraClusterRoleRules) (not .Values.rbac.useExistingClusterRole) }} +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + labels: + {{- include "grafana.labels" . | nindent 4 }} + {{- with .Values.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} + name: {{ include "grafana.fullname" . }}-clusterrole +{{- if or .Values.sidecar.dashboards.enabled .Values.rbac.extraClusterRoleRules .Values.sidecar.datasources.enabled .Values.sidecar.plugins.enabled .Values.sidecar.alerts.enabled }} +rules: + {{- if or .Values.sidecar.dashboards.enabled .Values.sidecar.datasources.enabled .Values.sidecar.plugins.enabled .Values.sidecar.alerts.enabled }} + - apiGroups: [""] # "" indicates the core API group + resources: ["configmaps", "secrets"] + verbs: ["get", "watch", "list"] + {{- end}} + {{- with .Values.rbac.extraClusterRoleRules }} + {{- toYaml . | nindent 2 }} + {{- end}} +{{- else }} +rules: [] +{{- end}} +{{- end}} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/clusterrolebinding.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/clusterrolebinding.yaml new file mode 100644 index 00000000000..bda9431a2c1 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/clusterrolebinding.yaml @@ -0,0 +1,24 @@ +{{- if and .Values.rbac.create (or (not .Values.rbac.namespaced) .Values.rbac.extraClusterRoleRules) }} +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: {{ include "grafana.fullname" . }}-clusterrolebinding + labels: + {{- include "grafana.labels" . | nindent 4 }} + {{- with .Values.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +subjects: + - kind: ServiceAccount + name: {{ include "grafana.serviceAccountName" . }} + namespace: {{ include "grafana.namespace" . }} +roleRef: + kind: ClusterRole + {{- if .Values.rbac.useExistingClusterRole }} + name: {{ .Values.rbac.useExistingClusterRole }} + {{- else }} + name: {{ include "grafana.fullname" . }}-clusterrole + {{- end }} + apiGroup: rbac.authorization.k8s.io +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/configSecret.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/configSecret.yaml new file mode 100644 index 00000000000..55574b9bbca --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/configSecret.yaml @@ -0,0 +1,43 @@ +{{- $createConfigSecret := eq (include "grafana.shouldCreateConfigSecret" .) "true" -}} +{{- if and .Values.createConfigmap $createConfigSecret }} +{{- $files := .Files }} +{{- $root := . -}} +apiVersion: v1 +kind: Secret +metadata: + name: "{{ include "grafana.fullname" . }}-config-secret" + namespace: {{ include "grafana.namespace" . }} + labels: + {{- include "grafana.labels" . | nindent 4 }} + {{- with .Values.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +data: +{{- range $key, $value := .Values.alerting }} + {{- if (hasKey $value "secretFile") }} + {{- $key | nindent 2 }}: + {{- toYaml ( $files.Get $value.secretFile ) | b64enc | nindent 4}} + {{/* as of https://helm.sh/docs/chart_template_guide/accessing_files/ this will only work if you fork this chart and add files to it*/}} + {{- end }} +{{- end }} +stringData: +{{- range $key, $value := .Values.datasources }} +{{- if (hasKey $value "secret") }} +{{- $key | nindent 2 }}: | + {{- tpl (toYaml $value.secret | nindent 4) $root }} +{{- end }} +{{- end }} +{{- range $key, $value := .Values.notifiers }} +{{- if (hasKey $value "secret") }} +{{- $key | nindent 2 }}: | + {{- tpl (toYaml $value.secret | nindent 4) $root }} +{{- end }} +{{- end }} +{{- range $key, $value := .Values.alerting }} +{{ if (hasKey $value "secret") }} + {{- $key | nindent 2 }}: | + {{- tpl (toYaml $value.secret | nindent 4) $root }} + {{- end }} +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/configmap-dashboard-provider.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/configmap-dashboard-provider.yaml new file mode 100644 index 00000000000..b412c4d1f0d --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/configmap-dashboard-provider.yaml @@ -0,0 +1,15 @@ +{{- if and .Values.sidecar.dashboards.enabled .Values.sidecar.dashboards.SCProvider }} +apiVersion: v1 +kind: ConfigMap +metadata: + labels: + {{- include "grafana.labels" . | nindent 4 }} + {{- with .Values.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} + name: {{ include "grafana.fullname" . }}-config-dashboards + namespace: {{ include "grafana.namespace" . }} +data: + {{- include "grafana.configDashboardProviderData" . | nindent 2 }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/configmap.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/configmap.yaml new file mode 100644 index 00000000000..7d7428be513 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/configmap.yaml @@ -0,0 +1,15 @@ +{{- if .Values.createConfigmap }} +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ include "grafana.fullname" . }} + namespace: {{ include "grafana.namespace" . }} + labels: + {{- include "grafana.labels" . | nindent 4 }} + {{- with .Values.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +data: + {{- include "grafana.configData" . | nindent 2 }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/dashboards-json-configmap.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/dashboards-json-configmap.yaml new file mode 100644 index 00000000000..b96ce720265 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/dashboards-json-configmap.yaml @@ -0,0 +1,38 @@ +{{- if .Values.dashboards }} +{{ $files := .Files }} +{{- range $provider, $dashboards := .Values.dashboards }} +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ include "grafana.fullname" $ }}-dashboards-{{ $provider }} + namespace: {{ include "grafana.namespace" $ }} + labels: + {{- include "grafana.labels" $ | nindent 4 }} + dashboard-provider: {{ $provider }} + {{- if $.Values.sidecar.dashboards.enabled }} + {{ $.Values.sidecar.dashboards.label }}: {{ $.Values.sidecar.dashboards.labelValue | quote }} + {{- end }} +{{- if $dashboards }} +data: +{{- $dashboardFound := false }} +{{- range $key, $value := $dashboards }} +{{- if (or (hasKey $value "json") (hasKey $value "file")) }} +{{- $dashboardFound = true }} + {{- print $key | nindent 2 }}.json: + {{- if hasKey $value "json" }} + |- + {{- $value.json | nindent 6 }} + {{- end }} + {{- if hasKey $value "file" }} + {{- toYaml ( $files.Get $value.file ) | nindent 4}} + {{- end }} +{{- end }} +{{- end }} +{{- if not $dashboardFound }} + {} +{{- end }} +{{- end }} +--- +{{- end }} + +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/deployment.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/deployment.yaml new file mode 100644 index 00000000000..46c016faa34 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/deployment.yaml @@ -0,0 +1,53 @@ +{{- if (and (not .Values.useStatefulSet) (or (not .Values.persistence.enabled) (eq .Values.persistence.type "pvc"))) }} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "grafana.fullname" . }} + namespace: {{ include "grafana.namespace" . }} + labels: + {{- include "grafana.labels" . | nindent 4 }} + {{- with .Values.labels }} + {{- toYaml . | nindent 4 }} + {{- end }} + {{- with .Values.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + {{- if and (not .Values.autoscaling.enabled) (.Values.replicas) }} + replicas: {{ .Values.replicas }} + {{- end }} + revisionHistoryLimit: {{ .Values.revisionHistoryLimit }} + selector: + matchLabels: + {{- include "grafana.selectorLabels" . | nindent 6 }} + {{- with .Values.deploymentStrategy }} + strategy: + {{- toYaml . | trim | nindent 4 }} + {{- end }} + template: + metadata: + labels: + {{- include "grafana.selectorLabels" . | nindent 8 }} + {{- with .Values.podLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + annotations: + checksum/config: {{ include "grafana.configData" . | sha256sum }} + {{- if .Values.dashboards }} + checksum/dashboards-json-config: {{ include (print $.Template.BasePath "/dashboards-json-configmap.yaml") . | sha256sum }} + {{- end }} + checksum/sc-dashboard-provider-config: {{ include "grafana.configDashboardProviderData" . | sha256sum }} + {{- if and (or (and (not .Values.admin.existingSecret) (not .Values.env.GF_SECURITY_ADMIN_PASSWORD__FILE) (not .Values.env.GF_SECURITY_ADMIN_PASSWORD)) (and .Values.ldap.enabled (not .Values.ldap.existingSecret))) (not .Values.env.GF_SECURITY_DISABLE_INITIAL_ADMIN_CREATION) }} + checksum/secret: {{ include "grafana.secretsData" . | sha256sum }} + {{- end }} + {{- if .Values.envRenderSecret }} + checksum/secret-env: {{ tpl (toYaml .Values.envRenderSecret) . | sha256sum }} + {{- end }} + kubectl.kubernetes.io/default-container: {{ .Chart.Name }} + {{- with .Values.podAnnotations }} + {{- toYaml . | nindent 8 }} + {{- end }} + spec: + {{- include "grafana.pod" . | nindent 6 }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/extra-manifests.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/extra-manifests.yaml new file mode 100644 index 00000000000..a9bb3b6ba8e --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/extra-manifests.yaml @@ -0,0 +1,4 @@ +{{ range .Values.extraObjects }} +--- +{{ tpl (toYaml .) $ }} +{{ end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/headless-service.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/headless-service.yaml new file mode 100644 index 00000000000..3028589d326 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/headless-service.yaml @@ -0,0 +1,22 @@ +{{- $sts := list "sts" "StatefulSet" "statefulset" -}} +{{- if or .Values.headlessService (and .Values.persistence.enabled (not .Values.persistence.existingClaim) (has .Values.persistence.type $sts)) }} +apiVersion: v1 +kind: Service +metadata: + name: {{ include "grafana.fullname" . }}-headless + namespace: {{ include "grafana.namespace" . }} + labels: + {{- include "grafana.labels" . | nindent 4 }} + {{- with .Values.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + clusterIP: None + selector: + {{- include "grafana.selectorLabels" . | nindent 4 }} + type: ClusterIP + ports: + - name: {{ .Values.gossipPortName }}-tcp + port: 9094 +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/hpa.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/hpa.yaml new file mode 100644 index 00000000000..46bbcb49a21 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/hpa.yaml @@ -0,0 +1,52 @@ +{{- $sts := list "sts" "StatefulSet" "statefulset" -}} +{{- if .Values.autoscaling.enabled }} +apiVersion: {{ include "grafana.hpa.apiVersion" . }} +kind: HorizontalPodAutoscaler +metadata: + name: {{ include "grafana.fullname" . }} + namespace: {{ include "grafana.namespace" . }} + labels: + app.kubernetes.io/name: {{ include "grafana.name" . }} + helm.sh/chart: {{ include "grafana.chart" . }} + app.kubernetes.io/managed-by: {{ .Release.Service }} + app.kubernetes.io/instance: {{ .Release.Name }} +spec: + scaleTargetRef: + apiVersion: apps/v1 + {{- if has .Values.persistence.type $sts }} + kind: StatefulSet + {{- else }} + kind: Deployment + {{- end }} + name: {{ include "grafana.fullname" . }} + minReplicas: {{ .Values.autoscaling.minReplicas }} + maxReplicas: {{ .Values.autoscaling.maxReplicas }} + metrics: + {{- if .Values.autoscaling.targetMemory }} + - type: Resource + resource: + name: memory + {{- if eq (include "grafana.hpa.apiVersion" .) "autoscaling/v2beta1" }} + targetAverageUtilization: {{ .Values.autoscaling.targetMemory }} + {{- else }} + target: + type: Utilization + averageUtilization: {{ .Values.autoscaling.targetMemory }} + {{- end }} + {{- end }} + {{- if .Values.autoscaling.targetCPU }} + - type: Resource + resource: + name: cpu + {{- if eq (include "grafana.hpa.apiVersion" .) "autoscaling/v2beta1" }} + targetAverageUtilization: {{ .Values.autoscaling.targetCPU }} + {{- else }} + target: + type: Utilization + averageUtilization: {{ .Values.autoscaling.targetCPU }} + {{- end }} + {{- end }} + {{- if .Values.autoscaling.behavior }} + behavior: {{ toYaml .Values.autoscaling.behavior | nindent 4 }} + {{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/image-renderer-deployment.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/image-renderer-deployment.yaml new file mode 100644 index 00000000000..28231b803ef --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/image-renderer-deployment.yaml @@ -0,0 +1,131 @@ +{{ if .Values.imageRenderer.enabled }} +{{- $root := . -}} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "grafana.fullname" . }}-image-renderer + namespace: {{ include "grafana.namespace" . }} + labels: + {{- include "grafana.imageRenderer.labels" . | nindent 4 }} + {{- with .Values.imageRenderer.labels }} + {{- toYaml . | nindent 4 }} + {{- end }} + {{- with .Values.imageRenderer.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + {{- if and (not .Values.imageRenderer.autoscaling.enabled) (.Values.imageRenderer.replicas) }} + replicas: {{ .Values.imageRenderer.replicas }} + {{- end }} + revisionHistoryLimit: {{ .Values.imageRenderer.revisionHistoryLimit }} + selector: + matchLabels: + {{- include "grafana.imageRenderer.selectorLabels" . | nindent 6 }} + + {{- with .Values.imageRenderer.deploymentStrategy }} + strategy: + {{- toYaml . | trim | nindent 4 }} + {{- end }} + template: + metadata: + labels: + {{- include "grafana.imageRenderer.selectorLabels" . | nindent 8 }} + {{- with .Values.imageRenderer.podLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + annotations: + checksum/config: {{ include (print $.Template.BasePath "/configmap.yaml") . | sha256sum }} + {{- with .Values.imageRenderer.podAnnotations }} + {{- toYaml . | nindent 8 }} + {{- end }} + spec: + {{- with .Values.imageRenderer.schedulerName }} + schedulerName: "{{ . }}" + {{- end }} + {{- with .Values.imageRenderer.serviceAccountName }} + serviceAccountName: "{{ . }}" + {{- end }} + {{- with .Values.imageRenderer.securityContext }} + securityContext: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.imageRenderer.hostAliases }} + hostAliases: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.imageRenderer.priorityClassName }} + priorityClassName: {{ . }} + {{- end }} + {{- with .Values.imageRenderer.image.pullSecrets }} + imagePullSecrets: + {{- range . }} + - name: {{ tpl . $root }} + {{- end}} + {{- end }} + containers: + - name: {{ .Chart.Name }}-image-renderer + {{- $registry := include "system_default_registry" | default .Values.imageRenderer.image.registry -}} + {{- if .Values.imageRenderer.image.sha }} + image: "{{ $registry }}{{ .Values.imageRenderer.image.repository }}:{{ .Values.imageRenderer.image.tag }}@sha256:{{ .Values.imageRenderer.image.sha }}" + {{- else }} + image: "{{ $registry }}{{ .Values.imageRenderer.image.repository }}:{{ .Values.imageRenderer.image.tag }}" + {{- end }} + imagePullPolicy: {{ .Values.imageRenderer.image.pullPolicy }} + {{- if .Values.imageRenderer.command }} + command: + {{- range .Values.imageRenderer.command }} + - {{ . }} + {{- end }} + {{- end}} + ports: + - name: {{ .Values.imageRenderer.service.portName }} + containerPort: {{ .Values.imageRenderer.service.targetPort }} + protocol: TCP + livenessProbe: + httpGet: + path: / + port: {{ .Values.imageRenderer.service.portName }} + env: + - name: HTTP_PORT + value: {{ .Values.imageRenderer.service.targetPort | quote }} + {{- if .Values.imageRenderer.serviceMonitor.enabled }} + - name: ENABLE_METRICS + value: "true" + {{- end }} + {{- range $key, $value := .Values.imageRenderer.envValueFrom }} + - name: {{ $key | quote }} + valueFrom: + {{- tpl (toYaml $value) $ | nindent 16 }} + {{- end }} + {{- range $key, $value := .Values.imageRenderer.env }} + - name: {{ $key | quote }} + value: {{ $value | quote }} + {{- end }} + {{- with .Values.imageRenderer.containerSecurityContext }} + securityContext: + {{- toYaml . | nindent 12 }} + {{- end }} + volumeMounts: + - mountPath: /tmp + name: image-renderer-tmpfs + {{- with .Values.imageRenderer.resources }} + resources: + {{- toYaml . | nindent 12 }} + {{- end }} + {{- with .Values.imageRenderer.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.imageRenderer.affinity }} + affinity: + {{- tpl (toYaml .) $root | nindent 8 }} + {{- end }} + {{- with .Values.imageRenderer.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} + volumes: + - name: image-renderer-tmpfs + emptyDir: {} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/image-renderer-hpa.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/image-renderer-hpa.yaml new file mode 100644 index 00000000000..b0f0059b790 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/image-renderer-hpa.yaml @@ -0,0 +1,47 @@ +{{- if and .Values.imageRenderer.enabled .Values.imageRenderer.autoscaling.enabled }} +apiVersion: {{ include "grafana.hpa.apiVersion" . }} +kind: HorizontalPodAutoscaler +metadata: + name: {{ include "grafana.fullname" . }}-image-renderer + namespace: {{ include "grafana.namespace" . }} + labels: + app.kubernetes.io/name: {{ include "grafana.name" . }}-image-renderer + helm.sh/chart: {{ include "grafana.chart" . }} + app.kubernetes.io/managed-by: {{ .Release.Service }} + app.kubernetes.io/instance: {{ .Release.Name }} +spec: + scaleTargetRef: + apiVersion: apps/v1 + kind: Deployment + name: {{ include "grafana.fullname" . }}-image-renderer + minReplicas: {{ .Values.imageRenderer.autoscaling.minReplicas }} + maxReplicas: {{ .Values.imageRenderer.autoscaling.maxReplicas }} + metrics: + {{- if .Values.imageRenderer.autoscaling.targetMemory }} + - type: Resource + resource: + name: memory + {{- if eq (include "grafana.hpa.apiVersion" .) "autoscaling/v2beta1" }} + targetAverageUtilization: {{ .Values.imageRenderer.autoscaling.targetMemory }} + {{- else }} + target: + type: Utilization + averageUtilization: {{ .Values.imageRenderer.autoscaling.targetMemory }} + {{- end }} + {{- end }} + {{- if .Values.imageRenderer.autoscaling.targetCPU }} + - type: Resource + resource: + name: cpu + {{- if eq (include "grafana.hpa.apiVersion" .) "autoscaling/v2beta1" }} + targetAverageUtilization: {{ .Values.imageRenderer.autoscaling.targetCPU }} + {{- else }} + target: + type: Utilization + averageUtilization: {{ .Values.imageRenderer.autoscaling.targetCPU }} + {{- end }} + {{- end }} + {{- if .Values.imageRenderer.autoscaling.behavior }} + behavior: {{ toYaml .Values.imageRenderer.autoscaling.behavior | nindent 4 }} + {{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/image-renderer-network-policy.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/image-renderer-network-policy.yaml new file mode 100644 index 00000000000..d1a0eb313db --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/image-renderer-network-policy.yaml @@ -0,0 +1,79 @@ +{{- if and .Values.imageRenderer.enabled .Values.imageRenderer.networkPolicy.limitIngress }} +--- +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: {{ include "grafana.fullname" . }}-image-renderer-ingress + namespace: {{ include "grafana.namespace" . }} + annotations: + comment: Limit image-renderer ingress traffic from grafana +spec: + podSelector: + matchLabels: + {{- include "grafana.imageRenderer.selectorLabels" . | nindent 6 }} + {{- with .Values.imageRenderer.podLabels }} + {{- toYaml . | nindent 6 }} + {{- end }} + + policyTypes: + - Ingress + ingress: + - ports: + - port: {{ .Values.imageRenderer.service.targetPort }} + protocol: TCP + from: + - namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: {{ include "grafana.namespace" . }} + podSelector: + matchLabels: + {{- include "grafana.selectorLabels" . | nindent 14 }} + {{- with .Values.podLabels }} + {{- toYaml . | nindent 14 }} + {{- end }} + {{- with .Values.imageRenderer.networkPolicy.extraIngressSelectors -}} + {{ toYaml . | nindent 8 }} + {{- end }} +{{- end }} + +{{- if and .Values.imageRenderer.enabled .Values.imageRenderer.networkPolicy.limitEgress }} +--- +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: {{ include "grafana.fullname" . }}-image-renderer-egress + namespace: {{ include "grafana.namespace" . }} + annotations: + comment: Limit image-renderer egress traffic to grafana +spec: + podSelector: + matchLabels: + {{- include "grafana.imageRenderer.selectorLabels" . | nindent 6 }} + {{- with .Values.imageRenderer.podLabels }} + {{- toYaml . | nindent 6 }} + {{- end }} + + policyTypes: + - Egress + egress: + # allow dns resolution + - ports: + - port: 53 + protocol: UDP + - port: 53 + protocol: TCP + # talk only to grafana + - ports: + - port: {{ .Values.service.targetPort }} + protocol: TCP + to: + - namespaceSelector: + matchLabels: + name: {{ include "grafana.namespace" . }} + podSelector: + matchLabels: + {{- include "grafana.selectorLabels" . | nindent 14 }} + {{- with .Values.podLabels }} + {{- toYaml . | nindent 14 }} + {{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/image-renderer-service.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/image-renderer-service.yaml new file mode 100644 index 00000000000..f8da127cf86 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/image-renderer-service.yaml @@ -0,0 +1,31 @@ +{{- if and .Values.imageRenderer.enabled .Values.imageRenderer.service.enabled }} +apiVersion: v1 +kind: Service +metadata: + name: {{ include "grafana.fullname" . }}-image-renderer + namespace: {{ include "grafana.namespace" . }} + labels: + {{- include "grafana.imageRenderer.labels" . | nindent 4 }} + {{- with .Values.imageRenderer.service.labels }} + {{- toYaml . | nindent 4 }} + {{- end }} + {{- with .Values.imageRenderer.service.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + type: ClusterIP + {{- with .Values.imageRenderer.service.clusterIP }} + clusterIP: {{ . }} + {{- end }} + ports: + - name: {{ .Values.imageRenderer.service.portName }} + port: {{ .Values.imageRenderer.service.port }} + protocol: TCP + targetPort: {{ .Values.imageRenderer.service.targetPort }} + {{- with .Values.imageRenderer.appProtocol }} + appProtocol: {{ . }} + {{- end }} + selector: + {{- include "grafana.imageRenderer.selectorLabels" . | nindent 4 }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/image-renderer-servicemonitor.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/image-renderer-servicemonitor.yaml new file mode 100644 index 00000000000..5d9f09d266b --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/image-renderer-servicemonitor.yaml @@ -0,0 +1,48 @@ +{{- if .Values.imageRenderer.serviceMonitor.enabled }} +--- +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ include "grafana.fullname" . }}-image-renderer + {{- if .Values.imageRenderer.serviceMonitor.namespace }} + namespace: {{ tpl .Values.imageRenderer.serviceMonitor.namespace . }} + {{- else }} + namespace: {{ include "grafana.namespace" . }} + {{- end }} + labels: + {{- include "grafana.imageRenderer.labels" . | nindent 4 }} + {{- with .Values.imageRenderer.serviceMonitor.labels }} + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + endpoints: + - port: {{ .Values.imageRenderer.service.portName }} + {{- with .Values.imageRenderer.serviceMonitor.interval }} + interval: {{ . }} + {{- end }} + {{- with .Values.imageRenderer.serviceMonitor.scrapeTimeout }} + scrapeTimeout: {{ . }} + {{- end }} + honorLabels: true + path: {{ .Values.imageRenderer.serviceMonitor.path }} + scheme: {{ .Values.imageRenderer.serviceMonitor.scheme }} + {{- with .Values.imageRenderer.serviceMonitor.tlsConfig }} + tlsConfig: + {{- toYaml . | nindent 6 }} + {{- end }} + {{- with .Values.imageRenderer.serviceMonitor.relabelings }} + relabelings: + {{- toYaml . | nindent 6 }} + {{- end }} + jobLabel: "{{ .Release.Name }}-image-renderer" + selector: + matchLabels: + {{- include "grafana.imageRenderer.selectorLabels" . | nindent 6 }} + namespaceSelector: + matchNames: + - {{ include "grafana.namespace" . }} + {{- with .Values.imageRenderer.serviceMonitor.targetLabels }} + targetLabels: + {{- toYaml . | nindent 4 }} + {{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/ingress.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/ingress.yaml new file mode 100644 index 00000000000..b2ffd810959 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/ingress.yaml @@ -0,0 +1,78 @@ +{{- if .Values.ingress.enabled -}} +{{- $ingressApiIsStable := eq (include "grafana.ingress.isStable" .) "true" -}} +{{- $ingressSupportsIngressClassName := eq (include "grafana.ingress.supportsIngressClassName" .) "true" -}} +{{- $ingressSupportsPathType := eq (include "grafana.ingress.supportsPathType" .) "true" -}} +{{- $fullName := include "grafana.fullname" . -}} +{{- $servicePort := .Values.service.port -}} +{{- $ingressPath := .Values.ingress.path -}} +{{- $ingressPathType := .Values.ingress.pathType -}} +{{- $extraPaths := .Values.ingress.extraPaths -}} +apiVersion: {{ include "grafana.ingress.apiVersion" . }} +kind: Ingress +metadata: + name: {{ $fullName }} + namespace: {{ include "grafana.namespace" . }} + labels: + {{- include "grafana.labels" . | nindent 4 }} + {{- with .Values.ingress.labels }} + {{- toYaml . | nindent 4 }} + {{- end }} + {{- with .Values.ingress.annotations }} + annotations: + {{- range $key, $value := . }} + {{ $key }}: {{ tpl $value $ | quote }} + {{- end }} + {{- end }} +spec: + {{- if and $ingressSupportsIngressClassName .Values.ingress.ingressClassName }} + ingressClassName: {{ .Values.ingress.ingressClassName }} + {{- end -}} + {{- with .Values.ingress.tls }} + tls: + {{- tpl (toYaml .) $ | nindent 4 }} + {{- end }} + rules: + {{- if .Values.ingress.hosts }} + {{- range .Values.ingress.hosts }} + - host: {{ tpl . $ | quote }} + http: + paths: + {{- with $extraPaths }} + {{- toYaml . | nindent 10 }} + {{- end }} + - path: {{ $ingressPath }} + {{- if $ingressSupportsPathType }} + pathType: {{ $ingressPathType }} + {{- end }} + backend: + {{- if $ingressApiIsStable }} + service: + name: {{ $fullName }} + port: + number: {{ $servicePort }} + {{- else }} + serviceName: {{ $fullName }} + servicePort: {{ $servicePort }} + {{- end }} + {{- end }} + {{- else }} + - http: + paths: + - backend: + {{- if $ingressApiIsStable }} + service: + name: {{ $fullName }} + port: + number: {{ $servicePort }} + {{- else }} + serviceName: {{ $fullName }} + servicePort: {{ $servicePort }} + {{- end }} + {{- with $ingressPath }} + path: {{ . }} + {{- end }} + {{- if $ingressSupportsPathType }} + pathType: {{ $ingressPathType }} + {{- end }} + {{- end -}} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/networkpolicy.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/networkpolicy.yaml new file mode 100644 index 00000000000..4cd3ed6976d --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/networkpolicy.yaml @@ -0,0 +1,61 @@ +{{- if .Values.networkPolicy.enabled }} +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: {{ include "grafana.fullname" . }} + namespace: {{ include "grafana.namespace" . }} + labels: + {{- include "grafana.labels" . | nindent 4 }} + {{- with .Values.labels }} + {{- toYaml . | nindent 4 }} + {{- end }} + {{- with .Values.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + policyTypes: + {{- if .Values.networkPolicy.ingress }} + - Ingress + {{- end }} + {{- if .Values.networkPolicy.egress.enabled }} + - Egress + {{- end }} + podSelector: + matchLabels: + {{- include "grafana.selectorLabels" . | nindent 6 }} + + {{- if .Values.networkPolicy.egress.enabled }} + egress: + {{- if not .Values.networkPolicy.egress.blockDNSResolution }} + - ports: + - port: 53 + protocol: UDP + {{- end }} + - ports: + {{ .Values.networkPolicy.egress.ports | toJson }} + {{- with .Values.networkPolicy.egress.to }} + to: + {{- toYaml . | nindent 12 }} + {{- end }} + {{- end }} + {{- if .Values.networkPolicy.ingress }} + ingress: + - ports: + - port: {{ .Values.service.targetPort }} + {{- if not .Values.networkPolicy.allowExternal }} + from: + - podSelector: + matchLabels: + {{ include "grafana.fullname" . }}-client: "true" + {{- with .Values.networkPolicy.explicitNamespacesSelector }} + - namespaceSelector: + {{- toYaml . | nindent 12 }} + {{- end }} + - podSelector: + matchLabels: + {{- include "grafana.labels" . | nindent 14 }} + role: read + {{- end }} + {{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/nginx-config.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/nginx-config.yaml new file mode 100644 index 00000000000..557471f6ff2 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/nginx-config.yaml @@ -0,0 +1,94 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: grafana-nginx-proxy-config + namespace: {{ template "grafana.namespace" . }} + labels: + {{- include "grafana.labels" . | nindent 4 }} +data: + nginx.conf: |- + worker_processes auto; + error_log /dev/stdout warn; + pid /var/cache/nginx/nginx.pid; + + events { + worker_connections 1024; + } + + http { + include /etc/nginx/mime.types; + log_format main '[$time_local - $status] $remote_addr - $remote_user $request ($http_referer)'; + + proxy_connect_timeout 10; + proxy_read_timeout 180; + proxy_send_timeout 5; + proxy_buffering off; + proxy_cache_path /var/cache/nginx/cache levels=1:2 keys_zone=my_zone:100m inactive=1d max_size=10g; + + map $http_upgrade $connection_upgrade { + default upgrade; + '' close; + } + + server { + listen 8080; + access_log off; + + gzip on; + gzip_min_length 1k; + gzip_comp_level 2; + gzip_types text/plain application/javascript application/x-javascript text/css application/xml text/javascript image/jpeg image/gif image/png; + gzip_vary on; + gzip_disable "MSIE [1-6]\."; + + proxy_set_header Host $host; + + location /api/dashboards { + proxy_pass http://localhost:3000; + } + + location /api/search { + proxy_pass http://localhost:3000; + + sub_filter_types application/json; + sub_filter_once off; + } + + location /api/live/ { + proxy_http_version 1.1; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection $connection_upgrade; + proxy_set_header Host $http_host; + proxy_pass http://localhost:3000; + } + + location / { + proxy_cache my_zone; + proxy_cache_valid 200 302 1d; + proxy_cache_valid 301 30d; + proxy_cache_valid any 5m; + proxy_cache_bypass $http_cache_control; + add_header X-Proxy-Cache $upstream_cache_status; + add_header Cache-Control "public"; + + proxy_pass http://localhost:3000/; + + sub_filter_once off; + + {{- if eq .Values.global.cattle.clusterId "local" -}} + sub_filter '"appSubUrl":""' '"appSubUrl":"/api/v1/namespaces/{{ template "grafana.namespace" . }}/services/http:{{ template "grafana.fullname" . }}:{{ .Values.service.port }}/proxy"'; + {{- else -}} + sub_filter '"appSubUrl":""' '"appSubUrl":"/k8s/clusters/{{ .Values.global.cattle.clusterId }}/api/v1/namespaces/{{ template "grafana.namespace" . }}/services/http:{{ template "grafana.fullname" . }}:{{ .Values.service.port }}/proxy"'; + {{- end -}} + + sub_filter ':"/avatar/' ':"avatar/'; + + if ($request_filename ~ .*\.(?:js|css|jpg|jpeg|gif|png|ico|cur|gz|svg|svgz|mp4|ogg|ogv|webm)$) { + expires 90d; + } + + rewrite ^/k8s/clusters/.*/proxy(.*) /$1 break; + + } + } + } diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/poddisruptionbudget.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/poddisruptionbudget.yaml new file mode 100644 index 00000000000..05251214acf --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/poddisruptionbudget.yaml @@ -0,0 +1,22 @@ +{{- if .Values.podDisruptionBudget }} +apiVersion: {{ include "grafana.podDisruptionBudget.apiVersion" . }} +kind: PodDisruptionBudget +metadata: + name: {{ include "grafana.fullname" . }} + namespace: {{ include "grafana.namespace" . }} + labels: + {{- include "grafana.labels" . | nindent 4 }} + {{- with .Values.labels }} + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + {{- with .Values.podDisruptionBudget.minAvailable }} + minAvailable: {{ . }} + {{- end }} + {{- with .Values.podDisruptionBudget.maxUnavailable }} + maxUnavailable: {{ . }} + {{- end }} + selector: + matchLabels: + {{- include "grafana.selectorLabels" . | nindent 6 }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/podsecuritypolicy.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/podsecuritypolicy.yaml new file mode 100644 index 00000000000..973caccd576 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/podsecuritypolicy.yaml @@ -0,0 +1,45 @@ +{{- if and (or .Values.global.cattle.psp.enabled .Values.rbac.pspEnabled) (.Capabilities.APIVersions.Has "policy/v1beta1/PodSecurityPolicy") }} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ include "grafana.fullname" . }} + labels: + {{- include "grafana.labels" . | nindent 4 }} +{{- if .Values.rbac.pspAnnotations }} + annotations: {{ toYaml .Values.rbac.pspAnnotations | nindent 4 }} +{{- end }} +spec: + privileged: false + allowPrivilegeEscalation: false + requiredDropCapabilities: + # Default set from Docker, with DAC_OVERRIDE and CHOWN + - ALL + volumes: + - 'configMap' + - 'emptyDir' + - 'projected' + - 'csi' + - 'secret' + - 'downwardAPI' + - 'persistentVolumeClaim' + hostNetwork: false + hostIPC: false + hostPID: false + runAsUser: + rule: 'RunAsAny' + seLinux: + rule: 'RunAsAny' + supplementalGroups: + rule: 'MustRunAs' + ranges: + # Forbid adding the root group. + - min: 1 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + # Forbid adding the root group. + - min: 1 + max: 65535 + readOnlyRootFilesystem: false +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/pvc.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/pvc.yaml new file mode 100644 index 00000000000..c9b234305f9 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/pvc.yaml @@ -0,0 +1,41 @@ +{{- if and .Values.persistence.enabled (not .Values.persistence.existingClaim) (eq .Values.persistence.type "pvc")}} +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: {{ include "grafana.fullname" . }} + namespace: {{ include "grafana.namespace" . }} + labels: + {{- include "grafana.labels" . | nindent 4 }} + {{- with .Values.persistence.extraPvcLabels }} + {{- toYaml . | nindent 4 }} + {{- end }} + {{- with .Values.persistence.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} + {{- with .Values.persistence.finalizers }} + finalizers: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + accessModes: +{{- $_ := required "Must provide at least one access mode for persistent volumes used by Grafana" .Values.persistence.accessModes }} +{{- $_ := required "Must provide at least one access mode for persistent volumes used by Grafana" (first .Values.persistence.accessModes) }} + {{- range .Values.persistence.accessModes }} + - {{ . | quote }} + {{- end }} + resources: + requests: + storage: {{ .Values.persistence.size | quote }} + {{- if (lookup "v1" "PersistentVolumeClaim" (include "grafana.namespace" .) (include "grafana.fullname" .)) }} + volumeName: {{ (lookup "v1" "PersistentVolumeClaim" (include "grafana.namespace" .) (include "grafana.fullname" .)).spec.volumeName }} + {{- end }} + {{- with .Values.persistence.storageClassName }} + storageClassName: {{ . }} + {{- end }} + {{- with .Values.persistence.selectorLabels }} + selector: + matchLabels: + {{- toYaml . | nindent 6 }} + {{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/role.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/role.yaml new file mode 100644 index 00000000000..469b6f4e6c9 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/role.yaml @@ -0,0 +1,32 @@ +{{- if and .Values.rbac.create (not .Values.rbac.useExistingRole) -}} +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: {{ include "grafana.fullname" . }} + namespace: {{ include "grafana.namespace" . }} + labels: + {{- include "grafana.labels" . | nindent 4 }} + {{- with .Values.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +{{- if or (or .Values.global.cattle.psp.enabled .Values.rbac.pspEnabled) (and .Values.rbac.namespaced (or .Values.sidecar.dashboards.enabled .Values.sidecar.datasources.enabled .Values.sidecar.plugins.enabled .Values.rbac.extraRoleRules)) }} +rules: + {{- if and (or .Values.global.cattle.psp.enabled .Values.rbac.pspEnabled) (.Capabilities.APIVersions.Has "policy/v1beta1/PodSecurityPolicy") }} + - apiGroups: ['extensions'] + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: [{{ include "grafana.fullname" . }}] + {{- end }} + {{- if and .Values.rbac.namespaced (or .Values.sidecar.dashboards.enabled .Values.sidecar.datasources.enabled .Values.sidecar.plugins.enabled) }} + - apiGroups: [""] # "" indicates the core API group + resources: ["configmaps", "secrets"] + verbs: ["get", "watch", "list"] + {{- end }} + {{- with .Values.rbac.extraRoleRules }} + {{- toYaml . | nindent 2 }} + {{- end}} +{{- else }} +rules: [] +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/rolebinding.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/rolebinding.yaml new file mode 100644 index 00000000000..58f77c6b0b6 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/rolebinding.yaml @@ -0,0 +1,25 @@ +{{- if .Values.rbac.create }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: {{ include "grafana.fullname" . }} + namespace: {{ include "grafana.namespace" . }} + labels: + {{- include "grafana.labels" . | nindent 4 }} + {{- with .Values.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + {{- if .Values.rbac.useExistingRole }} + name: {{ .Values.rbac.useExistingRole }} + {{- else }} + name: {{ include "grafana.fullname" . }} + {{- end }} +subjects: +- kind: ServiceAccount + name: {{ include "grafana.serviceAccountName" . }} + namespace: {{ include "grafana.namespace" . }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/secret-env.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/secret-env.yaml new file mode 100644 index 00000000000..eb14aac7071 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/secret-env.yaml @@ -0,0 +1,14 @@ +{{- if .Values.envRenderSecret }} +apiVersion: v1 +kind: Secret +metadata: + name: {{ include "grafana.fullname" . }}-env + namespace: {{ include "grafana.namespace" . }} + labels: + {{- include "grafana.labels" . | nindent 4 }} +type: Opaque +data: +{{- range $key, $val := .Values.envRenderSecret }} + {{ $key }}: {{ tpl ($val | toString) $ | b64enc | quote }} +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/secret.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/secret.yaml new file mode 100644 index 00000000000..fd2ca50f4be --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/secret.yaml @@ -0,0 +1,16 @@ +{{- if or (and (not .Values.admin.existingSecret) (not .Values.env.GF_SECURITY_ADMIN_PASSWORD__FILE) (not .Values.env.GF_SECURITY_ADMIN_PASSWORD) (not .Values.env.GF_SECURITY_DISABLE_INITIAL_ADMIN_CREATION)) (and .Values.ldap.enabled (not .Values.ldap.existingSecret)) }} +apiVersion: v1 +kind: Secret +metadata: + name: {{ include "grafana.fullname" . }} + namespace: {{ include "grafana.namespace" . }} + labels: + {{- include "grafana.labels" . | nindent 4 }} + {{- with .Values.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +type: Opaque +data: + {{- include "grafana.secretsData" . | nindent 2 }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/service.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/service.yaml new file mode 100644 index 00000000000..e9396a15c66 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/service.yaml @@ -0,0 +1,61 @@ +{{- if .Values.service.enabled }} +{{- $root := . }} +apiVersion: v1 +kind: Service +metadata: + name: {{ include "grafana.fullname" . }} + namespace: {{ include "grafana.namespace" . }} + labels: + {{- include "grafana.labels" . | nindent 4 }} + {{- with .Values.service.labels }} + {{- toYaml . | nindent 4 }} + {{- end }} + {{- with .Values.service.annotations }} + annotations: + {{- tpl (toYaml . | nindent 4) $root }} + {{- end }} +spec: + {{- if (or (eq .Values.service.type "ClusterIP") (empty .Values.service.type)) }} + type: ClusterIP + {{- with .Values.service.clusterIP }} + clusterIP: {{ . }} + {{- end }} + {{- else if eq .Values.service.type "LoadBalancer" }} + type: LoadBalancer + {{- with .Values.service.loadBalancerIP }} + loadBalancerIP: {{ . }} + {{- end }} + {{- with .Values.service.loadBalancerClass }} + loadBalancerClass: {{ . }} + {{- end }} + {{- with .Values.service.loadBalancerSourceRanges }} + loadBalancerSourceRanges: + {{- toYaml . | nindent 4 }} + {{- end }} + {{- else }} + type: {{ .Values.service.type }} + {{- end }} + {{- with .Values.service.externalIPs }} + externalIPs: + {{- toYaml . | nindent 4 }} + {{- end }} + {{- with .Values.service.externalTrafficPolicy }} + externalTrafficPolicy: {{ . }} + {{- end }} + ports: + - name: {{ .Values.service.portName }} + port: {{ .Values.service.port }} + protocol: TCP + targetPort: {{ .Values.service.targetPort }} + {{- with .Values.service.appProtocol }} + appProtocol: {{ . }} + {{- end }} + {{- if (and (eq .Values.service.type "NodePort") (not (empty .Values.service.nodePort))) }} + nodePort: {{ .Values.service.nodePort }} + {{- end }} + {{- with .Values.extraExposePorts }} + {{- tpl (toYaml . | nindent 4) $root }} + {{- end }} + selector: + {{- include "grafana.selectorLabels" . | nindent 4 }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/serviceaccount.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/serviceaccount.yaml new file mode 100644 index 00000000000..ffca0717ae3 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/serviceaccount.yaml @@ -0,0 +1,17 @@ +{{- if .Values.serviceAccount.create }} +apiVersion: v1 +kind: ServiceAccount +automountServiceAccountToken: {{ .Values.serviceAccount.autoMount | default .Values.serviceAccount.automountServiceAccountToken }} +metadata: + labels: + {{- include "grafana.labels" . | nindent 4 }} + {{- with .Values.serviceAccount.labels }} + {{- toYaml . | nindent 4 }} + {{- end }} + {{- with .Values.serviceAccount.annotations }} + annotations: + {{- tpl (toYaml . | nindent 4) $ }} + {{- end }} + name: {{ include "grafana.serviceAccountName" . }} + namespace: {{ include "grafana.namespace" . }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/servicemonitor.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/servicemonitor.yaml new file mode 100644 index 00000000000..b321b1269c2 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/servicemonitor.yaml @@ -0,0 +1,68 @@ +{{- if .Values.serviceMonitor.enabled }} +--- +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ include "grafana.fullname" . }} + {{- if .Values.serviceMonitor.namespace }} + namespace: {{ tpl .Values.serviceMonitor.namespace . }} + {{- else }} + namespace: {{ include "grafana.namespace" . }} + {{- end }} + labels: + {{- include "grafana.labels" . | nindent 4 }} + {{- with .Values.serviceMonitor.labels }} + {{- tpl (toYaml . | nindent 4) $ }} + {{- end }} +spec: + endpoints: + - port: {{ .Values.service.portName }} + {{- with .Values.serviceMonitor.interval }} + interval: {{ . }} + {{- end }} + {{- with .Values.serviceMonitor.scrapeTimeout }} + scrapeTimeout: {{ . }} + {{- end }} + honorLabels: true + path: {{ .Values.serviceMonitor.path }} + scheme: {{ .Values.serviceMonitor.scheme }} + {{- with .Values.serviceMonitor.tlsConfig }} + tlsConfig: + {{- toYaml . | nindent 6 }} + {{- end }} + metricRelabelings: + {{- if .Values.serviceMonitor.metricRelabelings }} + {{- toYaml .Values.serviceMonitor.metricRelabelings | nindent 6 }} + {{- end }} + {{ if .Values.global.cattle.clusterId }} + - sourceLabels: [__address__] + targetLabel: cluster_id + replacement: {{ .Values.global.cattle.clusterId }} + {{- end }} + {{ if .Values.global.cattle.clusterName }} + - sourceLabels: [__address__] + targetLabel: cluster_name + replacement: {{ .Values.global.cattle.clusterName }} + {{- end }} + {{- if .Values.serviceMonitor.relabelings }} + {{- with .Values.serviceMonitor.relabelings }} + relabelings: + {{- toYaml . | nindent 6 }} + {{- end }} + {{- end }} + {{- with .Values.serviceMonitor.metricRelabelings }} + metricRelabelings: + {{- toYaml . | nindent 6 }} + {{- end }} + jobLabel: "{{ .Release.Name }}" + selector: + matchLabels: + {{- include "grafana.selectorLabels" . | nindent 6 }} + namespaceSelector: + matchNames: + - {{ include "grafana.namespace" . }} + {{- with .Values.serviceMonitor.targetLabels }} + targetLabels: + {{- toYaml . | nindent 4 }} + {{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/statefulset.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/statefulset.yaml new file mode 100644 index 00000000000..49278083e8d --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/statefulset.yaml @@ -0,0 +1,58 @@ +{{- $sts := list "sts" "StatefulSet" "statefulset" -}} +{{- if (or (.Values.useStatefulSet) (and .Values.persistence.enabled (not .Values.persistence.existingClaim) (has .Values.persistence.type $sts)))}} +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: {{ include "grafana.fullname" . }} + namespace: {{ include "grafana.namespace" . }} + labels: + {{- include "grafana.labels" . | nindent 4 }} + {{- with .Values.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + replicas: {{ .Values.replicas }} + selector: + matchLabels: + {{- include "grafana.selectorLabels" . | nindent 6 }} + serviceName: {{ include "grafana.fullname" . }}-headless + template: + metadata: + labels: + {{- include "grafana.selectorLabels" . | nindent 8 }} + {{- with .Values.podLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + annotations: + checksum/config: {{ include (print $.Template.BasePath "/configmap.yaml") . | sha256sum }} + checksum/dashboards-json-config: {{ include (print $.Template.BasePath "/dashboards-json-configmap.yaml") . | sha256sum }} + checksum/sc-dashboard-provider-config: {{ include (print $.Template.BasePath "/configmap-dashboard-provider.yaml") . | sha256sum }} + {{- if and (or (and (not .Values.admin.existingSecret) (not .Values.env.GF_SECURITY_ADMIN_PASSWORD__FILE) (not .Values.env.GF_SECURITY_ADMIN_PASSWORD)) (and .Values.ldap.enabled (not .Values.ldap.existingSecret))) (not .Values.env.GF_SECURITY_DISABLE_INITIAL_ADMIN_CREATION) }} + checksum/secret: {{ include (print $.Template.BasePath "/secret.yaml") . | sha256sum }} + {{- end }} + kubectl.kubernetes.io/default-container: {{ .Chart.Name }} + {{- with .Values.podAnnotations }} + {{- toYaml . | nindent 8 }} + {{- end }} + spec: + {{- include "grafana.pod" . | nindent 6 }} + {{- if .Values.persistence.enabled}} + volumeClaimTemplates: + - metadata: + name: storage + spec: +{{- $_ := required "Must provide at least one access mode for persistent volumes used by Grafana" .Values.persistence.accessModes }} +{{- $_ := required "Must provide at least one access mode for persistent volumes used by Grafana" (first .Values.persistence.accessModes) }} + accessModes: {{ .Values.persistence.accessModes }} + storageClassName: {{ .Values.persistence.storageClassName }} + resources: + requests: + storage: {{ required "Must provide size for persistent volumes used by Grafana" .Values.persistence.size }} + {{- with .Values.persistence.selectorLabels }} + selector: + matchLabels: + {{- toYaml . | nindent 10 }} + {{- end }} + {{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/tests/test-configmap.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/tests/test-configmap.yaml new file mode 100644 index 00000000000..01c96c92434 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/tests/test-configmap.yaml @@ -0,0 +1,20 @@ +{{- if .Values.testFramework.enabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ include "grafana.fullname" . }}-test + namespace: {{ include "grafana.namespace" . }} + annotations: + "helm.sh/hook": test-success + "helm.sh/hook-delete-policy": "before-hook-creation,hook-succeeded" + labels: + {{- include "grafana.labels" . | nindent 4 }} +data: + run.sh: |- + @test "Test Health" { + url="http://{{ include "grafana.fullname" . }}/api/health" + + code=$(wget --server-response --spider --timeout 90 --tries 10 ${url} 2>&1 | awk '/^ HTTP/{print $2}') + [ "$code" == "200" ] + } +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/tests/test-podsecuritypolicy.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/tests/test-podsecuritypolicy.yaml new file mode 100644 index 00000000000..70a0a884c95 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/tests/test-podsecuritypolicy.yaml @@ -0,0 +1,32 @@ +{{- if and (.Capabilities.APIVersions.Has "policy/v1beta1/PodSecurityPolicy") .Values.testFramework.enabled (or .Values.global.cattle.psp.enabled .Values.rbac.pspEnabled) }} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ include "grafana.fullname" . }}-test + annotations: + "helm.sh/hook": test-success + "helm.sh/hook-delete-policy": "before-hook-creation,hook-succeeded" + labels: + {{- include "grafana.labels" . | nindent 4 }} +spec: + allowPrivilegeEscalation: true + privileged: false + hostNetwork: false + hostIPC: false + hostPID: false + fsGroup: + rule: RunAsAny + seLinux: + rule: RunAsAny + supplementalGroups: + rule: RunAsAny + runAsUser: + rule: RunAsAny + volumes: + - configMap + - downwardAPI + - emptyDir + - projected + - csi + - secret +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/tests/test-role.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/tests/test-role.yaml new file mode 100644 index 00000000000..976418b137a --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/tests/test-role.yaml @@ -0,0 +1,17 @@ +{{- if and (.Capabilities.APIVersions.Has "policy/v1beta1/PodSecurityPolicy") .Values.testFramework.enabled (or .Values.global.cattle.psp.enabled .Values.rbac.pspEnabled) }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: {{ include "grafana.fullname" . }}-test + namespace: {{ include "grafana.namespace" . }} + annotations: + "helm.sh/hook": test-success + "helm.sh/hook-delete-policy": "before-hook-creation,hook-succeeded" + labels: + {{- include "grafana.labels" . | nindent 4 }} +rules: + - apiGroups: ['policy'] + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: [{{ include "grafana.fullname" . }}-test] +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/tests/test-rolebinding.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/tests/test-rolebinding.yaml new file mode 100644 index 00000000000..509566eccdd --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/tests/test-rolebinding.yaml @@ -0,0 +1,20 @@ +{{- if and (.Capabilities.APIVersions.Has "policy/v1beta1/PodSecurityPolicy") .Values.testFramework.enabled (or .Values.global.cattle.psp.enabled .Values.rbac.pspEnabled) }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: {{ include "grafana.fullname" . }}-test + namespace: {{ include "grafana.namespace" . }} + annotations: + "helm.sh/hook": test-success + "helm.sh/hook-delete-policy": "before-hook-creation,hook-succeeded" + labels: + {{- include "grafana.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: {{ include "grafana.fullname" . }}-test +subjects: + - kind: ServiceAccount + name: {{ include "grafana.serviceAccountNameTest" . }} + namespace: {{ include "grafana.namespace" . }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/tests/test-serviceaccount.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/tests/test-serviceaccount.yaml new file mode 100644 index 00000000000..38fba3596ad --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/tests/test-serviceaccount.yaml @@ -0,0 +1,12 @@ +{{- if and .Values.testFramework.enabled .Values.serviceAccount.create }} +apiVersion: v1 +kind: ServiceAccount +metadata: + labels: + {{- include "grafana.labels" . | nindent 4 }} + name: {{ include "grafana.serviceAccountNameTest" . }} + namespace: {{ include "grafana.namespace" . }} + annotations: + "helm.sh/hook": test-success + "helm.sh/hook-delete-policy": "before-hook-creation,hook-succeeded" +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/tests/test.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/tests/test.yaml new file mode 100644 index 00000000000..83aaa185c25 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/templates/tests/test.yaml @@ -0,0 +1,53 @@ +{{- if .Values.testFramework.enabled }} +{{- $root := . }} +apiVersion: v1 +kind: Pod +metadata: + name: {{ include "grafana.fullname" . }}-test + labels: + {{- include "grafana.labels" . | nindent 4 }} + annotations: + "helm.sh/hook": test-success + "helm.sh/hook-delete-policy": "before-hook-creation,hook-succeeded" + namespace: {{ include "grafana.namespace" . }} +spec: + serviceAccountName: {{ include "grafana.serviceAccountNameTest" . }} + {{- with .Values.testFramework.securityContext }} + securityContext: + {{- toYaml . | nindent 4 }} + {{- end }} + {{- if or .Values.image.pullSecrets .Values.global.imagePullSecrets }} + imagePullSecrets: + {{- include "grafana.imagePullSecrets" (dict "root" $root "imagePullSecrets" .Values.image.pullSecrets) | nindent 4 }} + {{- end }} + {{- with .Values.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 4 }} + {{- end }} + {{- with .Values.affinity }} + affinity: + {{- tpl (toYaml .) $root | nindent 4 }} + {{- end }} + {{- with .Values.tolerations }} + tolerations: + {{- toYaml . | nindent 4 }} + {{- end }} + containers: + - name: {{ .Release.Name }}-test + image: "{{ template "system_default_registry" . | default .Values.testFramework.image.registry }}/{{ .Values.testFramework.image.repository }}:{{ .Values.testFramework.image.tag }}" + imagePullPolicy: "{{ .Values.testFramework.imagePullPolicy}}" + command: ["/opt/bats/bin/bats", "-t", "/tests/run.sh"] + volumeMounts: + - mountPath: /tests + name: tests + readOnly: true + {{- with .Values.testFramework.resources }} + resources: + {{- toYaml . | nindent 8 }} + {{- end }} + volumes: + - name: tests + configMap: + name: {{ include "grafana.fullname" . }}-test + restartPolicy: Never +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/values.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/values.yaml new file mode 100644 index 00000000000..45e3df9325f --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/grafana/values.yaml @@ -0,0 +1,1315 @@ +global: + cattle: + psp: + enabled: false + systemDefaultRegistry: "" + + # To help compatibility with other charts which use global.imagePullSecrets. + # Allow either an array of {name: pullSecret} maps (k8s-style), or an array of strings (more common helm-style). + # Can be tempalted. + # global: + # imagePullSecrets: + # - name: pullSecret1 + # - name: pullSecret2 + # or + # global: + # imagePullSecrets: + # - pullSecret1 + # - pullSecret2 + imagePullSecrets: [] + +rbac: + create: true + ## Use an existing ClusterRole/Role (depending on rbac.namespaced false/true) + # useExistingRole: name-of-some-role + # useExistingClusterRole: name-of-some-clusterRole + pspEnabled: false + pspUseAppArmor: false + namespaced: false + extraRoleRules: [] + # - apiGroups: [] + # resources: [] + # verbs: [] + extraClusterRoleRules: [] + # - apiGroups: [] + # resources: [] + # verbs: [] +serviceAccount: + create: true + name: + nameTest: + ## ServiceAccount labels. + labels: {} + ## Service account annotations. Can be templated. + # annotations: + # eks.amazonaws.com/role-arn: arn:aws:iam::123456789000:role/iam-role-name-here + + ## autoMount is deprecated in favor of automountServiceAccountToken + # autoMount: false + automountServiceAccountToken: true + +replicas: 1 + +## Create a headless service for the deployment +headlessService: false + +## Should the service account be auto mounted on the pod +automountServiceAccountToken: true + +## Create HorizontalPodAutoscaler object for deployment type +# +autoscaling: + enabled: false + minReplicas: 1 + maxReplicas: 5 + targetCPU: "60" + targetMemory: "" + behavior: {} + +## See `kubectl explain poddisruptionbudget.spec` for more +## ref: https://kubernetes.io/docs/tasks/run-application/configure-pdb/ +podDisruptionBudget: {} +# apiVersion: "" +# minAvailable: 1 +# maxUnavailable: 1 + +## See `kubectl explain deployment.spec.strategy` for more +## ref: https://kubernetes.io/docs/concepts/workloads/controllers/deployment/#strategy +deploymentStrategy: + type: RollingUpdate + +readinessProbe: + httpGet: + path: /api/health + port: 3000 + +livenessProbe: + httpGet: + path: /api/health + port: 3000 + initialDelaySeconds: 60 + timeoutSeconds: 30 + failureThreshold: 10 + +## Use an alternate scheduler, e.g. "stork". +## ref: https://kubernetes.io/docs/tasks/administer-cluster/configure-multiple-schedulers/ +## +# schedulerName: "default-scheduler" + +image: + repository: rancher/mirrored-grafana-grafana + # Overrides the Grafana image tag whose default is the chart appVersion + tag: 10.4.9 + sha: "" + pullPolicy: IfNotPresent + + ## Optionally specify an array of imagePullSecrets. + ## Secrets must be manually created in the namespace. + ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/ + ## Can be templated. + ## + pullSecrets: [] + # - myRegistrKeySecretName + +testFramework: + enabled: false + imagePullPolicy: IfNotPresent + securityContext: + runAsNonRoot: true + runAsUser: 1000 + # limits: + # cpu: 100m + # memory: 128Mi + # requests: + # cpu: 100m + # memory: 128Mi + +# dns configuration for pod +dnsPolicy: ~ +dnsConfig: {} + # nameservers: + # - 8.8.8.8 + # options: + # - name: ndots + # value: "2" + # - name: edns0 + +securityContext: + runAsNonRoot: true + runAsUser: 472 + runAsGroup: 472 + fsGroup: 472 + +containerSecurityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + seccompProfile: + type: RuntimeDefault + +# Enable creating the grafana configmap +createConfigmap: true + +# Extra configmaps to mount in grafana pods +# Values are templated. +extraConfigmapMounts: [] + # - name: certs-configmap + # mountPath: /etc/grafana/ssl/ + # subPath: certificates.crt # (optional) + # configMap: certs-configmap + # readOnly: true + + +extraEmptyDirMounts: [] + # - name: provisioning-notifiers + # mountPath: /etc/grafana/provisioning/notifiers + + +# Apply extra labels to common labels. +extraLabels: {} + +## Assign a PriorityClassName to pods if set +# priorityClassName: + +downloadDashboardsImage: + repository: rancher/mirrored-curlimages-curl + tag: 7.85.0 + sha: "" + pullPolicy: IfNotPresent + +downloadDashboards: + env: {} + envFromSecret: "" + resources: {} + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + seccompProfile: + type: RuntimeDefault + envValueFrom: {} + # ENV_NAME: + # configMapKeyRef: + # name: configmap-name + # key: value_key + +## Pod Annotations +# podAnnotations: {} + +## Pod Labels +# podLabels: {} + +podPortName: grafana +gossipPortName: gossip +## Deployment annotations +# annotations: {} + +## Expose the grafana service to be accessed from outside the cluster (LoadBalancer service). +## or access it from within the cluster (ClusterIP service). Set the service type and the port to serve it. +## ref: http://kubernetes.io/docs/user-guide/services/ +## +service: + enabled: true + type: ClusterIP + loadBalancerIP: "" + loadBalancerClass: "" + loadBalancerSourceRanges: [] + port: 80 + targetPort: 3000 + # targetPort: 4181 To be used with a proxy extraContainer + ## Service annotations. Can be templated. + annotations: {} + labels: {} + portName: service + # Adds the appProtocol field to the service. This allows to work with istio protocol selection. Ex: "http" or "tcp" + appProtocol: "" + +serviceMonitor: + ## If true, a ServiceMonitor CRD is created for a prometheus operator + ## https://github.com/coreos/prometheus-operator + ## + enabled: false + path: /metrics + # namespace: monitoring (defaults to use the namespace this chart is deployed to) + labels: {} + interval: 30s + scheme: http + tlsConfig: {} + scrapeTimeout: 30s + relabelings: [] + metricRelabelings: [] + targetLabels: [] + +extraExposePorts: [] + # - name: keycloak + # port: 8080 + # targetPort: 8080 + +# overrides pod.spec.hostAliases in the grafana deployment's pods +hostAliases: [] + # - ip: "1.2.3.4" + # hostnames: + # - "my.host.com" + +ingress: + enabled: false + # For Kubernetes >= 1.18 you should specify the ingress-controller via the field ingressClassName + # See https://kubernetes.io/blog/2020/04/02/improvements-to-the-ingress-api-in-kubernetes-1.18/#specifying-the-class-of-an-ingress + # ingressClassName: nginx + # Values can be templated + annotations: {} + # kubernetes.io/ingress.class: nginx + # kubernetes.io/tls-acme: "true" + labels: {} + path: / + + # pathType is only for k8s >= 1.1= + pathType: Prefix + + hosts: + - chart-example.local + ## Extra paths to prepend to every host configuration. This is useful when working with annotation based services. + extraPaths: [] + # - path: /* + # backend: + # serviceName: ssl-redirect + # servicePort: use-annotation + ## Or for k8s > 1.19 + # - path: /* + # pathType: Prefix + # backend: + # service: + # name: ssl-redirect + # port: + # name: use-annotation + + + tls: [] + # - secretName: chart-example-tls + # hosts: + # - chart-example.local + +resources: {} +# limits: +# cpu: 100m +# memory: 128Mi +# requests: +# cpu: 100m +# memory: 128Mi + +## Node labels for pod assignment +## ref: https://kubernetes.io/docs/user-guide/node-selection/ +# +nodeSelector: {} + +## Tolerations for pod assignment +## ref: https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/ +## +tolerations: [] + +## Affinity for pod assignment (evaluated as template) +## ref: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#affinity-and-anti-affinity +## +affinity: {} + +## Topology Spread Constraints +## ref: https://kubernetes.io/docs/concepts/workloads/pods/pod-topology-spread-constraints/ +## +topologySpreadConstraints: [] + +## Additional init containers (evaluated as template) +## ref: https://kubernetes.io/docs/concepts/workloads/pods/init-containers/ +## +extraInitContainers: [] + +## Enable an Specify container in extraContainers. This is meant to allow adding an authentication proxy to a grafana pod +extraContainers: "" +# extraContainers: | +# - name: proxy +# image: quay.io/gambol99/keycloak-proxy:latest +# args: +# - -provider=github +# - -client-id= +# - -client-secret= +# - -github-org= +# - -email-domain=* +# - -cookie-secret= +# - -http-address=http://0.0.0.0:4181 +# - -upstream-url=http://127.0.0.1:3000 +# ports: +# - name: proxy-web +# containerPort: 4181 + +## Volumes that can be used in init containers that will not be mounted to deployment pods +extraContainerVolumes: [] +# - name: volume-from-secret +# secret: +# secretName: secret-to-mount +# - name: empty-dir-volume +# emptyDir: {} + +## Enable persistence using Persistent Volume Claims +## ref: http://kubernetes.io/docs/user-guide/persistent-volumes/ +## +persistence: + type: pvc + enabled: false + # storageClassName: default + accessModes: + - ReadWriteOnce + size: 10Gi + # annotations: {} + finalizers: + - kubernetes.io/pvc-protection + # selectorLabels: {} + ## Sub-directory of the PV to mount. Can be templated. + # subPath: "" + ## Name of an existing PVC. Can be templated. + # existingClaim: + ## Extra labels to apply to a PVC. + extraPvcLabels: {} + + ## If persistence is not enabled, this allows to mount the + ## local storage in-memory to improve performance + ## + inMemory: + enabled: false + ## The maximum usage on memory medium EmptyDir would be + ## the minimum value between the SizeLimit specified + ## here and the sum of memory limits of all containers in a pod + ## + # sizeLimit: 300Mi + +initChownData: + ## If false, data ownership will not be reset at startup + ## This allows the grafana-server to be run with an arbitrary user + ## + enabled: true + + ## initChownData container image + ## + image: + repository: rancher/mirrored-library-busybox + tag: "1.31.1" + sha: "" + pullPolicy: IfNotPresent + + ## initChownData resource requests and limits + ## Ref: http://kubernetes.io/docs/user-guide/compute-resources/ + ## + resources: {} + # limits: + # cpu: 100m + # memory: 128Mi + # requests: + # cpu: 100m + # memory: 128Mi + securityContext: + runAsNonRoot: false + runAsUser: 0 + seccompProfile: + type: RuntimeDefault + capabilities: + add: + - CHOWN + +# Administrator credentials when not using an existing secret (see below) +adminUser: admin +# adminPassword: strongpassword + +# Use an existing secret for the admin user. +admin: + ## Name of the secret. Can be templated. + existingSecret: "" + userKey: admin-user + passwordKey: admin-password + +## Define command to be executed at startup by grafana container +## Needed if using `vault-env` to manage secrets (ref: https://banzaicloud.com/blog/inject-secrets-into-pods-vault/) +## Default is "run.sh" as defined in grafana's Dockerfile +# command: +# - "sh" +# - "/run.sh" + +## Optionally define args if command is used +## Needed if using `hashicorp/envconsul` to manage secrets +## By default no arguments are set +# args: +# - "-secret" +# - "secret/grafana" +# - "./grafana" + +## Extra environment variables that will be pass onto deployment pods +## +## to provide grafana with access to CloudWatch on AWS EKS: +## 1. create an iam role of type "Web identity" with provider oidc.eks.* (note the provider for later) +## 2. edit the "Trust relationships" of the role, add a line inside the StringEquals clause using the +## same oidc eks provider as noted before (same as the existing line) +## also, replace NAMESPACE and prometheus-operator-grafana with the service account namespace and name +## +## "oidc.eks.us-east-1.amazonaws.com/id/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX:sub": "system:serviceaccount:NAMESPACE:prometheus-operator-grafana", +## +## 3. attach a policy to the role, you can use a built in policy called CloudWatchReadOnlyAccess +## 4. use the following env: (replace 123456789000 and iam-role-name-here with your aws account number and role name) +## +## env: +## AWS_ROLE_ARN: arn:aws:iam::123456789000:role/iam-role-name-here +## AWS_WEB_IDENTITY_TOKEN_FILE: /var/run/secrets/eks.amazonaws.com/serviceaccount/token +## AWS_REGION: us-east-1 +## +## 5. uncomment the EKS section in extraSecretMounts: below +## 6. uncomment the annotation section in the serviceAccount: above +## make sure to replace arn:aws:iam::123456789000:role/iam-role-name-here with your role arn + +env: {} + +## "valueFrom" environment variable references that will be added to deployment pods. Name is templated. +## ref: https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.19/#envvarsource-v1-core +## Renders in container spec as: +## env: +## ... +## - name: +## valueFrom: +## +envValueFrom: {} + # ENV_NAME: + # configMapKeyRef: + # name: configmap-name + # key: value_key + +## The name of a secret in the same kubernetes namespace which contain values to be added to the environment +## This can be useful for auth tokens, etc. Value is templated. +envFromSecret: "" + +## Sensible environment variables that will be rendered as new secret object +## This can be useful for auth tokens, etc. +## If the secret values contains "{{", they'll need to be properly escaped so that they are not interpreted by Helm +## ref: https://helm.sh/docs/howto/charts_tips_and_tricks/#using-the-tpl-function +envRenderSecret: {} + +## The names of secrets in the same kubernetes namespace which contain values to be added to the environment +## Each entry should contain a name key, and can optionally specify whether the secret must be defined with an optional key. +## Name is templated. +envFromSecrets: [] +## - name: secret-name +## prefix: prefix +## optional: true + +## The names of conifgmaps in the same kubernetes namespace which contain values to be added to the environment +## Each entry should contain a name key, and can optionally specify whether the configmap must be defined with an optional key. +## Name is templated. +## ref: https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.23/#configmapenvsource-v1-core +envFromConfigMaps: [] +## - name: configmap-name +## prefix: prefix +## optional: true + +# Inject Kubernetes services as environment variables. +# See https://kubernetes.io/docs/concepts/services-networking/connect-applications-service/#environment-variables +enableServiceLinks: true + +## Additional grafana server secret mounts +# Defines additional mounts with secrets. Secrets must be manually created in the namespace. +extraSecretMounts: [] + # - name: secret-files + # mountPath: /etc/secrets + # secretName: grafana-secret-files + # readOnly: true + # subPath: "" + # + # for AWS EKS (cloudwatch) use the following (see also instruction in env: above) + # - name: aws-iam-token + # mountPath: /var/run/secrets/eks.amazonaws.com/serviceaccount + # readOnly: true + # projected: + # defaultMode: 420 + # sources: + # - serviceAccountToken: + # audience: sts.amazonaws.com + # expirationSeconds: 86400 + # path: token + # + # for CSI e.g. Azure Key Vault use the following + # - name: secrets-store-inline + # mountPath: /run/secrets + # readOnly: true + # csi: + # driver: secrets-store.csi.k8s.io + # readOnly: true + # volumeAttributes: + # secretProviderClass: "akv-grafana-spc" + # nodePublishSecretRef: # Only required when using service principal mode + # name: grafana-akv-creds # Only required when using service principal mode + +## Additional grafana server volume mounts +# Defines additional volume mounts. +extraVolumeMounts: [] + # - name: extra-volume-0 + # mountPath: /mnt/volume0 + # readOnly: true + # - name: extra-volume-1 + # mountPath: /mnt/volume1 + # readOnly: true + # - name: grafana-secrets + # mountPath: /mnt/volume2 + +## Additional Grafana server volumes +extraVolumes: [] + # - name: extra-volume-0 + # existingClaim: volume-claim + # - name: extra-volume-1 + # hostPath: + # path: /usr/shared/ + # type: "" + # - name: grafana-secrets + # csi: + # driver: secrets-store.csi.k8s.io + # readOnly: true + # volumeAttributes: + # secretProviderClass: "grafana-env-spc" + +## Container Lifecycle Hooks. Execute a specific bash command or make an HTTP request +lifecycleHooks: {} + # postStart: + # exec: + # command: [] + +## Pass the plugins you want installed as a list. +## +plugins: [] + # - digrich-bubblechart-panel + # - grafana-clock-panel + ## You can also use other plugin download URL, as long as they are valid zip files, + ## and specify the name of the plugin after the semicolon. Like this: + # - https://grafana.com/api/plugins/marcusolsson-json-datasource/versions/1.3.2/download;marcusolsson-json-datasource + +## Configure grafana datasources +## ref: http://docs.grafana.org/administration/provisioning/#datasources +## +datasources: {} +# datasources.yaml: +# apiVersion: 1 +# datasources: +# - name: Prometheus +# type: prometheus +# url: http://prometheus-prometheus-server +# access: proxy +# isDefault: true +# - name: CloudWatch +# type: cloudwatch +# access: proxy +# uid: cloudwatch +# editable: false +# jsonData: +# authType: default +# defaultRegion: us-east-1 +# deleteDatasources: [] +# - name: Prometheus + +## Configure grafana alerting (can be templated) +## ref: http://docs.grafana.org/administration/provisioning/#alerting +## +alerting: {} + # rules.yaml: + # apiVersion: 1 + # groups: + # - orgId: 1 + # name: '{{ .Chart.Name }}_my_rule_group' + # folder: my_first_folder + # interval: 60s + # rules: + # - uid: my_id_1 + # title: my_first_rule + # condition: A + # data: + # - refId: A + # datasourceUid: '-100' + # model: + # conditions: + # - evaluator: + # params: + # - 3 + # type: gt + # operator: + # type: and + # query: + # params: + # - A + # reducer: + # type: last + # type: query + # datasource: + # type: __expr__ + # uid: '-100' + # expression: 1==0 + # intervalMs: 1000 + # maxDataPoints: 43200 + # refId: A + # type: math + # dashboardUid: my_dashboard + # panelId: 123 + # noDataState: Alerting + # for: 60s + # annotations: + # some_key: some_value + # labels: + # team: sre_team_1 + # contactpoints.yaml: + # secret: + # apiVersion: 1 + # contactPoints: + # - orgId: 1 + # name: cp_1 + # receivers: + # - uid: first_uid + # type: pagerduty + # settings: + # integrationKey: XXX + # severity: critical + # class: ping failure + # component: Grafana + # group: app-stack + # summary: | + # {{ `{{ include "default.message" . }}` }} + +## Configure notifiers +## ref: http://docs.grafana.org/administration/provisioning/#alert-notification-channels +## +notifiers: {} +# notifiers.yaml: +# notifiers: +# - name: email-notifier +# type: email +# uid: email1 +# # either: +# org_id: 1 +# # or +# org_name: Main Org. +# is_default: true +# settings: +# addresses: an_email_address@example.com +# delete_notifiers: + +## Configure grafana dashboard providers +## ref: http://docs.grafana.org/administration/provisioning/#dashboards +## +## `path` must be /var/lib/grafana/dashboards/ +## +dashboardProviders: {} +# dashboardproviders.yaml: +# apiVersion: 1 +# providers: +# - name: 'default' +# orgId: 1 +# folder: '' +# type: file +# disableDeletion: false +# editable: true +# options: +# path: /var/lib/grafana/dashboards/default + +## Configure grafana dashboard to import +## NOTE: To use dashboards you must also enable/configure dashboardProviders +## ref: https://grafana.com/dashboards +## +## dashboards per provider, use provider name as key. +## +dashboards: {} + # default: + # some-dashboard: + # json: | + # $RAW_JSON + # custom-dashboard: + # file: dashboards/custom-dashboard.json + # prometheus-stats: + # gnetId: 2 + # revision: 2 + # datasource: Prometheus + # local-dashboard: + # url: https://example.com/repository/test.json + # token: '' + # local-dashboard-base64: + # url: https://example.com/repository/test-b64.json + # token: '' + # b64content: true + # local-dashboard-gitlab: + # url: https://example.com/repository/test-gitlab.json + # gitlabToken: '' + # local-dashboard-bitbucket: + # url: https://example.com/repository/test-bitbucket.json + # bearerToken: '' + # local-dashboard-azure: + # url: https://example.com/repository/test-azure.json + # basic: '' + # acceptHeader: '*/*' + +## Reference to external ConfigMap per provider. Use provider name as key and ConfigMap name as value. +## A provider dashboards must be defined either by external ConfigMaps or in values.yaml, not in both. +## ConfigMap data example: +## +## data: +## example-dashboard.json: | +## RAW_JSON +## +dashboardsConfigMaps: {} +# default: "" + +## Grafana's primary configuration +## NOTE: values in map will be converted to ini format +## ref: http://docs.grafana.org/installation/configuration/ +## +grafana.ini: + paths: + data: /var/lib/grafana/ + logs: /var/log/grafana + plugins: /var/lib/grafana/plugins + provisioning: /etc/grafana/provisioning + analytics: + check_for_updates: true + log: + mode: console + grafana_net: + url: https://grafana.net + server: + domain: "{{ if (and .Values.ingress.enabled .Values.ingress.hosts) }}{{ .Values.ingress.hosts | first }}{{ else }}''{{ end }}" +## grafana Authentication can be enabled with the following values on grafana.ini + # server: + # The full public facing url you use in browser, used for redirects and emails + # root_url: + # https://grafana.com/docs/grafana/latest/auth/github/#enable-github-in-grafana + # auth.github: + # enabled: false + # allow_sign_up: false + # scopes: user:email,read:org + # auth_url: https://github.com/login/oauth/authorize + # token_url: https://github.com/login/oauth/access_token + # api_url: https://api.github.com/user + # team_ids: + # allowed_organizations: + # client_id: + # client_secret: +## LDAP Authentication can be enabled with the following values on grafana.ini +## NOTE: Grafana will fail to start if the value for ldap.toml is invalid + # auth.ldap: + # enabled: true + # allow_sign_up: true + # config_file: /etc/grafana/ldap.toml + +## Grafana's LDAP configuration +## Templated by the template in _helpers.tpl +## NOTE: To enable the grafana.ini must be configured with auth.ldap.enabled +## ref: http://docs.grafana.org/installation/configuration/#auth-ldap +## ref: http://docs.grafana.org/installation/ldap/#configuration +ldap: + enabled: false + # `existingSecret` is a reference to an existing secret containing the ldap configuration + # for Grafana in a key `ldap-toml`. + existingSecret: "" + # `config` is the content of `ldap.toml` that will be stored in the created secret + config: "" + # config: |- + # verbose_logging = true + + # [[servers]] + # host = "my-ldap-server" + # port = 636 + # use_ssl = true + # start_tls = false + # ssl_skip_verify = false + # bind_dn = "uid=%s,ou=users,dc=myorg,dc=com" + +## Grafana's SMTP configuration +## NOTE: To enable, grafana.ini must be configured with smtp.enabled +## ref: http://docs.grafana.org/installation/configuration/#smtp +smtp: + # `existingSecret` is a reference to an existing secret containing the smtp configuration + # for Grafana. + existingSecret: "" + userKey: "user" + passwordKey: "password" + +## Sidecars that collect the configmaps with specified label and stores the included files them into the respective folders +## Requires at least Grafana 5 to work and can't be used together with parameters dashboardProviders, datasources and dashboards +sidecar: + image: + repository: rancher/mirrored-kiwigrid-k8s-sidecar + tag: 1.26.1 + sha: "" + imagePullPolicy: IfNotPresent + resources: {} +# limits: +# cpu: 100m +# memory: 100Mi +# requests: +# cpu: 50m +# memory: 50Mi + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + seccompProfile: + type: RuntimeDefault + # skipTlsVerify Set to true to skip tls verification for kube api calls + # skipTlsVerify: true + enableUniqueFilenames: false + readinessProbe: {} + livenessProbe: {} + # Log level default for all sidecars. Can be one of: DEBUG, INFO, WARN, ERROR, CRITICAL. Defaults to INFO + # logLevel: INFO + alerts: + enabled: false + # Additional environment variables for the alerts sidecar + env: {} + # Do not reprocess already processed unchanged resources on k8s API reconnect. + # ignoreAlreadyProcessed: true + # label that the configmaps with alert are marked with + label: grafana_alert + # value of label that the configmaps with alert are set to + labelValue: "" + # Log level. Can be one of: DEBUG, INFO, WARN, ERROR, CRITICAL. + # logLevel: INFO + # If specified, the sidecar will search for alert config-maps inside this namespace. + # Otherwise the namespace in which the sidecar is running will be used. + # It's also possible to specify ALL to search in all namespaces + searchNamespace: null + # Method to use to detect ConfigMap changes. With WATCH the sidecar will do a WATCH requests, with SLEEP it will list all ConfigMaps, then sleep for 60 seconds. + watchMethod: WATCH + # search in configmap, secret or both + resource: both + # watchServerTimeout: request to the server, asking it to cleanly close the connection after that. + # defaults to 60sec; much higher values like 3600 seconds (1h) are feasible for non-Azure K8S + # watchServerTimeout: 3600 + # + # watchClientTimeout: is a client-side timeout, configuring your local socket. + # If you have a network outage dropping all packets with no RST/FIN, + # this is how long your client waits before realizing & dropping the connection. + # defaults to 66sec (sic!) + # watchClientTimeout: 60 + # + # Endpoint to send request to reload alerts + reloadURL: "http://localhost:3000/api/admin/provisioning/alerting/reload" + # Absolute path to shell script to execute after a alert got reloaded + script: null + skipReload: false + # This is needed if skipReload is true, to load any alerts defined at startup time. + # Deploy the alert sidecar as an initContainer. + initAlerts: false + # Additional alert sidecar volume mounts + extraMounts: [] + # Sets the size limit of the alert sidecar emptyDir volume + sizeLimit: {} + dashboards: + enabled: false + # Additional environment variables for the dashboards sidecar + env: {} + # Do not reprocess already processed unchanged resources on k8s API reconnect. + # ignoreAlreadyProcessed: true + SCProvider: true + # label that the configmaps with dashboards are marked with + label: grafana_dashboard + # value of label that the configmaps with dashboards are set to + labelValue: "" + # Log level. Can be one of: DEBUG, INFO, WARN, ERROR, CRITICAL. + # logLevel: INFO + # folder in the pod that should hold the collected dashboards (unless `defaultFolderName` is set) + folder: /tmp/dashboards + # The default folder name, it will create a subfolder under the `folder` and put dashboards in there instead + defaultFolderName: null + # Namespaces list. If specified, the sidecar will search for config-maps/secrets inside these namespaces. + # Otherwise the namespace in which the sidecar is running will be used. + # It's also possible to specify ALL to search in all namespaces. + searchNamespace: null + # Method to use to detect ConfigMap changes. With WATCH the sidecar will do a WATCH requests, with SLEEP it will list all ConfigMaps, then sleep for 60 seconds. + watchMethod: WATCH + # search in configmap, secret or both + resource: both + # If specified, the sidecar will look for annotation with this name to create folder and put graph here. + # You can use this parameter together with `provider.foldersFromFilesStructure`to annotate configmaps and create folder structure. + folderAnnotation: null + # Endpoint to send request to reload alerts + reloadURL: "http://localhost:3000/api/admin/provisioning/dashboards/reload" + # Absolute path to shell script to execute after a configmap got reloaded + script: null + skipReload: false + # watchServerTimeout: request to the server, asking it to cleanly close the connection after that. + # defaults to 60sec; much higher values like 3600 seconds (1h) are feasible for non-Azure K8S + # watchServerTimeout: 3600 + # + # watchClientTimeout: is a client-side timeout, configuring your local socket. + # If you have a network outage dropping all packets with no RST/FIN, + # this is how long your client waits before realizing & dropping the connection. + # defaults to 66sec (sic!) + # watchClientTimeout: 60 + # + # provider configuration that lets grafana manage the dashboards + provider: + # name of the provider, should be unique + name: sidecarProvider + # orgid as configured in grafana + orgid: 1 + # folder in which the dashboards should be imported in grafana + folder: '' + # type of the provider + type: file + # disableDelete to activate a import-only behaviour + disableDelete: false + # allow updating provisioned dashboards from the UI + allowUiUpdates: false + # allow Grafana to replicate dashboard structure from filesystem + foldersFromFilesStructure: false + # Additional dashboard sidecar volume mounts + extraMounts: [] + # Sets the size limit of the dashboard sidecar emptyDir volume + sizeLimit: {} + datasources: + enabled: false + # Additional environment variables for the datasourcessidecar + env: {} + envValueFrom: {} + # Do not reprocess already processed unchanged resources on k8s API reconnect. + # ignoreAlreadyProcessed: true + # label that the configmaps with datasources are marked with + label: grafana_datasource + # value of label that the configmaps with datasources are set to + labelValue: "" + # Log level. Can be one of: DEBUG, INFO, WARN, ERROR, CRITICAL. + # logLevel: INFO + # If specified, the sidecar will search for datasource config-maps inside this namespace. + # Otherwise the namespace in which the sidecar is running will be used. + # It's also possible to specify ALL to search in all namespaces + searchNamespace: null + # Method to use to detect ConfigMap changes. With WATCH the sidecar will do a WATCH requests, with SLEEP it will list all ConfigMaps, then sleep for 60 seconds. + watchMethod: WATCH + # search in configmap, secret or both + resource: both + # watchServerTimeout: request to the server, asking it to cleanly close the connection after that. + # defaults to 60sec; much higher values like 3600 seconds (1h) are feasible for non-Azure K8S + # watchServerTimeout: 3600 + # + # watchClientTimeout: is a client-side timeout, configuring your local socket. + # If you have a network outage dropping all packets with no RST/FIN, + # this is how long your client waits before realizing & dropping the connection. + # defaults to 66sec (sic!) + # watchClientTimeout: 60 + # + # Endpoint to send request to reload datasources + reloadURL: "http://localhost:3000/api/admin/provisioning/datasources/reload" + # Absolute path to shell script to execute after a datasource got reloaded + script: null + skipReload: true + # This is needed if skipReload is true, to load any datasources defined at startup time. + # Deploy the datasources sidecar as an initContainer. + initDatasources: true + # Sets the size limit of the datasource sidecar emptyDir volume + sizeLimit: {} + plugins: + enabled: false + # Additional environment variables for the plugins sidecar + env: {} + # Do not reprocess already processed unchanged resources on k8s API reconnect. + # ignoreAlreadyProcessed: true + # label that the configmaps with plugins are marked with + label: grafana_plugin + # value of label that the configmaps with plugins are set to + labelValue: "" + # Log level. Can be one of: DEBUG, INFO, WARN, ERROR, CRITICAL. + # logLevel: INFO + # If specified, the sidecar will search for plugin config-maps inside this namespace. + # Otherwise the namespace in which the sidecar is running will be used. + # It's also possible to specify ALL to search in all namespaces + searchNamespace: null + # Method to use to detect ConfigMap changes. With WATCH the sidecar will do a WATCH requests, with SLEEP it will list all ConfigMaps, then sleep for 60 seconds. + watchMethod: WATCH + # search in configmap, secret or both + resource: both + # watchServerTimeout: request to the server, asking it to cleanly close the connection after that. + # defaults to 60sec; much higher values like 3600 seconds (1h) are feasible for non-Azure K8S + # watchServerTimeout: 3600 + # + # watchClientTimeout: is a client-side timeout, configuring your local socket. + # If you have a network outage dropping all packets with no RST/FIN, + # this is how long your client waits before realizing & dropping the connection. + # defaults to 66sec (sic!) + # watchClientTimeout: 60 + # + # Endpoint to send request to reload plugins + reloadURL: "http://localhost:3000/api/admin/provisioning/plugins/reload" + # Absolute path to shell script to execute after a plugin got reloaded + script: null + skipReload: false + # Deploy the datasource sidecar as an initContainer in addition to a container. + # This is needed if skipReload is true, to load any plugins defined at startup time. + initPlugins: false + # Sets the size limit of the plugin sidecar emptyDir volume + sizeLimit: {} + notifiers: + enabled: false + # Additional environment variables for the notifierssidecar + env: {} + # Do not reprocess already processed unchanged resources on k8s API reconnect. + # ignoreAlreadyProcessed: true + # label that the configmaps with notifiers are marked with + label: grafana_notifier + # value of label that the configmaps with notifiers are set to + labelValue: "" + # Log level. Can be one of: DEBUG, INFO, WARN, ERROR, CRITICAL. + # logLevel: INFO + # If specified, the sidecar will search for notifier config-maps inside this namespace. + # Otherwise the namespace in which the sidecar is running will be used. + # It's also possible to specify ALL to search in all namespaces + searchNamespace: null + # Method to use to detect ConfigMap changes. With WATCH the sidecar will do a WATCH requests, with SLEEP it will list all ConfigMaps, then sleep for 60 seconds. + watchMethod: WATCH + # search in configmap, secret or both + resource: both + # watchServerTimeout: request to the server, asking it to cleanly close the connection after that. + # defaults to 60sec; much higher values like 3600 seconds (1h) are feasible for non-Azure K8S + # watchServerTimeout: 3600 + # + # watchClientTimeout: is a client-side timeout, configuring your local socket. + # If you have a network outage dropping all packets with no RST/FIN, + # this is how long your client waits before realizing & dropping the connection. + # defaults to 66sec (sic!) + # watchClientTimeout: 60 + # + # Endpoint to send request to reload notifiers + reloadURL: "http://localhost:3000/api/admin/provisioning/notifications/reload" + # Absolute path to shell script to execute after a notifier got reloaded + script: null + skipReload: false + # Deploy the notifier sidecar as an initContainer in addition to a container. + # This is needed if skipReload is true, to load any notifiers defined at startup time. + initNotifiers: false + # Sets the size limit of the notifier sidecar emptyDir volume + sizeLimit: {} + +## Override the deployment namespace +## +namespaceOverride: "" + +## Number of old ReplicaSets to retain +## +revisionHistoryLimit: 10 + +## Add a seperate remote image renderer deployment/service +imageRenderer: + deploymentStrategy: {} + # Enable the image-renderer deployment & service + enabled: false + replicas: 1 + autoscaling: + enabled: false + minReplicas: 1 + maxReplicas: 5 + targetCPU: "60" + targetMemory: "" + behavior: {} + image: + # image-renderer Image repository + repository: rancher/mirrored-grafana-grafana-image-renderer + # image-renderer Image tag + tag: 3.10.5 + # image-renderer Image sha (optional) + sha: "" + # image-renderer ImagePullPolicy + pullPolicy: Always + # extra environment variables + env: + HTTP_HOST: "0.0.0.0" + # RENDERING_ARGS: --no-sandbox,--disable-gpu,--window-size=1280x758 + # RENDERING_MODE: clustered + # IGNORE_HTTPS_ERRORS: true + + ## "valueFrom" environment variable references that will be added to deployment pods. Name is templated. + ## ref: https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.19/#envvarsource-v1-core + ## Renders in container spec as: + ## env: + ## ... + ## - name: + ## valueFrom: + ## + envValueFrom: {} + # ENV_NAME: + # configMapKeyRef: + # name: configmap-name + # key: value_key + + # image-renderer deployment serviceAccount + serviceAccountName: "" + # image-renderer deployment securityContext + securityContext: {} + # image-renderer deployment container securityContext + containerSecurityContext: + seccompProfile: + type: RuntimeDefault + capabilities: + drop: ['ALL'] + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + ## image-renderer pod annotation + podAnnotations: {} + # image-renderer deployment Host Aliases + hostAliases: [] + # image-renderer deployment priority class + priorityClassName: '' + service: + # Enable the image-renderer service + enabled: true + # image-renderer service port name + portName: 'http' + # image-renderer service port used by both service and deployment + port: 8081 + targetPort: 8081 + # Adds the appProtocol field to the image-renderer service. This allows to work with istio protocol selection. Ex: "http" or "tcp" + appProtocol: "" + serviceMonitor: + ## If true, a ServiceMonitor CRD is created for a prometheus operator + ## https://github.com/coreos/prometheus-operator + ## + enabled: false + path: /metrics + # namespace: monitoring (defaults to use the namespace this chart is deployed to) + labels: {} + interval: 1m + scheme: http + tlsConfig: {} + scrapeTimeout: 30s + relabelings: [] + # See: https://doc.crds.dev/github.com/prometheus-operator/kube-prometheus/monitoring.coreos.com/ServiceMonitor/v1@v0.11.0#spec-targetLabels + targetLabels: [] + # - targetLabel1 + # - targetLabel2 + # If https is enabled in Grafana, this needs to be set as 'https' to correctly configure the callback used in Grafana + grafanaProtocol: http + # In case a sub_path is used this needs to be added to the image renderer callback + grafanaSubPath: "" + # name of the image-renderer port on the pod + podPortName: http + # number of image-renderer replica sets to keep + revisionHistoryLimit: 10 + networkPolicy: + # Enable a NetworkPolicy to limit inbound traffic to only the created grafana pods + limitIngress: true + # Enable a NetworkPolicy to limit outbound traffic to only the created grafana pods + limitEgress: false + # Allow additional services to access image-renderer (eg. Prometheus operator when ServiceMonitor is enabled) + extraIngressSelectors: [] + resources: {} +# limits: +# cpu: 100m +# memory: 100Mi +# requests: +# cpu: 50m +# memory: 50Mi + ## Node labels for pod assignment + ## ref: https://kubernetes.io/docs/user-guide/node-selection/ + # + nodeSelector: {} + + ## Tolerations for pod assignment + ## ref: https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/ + ## + tolerations: [] + + ## Affinity for pod assignment (evaluated as template) + ## ref: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#affinity-and-anti-affinity + ## + affinity: {} + + ## Use an alternate scheduler, e.g. "stork". + ## ref: https://kubernetes.io/docs/tasks/administer-cluster/configure-multiple-schedulers/ + ## + # schedulerName: "default-scheduler" + +networkPolicy: + ## @param networkPolicy.enabled Enable creation of NetworkPolicy resources. Only Ingress traffic is filtered for now. + ## + enabled: false + ## @param networkPolicy.allowExternal Don't require client label for connections + ## The Policy model to apply. When set to false, only pods with the correct + ## client label will have network access to grafana port defined. + ## When true, grafana will accept connections from any source + ## (with the correct destination port). + ## + ingress: true + ## @param networkPolicy.ingress When true enables the creation + ## an ingress network policy + ## + allowExternal: true + ## @param networkPolicy.explicitNamespacesSelector A Kubernetes LabelSelector to explicitly select namespaces from which traffic could be allowed + ## If explicitNamespacesSelector is missing or set to {}, only client Pods that are in the networkPolicy's namespace + ## and that match other criteria, the ones that have the good label, can reach the grafana. + ## But sometimes, we want the grafana to be accessible to clients from other namespaces, in this case, we can use this + ## LabelSelector to select these namespaces, note that the networkPolicy's namespace should also be explicitly added. + ## + ## Example: + ## explicitNamespacesSelector: + ## matchLabels: + ## role: frontend + ## matchExpressions: + ## - {key: role, operator: In, values: [frontend]} + ## + explicitNamespacesSelector: {} + ## + ## + ## + ## + ## + ## + egress: + ## @param networkPolicy.egress.enabled When enabled, an egress network policy will be + ## created allowing grafana to connect to external data sources from kubernetes cluster. + enabled: false + ## + ## @param networkPolicy.egress.blockDNSResolution When enabled, DNS resolution will be blocked + ## for all pods in the grafana namespace. + blockDNSResolution: false + ## + ## @param networkPolicy.egress.ports Add individual ports to be allowed by the egress + ports: [] + ## Add ports to the egress by specifying - port: + ## E.X. + ## - port: 80 + ## - port: 443 + ## + ## @param networkPolicy.egress.to Allow egress traffic to specific destinations + to: [] + ## Add destinations to the egress by specifying - ipBlock: + ## E.X. + ## to: + ## - namespaceSelector: + ## matchExpressions: + ## - {key: role, operator: In, values: [grafana]} + ## + ## + ## + ## + ## + +# Enable backward compatibility of kubernetes where version below 1.13 doesn't have the enableServiceLinks option +enableKubeBackwardCompatibility: false +useStatefulSet: false +# Create a dynamic manifests via values: +extraObjects: [] + # - apiVersion: "kubernetes-client.io/v1" + # kind: ExternalSecret + # metadata: + # name: grafana-secrets + # spec: + # backendType: gcpSecretsManager + # data: + # - key: grafana-admin-password + # name: adminPassword + +# assertNoLeakedSecrets is a helper function defined in _helpers.tpl that checks if secret +# values are not exposed in the rendered grafana.ini configmap. It is enabled by default. +# +# To pass values into grafana.ini without exposing them in a configmap, use variable expansion: +# https://grafana.com/docs/grafana/latest/setup-grafana/configure-grafana/#variable-expansion +# +# Alternatively, if you wish to allow secret values to be exposed in the rendered grafana.ini configmap, +# you can disable this check by setting assertNoLeakedSecrets to false. +assertNoLeakedSecrets: true diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/hardenedKubelet/.helmignore b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/hardenedKubelet/.helmignore new file mode 100644 index 00000000000..0e8a0eb36f4 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/hardenedKubelet/.helmignore @@ -0,0 +1,23 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*.orig +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/hardenedKubelet/Chart.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/hardenedKubelet/Chart.yaml new file mode 100644 index 00000000000..c81bc0093f0 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/hardenedKubelet/Chart.yaml @@ -0,0 +1,15 @@ +annotations: + catalog.cattle.io/hidden: "true" + catalog.cattle.io/kube-version: '>= 1.26.0-0 < 1.31.0-0' + catalog.cattle.io/os: linux + catalog.rancher.io/certified: rancher + catalog.rancher.io/namespace: cattle-monitoring-system + catalog.rancher.io/release-name: rancher-pushprox +apiVersion: v1 +appVersion: v0.1.4-rancher2 +description: Sets up a deployment of the PushProx proxy and a DaemonSet of PushProx + clients. +kubeVersion: '>=1.26.0-0' +name: hardenedKubelet +type: application +version: 0.2.1 diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/hardenedKubelet/README.md b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/hardenedKubelet/README.md new file mode 100644 index 00000000000..345002f48a2 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/hardenedKubelet/README.md @@ -0,0 +1,90 @@ +# rancher-pushprox + +A Rancher chart based on Rancher [PushProx](https://github.com/rancher/PushProx) that sets up a Deployment of a PushProx proxy and a DaemonSet of PushProx clients on a Kubernetes cluster. + +Installs [rancher-pushprox](https://github.com/rancher/charts/tree/gh-pages/packages/rancher-pushprox) to create PushProx clients that can access their host's network and register with a PushProx proxy. A [Prometheus Operator](https://github.com/coreos/prometheus-operator) ServiceMonitor CR is also included that is configured to scrape the metrics from each of the clients through the proxy. + +Using an instance of this chart is suitable for the following scenarios: +- You need to scrape metrics from a port that should not be accessible outside of the host (e.g. scraping `etcd` metrics in a hardened cluster) +- You need to scrape metrics on a host that are not exposed outside of 127.0.0.1 (e.g. scraping `kube-proxy` metrics) +- You need to scrape metrics through HTTPS using certs hosted directly on `hostPath` +- You need to scrape metrics from Kubernetes components that require authorization via a service account (e.g. permissions to make request to `/metrics`) +- You need to scrape metrics without access to cacerts (i.e. enable `insecureSkipVerify`) + +The clients and proxy are created based on a Rancher fork of the [prometheus-community/PushProx](https://github.com/prometheus-community/PushProx) project. + +## Upgrading to Kubernetes v1.25+ + +Starting in Kubernetes v1.25, [Pod Security Policies](https://kubernetes.io/docs/concepts/security/pod-security-policy/) have been removed from the Kubernetes API. + +As a result, **before upgrading to Kubernetes v1.25** (or on a fresh install in a Kubernetes v1.25+ cluster), users are expected to perform an in-place upgrade of this chart with `global.cattle.psp.enabled` set to `false` if it has been previously set to `true`. +​ +> **Note:** +> In this chart release, any previous field that was associated with any PSP resources have been removed in favor of a single global field: `global.cattle.psp.enabled`. + +> **Note:** +> If you upgrade your cluster to Kubernetes v1.25+ before removing PSPs via a `helm upgrade` (even if you manually clean up resources), **it will leave the Helm release in a broken state within the cluster such that further Helm operations will not work (`helm uninstall`, `helm upgrade`, etc.).** +> +> If your charts get stuck in this state, please consult the Rancher docs on how to clean up your Helm release secrets. + +Upon setting `global.cattle.psp.enabled` to false, the chart will remove any PSP resources deployed on its behalf from the cluster. This is the default setting for this chart. + +As a replacement for PSPs, [Pod Security Admission](https://kubernetes.io/docs/concepts/security/pod-security-admission/) should be used. Please consult the Rancher docs for more details on how to configure your chart release namespaces to work with the new Pod Security Admission and apply Pod Security Standards. + +## Configuration + +The following tables list the configurable parameters of the rancher-pushprox chart and their default values. + +### General + +#### Required +| Parameter | Description | Example | +| ----- | ----------- | ------ | +| `component` | The component that is being monitored | `kube-etcd` +| `metricsPort` | The port on the host that contains the metrics you want to scrape (e.g. `http://:/metrics`) | `2379` | +| `namespaceOverride` | The namespace to install the chart | `""` + +#### Optional +| Parameter | Description | Default | +| ----- | ----------- | ------ | +| `serviceMonitor.enabled` | Deploys a [Prometheus Operator](https://github.com/coreos/prometheus-operator/blob/master/Documentation/api.md#servicemonitor) ServiceMonitor CR that is configured to scrape metrics on the hosts that the clients are deployed on via the proxy. Also deploys a Service that points to all pods with the expected client name that exposes the `metricsPort` selected | `true` | +| `serviceMonitor.endpoints` | A list of endpoints that will be added to the ServiceMonitor based on the [Endpoint spec](https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#endpoint) | `[{port: metrics}]` | +| `service.selector` | The selector that is used to populate the Service's Endpoints object. The chart will error out on rendering templating if `.Values.clients.enabled` is set alongside this field, since it is expected that this service should point to the PushProx Clients Daemonset / Deployment | `{}` | +| `clients.enabled` | Deploys a DaemonSet of clients that are each capable of scraping endpoints on the hostNetwork it is deployed on | `true` | +| `clients.port` | The port where the client will publish PushProx client-specific metrics. If deploying multiple clients onto the same node, the clients should not have conflicting ports | `9369` | +| `clients.proxyUrl` | Overrides the default proxyUrl setting of `http://pushprox-{{ .Values.component }}-proxy.{{ . Release.Namespace }}.svc.cluster.local:{{ .Values.proxy.port }}"` with the `proxyUrl` specified | `""` | +| `clients.useLocalhost` | Sets a flag on each client deployment to redirect scrapes directed to `HOST_IP` to `127.0.0.1` | `false` | +| `clients.https.enabled` | Enables scraping metrics via HTTPS using the provided TLS certs that exist on each host | `false` | +| `clients.https.forceHTTPSScheme` | Forces scraping metrics via HTTPS using the provided TLS certs that exist on each host | `false` | +| `clients.https.useServiceAccountCredentials` | If set to true, the client will create a service account with permissions to scrape `/metrics` endpoint of Kubernetes components. The client will use the service account token provided to make authorized scrape requests to the Kubernetes API | `false` | +| `clients.https.authenticationMethod.bearerTokenFile.enabled` | If set to true, the client will use service account credentials mounted at the configured path `clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath`. This requires permissions to scrape `/metrics` endpoint of Kubernetes components. This method is deprecated by the prometheus operator and may be removed in a future release | `false` | +| `clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath` | This is a volume mount on the pod with permissions to scrape `/metrics` endpoint of Kubernetes components | `"/var/run/secrets/kubernetes.io/serviceaccount/token"` | +| `clients.https.authenticationMethod.bearerTokenSecret.enabled` | If set to true, the client will use service account credentials to scrape `/metrics` endpoint of Kubernetes components. This method is deprecated by the prometheus operator and may be removed in a future release | `false` | +| `clients.https.authenticationMethod.authorization.enabled` | If set to true, the client will use service account credentials to scrape `/metrics` endpoint of Kubernetes components | `false` | +| `clients.https.authenticationMethod.authorization.type` | If set, the client will use this type of authorization in its client requests for metrics | `"bearer"` | +| `clients.https.authenticationMethod.authorization.credentials.key` | If set, the client will use this key in the secret created by `clients.https.useServiceAccountCredentials` for authorization in its client requests for metrics | `"token"` | +| `clients.https.authenticationMethod.authorization.credentials.optional` | If set to false, the client will fail if the key in the secret created by `clients.https.useServiceAccountCredentials` does not exist | `false` | +| `clients.https.insecureSkipVerify` | If set to true, the client will disable SSL security checks | `false` | +| `clients.https.certDir` | A `hostPath` where TLS certs can be found. This path is mounted as a volume on an `initContainer` which copies only the necessary files over to an EmptyDir volume used by each client. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.certFile` | The path to the TLS cert file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.keyFile` | The path to the TLS key file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.caCertFile` | The path to the TLS cacert file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.seLinuxOptions` | seLinuxOptions to be passed into the container that copies certs. Should define a container with permissions to read the files in the certDir provided on the host. Required and only used if `clients.https.enabled` is set and `clients.https.certDir` is provided. | `""` | +| `clients.metrics.enabled` | Whether the client should publish PushProx client-specific metrics. | `false` | +| `clients.rbac.additionalRules` | Additional permissions to provide to the ServiceAccount bound to the client. This can be used to provide additional permissions for the client to scrape metrics from the k8s API. Only enabled if clients.https.enabled and clients.https.useServiceAccountCredentials are true | `[]` | +| `clients.deployment.enabled` | Deploys the client as a Deployment (generally used if the underlying hostNetwork Pod that is being scraped is managed by a Deployment) | `false` | +| `clients.deployment.replicas` | The number of pods the Deployment has, it should match the number of pod the hostNetwork Deployment has. Required and only used if `client.deployment.enable` is set | `0` | +| `clients.deployment.affinity` | The affinity rules that allocate the pod to the node in which the hostNetwork Deployment's pods run. Required and only used if `client.deployment.enable` is set | `{}` | +| `clients.resources` | Set resource limits and requests for the client container | `{}` | +| `clients.nodeSelector` | Select which nodes to deploy the clients on | `{}` | +| `clients.tolerations` | Specify tolerations for clients | `[]` | +| `proxy.enabled` | Deploys the proxy that each client will register with | `true` | +| `proxy.port` | The port exposed by the proxy that each client will register with to allow metrics to be scraped from the host | `8080` | +| `proxy.resources` | Set resource limits and requests for the proxy container | `{}` | +| `proxy.nodeSelector` | Select which nodes the proxy can be deployed on | `{}` | +| `proxy.tolerations` | Specify tolerations (if necessary) to allow the proxy to be deployed on the selected node | `[]` | +| `kubeVersionOverrides` | A list of Semver constraint strings (defined by https://github.com/Masterminds/semver) and values.yaml overrides. For each key in kubeVersionOverrides, this chart will check to see if the current Kubernetes cluster's version matches any of the semver constraints provided as keys on the map. On seeing a match, the default value for each values.yaml field overridden will be updated with the new value. If multiple matches are encountered (due to overlapping semver ranges), the matches will be applied in order. | `[]` + +*Tip: The filepaths set in `clients.https.File` can include wildcard characters*. + +See [rancher-monitoring](https://github.com/rancher/charts/tree/gh-pages/packages/rancher-monitoring) for examples of how this chart can be used. diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/hardenedKubelet/templates/_helpers.tpl b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/hardenedKubelet/templates/_helpers.tpl new file mode 100644 index 00000000000..1ba50939441 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/hardenedKubelet/templates/_helpers.tpl @@ -0,0 +1,170 @@ +# Rancher + +{{- define "system_default_registry" -}} +{{- if .Values.global.cattle.systemDefaultRegistry -}} +{{- printf "%s/" .Values.global.cattle.systemDefaultRegistry -}} +{{- end -}} +{{- end -}} + +# Windows Support + +{{/* +Windows cluster will add default taint for linux nodes, +add below linux tolerations to workloads could be scheduled to those linux nodes +*/}} + +{{- define "linux-node-tolerations" -}} +- key: "cattle.io/os" + value: "linux" + effect: "NoSchedule" + operator: "Equal" +{{- end -}} + +{{- define "linux-node-selector" -}} +{{- if semverCompare "<1.14-0" .Capabilities.KubeVersion.GitVersion -}} +beta.kubernetes.io/os: linux +{{- else -}} +kubernetes.io/os: linux +{{- end -}} +{{- end -}} + +# General + +{{- define "applyKubeVersionOverrides" -}} +{{- $overrides := dict -}} +{{- range $override := .Values.kubeVersionOverrides -}} +{{- if semverCompare $override.constraint $.Capabilities.KubeVersion.Version -}} +{{- $_ := mergeOverwrite $overrides $override.values -}} +{{- end -}} +{{- end -}} +{{- $_ := mergeOverwrite .Values $overrides -}} +{{- end -}} + +{{- define "pushprox.namespace" -}} + {{- if .Values.namespaceOverride -}} + {{- .Values.namespaceOverride -}} + {{- else -}} + {{- .Release.Namespace -}} + {{- end -}} +{{- end -}} + +{{- define "pushProxy.commonLabels" -}} +release: {{ .Release.Name }} +component: {{ .Values.component | quote }} +provider: kubernetes +{{- end -}} + +{{- define "pushProxy.proxyUrl" -}} +{{- $_ := (required "Template requires either .Values.proxy.port or .Values.client.proxyUrl to set proxyUrl for client" (or .Values.clients.proxyUrl .Values.proxy.port)) -}} +{{- if .Values.clients.proxyUrl -}} +{{ printf "%s" .Values.clients.proxyUrl }} +{{- else -}} +{{ printf "http://%s.%s.svc:%d" (include "pushProxy.proxy.name" .) (include "pushprox.namespace" .) (int .Values.proxy.port) }} +{{- end -}}{{- end -}} + +# Client + +{{- define "pushProxy.client.name" -}} +{{- printf "pushprox-%s-client" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.client.serviceAccountTokenName" -}} +{{- printf "pushprox-%s-client-service-account-token" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.client.labels" -}} +k8s-app: {{ template "pushProxy.client.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +# Proxy + +{{- define "pushProxy.proxy.name" -}} +{{- printf "pushprox-%s-proxy" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.proxy.labels" -}} +k8s-app: {{ template "pushProxy.proxy.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +# ServiceMonitor + +{{- define "pushprox.serviceMonitor.name" -}} +{{- printf "%s-%s" .Release.Name (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.serviceMonitor.labels" -}} +app: {{ template "pushprox.serviceMonitor.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +{{- define "pushProxy.serviceMonitor.endpoints" -}} +{{- $proxyURL := (include "pushProxy.proxyUrl" .) -}} +{{- $useHTTPS := .Values.clients.https.enabled -}} +{{- $setHTTPSScheme := .Values.clients.https.forceHTTPSScheme -}} +{{- $insecureSkipVerify := .Values.clients.https.insecureSkipVerify -}} +{{- $useServiceAccountCredentials := .Values.clients.https.useServiceAccountCredentials -}} +{{- $serviceAccountTokenName := (include "pushProxy.client.serviceAccountTokenName" . ) -}} +{{- $metricRelabelings := list }} +{{- $endpoints := .Values.serviceMonitor.endpoints }} +{{- if .Values.proxy.enabled }} +{{- $_ := set . "proxyUrl" $proxyURL }} +{{- end }} +{{- range $endpoints }} +{{- if $.Values.proxy.enabled }} +{{- $_ := set . "proxyUrl" $proxyURL }} +{{- end }} +{{- $clusterIdRelabel := dict }} +{{- $metricRelabelings := list }} +{{- if $.Values.global.cattle.clusterId }} +{{- $_ := set $clusterIdRelabel "action" "replace" }} +{{- $_ := set $clusterIdRelabel "sourceLabels" (list "__address__") }} +{{- $_ := set $clusterIdRelabel "targetLabel" "cluster_id" }} +{{- $_ := set $clusterIdRelabel "replacement" $.Values.global.cattle.clusterId }} +{{- $metricRelabelings = append $metricRelabelings $clusterIdRelabel }} +{{- end }} +{{- $clusterNameRelabel := dict }} +{{- if $.Values.global.cattle.clusterName }} +{{- $_ := set $clusterNameRelabel "action" "replace" }} +{{- $_ := set $clusterNameRelabel "sourceLabels" (list "__address__") }} +{{- $_ := set $clusterNameRelabel "targetLabel" "cluster_name" }} +{{- $_ := set $clusterNameRelabel "replacement" $.Values.global.cattle.clusterName }} +{{- $metricRelabelings = append $metricRelabelings $clusterNameRelabel }} +{{- end }} +{{- if not (empty $metricRelabelings) }} +{{- $_ := set . "metricRelabelings" ($metricRelabelings)}} +{{- end }} +{{- if $setHTTPSScheme -}} +{{- $_ := set . "scheme" "https" }} +{{- end -}} +{{- if $useHTTPS -}} +{{- if (hasKey . "params") }} +{{- $_ := set (get . "params") "_scheme" (list "https") }} +{{- else }} +{{- $_ := set . "params" (dict "_scheme" (list "https")) }} +{{- end }} +{{- end }} +{{- if (hasKey . "tlsConfig") }} +{{- $_ := set (get . "tlsConfig") "insecureSkipVerify" $insecureSkipVerify }} +{{- else }} +{{- $_ := set . "tlsConfig" (dict "insecureSkipVerify" $insecureSkipVerify) }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.bearerTokenFile.enabled }} +{{- $_ := set . "bearerTokenFile" $.Values.clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.bearerTokenSecret.enabled }} +{{- $_ := set . "bearerTokenSecret" $serviceAccountTokenName }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.authorization.enabled }} +{{- if (hasKey . "authorization") }} +{{- $_ := set (get . "authorization") "type" $.Values.clients.https.authenticationMethod.authorization.type }} +{{- $_ := set (get . "authorization") "credentials" (dict "name" $serviceAccountTokenName "key" $.Values.clients.https.authenticationMethod.authorization.credentials.key "optional" $.Values.clients.https.authenticationMethod.authorization.credentials.optional) }} +{{- else }} +{{- $_ := set . "authorization" (dict "type" $.Values.clients.https.authenticationMethod.authorization.type) }} +{{- $_ := set . "authorization" (dict "credentials" (dict "name" $serviceAccountTokenName "key" $.Values.clients.https.authenticationMethod.authorization.credentials.key "optional" $.Values.clients.https.authenticationMethod.authorization.credentials.optional)) }} +{{- end }} +{{- end }} +{{- end }} +{{- toYaml $endpoints }} +{{- end -}} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/hardenedKubelet/templates/pushprox-clients-rbac.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/hardenedKubelet/templates/pushprox-clients-rbac.yaml new file mode 100644 index 00000000000..a8e27c37356 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/hardenedKubelet/templates/pushprox-clients-rbac.yaml @@ -0,0 +1,97 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.clients }}{{- if .Values.clients.enabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ template "pushProxy.client.name" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +rules: +{{- if .Values.global.cattle.psp.enabled }} +- apiGroups: ['policy'] + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - {{ template "pushProxy.client.name" . }} +{{- end }} +{{- if and .Values.clients.https.enabled .Values.clients.https.useServiceAccountCredentials }} +- nonResourceURLs: ["/metrics"] + verbs: ["get"] +{{- if .Values.clients.rbac.additionalRules }} +{{ toYaml .Values.clients.rbac.additionalRules }} +{{- end }} +{{- end }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ template "pushProxy.client.name" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "pushProxy.client.name" . }} +subjects: + - kind: ServiceAccount + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +--- +{{- if .Values.clients.https.useServiceAccountCredentials }} +apiVersion: v1 +kind: Secret +type: kubernetes.io/service-account-token +metadata: + name: {{ template "pushProxy.client.serviceAccountTokenName" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} + annotations: + kubernetes.io/service-account.name: {{ template "pushProxy.client.name" . }} +{{- end }} +--- +{{- if .Values.global.cattle.psp.enabled }} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +spec: + privileged: false + hostNetwork: true + hostIPC: false + hostPID: false + runAsUser: + rule: 'RunAsAny' + seLinux: + rule: 'RunAsAny' +{{- if and .Values.clients.https.enabled .Values.clients.https.certDir .Values.global.seLinux.enabled .Values.clients.https.seLinuxOptions }} + seLinuxOptions: {{ .Values.clients.https.seLinuxOptions | toYaml | nindent 6 }} +{{- end }} + supplementalGroups: + rule: 'MustRunAs' + ranges: + - min: 0 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + - min: 0 + max: 65535 + readOnlyRootFilesystem: false + volumes: + - 'secret' +{{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + - 'emptyDir' + - 'hostPath' + allowedHostPaths: + - pathPrefix: {{ required "Need access to volume on host with the SSL cert files to use HTTPs" .Values.clients.https.certDir }} + readOnly: true +{{- end }} +{{- end }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/hardenedKubelet/templates/pushprox-clients.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/hardenedKubelet/templates/pushprox-clients.yaml new file mode 100644 index 00000000000..e8fcfb38836 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/hardenedKubelet/templates/pushprox-clients.yaml @@ -0,0 +1,157 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.clients }}{{- if .Values.clients.enabled }} +apiVersion: apps/v1 +{{- if .Values.clients.deployment.enabled }} +kind: Deployment +{{- else }} +kind: DaemonSet +{{- end }} +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} + pushprox-exporter: "client" +spec: + {{- if .Values.clients.deployment.enabled }} + replicas: {{ .Values.clients.deployment.replicas }} + {{- end }} + selector: + matchLabels: {{ include "pushProxy.client.labels" . | nindent 6 }} + template: + metadata: + labels: {{ include "pushProxy.client.labels" . | nindent 8 }} + spec: + {{- if .Values.clients.affinity }} + affinity: {{ toYaml .Values.clients.affinity | nindent 8 }} + {{- end }} + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} +{{- if .Values.clients.nodeSelector }} +{{ toYaml .Values.clients.nodeSelector | indent 8 }} +{{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} +{{- if .Values.clients.tolerations }} +{{ toYaml .Values.clients.tolerations | indent 8 }} +{{- end }} + hostNetwork: true + dnsPolicy: ClusterFirstWithHostNet + serviceAccountName: {{ template "pushProxy.client.name" . }} + {{- if .Values.global.imagePullSecretName }} + imagePullSecrets: + - name: {{ .Values.global.imagePullSecretName }} + {{- end }} + containers: + - name: pushprox-client + image: {{ template "system_default_registry" . }}{{ .Values.clients.image.repository }}:{{ .Values.clients.image.tag }} + command: + {{- range .Values.clients.command }} + - {{ . | quote }} + {{- end }} + args: + - --fqdn=$(HOST_IP) + - --proxy-url=$(PROXY_URL) + {{- if .Values.clients.metrics.enabled }} + - --metrics-addr=$(PORT) + {{- end }} + - --allow-port={{ required "Need .Values.metricsPort to configure client to be allowed to scrape metrics at port" .Values.metricsPort}} + {{- if .Values.clients.useLocalhost }} + - --use-localhost + {{- end }} + {{- if .Values.clients.https.enabled }} + {{- if .Values.clients.https.insecureSkipVerify }} + - --insecure-skip-verify + {{- end }} + {{- if .Values.clients.https.useServiceAccountCredentials }} + - --token-path=/var/run/secrets/kubernetes.io/serviceaccount/token + {{- end }} + {{- if .Values.clients.https.certDir }} + - --tls.cert=/etc/ssl/push-proxy/push-proxy.pem + - --tls.key=/etc/ssl/push-proxy/push-proxy-key.pem + - --tls.cacert=/etc/ssl/push-proxy/push-proxy-ca-cert.pem + {{- end }} + {{- end }} + env: + - name: HOST_IP + valueFrom: + fieldRef: + fieldPath: status.hostIP + {{- if .Values.clients.metrics.enabled }} + - name: PORT + value: :{{ .Values.clients.port }} + {{- end }} + - name: PROXY_URL + value: {{ template "pushProxy.proxyUrl" . }} + securityContext: + runAsNonRoot: true + runAsUser: 1000 + {{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + volumeMounts: + - name: metrics-cert-dir + mountPath: /etc/ssl/push-proxy + {{- end }} + {{- if .Values.clients.resources }} + resources: {{ toYaml .Values.clients.resources | nindent 10 }} + {{- end }} + {{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + initContainers: + - name: copy-certs + image: {{ template "system_default_registry" . }}{{ .Values.clients.copyCertsImage.repository }}:{{ .Values.clients.copyCertsImage.tag }} + command: + - sh + - -c + - | + echo "Searching for files to copy within the source volume" + echo "cert: ${CERT_FILE_NAME}" + echo "key: ${KEY_FILE_NAME}" + echo "cacert: ${CACERT_FILE_NAME}" + + CERT_FILE_SOURCE=$(find /etc/source/ -type f -name "${CERT_FILE_NAME}" | sort -r | head -n 1) + KEY_FILE_SOURCE=$(find /etc/source/ -type f -name "${KEY_FILE_NAME}" | sort -r | head -n 1) + CACERT_FILE_SOURCE=$(find /etc/source/ -type f -name "${CACERT_FILE_NAME}" | sort -r | head -n 1) + + test -z ${CERT_FILE_SOURCE} && echo "Failed to find cert file" && exit 1 + test -z ${KEY_FILE_SOURCE} && echo "Failed to find key file" && exit 1 + test -z ${CACERT_FILE_SOURCE} && echo "Failed to find cacert file" && exit 1 + + echo "Copying cert file from $CERT_FILE_SOURCE to $CERT_FILE_TARGET" + cp $CERT_FILE_SOURCE $CERT_FILE_TARGET || exit 1 + chmod 444 $CERT_FILE_TARGET || exit 1 + + echo "Copying key file from $KEY_FILE_SOURCE to $KEY_FILE_TARGET" + cp $KEY_FILE_SOURCE $KEY_FILE_TARGET || exit 1 + chmod 444 $KEY_FILE_TARGET || exit 1 + + echo "Copying cacert file from $CACERT_FILE_SOURCE to $CACERT_FILE_TARGET" + cp $CACERT_FILE_SOURCE $CACERT_FILE_TARGET || exit 1 + chmod 444 $CACERT_FILE_TARGET || exit 1 + env: + - name: CERT_FILE_NAME + value: {{ required "Need a TLS cert file for scraping metrics endpoint over HTTPs" .Values.clients.https.certFile }} + - name: KEY_FILE_NAME + value: {{ required "Need a TLS key file for scraping metrics endpoint over HTTPs" .Values.clients.https.keyFile }} + - name: CACERT_FILE_NAME + value: {{ required "Need a TLS CA cert file for scraping metrics endpoint over HTTPs" .Values.clients.https.caCertFile }} + - name: CERT_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy.pem + - name: KEY_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy-key.pem + - name: CACERT_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy-ca-cert.pem + securityContext: + runAsNonRoot: false +{{- if and .Values.global.seLinux.enabled .Values.clients.https.seLinuxOptions }} + seLinuxOptions: {{ .Values.clients.https.seLinuxOptions | toYaml | nindent 12 }} +{{- end }} + volumeMounts: + - name: metrics-cert-dir-source + mountPath: /etc/source + readOnly: true + - name: metrics-cert-dir + mountPath: /etc/ssl/push-proxy + volumes: + - name: metrics-cert-dir-source + hostPath: + path: {{ required "Need access to volume on host with the SSL cert files to use HTTPs" .Values.clients.https.certDir }} + - name: metrics-cert-dir + emptyDir: {} + {{- end }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/hardenedKubelet/templates/pushprox-proxy-rbac.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/hardenedKubelet/templates/pushprox-proxy-rbac.yaml new file mode 100644 index 00000000000..eefe609058c --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/hardenedKubelet/templates/pushprox-proxy-rbac.yaml @@ -0,0 +1,68 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if and .Values.proxy }}{{ if .Values.proxy.enabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ template "pushProxy.proxy.name" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +rules: +{{- if .Values.global.cattle.psp.enabled }} +- apiGroups: ['policy'] + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - {{ template "pushProxy.proxy.name" . }} +{{- end }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ template "pushProxy.proxy.name" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "pushProxy.proxy.name" . }} +subjects: + - kind: ServiceAccount + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +--- +{{- if .Values.global.cattle.psp.enabled }} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +spec: + privileged: false + hostNetwork: false + hostIPC: false + hostPID: false + runAsUser: + rule: 'MustRunAsNonRoot' + seLinux: + rule: 'RunAsAny' + supplementalGroups: + rule: 'MustRunAs' + ranges: + - min: 1 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + - min: 1 + max: 65535 + readOnlyRootFilesystem: false + volumes: + - 'secret' +{{- end }}{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/hardenedKubelet/templates/pushprox-proxy.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/hardenedKubelet/templates/pushprox-proxy.yaml new file mode 100644 index 00000000000..723bbd6c00b --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/hardenedKubelet/templates/pushprox-proxy.yaml @@ -0,0 +1,57 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if and .Values.proxy }}{{ if .Values.proxy.enabled }} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} + pushprox-exporter: "proxy" +spec: + selector: + matchLabels: {{ include "pushProxy.proxy.labels" . | nindent 6 }} + template: + metadata: + labels: {{ include "pushProxy.proxy.labels" . | nindent 8 }} + spec: + securityContext: + runAsNonRoot: true + runAsUser: 1000 + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} +{{- if .Values.proxy.nodeSelector }} +{{ toYaml .Values.proxy.nodeSelector | indent 8 }} +{{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} +{{- if .Values.proxy.tolerations }} +{{ toYaml .Values.proxy.tolerations | indent 8 }} +{{- end }} + serviceAccountName: {{ template "pushProxy.proxy.name" . }} + {{- if .Values.global.imagePullSecretName }} + imagePullSecrets: + - name: {{ .Values.global.imagePullSecretName }} + {{- end }} + containers: + - name: pushprox-proxy + image: {{ template "system_default_registry" . }}{{ .Values.proxy.image.repository }}:{{ .Values.proxy.image.tag }} + command: + {{- range .Values.proxy.command }} + - {{ . | quote }} + {{- end }} + {{- if .Values.proxy.resources }} + resources: {{ toYaml .Values.proxy.resources | nindent 10 }} + {{- end }} +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +spec: + ports: + - name: pp-proxy + port: {{ required "Need .Values.proxy.port to configure proxy" .Values.proxy.port }} + protocol: TCP + targetPort: {{ .Values.proxy.port }} + selector: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/hardenedKubelet/templates/pushprox-servicemonitor.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/hardenedKubelet/templates/pushprox-servicemonitor.yaml new file mode 100644 index 00000000000..67eb2216b5a --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/hardenedKubelet/templates/pushprox-servicemonitor.yaml @@ -0,0 +1,45 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.serviceMonitor }}{{- if .Values.serviceMonitor.enabled }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ template "pushprox.serviceMonitor.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.serviceMonitor.labels" . | nindent 4 }} +spec: + endpoints: {{include "pushProxy.serviceMonitor.endpoints" . | nindent 4 }} + jobLabel: component + podTargetLabels: + - component + - pushprox-exporter + namespaceSelector: + matchNames: + - {{ template "pushprox.namespace" . }} + selector: + matchLabels: {{ include "pushProxy.client.labels" . | nindent 6 }} +--- +{{- $selector := "" }} +{{- if not (kindIs "invalid" .Values.service) }} +{{- if not (kindIs "invalid" .Values.service.selector) }} +{{ if .Values.service.selector }} +{{- if .Values.clients.enabled }} +{{- required (printf "Cannot override .Values.service.selector=%s when .Values.clients.enabled=true" (toJson .Values.service.selector)) "" }} +{{- end }} +{{- $selector = (toYaml .Values.service.selector) }} +{{- end }} +{{- end }} +{{- end }} +apiVersion: v1 +kind: Service +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +spec: + ports: + - name: metrics + port: {{ required "Need .Values.metricsPort to configure client to listen to metrics at port" .Values.metricsPort}} + protocol: TCP + targetPort: {{ .Values.metricsPort }} + selector: {{ default (include "pushProxy.client.labels" .) $selector | nindent 4 }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/hardenedKubelet/templates/validate-install-crd.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/hardenedKubelet/templates/validate-install-crd.yaml new file mode 100644 index 00000000000..16abc2fa83a --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/hardenedKubelet/templates/validate-install-crd.yaml @@ -0,0 +1,14 @@ +#{{- if gt (len (lookup "rbac.authorization.k8s.io/v1" "ClusterRole" "" "")) 0 -}} +# {{- $found := dict -}} +# {{- set $found "monitoring.coreos.com/v1/ServiceMonitor" false -}} +# {{- range .Capabilities.APIVersions -}} +# {{- if hasKey $found (toString .) -}} +# {{- set $found (toString .) true -}} +# {{- end -}} +# {{- end -}} +# {{- range $_, $exists := $found -}} +# {{- if (eq $exists false) -}} +# {{- required "Required CRDs are missing. Please install Prometheus Operator CRDs before installing this chart." "" -}} +# {{- end -}} +# {{- end -}} +#{{- end -}} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/hardenedKubelet/templates/validate-psp-install.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/hardenedKubelet/templates/validate-psp-install.yaml new file mode 100644 index 00000000000..a30c59d3b7c --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/hardenedKubelet/templates/validate-psp-install.yaml @@ -0,0 +1,7 @@ +#{{- if gt (len (lookup "rbac.authorization.k8s.io/v1" "ClusterRole" "" "")) 0 -}} +#{{- if .Values.global.cattle.psp.enabled }} +#{{- if not (.Capabilities.APIVersions.Has "policy/v1beta1/PodSecurityPolicy") }} +#{{- fail "The target cluster does not have the PodSecurityPolicy API resource. Please disable PSPs in this chart before proceeding." -}} +#{{- end }} +#{{- end }} +#{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/hardenedKubelet/values.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/hardenedKubelet/values.yaml new file mode 100644 index 00000000000..7f38498a1c5 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/hardenedKubelet/values.yaml @@ -0,0 +1,166 @@ +# Default values for rancher-pushprox. +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. + +# Default image containing both the proxy and the client was generated from the following Dockerfile +# https://github.com/prometheus-community/PushProx/blob/eeadbe766641699129920ccfaaaa30a85c67fe81/Dockerfile#L1-L15 + +# Configuration + +global: + cattle: + psp: + enabled: false + systemDefaultRegistry: "" + seLinux: + enabled: false + +# A list of Semver constraint strings (defined by https://github.com/Masterminds/semver) and values.yaml overrides. +# +# For each key in kubeVersionOverrides, this chart will check to see if the current Kubernetes cluster's version matches +# any of the semver constraints provided as keys on the map. +# +# On seeing a match, the default value for each values.yaml field overridden will be updated with the new value. +# +# If multiple matches are encountered (due to overlapping semver ranges), the matches will be applied in order. +# +# Notes: +# - On running a helm template, Helm generally assumes the kubeVersion is v1.20.0 +# - On running a helm install --dry-run, the correct kubeVersion should be chosen. +kubeVersionOverrides: [] +# - constraint: "< 1.21" +# values: +# metricsPort: 10252 +# clients: +# https: +# enabled: false +# insecureSkipVerify: false +# useServiceAccountCredentials: false + +namespaceOverride: "" + +# The component that is being monitored (i.e. etcd) +component: "component" + +# The port containing the metrics that need to be scraped +metricsPort: 2739 + +# Configure ServiceMonitor that monitors metrics from the metricsPort endpoint +serviceMonitor: + enabled: true + # A list of endpoints that will be added to the ServiceMonitor based on the Endpoint spec + # Source: https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#endpoint + # By default, proxyUrl and params._scheme will be overridden based on other values + endpoints: + - port: metrics + +# Configure Service that grabs scrape targets +service: + # The selector that is used to populate the Service's Endpoints object. + # The chart will error out on rendering templating if .Values.clients.enabled is set alongside this field, + # since it is expected that this service should point to the PushProx Clients Daemonset / Deployment + selector: {} + +clients: + enabled: true + # The port which the PushProx client will post PushProx metrics to + port: 9369 + # If unset, this will default to the URL for the proxy service: http://pushprox-{{component}}-proxy.{{namepsace}}.svc.cluster.local:{{proxy.port}} + # Should be modified if the clients are being deployed outside the cluster where the proxy rests, otherwise leave it null + proxyUrl: "" + # If set to true, the client will forward any requests from the host IP to 127.0.0.1 + # It will only allow proxy requests to the metricsPort specified + useLocalhost: false + # Configuration for accessing metrics via HTTPS + https: + # Does the client require https to access the metrics? + enabled: false + # Does the client require requests be sent to http or https? + forceHTTPSScheme: false + # If set to true, the client will create a service account with adequate permissions and set a flag + # on the client to use the service account token provided by it to make authorized scrape requests + useServiceAccountCredentials: false + # Configuration for authentication to metrics via https endpoint + authenticationMethod: + # Reads token from defined file in container + # This function is deprecated in the prometheus operator api and may be removed in a future version + bearerTokenFile: + enabled: false + bearerTokenFilePath: "/var/run/secrets/kubernetes.io/serviceaccount/token" + # Reads token from defined secret in namespace + # This function is deprecated in the prometheus operator api and may be removed in a future version + bearerTokenSecret: + enabled: false + # Reads token from defined secret in namespace + authorization: + enabled: false + type: "bearer" + credentials: + key: "token" + optional: false + # If set to true, the client will disable SSL security checks + insecureSkipVerify: false + # Directory on host where necessary TLS cert and key to scrape metrics can be found + certDir: "" + # Filenames for files located in .Values.clients.https.certDir that correspond to TLS settings + certFile: "" + keyFile: "" + caCertFile: "" + # seLinuxOptions to be passed into the container that copies certs. Should define a container with permissions to read the files in the certDir provided on the host. + # Required and only used if `clients.https.enabled` is set and `clients.https.certDir` is provided. + seLinuxOptions: {} + + metrics: + # Whether the client should publish PushProx client-specific metrics to .Values.clients.port + enabled: false + + rbac: + # Additional permissions to provide to the ServiceAccount bound to the client + # This can be used to provide additional permissions for the client to scrape metrics from the k8s API + # Only enabled if clients.https.enabled and clients.https.useServiceAccountCredentials are true + additionalRules: [] + + # Resource limits + resources: {} + + # Options to select all nodes to deploy client DaemonSet on + nodeSelector: {} + tolerations: [] + affinity: {} + + image: + repository: rancher/pushprox-client + tag: v0.1.4-rancher2-client + command: ["pushprox-client"] + + copyCertsImage: + repository: rancher/mirrored-library-busybox + tag: 1.31.1 + + # The default intention of rancher-pushprox clients is to scrape hostNetwork metrics across all nodes. + # This can be used to scrape internal Kubernetes components or DaemonSets of hostNetwork Pods in + # situations where a cloud provider firewall prevents Pod-To-Host communication but not Pod-To-Pod. + # However, if the underlying hostNetwork Pod that is being scraped is managed by a Deployment, + # this advanced option enables users to deploy the client as a Deployment instead of a DaemonSet. + # If a user deploys this feature and the underlying Deployment's number of replicas changes, the user will + # be responsible for upgrading this chart accordingly to the right number of replicas. + deployment: + enabled: false + replicas: 0 + +proxy: + enabled: true + # The port through which PushProx clients will communicate to the proxy + port: 8080 + + # Resource limits + resources: {} + + # Options to select a node to run a single proxy deployment on + nodeSelector: {} + tolerations: [] + + image: + repository: rancher/pushprox-proxy + tag: v0.1.4-rancher2-proxy + command: ["pushprox-proxy"] diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/hardenedNodeExporter/.helmignore b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/hardenedNodeExporter/.helmignore new file mode 100644 index 00000000000..0e8a0eb36f4 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/hardenedNodeExporter/.helmignore @@ -0,0 +1,23 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*.orig +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/hardenedNodeExporter/Chart.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/hardenedNodeExporter/Chart.yaml new file mode 100644 index 00000000000..1efee00e2f6 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/hardenedNodeExporter/Chart.yaml @@ -0,0 +1,15 @@ +annotations: + catalog.cattle.io/hidden: "true" + catalog.cattle.io/kube-version: '>= 1.26.0-0 < 1.31.0-0' + catalog.cattle.io/os: linux + catalog.rancher.io/certified: rancher + catalog.rancher.io/namespace: cattle-monitoring-system + catalog.rancher.io/release-name: rancher-pushprox +apiVersion: v1 +appVersion: v0.1.4-rancher2 +description: Sets up a deployment of the PushProx proxy and a DaemonSet of PushProx + clients. +kubeVersion: '>=1.26.0-0' +name: hardenedNodeExporter +type: application +version: 0.2.1 diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/hardenedNodeExporter/README.md b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/hardenedNodeExporter/README.md new file mode 100644 index 00000000000..345002f48a2 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/hardenedNodeExporter/README.md @@ -0,0 +1,90 @@ +# rancher-pushprox + +A Rancher chart based on Rancher [PushProx](https://github.com/rancher/PushProx) that sets up a Deployment of a PushProx proxy and a DaemonSet of PushProx clients on a Kubernetes cluster. + +Installs [rancher-pushprox](https://github.com/rancher/charts/tree/gh-pages/packages/rancher-pushprox) to create PushProx clients that can access their host's network and register with a PushProx proxy. A [Prometheus Operator](https://github.com/coreos/prometheus-operator) ServiceMonitor CR is also included that is configured to scrape the metrics from each of the clients through the proxy. + +Using an instance of this chart is suitable for the following scenarios: +- You need to scrape metrics from a port that should not be accessible outside of the host (e.g. scraping `etcd` metrics in a hardened cluster) +- You need to scrape metrics on a host that are not exposed outside of 127.0.0.1 (e.g. scraping `kube-proxy` metrics) +- You need to scrape metrics through HTTPS using certs hosted directly on `hostPath` +- You need to scrape metrics from Kubernetes components that require authorization via a service account (e.g. permissions to make request to `/metrics`) +- You need to scrape metrics without access to cacerts (i.e. enable `insecureSkipVerify`) + +The clients and proxy are created based on a Rancher fork of the [prometheus-community/PushProx](https://github.com/prometheus-community/PushProx) project. + +## Upgrading to Kubernetes v1.25+ + +Starting in Kubernetes v1.25, [Pod Security Policies](https://kubernetes.io/docs/concepts/security/pod-security-policy/) have been removed from the Kubernetes API. + +As a result, **before upgrading to Kubernetes v1.25** (or on a fresh install in a Kubernetes v1.25+ cluster), users are expected to perform an in-place upgrade of this chart with `global.cattle.psp.enabled` set to `false` if it has been previously set to `true`. +​ +> **Note:** +> In this chart release, any previous field that was associated with any PSP resources have been removed in favor of a single global field: `global.cattle.psp.enabled`. + +> **Note:** +> If you upgrade your cluster to Kubernetes v1.25+ before removing PSPs via a `helm upgrade` (even if you manually clean up resources), **it will leave the Helm release in a broken state within the cluster such that further Helm operations will not work (`helm uninstall`, `helm upgrade`, etc.).** +> +> If your charts get stuck in this state, please consult the Rancher docs on how to clean up your Helm release secrets. + +Upon setting `global.cattle.psp.enabled` to false, the chart will remove any PSP resources deployed on its behalf from the cluster. This is the default setting for this chart. + +As a replacement for PSPs, [Pod Security Admission](https://kubernetes.io/docs/concepts/security/pod-security-admission/) should be used. Please consult the Rancher docs for more details on how to configure your chart release namespaces to work with the new Pod Security Admission and apply Pod Security Standards. + +## Configuration + +The following tables list the configurable parameters of the rancher-pushprox chart and their default values. + +### General + +#### Required +| Parameter | Description | Example | +| ----- | ----------- | ------ | +| `component` | The component that is being monitored | `kube-etcd` +| `metricsPort` | The port on the host that contains the metrics you want to scrape (e.g. `http://:/metrics`) | `2379` | +| `namespaceOverride` | The namespace to install the chart | `""` + +#### Optional +| Parameter | Description | Default | +| ----- | ----------- | ------ | +| `serviceMonitor.enabled` | Deploys a [Prometheus Operator](https://github.com/coreos/prometheus-operator/blob/master/Documentation/api.md#servicemonitor) ServiceMonitor CR that is configured to scrape metrics on the hosts that the clients are deployed on via the proxy. Also deploys a Service that points to all pods with the expected client name that exposes the `metricsPort` selected | `true` | +| `serviceMonitor.endpoints` | A list of endpoints that will be added to the ServiceMonitor based on the [Endpoint spec](https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#endpoint) | `[{port: metrics}]` | +| `service.selector` | The selector that is used to populate the Service's Endpoints object. The chart will error out on rendering templating if `.Values.clients.enabled` is set alongside this field, since it is expected that this service should point to the PushProx Clients Daemonset / Deployment | `{}` | +| `clients.enabled` | Deploys a DaemonSet of clients that are each capable of scraping endpoints on the hostNetwork it is deployed on | `true` | +| `clients.port` | The port where the client will publish PushProx client-specific metrics. If deploying multiple clients onto the same node, the clients should not have conflicting ports | `9369` | +| `clients.proxyUrl` | Overrides the default proxyUrl setting of `http://pushprox-{{ .Values.component }}-proxy.{{ . Release.Namespace }}.svc.cluster.local:{{ .Values.proxy.port }}"` with the `proxyUrl` specified | `""` | +| `clients.useLocalhost` | Sets a flag on each client deployment to redirect scrapes directed to `HOST_IP` to `127.0.0.1` | `false` | +| `clients.https.enabled` | Enables scraping metrics via HTTPS using the provided TLS certs that exist on each host | `false` | +| `clients.https.forceHTTPSScheme` | Forces scraping metrics via HTTPS using the provided TLS certs that exist on each host | `false` | +| `clients.https.useServiceAccountCredentials` | If set to true, the client will create a service account with permissions to scrape `/metrics` endpoint of Kubernetes components. The client will use the service account token provided to make authorized scrape requests to the Kubernetes API | `false` | +| `clients.https.authenticationMethod.bearerTokenFile.enabled` | If set to true, the client will use service account credentials mounted at the configured path `clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath`. This requires permissions to scrape `/metrics` endpoint of Kubernetes components. This method is deprecated by the prometheus operator and may be removed in a future release | `false` | +| `clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath` | This is a volume mount on the pod with permissions to scrape `/metrics` endpoint of Kubernetes components | `"/var/run/secrets/kubernetes.io/serviceaccount/token"` | +| `clients.https.authenticationMethod.bearerTokenSecret.enabled` | If set to true, the client will use service account credentials to scrape `/metrics` endpoint of Kubernetes components. This method is deprecated by the prometheus operator and may be removed in a future release | `false` | +| `clients.https.authenticationMethod.authorization.enabled` | If set to true, the client will use service account credentials to scrape `/metrics` endpoint of Kubernetes components | `false` | +| `clients.https.authenticationMethod.authorization.type` | If set, the client will use this type of authorization in its client requests for metrics | `"bearer"` | +| `clients.https.authenticationMethod.authorization.credentials.key` | If set, the client will use this key in the secret created by `clients.https.useServiceAccountCredentials` for authorization in its client requests for metrics | `"token"` | +| `clients.https.authenticationMethod.authorization.credentials.optional` | If set to false, the client will fail if the key in the secret created by `clients.https.useServiceAccountCredentials` does not exist | `false` | +| `clients.https.insecureSkipVerify` | If set to true, the client will disable SSL security checks | `false` | +| `clients.https.certDir` | A `hostPath` where TLS certs can be found. This path is mounted as a volume on an `initContainer` which copies only the necessary files over to an EmptyDir volume used by each client. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.certFile` | The path to the TLS cert file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.keyFile` | The path to the TLS key file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.caCertFile` | The path to the TLS cacert file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.seLinuxOptions` | seLinuxOptions to be passed into the container that copies certs. Should define a container with permissions to read the files in the certDir provided on the host. Required and only used if `clients.https.enabled` is set and `clients.https.certDir` is provided. | `""` | +| `clients.metrics.enabled` | Whether the client should publish PushProx client-specific metrics. | `false` | +| `clients.rbac.additionalRules` | Additional permissions to provide to the ServiceAccount bound to the client. This can be used to provide additional permissions for the client to scrape metrics from the k8s API. Only enabled if clients.https.enabled and clients.https.useServiceAccountCredentials are true | `[]` | +| `clients.deployment.enabled` | Deploys the client as a Deployment (generally used if the underlying hostNetwork Pod that is being scraped is managed by a Deployment) | `false` | +| `clients.deployment.replicas` | The number of pods the Deployment has, it should match the number of pod the hostNetwork Deployment has. Required and only used if `client.deployment.enable` is set | `0` | +| `clients.deployment.affinity` | The affinity rules that allocate the pod to the node in which the hostNetwork Deployment's pods run. Required and only used if `client.deployment.enable` is set | `{}` | +| `clients.resources` | Set resource limits and requests for the client container | `{}` | +| `clients.nodeSelector` | Select which nodes to deploy the clients on | `{}` | +| `clients.tolerations` | Specify tolerations for clients | `[]` | +| `proxy.enabled` | Deploys the proxy that each client will register with | `true` | +| `proxy.port` | The port exposed by the proxy that each client will register with to allow metrics to be scraped from the host | `8080` | +| `proxy.resources` | Set resource limits and requests for the proxy container | `{}` | +| `proxy.nodeSelector` | Select which nodes the proxy can be deployed on | `{}` | +| `proxy.tolerations` | Specify tolerations (if necessary) to allow the proxy to be deployed on the selected node | `[]` | +| `kubeVersionOverrides` | A list of Semver constraint strings (defined by https://github.com/Masterminds/semver) and values.yaml overrides. For each key in kubeVersionOverrides, this chart will check to see if the current Kubernetes cluster's version matches any of the semver constraints provided as keys on the map. On seeing a match, the default value for each values.yaml field overridden will be updated with the new value. If multiple matches are encountered (due to overlapping semver ranges), the matches will be applied in order. | `[]` + +*Tip: The filepaths set in `clients.https.File` can include wildcard characters*. + +See [rancher-monitoring](https://github.com/rancher/charts/tree/gh-pages/packages/rancher-monitoring) for examples of how this chart can be used. diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/hardenedNodeExporter/templates/_helpers.tpl b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/hardenedNodeExporter/templates/_helpers.tpl new file mode 100644 index 00000000000..1ba50939441 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/hardenedNodeExporter/templates/_helpers.tpl @@ -0,0 +1,170 @@ +# Rancher + +{{- define "system_default_registry" -}} +{{- if .Values.global.cattle.systemDefaultRegistry -}} +{{- printf "%s/" .Values.global.cattle.systemDefaultRegistry -}} +{{- end -}} +{{- end -}} + +# Windows Support + +{{/* +Windows cluster will add default taint for linux nodes, +add below linux tolerations to workloads could be scheduled to those linux nodes +*/}} + +{{- define "linux-node-tolerations" -}} +- key: "cattle.io/os" + value: "linux" + effect: "NoSchedule" + operator: "Equal" +{{- end -}} + +{{- define "linux-node-selector" -}} +{{- if semverCompare "<1.14-0" .Capabilities.KubeVersion.GitVersion -}} +beta.kubernetes.io/os: linux +{{- else -}} +kubernetes.io/os: linux +{{- end -}} +{{- end -}} + +# General + +{{- define "applyKubeVersionOverrides" -}} +{{- $overrides := dict -}} +{{- range $override := .Values.kubeVersionOverrides -}} +{{- if semverCompare $override.constraint $.Capabilities.KubeVersion.Version -}} +{{- $_ := mergeOverwrite $overrides $override.values -}} +{{- end -}} +{{- end -}} +{{- $_ := mergeOverwrite .Values $overrides -}} +{{- end -}} + +{{- define "pushprox.namespace" -}} + {{- if .Values.namespaceOverride -}} + {{- .Values.namespaceOverride -}} + {{- else -}} + {{- .Release.Namespace -}} + {{- end -}} +{{- end -}} + +{{- define "pushProxy.commonLabels" -}} +release: {{ .Release.Name }} +component: {{ .Values.component | quote }} +provider: kubernetes +{{- end -}} + +{{- define "pushProxy.proxyUrl" -}} +{{- $_ := (required "Template requires either .Values.proxy.port or .Values.client.proxyUrl to set proxyUrl for client" (or .Values.clients.proxyUrl .Values.proxy.port)) -}} +{{- if .Values.clients.proxyUrl -}} +{{ printf "%s" .Values.clients.proxyUrl }} +{{- else -}} +{{ printf "http://%s.%s.svc:%d" (include "pushProxy.proxy.name" .) (include "pushprox.namespace" .) (int .Values.proxy.port) }} +{{- end -}}{{- end -}} + +# Client + +{{- define "pushProxy.client.name" -}} +{{- printf "pushprox-%s-client" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.client.serviceAccountTokenName" -}} +{{- printf "pushprox-%s-client-service-account-token" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.client.labels" -}} +k8s-app: {{ template "pushProxy.client.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +# Proxy + +{{- define "pushProxy.proxy.name" -}} +{{- printf "pushprox-%s-proxy" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.proxy.labels" -}} +k8s-app: {{ template "pushProxy.proxy.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +# ServiceMonitor + +{{- define "pushprox.serviceMonitor.name" -}} +{{- printf "%s-%s" .Release.Name (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.serviceMonitor.labels" -}} +app: {{ template "pushprox.serviceMonitor.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +{{- define "pushProxy.serviceMonitor.endpoints" -}} +{{- $proxyURL := (include "pushProxy.proxyUrl" .) -}} +{{- $useHTTPS := .Values.clients.https.enabled -}} +{{- $setHTTPSScheme := .Values.clients.https.forceHTTPSScheme -}} +{{- $insecureSkipVerify := .Values.clients.https.insecureSkipVerify -}} +{{- $useServiceAccountCredentials := .Values.clients.https.useServiceAccountCredentials -}} +{{- $serviceAccountTokenName := (include "pushProxy.client.serviceAccountTokenName" . ) -}} +{{- $metricRelabelings := list }} +{{- $endpoints := .Values.serviceMonitor.endpoints }} +{{- if .Values.proxy.enabled }} +{{- $_ := set . "proxyUrl" $proxyURL }} +{{- end }} +{{- range $endpoints }} +{{- if $.Values.proxy.enabled }} +{{- $_ := set . "proxyUrl" $proxyURL }} +{{- end }} +{{- $clusterIdRelabel := dict }} +{{- $metricRelabelings := list }} +{{- if $.Values.global.cattle.clusterId }} +{{- $_ := set $clusterIdRelabel "action" "replace" }} +{{- $_ := set $clusterIdRelabel "sourceLabels" (list "__address__") }} +{{- $_ := set $clusterIdRelabel "targetLabel" "cluster_id" }} +{{- $_ := set $clusterIdRelabel "replacement" $.Values.global.cattle.clusterId }} +{{- $metricRelabelings = append $metricRelabelings $clusterIdRelabel }} +{{- end }} +{{- $clusterNameRelabel := dict }} +{{- if $.Values.global.cattle.clusterName }} +{{- $_ := set $clusterNameRelabel "action" "replace" }} +{{- $_ := set $clusterNameRelabel "sourceLabels" (list "__address__") }} +{{- $_ := set $clusterNameRelabel "targetLabel" "cluster_name" }} +{{- $_ := set $clusterNameRelabel "replacement" $.Values.global.cattle.clusterName }} +{{- $metricRelabelings = append $metricRelabelings $clusterNameRelabel }} +{{- end }} +{{- if not (empty $metricRelabelings) }} +{{- $_ := set . "metricRelabelings" ($metricRelabelings)}} +{{- end }} +{{- if $setHTTPSScheme -}} +{{- $_ := set . "scheme" "https" }} +{{- end -}} +{{- if $useHTTPS -}} +{{- if (hasKey . "params") }} +{{- $_ := set (get . "params") "_scheme" (list "https") }} +{{- else }} +{{- $_ := set . "params" (dict "_scheme" (list "https")) }} +{{- end }} +{{- end }} +{{- if (hasKey . "tlsConfig") }} +{{- $_ := set (get . "tlsConfig") "insecureSkipVerify" $insecureSkipVerify }} +{{- else }} +{{- $_ := set . "tlsConfig" (dict "insecureSkipVerify" $insecureSkipVerify) }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.bearerTokenFile.enabled }} +{{- $_ := set . "bearerTokenFile" $.Values.clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.bearerTokenSecret.enabled }} +{{- $_ := set . "bearerTokenSecret" $serviceAccountTokenName }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.authorization.enabled }} +{{- if (hasKey . "authorization") }} +{{- $_ := set (get . "authorization") "type" $.Values.clients.https.authenticationMethod.authorization.type }} +{{- $_ := set (get . "authorization") "credentials" (dict "name" $serviceAccountTokenName "key" $.Values.clients.https.authenticationMethod.authorization.credentials.key "optional" $.Values.clients.https.authenticationMethod.authorization.credentials.optional) }} +{{- else }} +{{- $_ := set . "authorization" (dict "type" $.Values.clients.https.authenticationMethod.authorization.type) }} +{{- $_ := set . "authorization" (dict "credentials" (dict "name" $serviceAccountTokenName "key" $.Values.clients.https.authenticationMethod.authorization.credentials.key "optional" $.Values.clients.https.authenticationMethod.authorization.credentials.optional)) }} +{{- end }} +{{- end }} +{{- end }} +{{- toYaml $endpoints }} +{{- end -}} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/hardenedNodeExporter/templates/pushprox-clients-rbac.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/hardenedNodeExporter/templates/pushprox-clients-rbac.yaml new file mode 100644 index 00000000000..a8e27c37356 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/hardenedNodeExporter/templates/pushprox-clients-rbac.yaml @@ -0,0 +1,97 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.clients }}{{- if .Values.clients.enabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ template "pushProxy.client.name" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +rules: +{{- if .Values.global.cattle.psp.enabled }} +- apiGroups: ['policy'] + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - {{ template "pushProxy.client.name" . }} +{{- end }} +{{- if and .Values.clients.https.enabled .Values.clients.https.useServiceAccountCredentials }} +- nonResourceURLs: ["/metrics"] + verbs: ["get"] +{{- if .Values.clients.rbac.additionalRules }} +{{ toYaml .Values.clients.rbac.additionalRules }} +{{- end }} +{{- end }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ template "pushProxy.client.name" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "pushProxy.client.name" . }} +subjects: + - kind: ServiceAccount + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +--- +{{- if .Values.clients.https.useServiceAccountCredentials }} +apiVersion: v1 +kind: Secret +type: kubernetes.io/service-account-token +metadata: + name: {{ template "pushProxy.client.serviceAccountTokenName" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} + annotations: + kubernetes.io/service-account.name: {{ template "pushProxy.client.name" . }} +{{- end }} +--- +{{- if .Values.global.cattle.psp.enabled }} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +spec: + privileged: false + hostNetwork: true + hostIPC: false + hostPID: false + runAsUser: + rule: 'RunAsAny' + seLinux: + rule: 'RunAsAny' +{{- if and .Values.clients.https.enabled .Values.clients.https.certDir .Values.global.seLinux.enabled .Values.clients.https.seLinuxOptions }} + seLinuxOptions: {{ .Values.clients.https.seLinuxOptions | toYaml | nindent 6 }} +{{- end }} + supplementalGroups: + rule: 'MustRunAs' + ranges: + - min: 0 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + - min: 0 + max: 65535 + readOnlyRootFilesystem: false + volumes: + - 'secret' +{{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + - 'emptyDir' + - 'hostPath' + allowedHostPaths: + - pathPrefix: {{ required "Need access to volume on host with the SSL cert files to use HTTPs" .Values.clients.https.certDir }} + readOnly: true +{{- end }} +{{- end }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/hardenedNodeExporter/templates/pushprox-clients.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/hardenedNodeExporter/templates/pushprox-clients.yaml new file mode 100644 index 00000000000..e8fcfb38836 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/hardenedNodeExporter/templates/pushprox-clients.yaml @@ -0,0 +1,157 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.clients }}{{- if .Values.clients.enabled }} +apiVersion: apps/v1 +{{- if .Values.clients.deployment.enabled }} +kind: Deployment +{{- else }} +kind: DaemonSet +{{- end }} +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} + pushprox-exporter: "client" +spec: + {{- if .Values.clients.deployment.enabled }} + replicas: {{ .Values.clients.deployment.replicas }} + {{- end }} + selector: + matchLabels: {{ include "pushProxy.client.labels" . | nindent 6 }} + template: + metadata: + labels: {{ include "pushProxy.client.labels" . | nindent 8 }} + spec: + {{- if .Values.clients.affinity }} + affinity: {{ toYaml .Values.clients.affinity | nindent 8 }} + {{- end }} + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} +{{- if .Values.clients.nodeSelector }} +{{ toYaml .Values.clients.nodeSelector | indent 8 }} +{{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} +{{- if .Values.clients.tolerations }} +{{ toYaml .Values.clients.tolerations | indent 8 }} +{{- end }} + hostNetwork: true + dnsPolicy: ClusterFirstWithHostNet + serviceAccountName: {{ template "pushProxy.client.name" . }} + {{- if .Values.global.imagePullSecretName }} + imagePullSecrets: + - name: {{ .Values.global.imagePullSecretName }} + {{- end }} + containers: + - name: pushprox-client + image: {{ template "system_default_registry" . }}{{ .Values.clients.image.repository }}:{{ .Values.clients.image.tag }} + command: + {{- range .Values.clients.command }} + - {{ . | quote }} + {{- end }} + args: + - --fqdn=$(HOST_IP) + - --proxy-url=$(PROXY_URL) + {{- if .Values.clients.metrics.enabled }} + - --metrics-addr=$(PORT) + {{- end }} + - --allow-port={{ required "Need .Values.metricsPort to configure client to be allowed to scrape metrics at port" .Values.metricsPort}} + {{- if .Values.clients.useLocalhost }} + - --use-localhost + {{- end }} + {{- if .Values.clients.https.enabled }} + {{- if .Values.clients.https.insecureSkipVerify }} + - --insecure-skip-verify + {{- end }} + {{- if .Values.clients.https.useServiceAccountCredentials }} + - --token-path=/var/run/secrets/kubernetes.io/serviceaccount/token + {{- end }} + {{- if .Values.clients.https.certDir }} + - --tls.cert=/etc/ssl/push-proxy/push-proxy.pem + - --tls.key=/etc/ssl/push-proxy/push-proxy-key.pem + - --tls.cacert=/etc/ssl/push-proxy/push-proxy-ca-cert.pem + {{- end }} + {{- end }} + env: + - name: HOST_IP + valueFrom: + fieldRef: + fieldPath: status.hostIP + {{- if .Values.clients.metrics.enabled }} + - name: PORT + value: :{{ .Values.clients.port }} + {{- end }} + - name: PROXY_URL + value: {{ template "pushProxy.proxyUrl" . }} + securityContext: + runAsNonRoot: true + runAsUser: 1000 + {{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + volumeMounts: + - name: metrics-cert-dir + mountPath: /etc/ssl/push-proxy + {{- end }} + {{- if .Values.clients.resources }} + resources: {{ toYaml .Values.clients.resources | nindent 10 }} + {{- end }} + {{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + initContainers: + - name: copy-certs + image: {{ template "system_default_registry" . }}{{ .Values.clients.copyCertsImage.repository }}:{{ .Values.clients.copyCertsImage.tag }} + command: + - sh + - -c + - | + echo "Searching for files to copy within the source volume" + echo "cert: ${CERT_FILE_NAME}" + echo "key: ${KEY_FILE_NAME}" + echo "cacert: ${CACERT_FILE_NAME}" + + CERT_FILE_SOURCE=$(find /etc/source/ -type f -name "${CERT_FILE_NAME}" | sort -r | head -n 1) + KEY_FILE_SOURCE=$(find /etc/source/ -type f -name "${KEY_FILE_NAME}" | sort -r | head -n 1) + CACERT_FILE_SOURCE=$(find /etc/source/ -type f -name "${CACERT_FILE_NAME}" | sort -r | head -n 1) + + test -z ${CERT_FILE_SOURCE} && echo "Failed to find cert file" && exit 1 + test -z ${KEY_FILE_SOURCE} && echo "Failed to find key file" && exit 1 + test -z ${CACERT_FILE_SOURCE} && echo "Failed to find cacert file" && exit 1 + + echo "Copying cert file from $CERT_FILE_SOURCE to $CERT_FILE_TARGET" + cp $CERT_FILE_SOURCE $CERT_FILE_TARGET || exit 1 + chmod 444 $CERT_FILE_TARGET || exit 1 + + echo "Copying key file from $KEY_FILE_SOURCE to $KEY_FILE_TARGET" + cp $KEY_FILE_SOURCE $KEY_FILE_TARGET || exit 1 + chmod 444 $KEY_FILE_TARGET || exit 1 + + echo "Copying cacert file from $CACERT_FILE_SOURCE to $CACERT_FILE_TARGET" + cp $CACERT_FILE_SOURCE $CACERT_FILE_TARGET || exit 1 + chmod 444 $CACERT_FILE_TARGET || exit 1 + env: + - name: CERT_FILE_NAME + value: {{ required "Need a TLS cert file for scraping metrics endpoint over HTTPs" .Values.clients.https.certFile }} + - name: KEY_FILE_NAME + value: {{ required "Need a TLS key file for scraping metrics endpoint over HTTPs" .Values.clients.https.keyFile }} + - name: CACERT_FILE_NAME + value: {{ required "Need a TLS CA cert file for scraping metrics endpoint over HTTPs" .Values.clients.https.caCertFile }} + - name: CERT_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy.pem + - name: KEY_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy-key.pem + - name: CACERT_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy-ca-cert.pem + securityContext: + runAsNonRoot: false +{{- if and .Values.global.seLinux.enabled .Values.clients.https.seLinuxOptions }} + seLinuxOptions: {{ .Values.clients.https.seLinuxOptions | toYaml | nindent 12 }} +{{- end }} + volumeMounts: + - name: metrics-cert-dir-source + mountPath: /etc/source + readOnly: true + - name: metrics-cert-dir + mountPath: /etc/ssl/push-proxy + volumes: + - name: metrics-cert-dir-source + hostPath: + path: {{ required "Need access to volume on host with the SSL cert files to use HTTPs" .Values.clients.https.certDir }} + - name: metrics-cert-dir + emptyDir: {} + {{- end }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/hardenedNodeExporter/templates/pushprox-proxy-rbac.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/hardenedNodeExporter/templates/pushprox-proxy-rbac.yaml new file mode 100644 index 00000000000..eefe609058c --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/hardenedNodeExporter/templates/pushprox-proxy-rbac.yaml @@ -0,0 +1,68 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if and .Values.proxy }}{{ if .Values.proxy.enabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ template "pushProxy.proxy.name" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +rules: +{{- if .Values.global.cattle.psp.enabled }} +- apiGroups: ['policy'] + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - {{ template "pushProxy.proxy.name" . }} +{{- end }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ template "pushProxy.proxy.name" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "pushProxy.proxy.name" . }} +subjects: + - kind: ServiceAccount + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +--- +{{- if .Values.global.cattle.psp.enabled }} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +spec: + privileged: false + hostNetwork: false + hostIPC: false + hostPID: false + runAsUser: + rule: 'MustRunAsNonRoot' + seLinux: + rule: 'RunAsAny' + supplementalGroups: + rule: 'MustRunAs' + ranges: + - min: 1 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + - min: 1 + max: 65535 + readOnlyRootFilesystem: false + volumes: + - 'secret' +{{- end }}{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/hardenedNodeExporter/templates/pushprox-proxy.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/hardenedNodeExporter/templates/pushprox-proxy.yaml new file mode 100644 index 00000000000..723bbd6c00b --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/hardenedNodeExporter/templates/pushprox-proxy.yaml @@ -0,0 +1,57 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if and .Values.proxy }}{{ if .Values.proxy.enabled }} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} + pushprox-exporter: "proxy" +spec: + selector: + matchLabels: {{ include "pushProxy.proxy.labels" . | nindent 6 }} + template: + metadata: + labels: {{ include "pushProxy.proxy.labels" . | nindent 8 }} + spec: + securityContext: + runAsNonRoot: true + runAsUser: 1000 + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} +{{- if .Values.proxy.nodeSelector }} +{{ toYaml .Values.proxy.nodeSelector | indent 8 }} +{{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} +{{- if .Values.proxy.tolerations }} +{{ toYaml .Values.proxy.tolerations | indent 8 }} +{{- end }} + serviceAccountName: {{ template "pushProxy.proxy.name" . }} + {{- if .Values.global.imagePullSecretName }} + imagePullSecrets: + - name: {{ .Values.global.imagePullSecretName }} + {{- end }} + containers: + - name: pushprox-proxy + image: {{ template "system_default_registry" . }}{{ .Values.proxy.image.repository }}:{{ .Values.proxy.image.tag }} + command: + {{- range .Values.proxy.command }} + - {{ . | quote }} + {{- end }} + {{- if .Values.proxy.resources }} + resources: {{ toYaml .Values.proxy.resources | nindent 10 }} + {{- end }} +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +spec: + ports: + - name: pp-proxy + port: {{ required "Need .Values.proxy.port to configure proxy" .Values.proxy.port }} + protocol: TCP + targetPort: {{ .Values.proxy.port }} + selector: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/hardenedNodeExporter/templates/pushprox-servicemonitor.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/hardenedNodeExporter/templates/pushprox-servicemonitor.yaml new file mode 100644 index 00000000000..67eb2216b5a --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/hardenedNodeExporter/templates/pushprox-servicemonitor.yaml @@ -0,0 +1,45 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.serviceMonitor }}{{- if .Values.serviceMonitor.enabled }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ template "pushprox.serviceMonitor.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.serviceMonitor.labels" . | nindent 4 }} +spec: + endpoints: {{include "pushProxy.serviceMonitor.endpoints" . | nindent 4 }} + jobLabel: component + podTargetLabels: + - component + - pushprox-exporter + namespaceSelector: + matchNames: + - {{ template "pushprox.namespace" . }} + selector: + matchLabels: {{ include "pushProxy.client.labels" . | nindent 6 }} +--- +{{- $selector := "" }} +{{- if not (kindIs "invalid" .Values.service) }} +{{- if not (kindIs "invalid" .Values.service.selector) }} +{{ if .Values.service.selector }} +{{- if .Values.clients.enabled }} +{{- required (printf "Cannot override .Values.service.selector=%s when .Values.clients.enabled=true" (toJson .Values.service.selector)) "" }} +{{- end }} +{{- $selector = (toYaml .Values.service.selector) }} +{{- end }} +{{- end }} +{{- end }} +apiVersion: v1 +kind: Service +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +spec: + ports: + - name: metrics + port: {{ required "Need .Values.metricsPort to configure client to listen to metrics at port" .Values.metricsPort}} + protocol: TCP + targetPort: {{ .Values.metricsPort }} + selector: {{ default (include "pushProxy.client.labels" .) $selector | nindent 4 }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/hardenedNodeExporter/templates/validate-install-crd.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/hardenedNodeExporter/templates/validate-install-crd.yaml new file mode 100644 index 00000000000..16abc2fa83a --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/hardenedNodeExporter/templates/validate-install-crd.yaml @@ -0,0 +1,14 @@ +#{{- if gt (len (lookup "rbac.authorization.k8s.io/v1" "ClusterRole" "" "")) 0 -}} +# {{- $found := dict -}} +# {{- set $found "monitoring.coreos.com/v1/ServiceMonitor" false -}} +# {{- range .Capabilities.APIVersions -}} +# {{- if hasKey $found (toString .) -}} +# {{- set $found (toString .) true -}} +# {{- end -}} +# {{- end -}} +# {{- range $_, $exists := $found -}} +# {{- if (eq $exists false) -}} +# {{- required "Required CRDs are missing. Please install Prometheus Operator CRDs before installing this chart." "" -}} +# {{- end -}} +# {{- end -}} +#{{- end -}} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/hardenedNodeExporter/templates/validate-psp-install.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/hardenedNodeExporter/templates/validate-psp-install.yaml new file mode 100644 index 00000000000..a30c59d3b7c --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/hardenedNodeExporter/templates/validate-psp-install.yaml @@ -0,0 +1,7 @@ +#{{- if gt (len (lookup "rbac.authorization.k8s.io/v1" "ClusterRole" "" "")) 0 -}} +#{{- if .Values.global.cattle.psp.enabled }} +#{{- if not (.Capabilities.APIVersions.Has "policy/v1beta1/PodSecurityPolicy") }} +#{{- fail "The target cluster does not have the PodSecurityPolicy API resource. Please disable PSPs in this chart before proceeding." -}} +#{{- end }} +#{{- end }} +#{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/hardenedNodeExporter/values.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/hardenedNodeExporter/values.yaml new file mode 100644 index 00000000000..7f38498a1c5 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/hardenedNodeExporter/values.yaml @@ -0,0 +1,166 @@ +# Default values for rancher-pushprox. +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. + +# Default image containing both the proxy and the client was generated from the following Dockerfile +# https://github.com/prometheus-community/PushProx/blob/eeadbe766641699129920ccfaaaa30a85c67fe81/Dockerfile#L1-L15 + +# Configuration + +global: + cattle: + psp: + enabled: false + systemDefaultRegistry: "" + seLinux: + enabled: false + +# A list of Semver constraint strings (defined by https://github.com/Masterminds/semver) and values.yaml overrides. +# +# For each key in kubeVersionOverrides, this chart will check to see if the current Kubernetes cluster's version matches +# any of the semver constraints provided as keys on the map. +# +# On seeing a match, the default value for each values.yaml field overridden will be updated with the new value. +# +# If multiple matches are encountered (due to overlapping semver ranges), the matches will be applied in order. +# +# Notes: +# - On running a helm template, Helm generally assumes the kubeVersion is v1.20.0 +# - On running a helm install --dry-run, the correct kubeVersion should be chosen. +kubeVersionOverrides: [] +# - constraint: "< 1.21" +# values: +# metricsPort: 10252 +# clients: +# https: +# enabled: false +# insecureSkipVerify: false +# useServiceAccountCredentials: false + +namespaceOverride: "" + +# The component that is being monitored (i.e. etcd) +component: "component" + +# The port containing the metrics that need to be scraped +metricsPort: 2739 + +# Configure ServiceMonitor that monitors metrics from the metricsPort endpoint +serviceMonitor: + enabled: true + # A list of endpoints that will be added to the ServiceMonitor based on the Endpoint spec + # Source: https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#endpoint + # By default, proxyUrl and params._scheme will be overridden based on other values + endpoints: + - port: metrics + +# Configure Service that grabs scrape targets +service: + # The selector that is used to populate the Service's Endpoints object. + # The chart will error out on rendering templating if .Values.clients.enabled is set alongside this field, + # since it is expected that this service should point to the PushProx Clients Daemonset / Deployment + selector: {} + +clients: + enabled: true + # The port which the PushProx client will post PushProx metrics to + port: 9369 + # If unset, this will default to the URL for the proxy service: http://pushprox-{{component}}-proxy.{{namepsace}}.svc.cluster.local:{{proxy.port}} + # Should be modified if the clients are being deployed outside the cluster where the proxy rests, otherwise leave it null + proxyUrl: "" + # If set to true, the client will forward any requests from the host IP to 127.0.0.1 + # It will only allow proxy requests to the metricsPort specified + useLocalhost: false + # Configuration for accessing metrics via HTTPS + https: + # Does the client require https to access the metrics? + enabled: false + # Does the client require requests be sent to http or https? + forceHTTPSScheme: false + # If set to true, the client will create a service account with adequate permissions and set a flag + # on the client to use the service account token provided by it to make authorized scrape requests + useServiceAccountCredentials: false + # Configuration for authentication to metrics via https endpoint + authenticationMethod: + # Reads token from defined file in container + # This function is deprecated in the prometheus operator api and may be removed in a future version + bearerTokenFile: + enabled: false + bearerTokenFilePath: "/var/run/secrets/kubernetes.io/serviceaccount/token" + # Reads token from defined secret in namespace + # This function is deprecated in the prometheus operator api and may be removed in a future version + bearerTokenSecret: + enabled: false + # Reads token from defined secret in namespace + authorization: + enabled: false + type: "bearer" + credentials: + key: "token" + optional: false + # If set to true, the client will disable SSL security checks + insecureSkipVerify: false + # Directory on host where necessary TLS cert and key to scrape metrics can be found + certDir: "" + # Filenames for files located in .Values.clients.https.certDir that correspond to TLS settings + certFile: "" + keyFile: "" + caCertFile: "" + # seLinuxOptions to be passed into the container that copies certs. Should define a container with permissions to read the files in the certDir provided on the host. + # Required and only used if `clients.https.enabled` is set and `clients.https.certDir` is provided. + seLinuxOptions: {} + + metrics: + # Whether the client should publish PushProx client-specific metrics to .Values.clients.port + enabled: false + + rbac: + # Additional permissions to provide to the ServiceAccount bound to the client + # This can be used to provide additional permissions for the client to scrape metrics from the k8s API + # Only enabled if clients.https.enabled and clients.https.useServiceAccountCredentials are true + additionalRules: [] + + # Resource limits + resources: {} + + # Options to select all nodes to deploy client DaemonSet on + nodeSelector: {} + tolerations: [] + affinity: {} + + image: + repository: rancher/pushprox-client + tag: v0.1.4-rancher2-client + command: ["pushprox-client"] + + copyCertsImage: + repository: rancher/mirrored-library-busybox + tag: 1.31.1 + + # The default intention of rancher-pushprox clients is to scrape hostNetwork metrics across all nodes. + # This can be used to scrape internal Kubernetes components or DaemonSets of hostNetwork Pods in + # situations where a cloud provider firewall prevents Pod-To-Host communication but not Pod-To-Pod. + # However, if the underlying hostNetwork Pod that is being scraped is managed by a Deployment, + # this advanced option enables users to deploy the client as a Deployment instead of a DaemonSet. + # If a user deploys this feature and the underlying Deployment's number of replicas changes, the user will + # be responsible for upgrading this chart accordingly to the right number of replicas. + deployment: + enabled: false + replicas: 0 + +proxy: + enabled: true + # The port through which PushProx clients will communicate to the proxy + port: 8080 + + # Resource limits + resources: {} + + # Options to select a node to run a single proxy deployment on + nodeSelector: {} + tolerations: [] + + image: + repository: rancher/pushprox-proxy + tag: v0.1.4-rancher2-proxy + command: ["pushprox-proxy"] diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/k3sServer/.helmignore b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/k3sServer/.helmignore new file mode 100644 index 00000000000..0e8a0eb36f4 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/k3sServer/.helmignore @@ -0,0 +1,23 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*.orig +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/k3sServer/Chart.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/k3sServer/Chart.yaml new file mode 100644 index 00000000000..725e8f31b0f --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/k3sServer/Chart.yaml @@ -0,0 +1,15 @@ +annotations: + catalog.cattle.io/hidden: "true" + catalog.cattle.io/kube-version: '>= 1.26.0-0 < 1.31.0-0' + catalog.cattle.io/os: linux + catalog.rancher.io/certified: rancher + catalog.rancher.io/namespace: cattle-monitoring-system + catalog.rancher.io/release-name: rancher-pushprox +apiVersion: v1 +appVersion: v0.1.4-rancher2 +description: Sets up a deployment of the PushProx proxy and a DaemonSet of PushProx + clients. +kubeVersion: '>=1.26.0-0' +name: k3sServer +type: application +version: 0.2.1 diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/k3sServer/README.md b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/k3sServer/README.md new file mode 100644 index 00000000000..345002f48a2 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/k3sServer/README.md @@ -0,0 +1,90 @@ +# rancher-pushprox + +A Rancher chart based on Rancher [PushProx](https://github.com/rancher/PushProx) that sets up a Deployment of a PushProx proxy and a DaemonSet of PushProx clients on a Kubernetes cluster. + +Installs [rancher-pushprox](https://github.com/rancher/charts/tree/gh-pages/packages/rancher-pushprox) to create PushProx clients that can access their host's network and register with a PushProx proxy. A [Prometheus Operator](https://github.com/coreos/prometheus-operator) ServiceMonitor CR is also included that is configured to scrape the metrics from each of the clients through the proxy. + +Using an instance of this chart is suitable for the following scenarios: +- You need to scrape metrics from a port that should not be accessible outside of the host (e.g. scraping `etcd` metrics in a hardened cluster) +- You need to scrape metrics on a host that are not exposed outside of 127.0.0.1 (e.g. scraping `kube-proxy` metrics) +- You need to scrape metrics through HTTPS using certs hosted directly on `hostPath` +- You need to scrape metrics from Kubernetes components that require authorization via a service account (e.g. permissions to make request to `/metrics`) +- You need to scrape metrics without access to cacerts (i.e. enable `insecureSkipVerify`) + +The clients and proxy are created based on a Rancher fork of the [prometheus-community/PushProx](https://github.com/prometheus-community/PushProx) project. + +## Upgrading to Kubernetes v1.25+ + +Starting in Kubernetes v1.25, [Pod Security Policies](https://kubernetes.io/docs/concepts/security/pod-security-policy/) have been removed from the Kubernetes API. + +As a result, **before upgrading to Kubernetes v1.25** (or on a fresh install in a Kubernetes v1.25+ cluster), users are expected to perform an in-place upgrade of this chart with `global.cattle.psp.enabled` set to `false` if it has been previously set to `true`. +​ +> **Note:** +> In this chart release, any previous field that was associated with any PSP resources have been removed in favor of a single global field: `global.cattle.psp.enabled`. + +> **Note:** +> If you upgrade your cluster to Kubernetes v1.25+ before removing PSPs via a `helm upgrade` (even if you manually clean up resources), **it will leave the Helm release in a broken state within the cluster such that further Helm operations will not work (`helm uninstall`, `helm upgrade`, etc.).** +> +> If your charts get stuck in this state, please consult the Rancher docs on how to clean up your Helm release secrets. + +Upon setting `global.cattle.psp.enabled` to false, the chart will remove any PSP resources deployed on its behalf from the cluster. This is the default setting for this chart. + +As a replacement for PSPs, [Pod Security Admission](https://kubernetes.io/docs/concepts/security/pod-security-admission/) should be used. Please consult the Rancher docs for more details on how to configure your chart release namespaces to work with the new Pod Security Admission and apply Pod Security Standards. + +## Configuration + +The following tables list the configurable parameters of the rancher-pushprox chart and their default values. + +### General + +#### Required +| Parameter | Description | Example | +| ----- | ----------- | ------ | +| `component` | The component that is being monitored | `kube-etcd` +| `metricsPort` | The port on the host that contains the metrics you want to scrape (e.g. `http://:/metrics`) | `2379` | +| `namespaceOverride` | The namespace to install the chart | `""` + +#### Optional +| Parameter | Description | Default | +| ----- | ----------- | ------ | +| `serviceMonitor.enabled` | Deploys a [Prometheus Operator](https://github.com/coreos/prometheus-operator/blob/master/Documentation/api.md#servicemonitor) ServiceMonitor CR that is configured to scrape metrics on the hosts that the clients are deployed on via the proxy. Also deploys a Service that points to all pods with the expected client name that exposes the `metricsPort` selected | `true` | +| `serviceMonitor.endpoints` | A list of endpoints that will be added to the ServiceMonitor based on the [Endpoint spec](https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#endpoint) | `[{port: metrics}]` | +| `service.selector` | The selector that is used to populate the Service's Endpoints object. The chart will error out on rendering templating if `.Values.clients.enabled` is set alongside this field, since it is expected that this service should point to the PushProx Clients Daemonset / Deployment | `{}` | +| `clients.enabled` | Deploys a DaemonSet of clients that are each capable of scraping endpoints on the hostNetwork it is deployed on | `true` | +| `clients.port` | The port where the client will publish PushProx client-specific metrics. If deploying multiple clients onto the same node, the clients should not have conflicting ports | `9369` | +| `clients.proxyUrl` | Overrides the default proxyUrl setting of `http://pushprox-{{ .Values.component }}-proxy.{{ . Release.Namespace }}.svc.cluster.local:{{ .Values.proxy.port }}"` with the `proxyUrl` specified | `""` | +| `clients.useLocalhost` | Sets a flag on each client deployment to redirect scrapes directed to `HOST_IP` to `127.0.0.1` | `false` | +| `clients.https.enabled` | Enables scraping metrics via HTTPS using the provided TLS certs that exist on each host | `false` | +| `clients.https.forceHTTPSScheme` | Forces scraping metrics via HTTPS using the provided TLS certs that exist on each host | `false` | +| `clients.https.useServiceAccountCredentials` | If set to true, the client will create a service account with permissions to scrape `/metrics` endpoint of Kubernetes components. The client will use the service account token provided to make authorized scrape requests to the Kubernetes API | `false` | +| `clients.https.authenticationMethod.bearerTokenFile.enabled` | If set to true, the client will use service account credentials mounted at the configured path `clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath`. This requires permissions to scrape `/metrics` endpoint of Kubernetes components. This method is deprecated by the prometheus operator and may be removed in a future release | `false` | +| `clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath` | This is a volume mount on the pod with permissions to scrape `/metrics` endpoint of Kubernetes components | `"/var/run/secrets/kubernetes.io/serviceaccount/token"` | +| `clients.https.authenticationMethod.bearerTokenSecret.enabled` | If set to true, the client will use service account credentials to scrape `/metrics` endpoint of Kubernetes components. This method is deprecated by the prometheus operator and may be removed in a future release | `false` | +| `clients.https.authenticationMethod.authorization.enabled` | If set to true, the client will use service account credentials to scrape `/metrics` endpoint of Kubernetes components | `false` | +| `clients.https.authenticationMethod.authorization.type` | If set, the client will use this type of authorization in its client requests for metrics | `"bearer"` | +| `clients.https.authenticationMethod.authorization.credentials.key` | If set, the client will use this key in the secret created by `clients.https.useServiceAccountCredentials` for authorization in its client requests for metrics | `"token"` | +| `clients.https.authenticationMethod.authorization.credentials.optional` | If set to false, the client will fail if the key in the secret created by `clients.https.useServiceAccountCredentials` does not exist | `false` | +| `clients.https.insecureSkipVerify` | If set to true, the client will disable SSL security checks | `false` | +| `clients.https.certDir` | A `hostPath` where TLS certs can be found. This path is mounted as a volume on an `initContainer` which copies only the necessary files over to an EmptyDir volume used by each client. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.certFile` | The path to the TLS cert file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.keyFile` | The path to the TLS key file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.caCertFile` | The path to the TLS cacert file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.seLinuxOptions` | seLinuxOptions to be passed into the container that copies certs. Should define a container with permissions to read the files in the certDir provided on the host. Required and only used if `clients.https.enabled` is set and `clients.https.certDir` is provided. | `""` | +| `clients.metrics.enabled` | Whether the client should publish PushProx client-specific metrics. | `false` | +| `clients.rbac.additionalRules` | Additional permissions to provide to the ServiceAccount bound to the client. This can be used to provide additional permissions for the client to scrape metrics from the k8s API. Only enabled if clients.https.enabled and clients.https.useServiceAccountCredentials are true | `[]` | +| `clients.deployment.enabled` | Deploys the client as a Deployment (generally used if the underlying hostNetwork Pod that is being scraped is managed by a Deployment) | `false` | +| `clients.deployment.replicas` | The number of pods the Deployment has, it should match the number of pod the hostNetwork Deployment has. Required and only used if `client.deployment.enable` is set | `0` | +| `clients.deployment.affinity` | The affinity rules that allocate the pod to the node in which the hostNetwork Deployment's pods run. Required and only used if `client.deployment.enable` is set | `{}` | +| `clients.resources` | Set resource limits and requests for the client container | `{}` | +| `clients.nodeSelector` | Select which nodes to deploy the clients on | `{}` | +| `clients.tolerations` | Specify tolerations for clients | `[]` | +| `proxy.enabled` | Deploys the proxy that each client will register with | `true` | +| `proxy.port` | The port exposed by the proxy that each client will register with to allow metrics to be scraped from the host | `8080` | +| `proxy.resources` | Set resource limits and requests for the proxy container | `{}` | +| `proxy.nodeSelector` | Select which nodes the proxy can be deployed on | `{}` | +| `proxy.tolerations` | Specify tolerations (if necessary) to allow the proxy to be deployed on the selected node | `[]` | +| `kubeVersionOverrides` | A list of Semver constraint strings (defined by https://github.com/Masterminds/semver) and values.yaml overrides. For each key in kubeVersionOverrides, this chart will check to see if the current Kubernetes cluster's version matches any of the semver constraints provided as keys on the map. On seeing a match, the default value for each values.yaml field overridden will be updated with the new value. If multiple matches are encountered (due to overlapping semver ranges), the matches will be applied in order. | `[]` + +*Tip: The filepaths set in `clients.https.File` can include wildcard characters*. + +See [rancher-monitoring](https://github.com/rancher/charts/tree/gh-pages/packages/rancher-monitoring) for examples of how this chart can be used. diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/k3sServer/templates/_helpers.tpl b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/k3sServer/templates/_helpers.tpl new file mode 100644 index 00000000000..1ba50939441 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/k3sServer/templates/_helpers.tpl @@ -0,0 +1,170 @@ +# Rancher + +{{- define "system_default_registry" -}} +{{- if .Values.global.cattle.systemDefaultRegistry -}} +{{- printf "%s/" .Values.global.cattle.systemDefaultRegistry -}} +{{- end -}} +{{- end -}} + +# Windows Support + +{{/* +Windows cluster will add default taint for linux nodes, +add below linux tolerations to workloads could be scheduled to those linux nodes +*/}} + +{{- define "linux-node-tolerations" -}} +- key: "cattle.io/os" + value: "linux" + effect: "NoSchedule" + operator: "Equal" +{{- end -}} + +{{- define "linux-node-selector" -}} +{{- if semverCompare "<1.14-0" .Capabilities.KubeVersion.GitVersion -}} +beta.kubernetes.io/os: linux +{{- else -}} +kubernetes.io/os: linux +{{- end -}} +{{- end -}} + +# General + +{{- define "applyKubeVersionOverrides" -}} +{{- $overrides := dict -}} +{{- range $override := .Values.kubeVersionOverrides -}} +{{- if semverCompare $override.constraint $.Capabilities.KubeVersion.Version -}} +{{- $_ := mergeOverwrite $overrides $override.values -}} +{{- end -}} +{{- end -}} +{{- $_ := mergeOverwrite .Values $overrides -}} +{{- end -}} + +{{- define "pushprox.namespace" -}} + {{- if .Values.namespaceOverride -}} + {{- .Values.namespaceOverride -}} + {{- else -}} + {{- .Release.Namespace -}} + {{- end -}} +{{- end -}} + +{{- define "pushProxy.commonLabels" -}} +release: {{ .Release.Name }} +component: {{ .Values.component | quote }} +provider: kubernetes +{{- end -}} + +{{- define "pushProxy.proxyUrl" -}} +{{- $_ := (required "Template requires either .Values.proxy.port or .Values.client.proxyUrl to set proxyUrl for client" (or .Values.clients.proxyUrl .Values.proxy.port)) -}} +{{- if .Values.clients.proxyUrl -}} +{{ printf "%s" .Values.clients.proxyUrl }} +{{- else -}} +{{ printf "http://%s.%s.svc:%d" (include "pushProxy.proxy.name" .) (include "pushprox.namespace" .) (int .Values.proxy.port) }} +{{- end -}}{{- end -}} + +# Client + +{{- define "pushProxy.client.name" -}} +{{- printf "pushprox-%s-client" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.client.serviceAccountTokenName" -}} +{{- printf "pushprox-%s-client-service-account-token" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.client.labels" -}} +k8s-app: {{ template "pushProxy.client.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +# Proxy + +{{- define "pushProxy.proxy.name" -}} +{{- printf "pushprox-%s-proxy" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.proxy.labels" -}} +k8s-app: {{ template "pushProxy.proxy.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +# ServiceMonitor + +{{- define "pushprox.serviceMonitor.name" -}} +{{- printf "%s-%s" .Release.Name (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.serviceMonitor.labels" -}} +app: {{ template "pushprox.serviceMonitor.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +{{- define "pushProxy.serviceMonitor.endpoints" -}} +{{- $proxyURL := (include "pushProxy.proxyUrl" .) -}} +{{- $useHTTPS := .Values.clients.https.enabled -}} +{{- $setHTTPSScheme := .Values.clients.https.forceHTTPSScheme -}} +{{- $insecureSkipVerify := .Values.clients.https.insecureSkipVerify -}} +{{- $useServiceAccountCredentials := .Values.clients.https.useServiceAccountCredentials -}} +{{- $serviceAccountTokenName := (include "pushProxy.client.serviceAccountTokenName" . ) -}} +{{- $metricRelabelings := list }} +{{- $endpoints := .Values.serviceMonitor.endpoints }} +{{- if .Values.proxy.enabled }} +{{- $_ := set . "proxyUrl" $proxyURL }} +{{- end }} +{{- range $endpoints }} +{{- if $.Values.proxy.enabled }} +{{- $_ := set . "proxyUrl" $proxyURL }} +{{- end }} +{{- $clusterIdRelabel := dict }} +{{- $metricRelabelings := list }} +{{- if $.Values.global.cattle.clusterId }} +{{- $_ := set $clusterIdRelabel "action" "replace" }} +{{- $_ := set $clusterIdRelabel "sourceLabels" (list "__address__") }} +{{- $_ := set $clusterIdRelabel "targetLabel" "cluster_id" }} +{{- $_ := set $clusterIdRelabel "replacement" $.Values.global.cattle.clusterId }} +{{- $metricRelabelings = append $metricRelabelings $clusterIdRelabel }} +{{- end }} +{{- $clusterNameRelabel := dict }} +{{- if $.Values.global.cattle.clusterName }} +{{- $_ := set $clusterNameRelabel "action" "replace" }} +{{- $_ := set $clusterNameRelabel "sourceLabels" (list "__address__") }} +{{- $_ := set $clusterNameRelabel "targetLabel" "cluster_name" }} +{{- $_ := set $clusterNameRelabel "replacement" $.Values.global.cattle.clusterName }} +{{- $metricRelabelings = append $metricRelabelings $clusterNameRelabel }} +{{- end }} +{{- if not (empty $metricRelabelings) }} +{{- $_ := set . "metricRelabelings" ($metricRelabelings)}} +{{- end }} +{{- if $setHTTPSScheme -}} +{{- $_ := set . "scheme" "https" }} +{{- end -}} +{{- if $useHTTPS -}} +{{- if (hasKey . "params") }} +{{- $_ := set (get . "params") "_scheme" (list "https") }} +{{- else }} +{{- $_ := set . "params" (dict "_scheme" (list "https")) }} +{{- end }} +{{- end }} +{{- if (hasKey . "tlsConfig") }} +{{- $_ := set (get . "tlsConfig") "insecureSkipVerify" $insecureSkipVerify }} +{{- else }} +{{- $_ := set . "tlsConfig" (dict "insecureSkipVerify" $insecureSkipVerify) }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.bearerTokenFile.enabled }} +{{- $_ := set . "bearerTokenFile" $.Values.clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.bearerTokenSecret.enabled }} +{{- $_ := set . "bearerTokenSecret" $serviceAccountTokenName }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.authorization.enabled }} +{{- if (hasKey . "authorization") }} +{{- $_ := set (get . "authorization") "type" $.Values.clients.https.authenticationMethod.authorization.type }} +{{- $_ := set (get . "authorization") "credentials" (dict "name" $serviceAccountTokenName "key" $.Values.clients.https.authenticationMethod.authorization.credentials.key "optional" $.Values.clients.https.authenticationMethod.authorization.credentials.optional) }} +{{- else }} +{{- $_ := set . "authorization" (dict "type" $.Values.clients.https.authenticationMethod.authorization.type) }} +{{- $_ := set . "authorization" (dict "credentials" (dict "name" $serviceAccountTokenName "key" $.Values.clients.https.authenticationMethod.authorization.credentials.key "optional" $.Values.clients.https.authenticationMethod.authorization.credentials.optional)) }} +{{- end }} +{{- end }} +{{- end }} +{{- toYaml $endpoints }} +{{- end -}} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/k3sServer/templates/pushprox-clients-rbac.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/k3sServer/templates/pushprox-clients-rbac.yaml new file mode 100644 index 00000000000..a8e27c37356 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/k3sServer/templates/pushprox-clients-rbac.yaml @@ -0,0 +1,97 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.clients }}{{- if .Values.clients.enabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ template "pushProxy.client.name" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +rules: +{{- if .Values.global.cattle.psp.enabled }} +- apiGroups: ['policy'] + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - {{ template "pushProxy.client.name" . }} +{{- end }} +{{- if and .Values.clients.https.enabled .Values.clients.https.useServiceAccountCredentials }} +- nonResourceURLs: ["/metrics"] + verbs: ["get"] +{{- if .Values.clients.rbac.additionalRules }} +{{ toYaml .Values.clients.rbac.additionalRules }} +{{- end }} +{{- end }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ template "pushProxy.client.name" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "pushProxy.client.name" . }} +subjects: + - kind: ServiceAccount + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +--- +{{- if .Values.clients.https.useServiceAccountCredentials }} +apiVersion: v1 +kind: Secret +type: kubernetes.io/service-account-token +metadata: + name: {{ template "pushProxy.client.serviceAccountTokenName" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} + annotations: + kubernetes.io/service-account.name: {{ template "pushProxy.client.name" . }} +{{- end }} +--- +{{- if .Values.global.cattle.psp.enabled }} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +spec: + privileged: false + hostNetwork: true + hostIPC: false + hostPID: false + runAsUser: + rule: 'RunAsAny' + seLinux: + rule: 'RunAsAny' +{{- if and .Values.clients.https.enabled .Values.clients.https.certDir .Values.global.seLinux.enabled .Values.clients.https.seLinuxOptions }} + seLinuxOptions: {{ .Values.clients.https.seLinuxOptions | toYaml | nindent 6 }} +{{- end }} + supplementalGroups: + rule: 'MustRunAs' + ranges: + - min: 0 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + - min: 0 + max: 65535 + readOnlyRootFilesystem: false + volumes: + - 'secret' +{{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + - 'emptyDir' + - 'hostPath' + allowedHostPaths: + - pathPrefix: {{ required "Need access to volume on host with the SSL cert files to use HTTPs" .Values.clients.https.certDir }} + readOnly: true +{{- end }} +{{- end }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/k3sServer/templates/pushprox-clients.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/k3sServer/templates/pushprox-clients.yaml new file mode 100644 index 00000000000..e8fcfb38836 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/k3sServer/templates/pushprox-clients.yaml @@ -0,0 +1,157 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.clients }}{{- if .Values.clients.enabled }} +apiVersion: apps/v1 +{{- if .Values.clients.deployment.enabled }} +kind: Deployment +{{- else }} +kind: DaemonSet +{{- end }} +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} + pushprox-exporter: "client" +spec: + {{- if .Values.clients.deployment.enabled }} + replicas: {{ .Values.clients.deployment.replicas }} + {{- end }} + selector: + matchLabels: {{ include "pushProxy.client.labels" . | nindent 6 }} + template: + metadata: + labels: {{ include "pushProxy.client.labels" . | nindent 8 }} + spec: + {{- if .Values.clients.affinity }} + affinity: {{ toYaml .Values.clients.affinity | nindent 8 }} + {{- end }} + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} +{{- if .Values.clients.nodeSelector }} +{{ toYaml .Values.clients.nodeSelector | indent 8 }} +{{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} +{{- if .Values.clients.tolerations }} +{{ toYaml .Values.clients.tolerations | indent 8 }} +{{- end }} + hostNetwork: true + dnsPolicy: ClusterFirstWithHostNet + serviceAccountName: {{ template "pushProxy.client.name" . }} + {{- if .Values.global.imagePullSecretName }} + imagePullSecrets: + - name: {{ .Values.global.imagePullSecretName }} + {{- end }} + containers: + - name: pushprox-client + image: {{ template "system_default_registry" . }}{{ .Values.clients.image.repository }}:{{ .Values.clients.image.tag }} + command: + {{- range .Values.clients.command }} + - {{ . | quote }} + {{- end }} + args: + - --fqdn=$(HOST_IP) + - --proxy-url=$(PROXY_URL) + {{- if .Values.clients.metrics.enabled }} + - --metrics-addr=$(PORT) + {{- end }} + - --allow-port={{ required "Need .Values.metricsPort to configure client to be allowed to scrape metrics at port" .Values.metricsPort}} + {{- if .Values.clients.useLocalhost }} + - --use-localhost + {{- end }} + {{- if .Values.clients.https.enabled }} + {{- if .Values.clients.https.insecureSkipVerify }} + - --insecure-skip-verify + {{- end }} + {{- if .Values.clients.https.useServiceAccountCredentials }} + - --token-path=/var/run/secrets/kubernetes.io/serviceaccount/token + {{- end }} + {{- if .Values.clients.https.certDir }} + - --tls.cert=/etc/ssl/push-proxy/push-proxy.pem + - --tls.key=/etc/ssl/push-proxy/push-proxy-key.pem + - --tls.cacert=/etc/ssl/push-proxy/push-proxy-ca-cert.pem + {{- end }} + {{- end }} + env: + - name: HOST_IP + valueFrom: + fieldRef: + fieldPath: status.hostIP + {{- if .Values.clients.metrics.enabled }} + - name: PORT + value: :{{ .Values.clients.port }} + {{- end }} + - name: PROXY_URL + value: {{ template "pushProxy.proxyUrl" . }} + securityContext: + runAsNonRoot: true + runAsUser: 1000 + {{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + volumeMounts: + - name: metrics-cert-dir + mountPath: /etc/ssl/push-proxy + {{- end }} + {{- if .Values.clients.resources }} + resources: {{ toYaml .Values.clients.resources | nindent 10 }} + {{- end }} + {{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + initContainers: + - name: copy-certs + image: {{ template "system_default_registry" . }}{{ .Values.clients.copyCertsImage.repository }}:{{ .Values.clients.copyCertsImage.tag }} + command: + - sh + - -c + - | + echo "Searching for files to copy within the source volume" + echo "cert: ${CERT_FILE_NAME}" + echo "key: ${KEY_FILE_NAME}" + echo "cacert: ${CACERT_FILE_NAME}" + + CERT_FILE_SOURCE=$(find /etc/source/ -type f -name "${CERT_FILE_NAME}" | sort -r | head -n 1) + KEY_FILE_SOURCE=$(find /etc/source/ -type f -name "${KEY_FILE_NAME}" | sort -r | head -n 1) + CACERT_FILE_SOURCE=$(find /etc/source/ -type f -name "${CACERT_FILE_NAME}" | sort -r | head -n 1) + + test -z ${CERT_FILE_SOURCE} && echo "Failed to find cert file" && exit 1 + test -z ${KEY_FILE_SOURCE} && echo "Failed to find key file" && exit 1 + test -z ${CACERT_FILE_SOURCE} && echo "Failed to find cacert file" && exit 1 + + echo "Copying cert file from $CERT_FILE_SOURCE to $CERT_FILE_TARGET" + cp $CERT_FILE_SOURCE $CERT_FILE_TARGET || exit 1 + chmod 444 $CERT_FILE_TARGET || exit 1 + + echo "Copying key file from $KEY_FILE_SOURCE to $KEY_FILE_TARGET" + cp $KEY_FILE_SOURCE $KEY_FILE_TARGET || exit 1 + chmod 444 $KEY_FILE_TARGET || exit 1 + + echo "Copying cacert file from $CACERT_FILE_SOURCE to $CACERT_FILE_TARGET" + cp $CACERT_FILE_SOURCE $CACERT_FILE_TARGET || exit 1 + chmod 444 $CACERT_FILE_TARGET || exit 1 + env: + - name: CERT_FILE_NAME + value: {{ required "Need a TLS cert file for scraping metrics endpoint over HTTPs" .Values.clients.https.certFile }} + - name: KEY_FILE_NAME + value: {{ required "Need a TLS key file for scraping metrics endpoint over HTTPs" .Values.clients.https.keyFile }} + - name: CACERT_FILE_NAME + value: {{ required "Need a TLS CA cert file for scraping metrics endpoint over HTTPs" .Values.clients.https.caCertFile }} + - name: CERT_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy.pem + - name: KEY_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy-key.pem + - name: CACERT_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy-ca-cert.pem + securityContext: + runAsNonRoot: false +{{- if and .Values.global.seLinux.enabled .Values.clients.https.seLinuxOptions }} + seLinuxOptions: {{ .Values.clients.https.seLinuxOptions | toYaml | nindent 12 }} +{{- end }} + volumeMounts: + - name: metrics-cert-dir-source + mountPath: /etc/source + readOnly: true + - name: metrics-cert-dir + mountPath: /etc/ssl/push-proxy + volumes: + - name: metrics-cert-dir-source + hostPath: + path: {{ required "Need access to volume on host with the SSL cert files to use HTTPs" .Values.clients.https.certDir }} + - name: metrics-cert-dir + emptyDir: {} + {{- end }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/k3sServer/templates/pushprox-proxy-rbac.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/k3sServer/templates/pushprox-proxy-rbac.yaml new file mode 100644 index 00000000000..eefe609058c --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/k3sServer/templates/pushprox-proxy-rbac.yaml @@ -0,0 +1,68 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if and .Values.proxy }}{{ if .Values.proxy.enabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ template "pushProxy.proxy.name" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +rules: +{{- if .Values.global.cattle.psp.enabled }} +- apiGroups: ['policy'] + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - {{ template "pushProxy.proxy.name" . }} +{{- end }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ template "pushProxy.proxy.name" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "pushProxy.proxy.name" . }} +subjects: + - kind: ServiceAccount + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +--- +{{- if .Values.global.cattle.psp.enabled }} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +spec: + privileged: false + hostNetwork: false + hostIPC: false + hostPID: false + runAsUser: + rule: 'MustRunAsNonRoot' + seLinux: + rule: 'RunAsAny' + supplementalGroups: + rule: 'MustRunAs' + ranges: + - min: 1 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + - min: 1 + max: 65535 + readOnlyRootFilesystem: false + volumes: + - 'secret' +{{- end }}{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/k3sServer/templates/pushprox-proxy.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/k3sServer/templates/pushprox-proxy.yaml new file mode 100644 index 00000000000..723bbd6c00b --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/k3sServer/templates/pushprox-proxy.yaml @@ -0,0 +1,57 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if and .Values.proxy }}{{ if .Values.proxy.enabled }} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} + pushprox-exporter: "proxy" +spec: + selector: + matchLabels: {{ include "pushProxy.proxy.labels" . | nindent 6 }} + template: + metadata: + labels: {{ include "pushProxy.proxy.labels" . | nindent 8 }} + spec: + securityContext: + runAsNonRoot: true + runAsUser: 1000 + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} +{{- if .Values.proxy.nodeSelector }} +{{ toYaml .Values.proxy.nodeSelector | indent 8 }} +{{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} +{{- if .Values.proxy.tolerations }} +{{ toYaml .Values.proxy.tolerations | indent 8 }} +{{- end }} + serviceAccountName: {{ template "pushProxy.proxy.name" . }} + {{- if .Values.global.imagePullSecretName }} + imagePullSecrets: + - name: {{ .Values.global.imagePullSecretName }} + {{- end }} + containers: + - name: pushprox-proxy + image: {{ template "system_default_registry" . }}{{ .Values.proxy.image.repository }}:{{ .Values.proxy.image.tag }} + command: + {{- range .Values.proxy.command }} + - {{ . | quote }} + {{- end }} + {{- if .Values.proxy.resources }} + resources: {{ toYaml .Values.proxy.resources | nindent 10 }} + {{- end }} +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +spec: + ports: + - name: pp-proxy + port: {{ required "Need .Values.proxy.port to configure proxy" .Values.proxy.port }} + protocol: TCP + targetPort: {{ .Values.proxy.port }} + selector: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/k3sServer/templates/pushprox-servicemonitor.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/k3sServer/templates/pushprox-servicemonitor.yaml new file mode 100644 index 00000000000..67eb2216b5a --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/k3sServer/templates/pushprox-servicemonitor.yaml @@ -0,0 +1,45 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.serviceMonitor }}{{- if .Values.serviceMonitor.enabled }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ template "pushprox.serviceMonitor.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.serviceMonitor.labels" . | nindent 4 }} +spec: + endpoints: {{include "pushProxy.serviceMonitor.endpoints" . | nindent 4 }} + jobLabel: component + podTargetLabels: + - component + - pushprox-exporter + namespaceSelector: + matchNames: + - {{ template "pushprox.namespace" . }} + selector: + matchLabels: {{ include "pushProxy.client.labels" . | nindent 6 }} +--- +{{- $selector := "" }} +{{- if not (kindIs "invalid" .Values.service) }} +{{- if not (kindIs "invalid" .Values.service.selector) }} +{{ if .Values.service.selector }} +{{- if .Values.clients.enabled }} +{{- required (printf "Cannot override .Values.service.selector=%s when .Values.clients.enabled=true" (toJson .Values.service.selector)) "" }} +{{- end }} +{{- $selector = (toYaml .Values.service.selector) }} +{{- end }} +{{- end }} +{{- end }} +apiVersion: v1 +kind: Service +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +spec: + ports: + - name: metrics + port: {{ required "Need .Values.metricsPort to configure client to listen to metrics at port" .Values.metricsPort}} + protocol: TCP + targetPort: {{ .Values.metricsPort }} + selector: {{ default (include "pushProxy.client.labels" .) $selector | nindent 4 }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/k3sServer/templates/validate-install-crd.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/k3sServer/templates/validate-install-crd.yaml new file mode 100644 index 00000000000..16abc2fa83a --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/k3sServer/templates/validate-install-crd.yaml @@ -0,0 +1,14 @@ +#{{- if gt (len (lookup "rbac.authorization.k8s.io/v1" "ClusterRole" "" "")) 0 -}} +# {{- $found := dict -}} +# {{- set $found "monitoring.coreos.com/v1/ServiceMonitor" false -}} +# {{- range .Capabilities.APIVersions -}} +# {{- if hasKey $found (toString .) -}} +# {{- set $found (toString .) true -}} +# {{- end -}} +# {{- end -}} +# {{- range $_, $exists := $found -}} +# {{- if (eq $exists false) -}} +# {{- required "Required CRDs are missing. Please install Prometheus Operator CRDs before installing this chart." "" -}} +# {{- end -}} +# {{- end -}} +#{{- end -}} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/k3sServer/templates/validate-psp-install.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/k3sServer/templates/validate-psp-install.yaml new file mode 100644 index 00000000000..a30c59d3b7c --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/k3sServer/templates/validate-psp-install.yaml @@ -0,0 +1,7 @@ +#{{- if gt (len (lookup "rbac.authorization.k8s.io/v1" "ClusterRole" "" "")) 0 -}} +#{{- if .Values.global.cattle.psp.enabled }} +#{{- if not (.Capabilities.APIVersions.Has "policy/v1beta1/PodSecurityPolicy") }} +#{{- fail "The target cluster does not have the PodSecurityPolicy API resource. Please disable PSPs in this chart before proceeding." -}} +#{{- end }} +#{{- end }} +#{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/k3sServer/values.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/k3sServer/values.yaml new file mode 100644 index 00000000000..7f38498a1c5 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/k3sServer/values.yaml @@ -0,0 +1,166 @@ +# Default values for rancher-pushprox. +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. + +# Default image containing both the proxy and the client was generated from the following Dockerfile +# https://github.com/prometheus-community/PushProx/blob/eeadbe766641699129920ccfaaaa30a85c67fe81/Dockerfile#L1-L15 + +# Configuration + +global: + cattle: + psp: + enabled: false + systemDefaultRegistry: "" + seLinux: + enabled: false + +# A list of Semver constraint strings (defined by https://github.com/Masterminds/semver) and values.yaml overrides. +# +# For each key in kubeVersionOverrides, this chart will check to see if the current Kubernetes cluster's version matches +# any of the semver constraints provided as keys on the map. +# +# On seeing a match, the default value for each values.yaml field overridden will be updated with the new value. +# +# If multiple matches are encountered (due to overlapping semver ranges), the matches will be applied in order. +# +# Notes: +# - On running a helm template, Helm generally assumes the kubeVersion is v1.20.0 +# - On running a helm install --dry-run, the correct kubeVersion should be chosen. +kubeVersionOverrides: [] +# - constraint: "< 1.21" +# values: +# metricsPort: 10252 +# clients: +# https: +# enabled: false +# insecureSkipVerify: false +# useServiceAccountCredentials: false + +namespaceOverride: "" + +# The component that is being monitored (i.e. etcd) +component: "component" + +# The port containing the metrics that need to be scraped +metricsPort: 2739 + +# Configure ServiceMonitor that monitors metrics from the metricsPort endpoint +serviceMonitor: + enabled: true + # A list of endpoints that will be added to the ServiceMonitor based on the Endpoint spec + # Source: https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#endpoint + # By default, proxyUrl and params._scheme will be overridden based on other values + endpoints: + - port: metrics + +# Configure Service that grabs scrape targets +service: + # The selector that is used to populate the Service's Endpoints object. + # The chart will error out on rendering templating if .Values.clients.enabled is set alongside this field, + # since it is expected that this service should point to the PushProx Clients Daemonset / Deployment + selector: {} + +clients: + enabled: true + # The port which the PushProx client will post PushProx metrics to + port: 9369 + # If unset, this will default to the URL for the proxy service: http://pushprox-{{component}}-proxy.{{namepsace}}.svc.cluster.local:{{proxy.port}} + # Should be modified if the clients are being deployed outside the cluster where the proxy rests, otherwise leave it null + proxyUrl: "" + # If set to true, the client will forward any requests from the host IP to 127.0.0.1 + # It will only allow proxy requests to the metricsPort specified + useLocalhost: false + # Configuration for accessing metrics via HTTPS + https: + # Does the client require https to access the metrics? + enabled: false + # Does the client require requests be sent to http or https? + forceHTTPSScheme: false + # If set to true, the client will create a service account with adequate permissions and set a flag + # on the client to use the service account token provided by it to make authorized scrape requests + useServiceAccountCredentials: false + # Configuration for authentication to metrics via https endpoint + authenticationMethod: + # Reads token from defined file in container + # This function is deprecated in the prometheus operator api and may be removed in a future version + bearerTokenFile: + enabled: false + bearerTokenFilePath: "/var/run/secrets/kubernetes.io/serviceaccount/token" + # Reads token from defined secret in namespace + # This function is deprecated in the prometheus operator api and may be removed in a future version + bearerTokenSecret: + enabled: false + # Reads token from defined secret in namespace + authorization: + enabled: false + type: "bearer" + credentials: + key: "token" + optional: false + # If set to true, the client will disable SSL security checks + insecureSkipVerify: false + # Directory on host where necessary TLS cert and key to scrape metrics can be found + certDir: "" + # Filenames for files located in .Values.clients.https.certDir that correspond to TLS settings + certFile: "" + keyFile: "" + caCertFile: "" + # seLinuxOptions to be passed into the container that copies certs. Should define a container with permissions to read the files in the certDir provided on the host. + # Required and only used if `clients.https.enabled` is set and `clients.https.certDir` is provided. + seLinuxOptions: {} + + metrics: + # Whether the client should publish PushProx client-specific metrics to .Values.clients.port + enabled: false + + rbac: + # Additional permissions to provide to the ServiceAccount bound to the client + # This can be used to provide additional permissions for the client to scrape metrics from the k8s API + # Only enabled if clients.https.enabled and clients.https.useServiceAccountCredentials are true + additionalRules: [] + + # Resource limits + resources: {} + + # Options to select all nodes to deploy client DaemonSet on + nodeSelector: {} + tolerations: [] + affinity: {} + + image: + repository: rancher/pushprox-client + tag: v0.1.4-rancher2-client + command: ["pushprox-client"] + + copyCertsImage: + repository: rancher/mirrored-library-busybox + tag: 1.31.1 + + # The default intention of rancher-pushprox clients is to scrape hostNetwork metrics across all nodes. + # This can be used to scrape internal Kubernetes components or DaemonSets of hostNetwork Pods in + # situations where a cloud provider firewall prevents Pod-To-Host communication but not Pod-To-Pod. + # However, if the underlying hostNetwork Pod that is being scraped is managed by a Deployment, + # this advanced option enables users to deploy the client as a Deployment instead of a DaemonSet. + # If a user deploys this feature and the underlying Deployment's number of replicas changes, the user will + # be responsible for upgrading this chart accordingly to the right number of replicas. + deployment: + enabled: false + replicas: 0 + +proxy: + enabled: true + # The port through which PushProx clients will communicate to the proxy + port: 8080 + + # Resource limits + resources: {} + + # Options to select a node to run a single proxy deployment on + nodeSelector: {} + tolerations: [] + + image: + repository: rancher/pushprox-proxy + tag: v0.1.4-rancher2-proxy + command: ["pushprox-proxy"] diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kube-state-metrics/.helmignore b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kube-state-metrics/.helmignore new file mode 100644 index 00000000000..f0c13194444 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kube-state-metrics/.helmignore @@ -0,0 +1,21 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*~ +# Various IDEs +.project +.idea/ +*.tmproj diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kube-state-metrics/Chart.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kube-state-metrics/Chart.yaml new file mode 100644 index 00000000000..002a6a180d6 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kube-state-metrics/Chart.yaml @@ -0,0 +1,32 @@ +annotations: + artifacthub.io/license: Apache-2.0 + artifacthub.io/links: | + - name: Chart Source + url: https://github.com/prometheus-community/helm-charts + catalog.cattle.io/hidden: "true" + catalog.cattle.io/kube-version: '>= 1.26.0-0 < 1.31.0-0' + catalog.cattle.io/os: linux + catalog.rancher.io/certified: rancher + catalog.rancher.io/namespace: cattle-monitoring-system + catalog.rancher.io/release-name: rancher-kube-state-metrics +apiVersion: v2 +appVersion: 2.10.1 +description: Install kube-state-metrics to generate and expose cluster-level metrics +home: https://github.com/kubernetes/kube-state-metrics/ +keywords: +- metric +- monitoring +- prometheus +- kubernetes +maintainers: +- email: tariq.ibrahim@mulesoft.com + name: tariq1890 +- email: manuel@rueg.eu + name: mrueg +- email: david@0xdc.me + name: dotdc +name: kube-state-metrics +sources: +- https://github.com/kubernetes/kube-state-metrics/ +type: application +version: 5.16.4 diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kube-state-metrics/README.md b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kube-state-metrics/README.md new file mode 100644 index 00000000000..843be89e69f --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kube-state-metrics/README.md @@ -0,0 +1,85 @@ +# kube-state-metrics Helm Chart + +Installs the [kube-state-metrics agent](https://github.com/kubernetes/kube-state-metrics). + +## Get Repository Info + +```console +helm repo add prometheus-community https://prometheus-community.github.io/helm-charts +helm repo update +``` + +_See [helm repo](https://helm.sh/docs/helm/helm_repo/) for command documentation._ + + +## Install Chart + +```console +helm install [RELEASE_NAME] prometheus-community/kube-state-metrics [flags] +``` + +_See [configuration](#configuration) below._ + +_See [helm install](https://helm.sh/docs/helm/helm_install/) for command documentation._ + +## Uninstall Chart + +```console +helm uninstall [RELEASE_NAME] +``` + +This removes all the Kubernetes components associated with the chart and deletes the release. + +_See [helm uninstall](https://helm.sh/docs/helm/helm_uninstall/) for command documentation._ + +## Upgrading Chart + +```console +helm upgrade [RELEASE_NAME] prometheus-community/kube-state-metrics [flags] +``` + +_See [helm upgrade](https://helm.sh/docs/helm/helm_upgrade/) for command documentation._ + +### Migrating from stable/kube-state-metrics and kubernetes/kube-state-metrics + +You can upgrade in-place: + +1. [get repository info](#get-repository-info) +1. [upgrade](#upgrading-chart) your existing release name using the new chart repository + +## Upgrading to v3.0.0 + +v3.0.0 includes kube-state-metrics v2.0, see the [changelog](https://github.com/kubernetes/kube-state-metrics/blob/release-2.0/CHANGELOG.md) for major changes on the application-side. + +The upgraded chart now the following changes: + +* Dropped support for helm v2 (helm v3 or later is required) +* collectors key was renamed to resources +* namespace key was renamed to namespaces + +## Configuration + +See [Customizing the Chart Before Installing](https://helm.sh/docs/intro/using_helm/#customizing-the-chart-before-installing). To see all configurable options with detailed comments: + +```console +helm show values prometheus-community/kube-state-metrics +``` + +### kube-rbac-proxy + +You can enable `kube-state-metrics` endpoint protection using `kube-rbac-proxy`. By setting `kubeRBACProxy.enabled: true`, this chart will deploy one RBAC proxy container per endpoint (metrics & telemetry). +To authorize access, authenticate your requests (via a `ServiceAccount` for example) with a `ClusterRole` attached such as: + +```yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: kube-state-metrics-read +rules: + - apiGroups: [ "" ] + resources: ["services/kube-state-metrics"] + verbs: + - get +``` + +See [kube-rbac-proxy examples](https://github.com/brancz/kube-rbac-proxy/tree/master/examples/resource-attributes) for more details. diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kube-state-metrics/templates/NOTES.txt b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kube-state-metrics/templates/NOTES.txt new file mode 100644 index 00000000000..3589c24ec39 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kube-state-metrics/templates/NOTES.txt @@ -0,0 +1,23 @@ +kube-state-metrics is a simple service that listens to the Kubernetes API server and generates metrics about the state of the objects. +The exposed metrics can be found here: +https://github.com/kubernetes/kube-state-metrics/blob/master/docs/README.md#exposed-metrics + +The metrics are exported on the HTTP endpoint /metrics on the listening port. +In your case, {{ template "kube-state-metrics.fullname" . }}.{{ template "kube-state-metrics.namespace" . }}.svc.cluster.local:{{ .Values.service.port }}/metrics + +They are served either as plaintext or protobuf depending on the Accept header. +They are designed to be consumed either by Prometheus itself or by a scraper that is compatible with scraping a Prometheus client endpoint. + +{{- if .Values.kubeRBACProxy.enabled}} + +kube-rbac-proxy endpoint protections is enabled: +- Metrics endpoints are now HTTPS +- Ensure that the client authenticates the requests (e.g. via service account) with the following role permissions: +``` +rules: + - apiGroups: [ "" ] + resources: ["services/{{ template "kube-state-metrics.fullname" . }}"] + verbs: + - get +``` +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kube-state-metrics/templates/_helpers.tpl b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kube-state-metrics/templates/_helpers.tpl new file mode 100644 index 00000000000..ed277fbb53a --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kube-state-metrics/templates/_helpers.tpl @@ -0,0 +1,196 @@ +# Rancher +{{- define "system_default_registry" -}} +{{- if .Values.global.cattle.systemDefaultRegistry -}} +{{- printf "%s/" .Values.global.cattle.systemDefaultRegistry -}} +{{- end -}} +{{- end -}} + +{{- define "monitoring_registry" -}} + {{- $temp_registry := (include "system_default_registry" .) -}} + {{- if $temp_registry -}} + {{- trimSuffix "/" $temp_registry -}} + {{- else -}} + {{- .Values.global.imageRegistry -}} + {{- end -}} +{{- end -}} + +# Windows Support + +{{/* +Windows cluster will add default taint for linux nodes, +add below linux tolerations to workloads could be scheduled to those linux nodes +*/}} + +{{- define "linux-node-tolerations" -}} +- key: "cattle.io/os" + value: "linux" + effect: "NoSchedule" + operator: "Equal" +{{- end -}} + +{{- define "linux-node-selector" -}} +{{- if semverCompare "<1.14-0" .Capabilities.KubeVersion.GitVersion -}} +beta.kubernetes.io/os: linux +{{- else -}} +kubernetes.io/os: linux +{{- end -}} +{{- end -}} + +{{/* vim: set filetype=mustache: */}} +{{/* +Expand the name of the chart. +*/}} +{{- define "kube-state-metrics.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "kube-state-metrics.fullname" -}} +{{- if .Values.fullnameOverride -}} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- $name := default .Chart.Name .Values.nameOverride -}} +{{- if contains $name .Release.Name -}} +{{- .Release.Name | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}} +{{- end -}} +{{- end -}} +{{- end -}} + +{{/* +Create the name of the service account to use +*/}} +{{- define "kube-state-metrics.serviceAccountName" -}} +{{- if .Values.serviceAccount.create -}} + {{ default (include "kube-state-metrics.fullname" .) .Values.serviceAccount.name }} +{{- else -}} + {{ default "default" .Values.serviceAccount.name }} +{{- end -}} +{{- end -}} + +{{/* +Allow the release namespace to be overridden for multi-namespace deployments in combined charts +*/}} +{{- define "kube-state-metrics.namespace" -}} + {{- if .Values.namespaceOverride -}} + {{- .Values.namespaceOverride -}} + {{- else -}} + {{- .Release.Namespace -}} + {{- end -}} +{{- end -}} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "kube-state-metrics.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{/* +Generate basic labels +*/}} +{{- define "kube-state-metrics.labels" }} +helm.sh/chart: {{ template "kube-state-metrics.chart" . }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +app.kubernetes.io/component: metrics +app.kubernetes.io/part-of: {{ template "kube-state-metrics.name" . }} +{{- include "kube-state-metrics.selectorLabels" . }} +{{- if .Chart.AppVersion }} +app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} +{{- end }} +{{- if .Values.customLabels }} +{{ toYaml .Values.customLabels }} +{{- end }} +{{- if .Values.releaseLabel }} +release: {{ .Release.Name }} +{{- end }} +{{- end }} + +{{/* +Selector labels +*/}} +{{- define "kube-state-metrics.selectorLabels" }} +{{- if .Values.selectorOverride }} +{{ toYaml .Values.selectorOverride }} +{{- else }} +app.kubernetes.io/name: {{ include "kube-state-metrics.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end }} +{{- end }} + +{{/* Sets default scrape limits for servicemonitor */}} +{{- define "servicemonitor.scrapeLimits" -}} +{{- with .sampleLimit }} +sampleLimit: {{ . }} +{{- end }} +{{- with .targetLimit }} +targetLimit: {{ . }} +{{- end }} +{{- with .labelLimit }} +labelLimit: {{ . }} +{{- end }} +{{- with .labelNameLengthLimit }} +labelNameLengthLimit: {{ . }} +{{- end }} +{{- with .labelValueLengthLimit }} +labelValueLengthLimit: {{ . }} +{{- end }} +{{- end -}} + +{{/* +Formats imagePullSecrets. Input is (dict "Values" .Values "imagePullSecrets" .{specific imagePullSecrets}) +*/}} +{{- define "kube-state-metrics.imagePullSecrets" -}} +{{- range (concat .Values.global.imagePullSecrets .imagePullSecrets) }} + {{- if eq (typeOf .) "map[string]interface {}" }} +- {{ toYaml . | trim }} + {{- else }} +- name: {{ . }} + {{- end }} +{{- end }} +{{- end -}} + +{{/* +The image to use for kube-state-metrics +*/}} +{{- define "kube-state-metrics.image" -}} +{{- $registry := (include "monitoring_registry" .) }} +{{- if .Values.image.sha }} +{{- if $registry }} +{{- printf "%s/%s:%s@%s" $registry .Values.image.repository (default (printf "v%s" .Chart.AppVersion) .Values.image.tag) .Values.image.sha }} +{{- else }} +{{- printf "%s/%s:%s@%s" .Values.image.registry .Values.image.repository (default (printf "v%s" .Chart.AppVersion) .Values.image.tag) .Values.image.sha }} +{{- end }} +{{- else }} +{{- if $registry }} +{{- printf "%s/%s:%s" $registry .Values.image.repository (default (printf "v%s" .Chart.AppVersion) .Values.image.tag) }} +{{- else }} +{{- printf "%s/%s:%s" .Values.image.registry .Values.image.repository (default (printf "v%s" .Chart.AppVersion) .Values.image.tag) }} +{{- end }} +{{- end }} +{{- end }} + +{{/* +The image to use for kubeRBACProxy +*/}} +{{- define "kubeRBACProxy.image" -}} +{{- $registry := (include "monitoring_registry" .) }} +{{- if .Values.kubeRBACProxy.image.sha }} +{{- if $registry }} +{{- printf "%s/%s:%s@%s" $registry .Values.kubeRBACProxy.image.repository (default (printf "v%s" .Chart.AppVersion) .Values.kubeRBACProxy.image.tag) .Values.kubeRBACProxy.image.sha }} +{{- else }} +{{- printf "%s/%s:%s@%s" .Values.kubeRBACProxy.image.registry .Values.kubeRBACProxy.image.repository (default (printf "v%s" .Chart.AppVersion) .Values.kubeRBACProxy.image.tag) .Values.kubeRBACProxy.image.sha }} +{{- end }} +{{- else }} +{{- if $registry }} +{{- printf "%s/%s:%s" $registry .Values.kubeRBACProxy.image.repository (default (printf "v%s" .Chart.AppVersion) .Values.kubeRBACProxy.image.tag) }} +{{- else }} +{{- printf "%s/%s:%s" .Values.kubeRBACProxy.image.registry .Values.kubeRBACProxy.image.repository (default (printf "v%s" .Chart.AppVersion) .Values.kubeRBACProxy.image.tag) }} +{{- end }} +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kube-state-metrics/templates/ciliumnetworkpolicy.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kube-state-metrics/templates/ciliumnetworkpolicy.yaml new file mode 100644 index 00000000000..025cd47a880 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kube-state-metrics/templates/ciliumnetworkpolicy.yaml @@ -0,0 +1,33 @@ +{{- if and .Values.networkPolicy.enabled (eq .Values.networkPolicy.flavor "cilium") }} +apiVersion: cilium.io/v2 +kind: CiliumNetworkPolicy +metadata: + {{- if .Values.annotations }} + annotations: + {{ toYaml .Values.annotations | nindent 4 }} + {{- end }} + labels: + {{- include "kube-state-metrics.labels" . | indent 4 }} + name: {{ template "kube-state-metrics.fullname" . }} + namespace: {{ template "kube-state-metrics.namespace" . }} +spec: + endpointSelector: + matchLabels: + {{- include "kube-state-metrics.selectorLabels" . | indent 6 }} + egress: + {{- if and .Values.networkPolicy.cilium .Values.networkPolicy.cilium.kubeApiServerSelector }} + {{ toYaml .Values.networkPolicy.cilium.kubeApiServerSelector | nindent 6 }} + {{- else }} + - toEntities: + - kube-apiserver + {{- end }} + ingress: + - toPorts: + - ports: + - port: {{ .Values.service.port | quote }} + protocol: TCP + {{- if .Values.selfMonitor.enabled }} + - port: {{ .Values.selfMonitor.telemetryPort | default 8081 | quote }} + protocol: TCP + {{ end }} +{{ end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kube-state-metrics/templates/clusterrolebinding.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kube-state-metrics/templates/clusterrolebinding.yaml new file mode 100644 index 00000000000..cf9f628d041 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kube-state-metrics/templates/clusterrolebinding.yaml @@ -0,0 +1,20 @@ +{{- if and .Values.rbac.create .Values.rbac.useClusterRole -}} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + labels: + {{- include "kube-state-metrics.labels" . | indent 4 }} + name: {{ template "kube-state-metrics.fullname" . }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole +{{- if .Values.rbac.useExistingRole }} + name: {{ .Values.rbac.useExistingRole }} +{{- else }} + name: {{ template "kube-state-metrics.fullname" . }} +{{- end }} +subjects: +- kind: ServiceAccount + name: {{ template "kube-state-metrics.serviceAccountName" . }} + namespace: {{ template "kube-state-metrics.namespace" . }} +{{- end -}} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kube-state-metrics/templates/crs-configmap.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kube-state-metrics/templates/crs-configmap.yaml new file mode 100644 index 00000000000..d38a75a51df --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kube-state-metrics/templates/crs-configmap.yaml @@ -0,0 +1,16 @@ +{{- if .Values.customResourceState.enabled}} +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ template "kube-state-metrics.fullname" . }}-customresourcestate-config + namespace: {{ template "kube-state-metrics.namespace" . }} + labels: + {{- include "kube-state-metrics.labels" . | indent 4 }} + {{- if .Values.annotations }} + annotations: + {{ toYaml .Values.annotations | nindent 4 }} + {{- end }} +data: + config.yaml: | + {{- toYaml .Values.customResourceState.config | nindent 4 }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kube-state-metrics/templates/deployment.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kube-state-metrics/templates/deployment.yaml new file mode 100644 index 00000000000..03158eb948c --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kube-state-metrics/templates/deployment.yaml @@ -0,0 +1,314 @@ +apiVersion: apps/v1 +{{- if .Values.autosharding.enabled }} +kind: StatefulSet +{{- else }} +kind: Deployment +{{- end }} +metadata: + name: {{ template "kube-state-metrics.fullname" . }} + namespace: {{ template "kube-state-metrics.namespace" . }} + labels: + {{- include "kube-state-metrics.labels" . | indent 4 }} + {{- if .Values.annotations }} + annotations: +{{ toYaml .Values.annotations | indent 4 }} + {{- end }} +spec: + selector: + matchLabels: + {{- include "kube-state-metrics.selectorLabels" . | indent 6 }} + replicas: {{ .Values.replicas }} + {{- if not .Values.autosharding.enabled }} + strategy: + type: {{ .Values.updateStrategy | default "RollingUpdate" }} + {{- end }} + revisionHistoryLimit: {{ .Values.revisionHistoryLimit }} + {{- if .Values.autosharding.enabled }} + serviceName: {{ template "kube-state-metrics.fullname" . }} + volumeClaimTemplates: [] + {{- end }} + template: + metadata: + labels: + {{- include "kube-state-metrics.labels" . | indent 8 }} + {{- if .Values.podAnnotations }} + annotations: +{{ toYaml .Values.podAnnotations | indent 8 }} + {{- end }} + spec: + hostNetwork: {{ .Values.hostNetwork }} + serviceAccountName: {{ template "kube-state-metrics.serviceAccountName" . }} + {{- if .Values.securityContext.enabled }} + securityContext: {{- omit .Values.securityContext "enabled" | toYaml | nindent 8 }} + {{- end }} + {{- if .Values.priorityClassName }} + priorityClassName: {{ .Values.priorityClassName }} + {{- end }} + {{- with .Values.initContainers }} + initContainers: + {{- toYaml . | nindent 6 }} + {{- end }} + containers: + {{- $servicePort := ternary 9090 (.Values.service.port | default 8080) .Values.kubeRBACProxy.enabled}} + {{- $telemetryPort := ternary 9091 (.Values.selfMonitor.telemetryPort | default 8081) .Values.kubeRBACProxy.enabled}} + - name: {{ template "kube-state-metrics.name" . }} + {{- if .Values.autosharding.enabled }} + env: + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + {{- end }} + args: + {{- if .Values.extraArgs }} + {{- .Values.extraArgs | toYaml | nindent 8 }} + {{- end }} + - --port={{ $servicePort }} + {{- if .Values.collectors }} + - --resources={{ .Values.collectors | join "," }} + {{- end }} + {{- if .Values.metricLabelsAllowlist }} + - --metric-labels-allowlist={{ .Values.metricLabelsAllowlist | join "," }} + {{- end }} + {{- if .Values.metricAnnotationsAllowList }} + - --metric-annotations-allowlist={{ .Values.metricAnnotationsAllowList | join "," }} + {{- end }} + {{- if .Values.metricAllowlist }} + - --metric-allowlist={{ .Values.metricAllowlist | join "," }} + {{- end }} + {{- if .Values.metricDenylist }} + - --metric-denylist={{ .Values.metricDenylist | join "," }} + {{- end }} + {{- $namespaces := list }} + {{- if .Values.namespaces }} + {{- range $ns := join "," .Values.namespaces | split "," }} + {{- $namespaces = append $namespaces (tpl $ns $) }} + {{- end }} + {{- end }} + {{- if .Values.releaseNamespace }} + {{- $namespaces = append $namespaces ( include "kube-state-metrics.namespace" . ) }} + {{- end }} + {{- if $namespaces }} + - --namespaces={{ $namespaces | mustUniq | join "," }} + {{- end }} + {{- if .Values.namespacesDenylist }} + - --namespaces-denylist={{ tpl (.Values.namespacesDenylist | join ",") $ }} + {{- end }} + {{- if .Values.autosharding.enabled }} + - --pod=$(POD_NAME) + - --pod-namespace=$(POD_NAMESPACE) + {{- end }} + {{- if .Values.kubeconfig.enabled }} + - --kubeconfig=/opt/k8s/.kube/config + {{- end }} + {{- if .Values.kubeRBACProxy.enabled }} + - --telemetry-host=127.0.0.1 + - --telemetry-port={{ $telemetryPort }} + {{- else }} + {{- if .Values.selfMonitor.telemetryHost }} + - --telemetry-host={{ .Values.selfMonitor.telemetryHost }} + {{- end }} + {{- if .Values.selfMonitor.telemetryPort }} + - --telemetry-port={{ $telemetryPort }} + {{- end }} + {{- end }} + {{- if .Values.customResourceState.enabled }} + - --custom-resource-state-config-file=/etc/customresourcestate/config.yaml + {{- end }} + {{- if or (.Values.kubeconfig.enabled) (.Values.customResourceState.enabled) (.Values.volumeMounts) }} + volumeMounts: + {{- if .Values.kubeconfig.enabled }} + - name: kubeconfig + mountPath: /opt/k8s/.kube/ + readOnly: true + {{- end }} + {{- if .Values.customResourceState.enabled }} + - name: customresourcestate-config + mountPath: /etc/customresourcestate + readOnly: true + {{- end }} + {{- if .Values.volumeMounts }} +{{ toYaml .Values.volumeMounts | indent 8 }} + {{- end }} + {{- end }} + imagePullPolicy: {{ .Values.image.pullPolicy }} + image: {{ include "kube-state-metrics.image" . }} + {{- if eq .Values.kubeRBACProxy.enabled false }} + ports: + - containerPort: {{ .Values.service.port | default 8080}} + name: "http" + {{- if .Values.selfMonitor.enabled }} + - containerPort: {{ $telemetryPort }} + name: "metrics" + {{- end }} + {{- end }} + livenessProbe: + failureThreshold: {{ .Values.livenessProbe.failureThreshold }} + httpGet: + {{- if .Values.hostNetwork }} + host: 127.0.0.1 + {{- end }} + httpHeaders: + {{- range $_, $header := .Values.livenessProbe.httpGet.httpHeaders }} + - name: {{ $header.name }} + value: {{ $header.value }} + {{- end }} + path: /healthz + port: {{ $servicePort }} + scheme: {{ upper .Values.livenessProbe.httpGet.scheme }} + initialDelaySeconds: {{ .Values.livenessProbe.initialDelaySeconds }} + periodSeconds: {{ .Values.livenessProbe.periodSeconds }} + successThreshold: {{ .Values.livenessProbe.successThreshold }} + timeoutSeconds: {{ .Values.livenessProbe.timeoutSeconds }} + readinessProbe: + failureThreshold: {{ .Values.readinessProbe.failureThreshold }} + httpGet: + {{- if .Values.hostNetwork }} + host: 127.0.0.1 + {{- end }} + httpHeaders: + {{- range $_, $header := .Values.readinessProbe.httpGet.httpHeaders }} + - name: {{ $header.name }} + value: {{ $header.value }} + {{- end }} + path: / + port: {{ $servicePort }} + scheme: {{ upper .Values.readinessProbe.httpGet.scheme }} + initialDelaySeconds: {{ .Values.readinessProbe.initialDelaySeconds }} + periodSeconds: {{ .Values.readinessProbe.periodSeconds }} + successThreshold: {{ .Values.readinessProbe.successThreshold }} + timeoutSeconds: {{ .Values.readinessProbe.timeoutSeconds }} + {{- if .Values.resources }} + resources: +{{ toYaml .Values.resources | indent 10 }} +{{- end }} +{{- if .Values.containerSecurityContext }} + securityContext: +{{ toYaml .Values.containerSecurityContext | indent 10 }} +{{- end }} + {{- if .Values.kubeRBACProxy.enabled }} + - name: kube-rbac-proxy-http + args: + {{- if .Values.kubeRBACProxy.extraArgs }} + {{- .Values.kubeRBACProxy.extraArgs | toYaml | nindent 8 }} + {{- end }} + - --secure-listen-address=:{{ .Values.service.port | default 8080}} + - --upstream=http://127.0.0.1:{{ $servicePort }}/ + - --proxy-endpoints-port=8888 + - --config-file=/etc/kube-rbac-proxy-config/config-file.yaml + volumeMounts: + - name: kube-rbac-proxy-config + mountPath: /etc/kube-rbac-proxy-config + {{- with .Values.kubeRBACProxy.volumeMounts }} + {{- toYaml . | nindent 10 }} + {{- end }} + imagePullPolicy: {{ .Values.kubeRBACProxy.image.pullPolicy }} + image: {{ include "kubeRBACProxy.image" . }} + ports: + - containerPort: {{ .Values.service.port | default 8080}} + name: "http" + - containerPort: 8888 + name: "http-healthz" + readinessProbe: + httpGet: + scheme: HTTPS + port: 8888 + path: healthz + initialDelaySeconds: 5 + timeoutSeconds: 5 + {{- if .Values.kubeRBACProxy.resources }} + resources: +{{ toYaml .Values.kubeRBACProxy.resources | indent 10 }} +{{- end }} +{{- if .Values.kubeRBACProxy.containerSecurityContext }} + securityContext: +{{ toYaml .Values.kubeRBACProxy.containerSecurityContext | indent 10 }} +{{- end }} + {{- if .Values.selfMonitor.enabled }} + - name: kube-rbac-proxy-telemetry + args: + {{- if .Values.kubeRBACProxy.extraArgs }} + {{- .Values.kubeRBACProxy.extraArgs | toYaml | nindent 8 }} + {{- end }} + - --secure-listen-address=:{{ .Values.selfMonitor.telemetryPort | default 8081 }} + - --upstream=http://127.0.0.1:{{ $telemetryPort }}/ + - --proxy-endpoints-port=8889 + - --config-file=/etc/kube-rbac-proxy-config/config-file.yaml + volumeMounts: + - name: kube-rbac-proxy-config + mountPath: /etc/kube-rbac-proxy-config + {{- with .Values.kubeRBACProxy.volumeMounts }} + {{- toYaml . | nindent 10 }} + {{- end }} + imagePullPolicy: {{ .Values.kubeRBACProxy.image.pullPolicy }} + image: {{ include "kubeRBACProxy.image" . }} + ports: + - containerPort: {{ .Values.selfMonitor.telemetryPort | default 8081 }} + name: "metrics" + - containerPort: 8889 + name: "metrics-healthz" + readinessProbe: + httpGet: + scheme: HTTPS + port: 8889 + path: healthz + initialDelaySeconds: 5 + timeoutSeconds: 5 + {{- if .Values.kubeRBACProxy.resources }} + resources: +{{ toYaml .Values.kubeRBACProxy.resources | indent 10 }} +{{- end }} +{{- if .Values.kubeRBACProxy.containerSecurityContext }} + securityContext: +{{ toYaml .Values.kubeRBACProxy.containerSecurityContext | indent 10 }} +{{- end }} + {{- end }} + {{- end }} + {{- with .Values.containers }} + {{- toYaml . | nindent 6 }} + {{- end }} +{{- if or .Values.imagePullSecrets .Values.global.imagePullSecrets }} + imagePullSecrets: + {{- include "kube-state-metrics.imagePullSecrets" (dict "Values" .Values "imagePullSecrets" .Values.imagePullSecrets) | indent 8 }} + {{- end }} + {{- if .Values.affinity }} + affinity: +{{ toYaml .Values.affinity | indent 8 }} + {{- end }} + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} + {{- if .Values.nodeSelector }} +{{ toYaml .Values.nodeSelector | indent 8 }} + {{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} + {{- if .Values.tolerations }} +{{ toYaml .Values.tolerations | indent 8 }} + {{- end }} + {{- if .Values.topologySpreadConstraints }} + topologySpreadConstraints: +{{ toYaml .Values.topologySpreadConstraints | indent 8 }} + {{- end }} + {{- if or (.Values.kubeconfig.enabled) (.Values.customResourceState.enabled) (.Values.volumes) (.Values.kubeRBACProxy.enabled) }} + volumes: + {{- if .Values.kubeconfig.enabled}} + - name: kubeconfig + secret: + secretName: {{ template "kube-state-metrics.fullname" . }}-kubeconfig + {{- end }} + {{- if .Values.kubeRBACProxy.enabled}} + - name: kube-rbac-proxy-config + configMap: + name: {{ template "kube-state-metrics.fullname" . }}-rbac-config + {{- end }} + {{- if .Values.customResourceState.enabled}} + - name: customresourcestate-config + configMap: + name: {{ template "kube-state-metrics.fullname" . }}-customresourcestate-config + {{- end }} + {{- if .Values.volumes }} +{{ toYaml .Values.volumes | indent 8 }} + {{- end }} + {{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kube-state-metrics/templates/extra-manifests.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kube-state-metrics/templates/extra-manifests.yaml new file mode 100644 index 00000000000..567f7bf3297 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kube-state-metrics/templates/extra-manifests.yaml @@ -0,0 +1,4 @@ +{{ range .Values.extraManifests }} +--- +{{ tpl (toYaml .) $ }} +{{ end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kube-state-metrics/templates/kubeconfig-secret.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kube-state-metrics/templates/kubeconfig-secret.yaml new file mode 100644 index 00000000000..6af00845029 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kube-state-metrics/templates/kubeconfig-secret.yaml @@ -0,0 +1,12 @@ +{{- if .Values.kubeconfig.enabled -}} +apiVersion: v1 +kind: Secret +metadata: + name: {{ template "kube-state-metrics.fullname" . }}-kubeconfig + namespace: {{ template "kube-state-metrics.namespace" . }} + labels: + {{- include "kube-state-metrics.labels" . | indent 4 }} +type: Opaque +data: + config: '{{ .Values.kubeconfig.secret }}' +{{- end -}} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kube-state-metrics/templates/networkpolicy.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kube-state-metrics/templates/networkpolicy.yaml new file mode 100644 index 00000000000..309b38ec54e --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kube-state-metrics/templates/networkpolicy.yaml @@ -0,0 +1,43 @@ +{{- if and .Values.networkPolicy.enabled (eq .Values.networkPolicy.flavor "kubernetes") }} +kind: NetworkPolicy +apiVersion: networking.k8s.io/v1 +metadata: + {{- if .Values.annotations }} + annotations: + {{ toYaml .Values.annotations | nindent 4 }} + {{- end }} + labels: + {{- include "kube-state-metrics.labels" . | indent 4 }} + name: {{ template "kube-state-metrics.fullname" . }} + namespace: {{ template "kube-state-metrics.namespace" . }} +spec: + {{- if .Values.networkPolicy.egress }} + ## Deny all egress by default + egress: + {{- toYaml .Values.networkPolicy.egress | nindent 4 }} + {{- end }} + ingress: + {{- if .Values.networkPolicy.ingress }} + {{- toYaml .Values.networkPolicy.ingress | nindent 4 }} + {{- else }} + ## Allow ingress on default ports by default + - ports: + - port: {{ .Values.service.port | default 8080 }} + protocol: TCP + {{- if .Values.selfMonitor.enabled }} + {{- $telemetryPort := ternary 9091 (.Values.selfMonitor.telemetryPort | default 8081) .Values.kubeRBACProxy.enabled}} + - port: {{ $telemetryPort }} + protocol: TCP + {{- end }} + {{- end }} + podSelector: + {{- if .Values.networkPolicy.podSelector }} + {{- toYaml .Values.networkPolicy.podSelector | nindent 4 }} + {{- else }} + matchLabels: + {{- include "kube-state-metrics.selectorLabels" . | indent 6 }} + {{- end }} + policyTypes: + - Ingress + - Egress +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kube-state-metrics/templates/pdb.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kube-state-metrics/templates/pdb.yaml new file mode 100644 index 00000000000..3771b511de0 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kube-state-metrics/templates/pdb.yaml @@ -0,0 +1,18 @@ +{{- if .Values.podDisruptionBudget -}} +{{ if $.Capabilities.APIVersions.Has "policy/v1/PodDisruptionBudget" -}} +apiVersion: policy/v1 +{{- else -}} +apiVersion: policy/v1beta1 +{{- end }} +kind: PodDisruptionBudget +metadata: + name: {{ template "kube-state-metrics.fullname" . }} + namespace: {{ template "kube-state-metrics.namespace" . }} + labels: + {{- include "kube-state-metrics.labels" . | indent 4 }} +spec: + selector: + matchLabels: + app.kubernetes.io/name: {{ template "kube-state-metrics.name" . }} +{{ toYaml .Values.podDisruptionBudget | indent 2 }} +{{- end -}} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kube-state-metrics/templates/podsecuritypolicy.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kube-state-metrics/templates/podsecuritypolicy.yaml new file mode 100644 index 00000000000..d9d944d7401 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kube-state-metrics/templates/podsecuritypolicy.yaml @@ -0,0 +1,39 @@ +{{- if and .Values.rbac.create (and (or .Values.global.cattle.psp.enabled .Values.podSecurityPolicy.enabled) (.Capabilities.APIVersions.Has "policy/v1beta1/PodSecurityPolicy")) }} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ template "kube-state-metrics.fullname" . }} + labels: + {{- include "kube-state-metrics.labels" . | indent 4 }} +{{- if .Values.podSecurityPolicy.annotations }} + annotations: +{{ toYaml .Values.podSecurityPolicy.annotations | indent 4 }} +{{- end }} +spec: + privileged: false + volumes: + - 'secret' +{{- if .Values.podSecurityPolicy.additionalVolumes }} +{{ toYaml .Values.podSecurityPolicy.additionalVolumes | indent 4 }} +{{- end }} + hostNetwork: false + hostIPC: false + hostPID: false + runAsUser: + rule: 'MustRunAsNonRoot' + seLinux: + rule: 'RunAsAny' + supplementalGroups: + rule: 'MustRunAs' + ranges: + # Forbid adding the root group. + - min: 1 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + # Forbid adding the root group. + - min: 1 + max: 65535 + readOnlyRootFilesystem: false +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kube-state-metrics/templates/psp-clusterrole.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kube-state-metrics/templates/psp-clusterrole.yaml new file mode 100644 index 00000000000..c69e01a7163 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kube-state-metrics/templates/psp-clusterrole.yaml @@ -0,0 +1,19 @@ +{{- if and .Values.rbac.create (and (or .Values.global.cattle.psp.enabled .Values.podSecurityPolicy.enabled) (.Capabilities.APIVersions.Has "policy/v1beta1/PodSecurityPolicy")) }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + {{- include "kube-state-metrics.labels" . | indent 4 }} + name: psp-{{ template "kube-state-metrics.fullname" . }} +rules: +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if semverCompare "> 1.15.0-0" $kubeTargetVersion }} +- apiGroups: ['policy'] +{{- else }} +- apiGroups: ['extensions'] +{{- end }} + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - {{ template "kube-state-metrics.fullname" . }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kube-state-metrics/templates/psp-clusterrolebinding.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kube-state-metrics/templates/psp-clusterrolebinding.yaml new file mode 100644 index 00000000000..df81c490289 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kube-state-metrics/templates/psp-clusterrolebinding.yaml @@ -0,0 +1,16 @@ +{{- if and .Values.rbac.create (and (or .Values.global.cattle.psp.enabled .Values.podSecurityPolicy.enabled) (.Capabilities.APIVersions.Has "policy/v1beta1/PodSecurityPolicy")) }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + labels: + {{- include "kube-state-metrics.labels" . | indent 4 }} + name: psp-{{ template "kube-state-metrics.fullname" . }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: psp-{{ template "kube-state-metrics.fullname" . }} +subjects: + - kind: ServiceAccount + name: {{ template "kube-state-metrics.serviceAccountName" . }} + namespace: {{ template "kube-state-metrics.namespace" . }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kube-state-metrics/templates/rbac-configmap.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kube-state-metrics/templates/rbac-configmap.yaml new file mode 100644 index 00000000000..671dc9d6604 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kube-state-metrics/templates/rbac-configmap.yaml @@ -0,0 +1,22 @@ +{{- if .Values.kubeRBACProxy.enabled}} +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ template "kube-state-metrics.fullname" . }}-rbac-config + namespace: {{ template "kube-state-metrics.namespace" . }} + labels: + {{- include "kube-state-metrics.labels" . | indent 4 }} + {{- if .Values.annotations }} + annotations: + {{ toYaml .Values.annotations | nindent 4 }} + {{- end }} +data: + config-file.yaml: |+ + authorization: + resourceAttributes: + namespace: {{ template "kube-state-metrics.namespace" . }} + apiVersion: v1 + resource: services + subresource: {{ template "kube-state-metrics.fullname" . }} + name: {{ template "kube-state-metrics.fullname" . }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kube-state-metrics/templates/role.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kube-state-metrics/templates/role.yaml new file mode 100644 index 00000000000..0170878376e --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kube-state-metrics/templates/role.yaml @@ -0,0 +1,215 @@ +{{- if not (kindIs "slice" .Values.collectors) }} +{{- fail "Collectors need to be a List since kube-state-metrics chart 3.2.2. Please check README for more information."}} +{{- end }} +{{- if and (eq .Values.rbac.create true) (not .Values.rbac.useExistingRole) -}} +{{- range (ternary (join "," .Values.namespaces | split "," ) (list "") (eq $.Values.rbac.useClusterRole false)) }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +{{- if eq $.Values.rbac.useClusterRole false }} +kind: Role +{{- else }} +kind: ClusterRole +{{- end }} +metadata: + labels: + {{- include "kube-state-metrics.labels" $ | indent 4 }} + name: {{ template "kube-state-metrics.fullname" $ }} +{{- if eq $.Values.rbac.useClusterRole false }} + namespace: {{ . }} +{{- end }} +rules: +{{ if has "certificatesigningrequests" $.Values.collectors }} +- apiGroups: ["certificates.k8s.io"] + resources: + - certificatesigningrequests + verbs: ["list", "watch"] +{{ end -}} +{{ if has "configmaps" $.Values.collectors }} +- apiGroups: [""] + resources: + - configmaps + verbs: ["list", "watch"] +{{ end -}} +{{ if has "cronjobs" $.Values.collectors }} +- apiGroups: ["batch"] + resources: + - cronjobs + verbs: ["list", "watch"] +{{ end -}} +{{ if has "daemonsets" $.Values.collectors }} +- apiGroups: ["extensions", "apps"] + resources: + - daemonsets + verbs: ["list", "watch"] +{{ end -}} +{{ if has "deployments" $.Values.collectors }} +- apiGroups: ["extensions", "apps"] + resources: + - deployments + verbs: ["list", "watch"] +{{ end -}} +{{ if has "endpoints" $.Values.collectors }} +- apiGroups: [""] + resources: + - endpoints + verbs: ["list", "watch"] +{{ end -}} +{{ if has "endpointslices" $.Values.collectors }} +- apiGroups: ["discovery.k8s.io"] + resources: + - endpointslices + verbs: ["list", "watch"] +{{ end -}} +{{ if has "horizontalpodautoscalers" $.Values.collectors }} +- apiGroups: ["autoscaling"] + resources: + - horizontalpodautoscalers + verbs: ["list", "watch"] +{{ end -}} +{{ if has "ingresses" $.Values.collectors }} +- apiGroups: ["extensions", "networking.k8s.io"] + resources: + - ingresses + verbs: ["list", "watch"] +{{ end -}} +{{ if has "jobs" $.Values.collectors }} +- apiGroups: ["batch"] + resources: + - jobs + verbs: ["list", "watch"] +{{ end -}} +{{ if has "leases" $.Values.collectors }} +- apiGroups: ["coordination.k8s.io"] + resources: + - leases + verbs: ["list", "watch"] +{{ end -}} +{{ if has "limitranges" $.Values.collectors }} +- apiGroups: [""] + resources: + - limitranges + verbs: ["list", "watch"] +{{ end -}} +{{ if has "mutatingwebhookconfigurations" $.Values.collectors }} +- apiGroups: ["admissionregistration.k8s.io"] + resources: + - mutatingwebhookconfigurations + verbs: ["list", "watch"] +{{ end -}} +{{ if has "namespaces" $.Values.collectors }} +- apiGroups: [""] + resources: + - namespaces + verbs: ["list", "watch"] +{{ end -}} +{{ if has "networkpolicies" $.Values.collectors }} +- apiGroups: ["networking.k8s.io"] + resources: + - networkpolicies + verbs: ["list", "watch"] +{{ end -}} +{{ if has "nodes" $.Values.collectors }} +- apiGroups: [""] + resources: + - nodes + verbs: ["list", "watch"] +{{ end -}} +{{ if has "persistentvolumeclaims" $.Values.collectors }} +- apiGroups: [""] + resources: + - persistentvolumeclaims + verbs: ["list", "watch"] +{{ end -}} +{{ if has "persistentvolumes" $.Values.collectors }} +- apiGroups: [""] + resources: + - persistentvolumes + verbs: ["list", "watch"] +{{ end -}} +{{ if has "poddisruptionbudgets" $.Values.collectors }} +- apiGroups: ["policy"] + resources: + - poddisruptionbudgets + verbs: ["list", "watch"] +{{ end -}} +{{ if has "pods" $.Values.collectors }} +- apiGroups: [""] + resources: + - pods + verbs: ["list", "watch"] +{{ end -}} +{{ if has "replicasets" $.Values.collectors }} +- apiGroups: ["extensions", "apps"] + resources: + - replicasets + verbs: ["list", "watch"] +{{ end -}} +{{ if has "replicationcontrollers" $.Values.collectors }} +- apiGroups: [""] + resources: + - replicationcontrollers + verbs: ["list", "watch"] +{{ end -}} +{{ if has "resourcequotas" $.Values.collectors }} +- apiGroups: [""] + resources: + - resourcequotas + verbs: ["list", "watch"] +{{ end -}} +{{ if has "secrets" $.Values.collectors }} +- apiGroups: [""] + resources: + - secrets + verbs: ["list", "watch"] +{{ end -}} +{{ if has "services" $.Values.collectors }} +- apiGroups: [""] + resources: + - services + verbs: ["list", "watch"] +{{ end -}} +{{ if has "statefulsets" $.Values.collectors }} +- apiGroups: ["apps"] + resources: + - statefulsets + verbs: ["list", "watch"] +{{ end -}} +{{ if has "storageclasses" $.Values.collectors }} +- apiGroups: ["storage.k8s.io"] + resources: + - storageclasses + verbs: ["list", "watch"] +{{ end -}} +{{ if has "validatingwebhookconfigurations" $.Values.collectors }} +- apiGroups: ["admissionregistration.k8s.io"] + resources: + - validatingwebhookconfigurations + verbs: ["list", "watch"] +{{ end -}} +{{ if has "volumeattachments" $.Values.collectors }} +- apiGroups: ["storage.k8s.io"] + resources: + - volumeattachments + verbs: ["list", "watch"] +{{ end -}} +{{- if $.Values.kubeRBACProxy.enabled }} +- apiGroups: ["authentication.k8s.io"] + resources: + - tokenreviews + verbs: ["create"] +- apiGroups: ["authorization.k8s.io"] + resources: + - subjectaccessreviews + verbs: ["create"] +{{- end }} +{{- if $.Values.customResourceState.enabled }} +- apiGroups: ["apiextensions.k8s.io"] + resources: + - customresourcedefinitions + verbs: ["list", "watch"] +{{- end }} +{{ if $.Values.rbac.extraRules }} +{{ toYaml $.Values.rbac.extraRules }} +{{ end }} +{{- end -}} +{{- end -}} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kube-state-metrics/templates/rolebinding.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kube-state-metrics/templates/rolebinding.yaml new file mode 100644 index 00000000000..330651b73f4 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kube-state-metrics/templates/rolebinding.yaml @@ -0,0 +1,24 @@ +{{- if and (eq .Values.rbac.create true) (eq .Values.rbac.useClusterRole false) -}} +{{- range (join "," $.Values.namespaces) | split "," }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + labels: + {{- include "kube-state-metrics.labels" $ | indent 4 }} + name: {{ template "kube-state-metrics.fullname" $ }} + namespace: {{ . }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role +{{- if (not $.Values.rbac.useExistingRole) }} + name: {{ template "kube-state-metrics.fullname" $ }} +{{- else }} + name: {{ $.Values.rbac.useExistingRole }} +{{- end }} +subjects: +- kind: ServiceAccount + name: {{ template "kube-state-metrics.serviceAccountName" $ }} + namespace: {{ template "kube-state-metrics.namespace" $ }} +{{- end -}} +{{- end -}} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kube-state-metrics/templates/service.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kube-state-metrics/templates/service.yaml new file mode 100644 index 00000000000..6c486a662ab --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kube-state-metrics/templates/service.yaml @@ -0,0 +1,49 @@ +apiVersion: v1 +kind: Service +metadata: + name: {{ template "kube-state-metrics.fullname" . }} + namespace: {{ template "kube-state-metrics.namespace" . }} + labels: + {{- include "kube-state-metrics.labels" . | indent 4 }} + annotations: + {{- if .Values.prometheusScrape }} + prometheus.io/scrape: '{{ .Values.prometheusScrape }}' + {{- end }} + {{- if .Values.service.annotations }} + {{- toYaml .Values.service.annotations | nindent 4 }} + {{- end }} +spec: + type: "{{ .Values.service.type }}" + ports: + - name: "http" + protocol: TCP + port: {{ .Values.service.port | default 8080}} + {{- if .Values.service.nodePort }} + nodePort: {{ .Values.service.nodePort }} + {{- end }} + targetPort: {{ .Values.service.port | default 8080}} + {{ if .Values.selfMonitor.enabled }} + - name: "metrics" + protocol: TCP + port: {{ .Values.selfMonitor.telemetryPort | default 8081 }} + targetPort: {{ .Values.selfMonitor.telemetryPort | default 8081 }} + {{- if .Values.selfMonitor.telemetryNodePort }} + nodePort: {{ .Values.selfMonitor.telemetryNodePort }} + {{- end }} + {{ end }} +{{- if .Values.service.loadBalancerIP }} + loadBalancerIP: "{{ .Values.service.loadBalancerIP }}" +{{- end }} +{{- if .Values.service.loadBalancerSourceRanges }} + loadBalancerSourceRanges: + {{- range $cidr := .Values.service.loadBalancerSourceRanges }} + - {{ $cidr }} + {{- end }} +{{- end }} +{{- if .Values.autosharding.enabled }} + clusterIP: None +{{- else if .Values.service.clusterIP }} + clusterIP: "{{ .Values.service.clusterIP }}" +{{- end }} + selector: + {{- include "kube-state-metrics.selectorLabels" . | indent 4 }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kube-state-metrics/templates/serviceaccount.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kube-state-metrics/templates/serviceaccount.yaml new file mode 100644 index 00000000000..38a93b31d1e --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kube-state-metrics/templates/serviceaccount.yaml @@ -0,0 +1,17 @@ +{{- if .Values.serviceAccount.create -}} +apiVersion: v1 +kind: ServiceAccount +metadata: + labels: + {{- include "kube-state-metrics.labels" . | indent 4 }} + name: {{ template "kube-state-metrics.serviceAccountName" . }} + namespace: {{ template "kube-state-metrics.namespace" . }} +{{- if .Values.serviceAccount.annotations }} + annotations: +{{ toYaml .Values.serviceAccount.annotations | indent 4 }} +{{- end }} +{{- if or .Values.serviceAccount.imagePullSecrets .Values.global.imagePullSecrets }} +imagePullSecrets: + {{- include "kube-state-metrics.imagePullSecrets" (dict "Values" .Values "imagePullSecrets" .Values.serviceAccount.imagePullSecrets) | indent 2 }} +{{- end }} +{{- end -}} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kube-state-metrics/templates/servicemonitor.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kube-state-metrics/templates/servicemonitor.yaml new file mode 100644 index 00000000000..01ec44e0676 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kube-state-metrics/templates/servicemonitor.yaml @@ -0,0 +1,126 @@ +{{- if .Values.prometheus.monitor.enabled }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ template "kube-state-metrics.fullname" . }} + namespace: {{ template "kube-state-metrics.namespace" . }} + labels: + {{- include "kube-state-metrics.labels" . | indent 4 }} + {{- with .Values.prometheus.monitor.additionalLabels }} + {{- tpl (toYaml . | nindent 4) $ }} + {{- end }} + {{- with .Values.prometheus.monitor.annotations }} + annotations: + {{- tpl (toYaml . | nindent 4) $ }} + {{- end }} +spec: + jobLabel: {{ default "app.kubernetes.io/name" .Values.prometheus.monitor.jobLabel }} + {{- with .Values.prometheus.monitor.targetLabels }} + targetLabels: + {{- toYaml . | trim | nindent 4 }} + {{- end }} + {{- with .Values.prometheus.monitor.podTargetLabels }} + podTargetLabels: + {{- toYaml . | trim | nindent 4 }} + {{- end }} + {{- include "servicemonitor.scrapeLimits" .Values.prometheus.monitor | indent 2 }} + {{- if .Values.prometheus.monitor.namespaceSelector }} + namespaceSelector: + matchNames: + {{- with .Values.prometheus.monitor.namespaceSelector }} + {{- toYaml . | nindent 6 }} + {{- end }} + {{- end }} + selector: + matchLabels: + {{- with .Values.prometheus.monitor.selectorOverride }} + {{- toYaml . | nindent 6 }} + {{- else }} + {{- include "kube-state-metrics.selectorLabels" . | indent 6 }} + {{- end }} + endpoints: + - port: http + {{- if .Values.prometheus.monitor.interval }} + interval: {{ .Values.prometheus.monitor.interval }} + {{- end }} + {{- if .Values.prometheus.monitor.scrapeTimeout }} + scrapeTimeout: {{ .Values.prometheus.monitor.scrapeTimeout }} + {{- end }} + {{- if .Values.prometheus.monitor.proxyUrl }} + proxyUrl: {{ .Values.prometheus.monitor.proxyUrl}} + {{- end }} + {{- if .Values.prometheus.monitor.enableHttp2 }} + enableHttp2: {{ .Values.prometheus.monitor.enableHttp2}} + {{- end }} + {{- if .Values.prometheus.monitor.honorLabels }} + honorLabels: true + {{- end }} + metricRelabelings: + {{- if .Values.prometheus.monitor.metricRelabelings }} + {{- toYaml .Values.prometheus.monitor.metricRelabelings | nindent 6 }} + {{- end }} + {{ if .Values.global.cattle.clusterId }} + - sourceLabels: [__address__] + targetLabel: cluster_id + replacement: {{ .Values.global.cattle.clusterId }} + {{- end }} + {{ if .Values.global.cattle.clusterName }} + - sourceLabels: [__address__] + targetLabel: cluster_name + replacement: {{ .Values.global.cattle.clusterName }} + {{- end }} + {{- if .Values.prometheus.monitor.relabelings }} + relabelings: + {{- toYaml .Values.prometheus.monitor.relabelings | nindent 8 }} + {{- end }} + {{- if .Values.prometheus.monitor.scheme }} + scheme: {{ .Values.prometheus.monitor.scheme }} + {{- end }} + {{- if .Values.prometheus.monitor.tlsConfig }} + tlsConfig: + {{- toYaml .Values.prometheus.monitor.tlsConfig | nindent 8 }} + {{- end }} + {{- if .Values.prometheus.monitor.bearerTokenFile }} + bearerTokenFile: {{ .Values.prometheus.monitor.bearerTokenFile }} + {{- end }} + {{- with .Values.prometheus.monitor.bearerTokenSecret }} + bearerTokenSecret: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- if .Values.selfMonitor.enabled }} + - port: metrics + {{- if .Values.prometheus.monitor.interval }} + interval: {{ .Values.prometheus.monitor.interval }} + {{- end }} + {{- if .Values.prometheus.monitor.scrapeTimeout }} + scrapeTimeout: {{ .Values.prometheus.monitor.scrapeTimeout }} + {{- end }} + {{- if .Values.prometheus.monitor.proxyUrl }} + proxyUrl: {{ .Values.prometheus.monitor.proxyUrl}} + {{- end }} + {{- if .Values.prometheus.monitor.enableHttp2 }} + enableHttp2: {{ .Values.prometheus.monitor.enableHttp2}} + {{- end }} + {{- if .Values.prometheus.monitor.honorLabels }} + honorLabels: true + {{- end }} + {{- if .Values.prometheus.monitor.relabelings }} + relabelings: + {{- toYaml .Values.prometheus.monitor.relabelings | nindent 8 }} + {{- end }} + {{- if .Values.prometheus.monitor.scheme }} + scheme: {{ .Values.prometheus.monitor.scheme }} + {{- end }} + {{- if .Values.prometheus.monitor.tlsConfig }} + tlsConfig: + {{- toYaml .Values.prometheus.monitor.tlsConfig | nindent 8 }} + {{- end }} + {{- if .Values.prometheus.monitor.bearerTokenFile }} + bearerTokenFile: {{ .Values.prometheus.monitor.bearerTokenFile }} + {{- end }} + {{- with .Values.prometheus.monitor.bearerTokenSecret }} + bearerTokenSecret: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kube-state-metrics/templates/stsdiscovery-role.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kube-state-metrics/templates/stsdiscovery-role.yaml new file mode 100644 index 00000000000..489de147c15 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kube-state-metrics/templates/stsdiscovery-role.yaml @@ -0,0 +1,26 @@ +{{- if and .Values.autosharding.enabled .Values.rbac.create -}} +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: stsdiscovery-{{ template "kube-state-metrics.fullname" . }} + namespace: {{ template "kube-state-metrics.namespace" . }} + labels: + {{- include "kube-state-metrics.labels" . | indent 4 }} +rules: +- apiGroups: + - "" + resources: + - pods + verbs: + - get +- apiGroups: + - apps + resourceNames: + - {{ template "kube-state-metrics.fullname" . }} + resources: + - statefulsets + verbs: + - get + - list + - watch +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kube-state-metrics/templates/stsdiscovery-rolebinding.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kube-state-metrics/templates/stsdiscovery-rolebinding.yaml new file mode 100644 index 00000000000..73b37a4f647 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kube-state-metrics/templates/stsdiscovery-rolebinding.yaml @@ -0,0 +1,17 @@ +{{- if and .Values.autosharding.enabled .Values.rbac.create -}} +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: stsdiscovery-{{ template "kube-state-metrics.fullname" . }} + namespace: {{ template "kube-state-metrics.namespace" . }} + labels: + {{- include "kube-state-metrics.labels" . | indent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: stsdiscovery-{{ template "kube-state-metrics.fullname" . }} +subjects: + - kind: ServiceAccount + name: {{ template "kube-state-metrics.serviceAccountName" . }} + namespace: {{ template "kube-state-metrics.namespace" . }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kube-state-metrics/templates/verticalpodautoscaler.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kube-state-metrics/templates/verticalpodautoscaler.yaml new file mode 100644 index 00000000000..f46305b517a --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kube-state-metrics/templates/verticalpodautoscaler.yaml @@ -0,0 +1,44 @@ +{{- if and (.Capabilities.APIVersions.Has "autoscaling.k8s.io/v1") (.Values.verticalPodAutoscaler.enabled) }} +apiVersion: autoscaling.k8s.io/v1 +kind: VerticalPodAutoscaler +metadata: + name: {{ template "kube-state-metrics.fullname" . }} + namespace: {{ template "kube-state-metrics.namespace" . }} + labels: + {{- include "kube-state-metrics.labels" . | indent 4 }} +spec: + {{- with .Values.verticalPodAutoscaler.recommenders }} + recommenders: + {{- toYaml . | nindent 4 }} + {{- end }} + resourcePolicy: + containerPolicies: + - containerName: {{ template "kube-state-metrics.name" . }} + {{- with .Values.verticalPodAutoscaler.controlledResources }} + controlledResources: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- if .Values.verticalPodAutoscaler.controlledValues }} + controlledValues: {{ .Values.verticalPodAutoscaler.controlledValues }} + {{- end }} + {{- if .Values.verticalPodAutoscaler.maxAllowed }} + maxAllowed: + {{ toYaml .Values.verticalPodAutoscaler.maxAllowed | nindent 8 }} + {{- end }} + {{- if .Values.verticalPodAutoscaler.minAllowed }} + minAllowed: + {{ toYaml .Values.verticalPodAutoscaler.minAllowed | nindent 8 }} + {{- end }} + targetRef: + apiVersion: apps/v1 + {{- if .Values.autosharding.enabled }} + kind: StatefulSet + {{- else }} + kind: Deployment + {{- end }} + name: {{ template "kube-state-metrics.fullname" . }} + {{- with .Values.verticalPodAutoscaler.updatePolicy }} + updatePolicy: + {{- toYaml . | nindent 4 }} + {{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kube-state-metrics/values.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kube-state-metrics/values.yaml new file mode 100644 index 00000000000..bc8ee28fda4 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kube-state-metrics/values.yaml @@ -0,0 +1,491 @@ +# Default values for kube-state-metrics. +prometheusScrape: true +image: + registry: docker.io + repository: rancher/mirrored-kube-state-metrics-kube-state-metrics + tag: v2.10.1 + sha: "" + pullPolicy: IfNotPresent + +imagePullSecrets: [] +# - name: "image-pull-secret" + +global: + cattle: + psp: + enabled: false + systemDefaultRegistry: "" + + # To help compatibility with other charts which use global.imagePullSecrets. + # Allow either an array of {name: pullSecret} maps (k8s-style), or an array of strings (more common helm-style). + # global: + # imagePullSecrets: + # - name: pullSecret1 + # - name: pullSecret2 + # or + # global: + # imagePullSecrets: + # - pullSecret1 + # - pullSecret2 + imagePullSecrets: [] + # + # Allow parent charts to override registry hostname + imageRegistry: "" + +# If set to true, this will deploy kube-state-metrics as a StatefulSet and the data +# will be automatically sharded across <.Values.replicas> pods using the built-in +# autodiscovery feature: https://github.com/kubernetes/kube-state-metrics#automated-sharding +# This is an experimental feature and there are no stability guarantees. +autosharding: + enabled: false + +replicas: 1 + +# Change the deployment strategy when autosharding is disabled. +# ref: https://kubernetes.io/docs/concepts/workloads/controllers/deployment/#strategy +# The default is "RollingUpdate" as per Kubernetes defaults. +# During a release, 'RollingUpdate' can lead to two running instances for a short period of time while 'Recreate' can create a small gap in data. +# updateStrategy: Recreate + +# Number of old history to retain to allow rollback +# Default Kubernetes value is set to 10 +revisionHistoryLimit: 10 + +# List of additional cli arguments to configure kube-state-metrics +# for example: --enable-gzip-encoding, --log-file, etc. +# all the possible args can be found here: https://github.com/kubernetes/kube-state-metrics/blob/master/docs/cli-arguments.md +extraArgs: [] + +service: + port: 8080 + # Default to clusterIP for backward compatibility + type: ClusterIP + nodePort: 0 + loadBalancerIP: "" + # Only allow access to the loadBalancerIP from these IPs + loadBalancerSourceRanges: [] + clusterIP: "" + annotations: {} + +## Additional labels to add to all resources +customLabels: {} + # app: kube-state-metrics + +## Override selector labels +selectorOverride: {} + +## set to true to add the release label so scraping of the servicemonitor with kube-prometheus-stack works out of the box +releaseLabel: false + +hostNetwork: false + +rbac: + # If true, create & use RBAC resources + create: true + + # Set to a rolename to use existing role - skipping role creating - but still doing serviceaccount and rolebinding to it, rolename set here. + # useExistingRole: your-existing-role + + # If set to false - Run without Cluteradmin privs needed - ONLY works if namespace is also set (if useExistingRole is set this name is used as ClusterRole or Role to bind to) + useClusterRole: true + + # Add permissions for CustomResources' apiGroups in Role/ClusterRole. Should be used in conjunction with Custom Resource State Metrics configuration + # Example: + # - apiGroups: ["monitoring.coreos.com"] + # resources: ["prometheuses"] + # verbs: ["list", "watch"] + extraRules: [] + +# Configure kube-rbac-proxy. When enabled, creates one kube-rbac-proxy container per exposed HTTP endpoint (metrics and telemetry if enabled). +# The requests are served through the same service but requests are then HTTPS. +kubeRBACProxy: + enabled: false + image: + repository: rancher/mirrored-kube-rbac-proxy + tag: v0.14.0 + sha: "" + pullPolicy: IfNotPresent + + # List of additional cli arguments to configure kube-rbac-prxy + # for example: --tls-cipher-suites, --log-file, etc. + # all the possible args can be found here: https://github.com/brancz/kube-rbac-proxy#usage + extraArgs: [] + + ## Specify security settings for a Container + ## Allows overrides and additional options compared to (Pod) securityContext + ## Ref: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/#set-the-security-context-for-a-container + containerSecurityContext: + readOnlyRootFilesystem: true + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + + resources: {} + # We usually recommend not to specify default resources and to leave this as a conscious + # choice for the user. This also increases chances charts run on environments with little + # resources, such as Minikube. If you do want to specify resources, uncomment the following + # lines, adjust them as necessary, and remove the curly braces after 'resources:'. + # limits: + # cpu: 100m + # memory: 64Mi + # requests: + # cpu: 10m + # memory: 32Mi + + ## volumeMounts enables mounting custom volumes in rbac-proxy containers + ## Useful for TLS certificates and keys + volumeMounts: [] + # - mountPath: /etc/tls + # name: kube-rbac-proxy-tls + # readOnly: true + +serviceAccount: + # Specifies whether a ServiceAccount should be created, require rbac true + create: true + # The name of the ServiceAccount to use. + # If not set and create is true, a name is generated using the fullname template + name: + # Reference to one or more secrets to be used when pulling images + # ref: https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/ + imagePullSecrets: [] + # ServiceAccount annotations. + # Use case: AWS EKS IAM roles for service accounts + # ref: https://docs.aws.amazon.com/eks/latest/userguide/specify-service-account-role.html + annotations: {} + +prometheus: + monitor: + enabled: false + annotations: {} + additionalLabels: {} + namespace: "" + namespaceSelector: [] + jobLabel: "" + targetLabels: [] + podTargetLabels: [] + interval: "" + ## SampleLimit defines per-scrape limit on number of scraped samples that will be accepted. + ## + sampleLimit: 0 + + ## TargetLimit defines a limit on the number of scraped targets that will be accepted. + ## + targetLimit: 0 + + ## Per-scrape limit on number of labels that will be accepted for a sample. Only valid in Prometheus versions 2.27.0 and newer. + ## + labelLimit: 0 + + ## Per-scrape limit on length of labels name that will be accepted for a sample. Only valid in Prometheus versions 2.27.0 and newer. + ## + labelNameLengthLimit: 0 + + ## Per-scrape limit on length of labels value that will be accepted for a sample. Only valid in Prometheus versions 2.27.0 and newer. + ## + labelValueLengthLimit: 0 + scrapeTimeout: "" + proxyUrl: "" + ## Whether to enable HTTP2 for servicemonitor + # enableHttp2: false + selectorOverride: {} + honorLabels: false + metricRelabelings: [] + relabelings: [] + scheme: "" + ## File to read bearer token for scraping targets + bearerTokenFile: "" + ## Secret to mount to read bearer token for scraping targets. The secret needs + ## to be in the same namespace as the service monitor and accessible by the + ## Prometheus Operator + bearerTokenSecret: {} + # name: secret-name + # key: key-name + tlsConfig: {} + +## Specify if a Pod Security Policy for kube-state-metrics must be created +## Ref: https://kubernetes.io/docs/concepts/policy/pod-security-policy/ +## +podSecurityPolicy: + annotations: {} + ## Specify pod annotations + ## Ref: https://kubernetes.io/docs/concepts/policy/pod-security-policy/#apparmor + ## Ref: https://kubernetes.io/docs/concepts/policy/pod-security-policy/#seccomp + ## Ref: https://kubernetes.io/docs/concepts/policy/pod-security-policy/#sysctl + ## + # seccomp.security.alpha.kubernetes.io/allowedProfileNames: '*' + # seccomp.security.alpha.kubernetes.io/defaultProfileName: 'docker/default' + # apparmor.security.beta.kubernetes.io/defaultProfileName: 'runtime/default' + + additionalVolumes: [] + +## Configure network policy for kube-state-metrics +networkPolicy: + enabled: false + # networkPolicy.flavor -- Flavor of the network policy to use. + # Can be: + # * kubernetes for networking.k8s.io/v1/NetworkPolicy + # * cilium for cilium.io/v2/CiliumNetworkPolicy + flavor: kubernetes + + ## Configure the cilium network policy kube-apiserver selector + # cilium: + # kubeApiServerSelector: + # - toEntities: + # - kube-apiserver + + # egress: + # - {} + # ingress: + # - {} + # podSelector: + # matchLabels: + # app.kubernetes.io/name: kube-state-metrics + +securityContext: + enabled: true + runAsGroup: 65534 + runAsUser: 65534 + fsGroup: 65534 + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault + +## Specify security settings for a Container +## Allows overrides and additional options compared to (Pod) securityContext +## Ref: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/#set-the-security-context-for-a-container +containerSecurityContext: + readOnlyRootFilesystem: true + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + +## Node labels for pod assignment +## Ref: https://kubernetes.io/docs/user-guide/node-selection/ +nodeSelector: {} + +## Affinity settings for pod assignment +## Ref: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/ +affinity: {} + +## Tolerations for pod assignment +## Ref: https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/ +tolerations: [] + +## Topology spread constraints for pod assignment +## Ref: https://kubernetes.io/docs/concepts/workloads/pods/pod-topology-spread-constraints/ +topologySpreadConstraints: [] + +# Annotations to be added to the deployment/statefulset +annotations: {} + +# Annotations to be added to the pod +podAnnotations: {} + +## Assign a PriorityClassName to pods if set +# priorityClassName: "" + +# Ref: https://kubernetes.io/docs/tasks/run-application/configure-pdb/ +podDisruptionBudget: {} + +# Comma-separated list of metrics to be exposed. +# This list comprises of exact metric names and/or regex patterns. +# The allowlist and denylist are mutually exclusive. +metricAllowlist: [] + +# Comma-separated list of metrics not to be enabled. +# This list comprises of exact metric names and/or regex patterns. +# The allowlist and denylist are mutually exclusive. +metricDenylist: [] + +# Comma-separated list of additional Kubernetes label keys that will be used in the resource's +# labels metric. By default the metric contains only name and namespace labels. +# To include additional labels, provide a list of resource names in their plural form and Kubernetes +# label keys you would like to allow for them (Example: '=namespaces=[k8s-label-1,k8s-label-n,...],pods=[app],...)'. +# A single '*' can be provided per resource instead to allow any labels, but that has +# severe performance implications (Example: '=pods=[*]'). +metricLabelsAllowlist: [] + # - namespaces=[k8s-label-1,k8s-label-n] + +# Comma-separated list of Kubernetes annotations keys that will be used in the resource' +# labels metric. By default the metric contains only name and namespace labels. +# To include additional annotations provide a list of resource names in their plural form and Kubernetes +# annotation keys you would like to allow for them (Example: '=namespaces=[kubernetes.io/team,...],pods=[kubernetes.io/team],...)'. +# A single '*' can be provided per resource instead to allow any annotations, but that has +# severe performance implications (Example: '=pods=[*]'). +metricAnnotationsAllowList: [] + # - pods=[k8s-annotation-1,k8s-annotation-n] + +# Available collectors for kube-state-metrics. +# By default, all available resources are enabled, comment out to disable. +collectors: + - certificatesigningrequests + - configmaps + - cronjobs + - daemonsets + - deployments + - endpoints + - horizontalpodautoscalers + - ingresses + - jobs + - leases + - limitranges + - mutatingwebhookconfigurations + - namespaces + - networkpolicies + - nodes + - persistentvolumeclaims + - persistentvolumes + - poddisruptionbudgets + - pods + - replicasets + - replicationcontrollers + - resourcequotas + - secrets + - services + - statefulsets + - storageclasses + - validatingwebhookconfigurations + - volumeattachments + +# Enabling kubeconfig will pass the --kubeconfig argument to the container +kubeconfig: + enabled: false + # base64 encoded kube-config file + secret: + +# Enabling support for customResourceState, will create a configMap including your config that will be read from kube-state-metrics +customResourceState: + enabled: false + # Add (Cluster)Role permissions to list/watch the customResources defined in the config to rbac.extraRules + config: {} + +# Enable only the release namespace for collecting resources. By default all namespaces are collected. +# If releaseNamespace and namespaces are both set a merged list will be collected. +releaseNamespace: false + +# Comma-separated list(string) or yaml list of namespaces to be enabled for collecting resources. By default all namespaces are collected. +namespaces: "" + +# Comma-separated list of namespaces not to be enabled. If namespaces and namespaces-denylist are both set, +# only namespaces that are excluded in namespaces-denylist will be used. +namespacesDenylist: "" + +## Override the deployment namespace +## +namespaceOverride: "" + +resources: {} + # We usually recommend not to specify default resources and to leave this as a conscious + # choice for the user. This also increases chances charts run on environments with little + # resources, such as Minikube. If you do want to specify resources, uncomment the following + # lines, adjust them as necessary, and remove the curly braces after 'resources:'. + # limits: + # cpu: 100m + # memory: 64Mi + # requests: + # cpu: 10m + # memory: 32Mi + +## Provide a k8s version to define apiGroups for podSecurityPolicy Cluster Role. +## For example: kubeTargetVersionOverride: 1.14.9 +## +kubeTargetVersionOverride: "" + +# Enable self metrics configuration for service and Service Monitor +# Default values for telemetry configuration can be overridden +# If you set telemetryNodePort, you must also set service.type to NodePort +selfMonitor: + enabled: false + # telemetryHost: 0.0.0.0 + # telemetryPort: 8081 + # telemetryNodePort: 0 + +# Enable vertical pod autoscaler support for kube-state-metrics +verticalPodAutoscaler: + enabled: false + + # Recommender responsible for generating recommendation for the object. + # List should be empty (then the default recommender will generate the recommendation) + # or contain exactly one recommender. + # recommenders: [] + # - name: custom-recommender-performance + + # List of resources that the vertical pod autoscaler can control. Defaults to cpu and memory + controlledResources: [] + # Specifies which resource values should be controlled: RequestsOnly or RequestsAndLimits. + # controlledValues: RequestsAndLimits + + # Define the max allowed resources for the pod + maxAllowed: {} + # cpu: 200m + # memory: 100Mi + # Define the min allowed resources for the pod + minAllowed: {} + # cpu: 200m + # memory: 100Mi + + # updatePolicy: + # Specifies minimal number of replicas which need to be alive for VPA Updater to attempt pod eviction + # minReplicas: 1 + # Specifies whether recommended updates are applied when a Pod is started and whether recommended updates + # are applied during the life of a Pod. Possible values are "Off", "Initial", "Recreate", and "Auto". + # updateMode: Auto + +# volumeMounts are used to add custom volume mounts to deployment. +# See example below +volumeMounts: [] +# - mountPath: /etc/config +# name: config-volume + +# volumes are used to add custom volumes to deployment +# See example below +volumes: [] +# - configMap: +# name: cm-for-volume +# name: config-volume + +# Extra manifests to deploy as an array +extraManifests: [] + # - apiVersion: v1 + # kind: ConfigMap + # metadata: + # labels: + # name: prometheus-extra + # data: + # extra-data: "value" + +## Containers allows injecting additional containers. +containers: [] + # - name: crd-init + # image: kiwigrid/k8s-sidecar:latest + +## InitContainers allows injecting additional initContainers. +initContainers: [] + # - name: crd-sidecar + # image: kiwigrid/k8s-sidecar:latest + +## Liveness probe +## +livenessProbe: + failureThreshold: 3 + httpGet: + httpHeaders: [] + scheme: http + initialDelaySeconds: 5 + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 5 + +## Readiness probe +## +readinessProbe: + failureThreshold: 3 + httpGet: + httpHeaders: [] + scheme: http + initialDelaySeconds: 5 + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 5 diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmControllerManager/.helmignore b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmControllerManager/.helmignore new file mode 100644 index 00000000000..0e8a0eb36f4 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmControllerManager/.helmignore @@ -0,0 +1,23 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*.orig +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmControllerManager/Chart.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmControllerManager/Chart.yaml new file mode 100644 index 00000000000..0b753af5db9 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmControllerManager/Chart.yaml @@ -0,0 +1,15 @@ +annotations: + catalog.cattle.io/hidden: "true" + catalog.cattle.io/kube-version: '>= 1.26.0-0 < 1.31.0-0' + catalog.cattle.io/os: linux + catalog.rancher.io/certified: rancher + catalog.rancher.io/namespace: cattle-monitoring-system + catalog.rancher.io/release-name: rancher-pushprox +apiVersion: v1 +appVersion: v0.1.4-rancher2 +description: Sets up a deployment of the PushProx proxy and a DaemonSet of PushProx + clients. +kubeVersion: '>=1.26.0-0' +name: kubeAdmControllerManager +type: application +version: 0.2.1 diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmControllerManager/README.md b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmControllerManager/README.md new file mode 100644 index 00000000000..345002f48a2 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmControllerManager/README.md @@ -0,0 +1,90 @@ +# rancher-pushprox + +A Rancher chart based on Rancher [PushProx](https://github.com/rancher/PushProx) that sets up a Deployment of a PushProx proxy and a DaemonSet of PushProx clients on a Kubernetes cluster. + +Installs [rancher-pushprox](https://github.com/rancher/charts/tree/gh-pages/packages/rancher-pushprox) to create PushProx clients that can access their host's network and register with a PushProx proxy. A [Prometheus Operator](https://github.com/coreos/prometheus-operator) ServiceMonitor CR is also included that is configured to scrape the metrics from each of the clients through the proxy. + +Using an instance of this chart is suitable for the following scenarios: +- You need to scrape metrics from a port that should not be accessible outside of the host (e.g. scraping `etcd` metrics in a hardened cluster) +- You need to scrape metrics on a host that are not exposed outside of 127.0.0.1 (e.g. scraping `kube-proxy` metrics) +- You need to scrape metrics through HTTPS using certs hosted directly on `hostPath` +- You need to scrape metrics from Kubernetes components that require authorization via a service account (e.g. permissions to make request to `/metrics`) +- You need to scrape metrics without access to cacerts (i.e. enable `insecureSkipVerify`) + +The clients and proxy are created based on a Rancher fork of the [prometheus-community/PushProx](https://github.com/prometheus-community/PushProx) project. + +## Upgrading to Kubernetes v1.25+ + +Starting in Kubernetes v1.25, [Pod Security Policies](https://kubernetes.io/docs/concepts/security/pod-security-policy/) have been removed from the Kubernetes API. + +As a result, **before upgrading to Kubernetes v1.25** (or on a fresh install in a Kubernetes v1.25+ cluster), users are expected to perform an in-place upgrade of this chart with `global.cattle.psp.enabled` set to `false` if it has been previously set to `true`. +​ +> **Note:** +> In this chart release, any previous field that was associated with any PSP resources have been removed in favor of a single global field: `global.cattle.psp.enabled`. + +> **Note:** +> If you upgrade your cluster to Kubernetes v1.25+ before removing PSPs via a `helm upgrade` (even if you manually clean up resources), **it will leave the Helm release in a broken state within the cluster such that further Helm operations will not work (`helm uninstall`, `helm upgrade`, etc.).** +> +> If your charts get stuck in this state, please consult the Rancher docs on how to clean up your Helm release secrets. + +Upon setting `global.cattle.psp.enabled` to false, the chart will remove any PSP resources deployed on its behalf from the cluster. This is the default setting for this chart. + +As a replacement for PSPs, [Pod Security Admission](https://kubernetes.io/docs/concepts/security/pod-security-admission/) should be used. Please consult the Rancher docs for more details on how to configure your chart release namespaces to work with the new Pod Security Admission and apply Pod Security Standards. + +## Configuration + +The following tables list the configurable parameters of the rancher-pushprox chart and their default values. + +### General + +#### Required +| Parameter | Description | Example | +| ----- | ----------- | ------ | +| `component` | The component that is being monitored | `kube-etcd` +| `metricsPort` | The port on the host that contains the metrics you want to scrape (e.g. `http://:/metrics`) | `2379` | +| `namespaceOverride` | The namespace to install the chart | `""` + +#### Optional +| Parameter | Description | Default | +| ----- | ----------- | ------ | +| `serviceMonitor.enabled` | Deploys a [Prometheus Operator](https://github.com/coreos/prometheus-operator/blob/master/Documentation/api.md#servicemonitor) ServiceMonitor CR that is configured to scrape metrics on the hosts that the clients are deployed on via the proxy. Also deploys a Service that points to all pods with the expected client name that exposes the `metricsPort` selected | `true` | +| `serviceMonitor.endpoints` | A list of endpoints that will be added to the ServiceMonitor based on the [Endpoint spec](https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#endpoint) | `[{port: metrics}]` | +| `service.selector` | The selector that is used to populate the Service's Endpoints object. The chart will error out on rendering templating if `.Values.clients.enabled` is set alongside this field, since it is expected that this service should point to the PushProx Clients Daemonset / Deployment | `{}` | +| `clients.enabled` | Deploys a DaemonSet of clients that are each capable of scraping endpoints on the hostNetwork it is deployed on | `true` | +| `clients.port` | The port where the client will publish PushProx client-specific metrics. If deploying multiple clients onto the same node, the clients should not have conflicting ports | `9369` | +| `clients.proxyUrl` | Overrides the default proxyUrl setting of `http://pushprox-{{ .Values.component }}-proxy.{{ . Release.Namespace }}.svc.cluster.local:{{ .Values.proxy.port }}"` with the `proxyUrl` specified | `""` | +| `clients.useLocalhost` | Sets a flag on each client deployment to redirect scrapes directed to `HOST_IP` to `127.0.0.1` | `false` | +| `clients.https.enabled` | Enables scraping metrics via HTTPS using the provided TLS certs that exist on each host | `false` | +| `clients.https.forceHTTPSScheme` | Forces scraping metrics via HTTPS using the provided TLS certs that exist on each host | `false` | +| `clients.https.useServiceAccountCredentials` | If set to true, the client will create a service account with permissions to scrape `/metrics` endpoint of Kubernetes components. The client will use the service account token provided to make authorized scrape requests to the Kubernetes API | `false` | +| `clients.https.authenticationMethod.bearerTokenFile.enabled` | If set to true, the client will use service account credentials mounted at the configured path `clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath`. This requires permissions to scrape `/metrics` endpoint of Kubernetes components. This method is deprecated by the prometheus operator and may be removed in a future release | `false` | +| `clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath` | This is a volume mount on the pod with permissions to scrape `/metrics` endpoint of Kubernetes components | `"/var/run/secrets/kubernetes.io/serviceaccount/token"` | +| `clients.https.authenticationMethod.bearerTokenSecret.enabled` | If set to true, the client will use service account credentials to scrape `/metrics` endpoint of Kubernetes components. This method is deprecated by the prometheus operator and may be removed in a future release | `false` | +| `clients.https.authenticationMethod.authorization.enabled` | If set to true, the client will use service account credentials to scrape `/metrics` endpoint of Kubernetes components | `false` | +| `clients.https.authenticationMethod.authorization.type` | If set, the client will use this type of authorization in its client requests for metrics | `"bearer"` | +| `clients.https.authenticationMethod.authorization.credentials.key` | If set, the client will use this key in the secret created by `clients.https.useServiceAccountCredentials` for authorization in its client requests for metrics | `"token"` | +| `clients.https.authenticationMethod.authorization.credentials.optional` | If set to false, the client will fail if the key in the secret created by `clients.https.useServiceAccountCredentials` does not exist | `false` | +| `clients.https.insecureSkipVerify` | If set to true, the client will disable SSL security checks | `false` | +| `clients.https.certDir` | A `hostPath` where TLS certs can be found. This path is mounted as a volume on an `initContainer` which copies only the necessary files over to an EmptyDir volume used by each client. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.certFile` | The path to the TLS cert file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.keyFile` | The path to the TLS key file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.caCertFile` | The path to the TLS cacert file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.seLinuxOptions` | seLinuxOptions to be passed into the container that copies certs. Should define a container with permissions to read the files in the certDir provided on the host. Required and only used if `clients.https.enabled` is set and `clients.https.certDir` is provided. | `""` | +| `clients.metrics.enabled` | Whether the client should publish PushProx client-specific metrics. | `false` | +| `clients.rbac.additionalRules` | Additional permissions to provide to the ServiceAccount bound to the client. This can be used to provide additional permissions for the client to scrape metrics from the k8s API. Only enabled if clients.https.enabled and clients.https.useServiceAccountCredentials are true | `[]` | +| `clients.deployment.enabled` | Deploys the client as a Deployment (generally used if the underlying hostNetwork Pod that is being scraped is managed by a Deployment) | `false` | +| `clients.deployment.replicas` | The number of pods the Deployment has, it should match the number of pod the hostNetwork Deployment has. Required and only used if `client.deployment.enable` is set | `0` | +| `clients.deployment.affinity` | The affinity rules that allocate the pod to the node in which the hostNetwork Deployment's pods run. Required and only used if `client.deployment.enable` is set | `{}` | +| `clients.resources` | Set resource limits and requests for the client container | `{}` | +| `clients.nodeSelector` | Select which nodes to deploy the clients on | `{}` | +| `clients.tolerations` | Specify tolerations for clients | `[]` | +| `proxy.enabled` | Deploys the proxy that each client will register with | `true` | +| `proxy.port` | The port exposed by the proxy that each client will register with to allow metrics to be scraped from the host | `8080` | +| `proxy.resources` | Set resource limits and requests for the proxy container | `{}` | +| `proxy.nodeSelector` | Select which nodes the proxy can be deployed on | `{}` | +| `proxy.tolerations` | Specify tolerations (if necessary) to allow the proxy to be deployed on the selected node | `[]` | +| `kubeVersionOverrides` | A list of Semver constraint strings (defined by https://github.com/Masterminds/semver) and values.yaml overrides. For each key in kubeVersionOverrides, this chart will check to see if the current Kubernetes cluster's version matches any of the semver constraints provided as keys on the map. On seeing a match, the default value for each values.yaml field overridden will be updated with the new value. If multiple matches are encountered (due to overlapping semver ranges), the matches will be applied in order. | `[]` + +*Tip: The filepaths set in `clients.https.File` can include wildcard characters*. + +See [rancher-monitoring](https://github.com/rancher/charts/tree/gh-pages/packages/rancher-monitoring) for examples of how this chart can be used. diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmControllerManager/templates/_helpers.tpl b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmControllerManager/templates/_helpers.tpl new file mode 100644 index 00000000000..1ba50939441 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmControllerManager/templates/_helpers.tpl @@ -0,0 +1,170 @@ +# Rancher + +{{- define "system_default_registry" -}} +{{- if .Values.global.cattle.systemDefaultRegistry -}} +{{- printf "%s/" .Values.global.cattle.systemDefaultRegistry -}} +{{- end -}} +{{- end -}} + +# Windows Support + +{{/* +Windows cluster will add default taint for linux nodes, +add below linux tolerations to workloads could be scheduled to those linux nodes +*/}} + +{{- define "linux-node-tolerations" -}} +- key: "cattle.io/os" + value: "linux" + effect: "NoSchedule" + operator: "Equal" +{{- end -}} + +{{- define "linux-node-selector" -}} +{{- if semverCompare "<1.14-0" .Capabilities.KubeVersion.GitVersion -}} +beta.kubernetes.io/os: linux +{{- else -}} +kubernetes.io/os: linux +{{- end -}} +{{- end -}} + +# General + +{{- define "applyKubeVersionOverrides" -}} +{{- $overrides := dict -}} +{{- range $override := .Values.kubeVersionOverrides -}} +{{- if semverCompare $override.constraint $.Capabilities.KubeVersion.Version -}} +{{- $_ := mergeOverwrite $overrides $override.values -}} +{{- end -}} +{{- end -}} +{{- $_ := mergeOverwrite .Values $overrides -}} +{{- end -}} + +{{- define "pushprox.namespace" -}} + {{- if .Values.namespaceOverride -}} + {{- .Values.namespaceOverride -}} + {{- else -}} + {{- .Release.Namespace -}} + {{- end -}} +{{- end -}} + +{{- define "pushProxy.commonLabels" -}} +release: {{ .Release.Name }} +component: {{ .Values.component | quote }} +provider: kubernetes +{{- end -}} + +{{- define "pushProxy.proxyUrl" -}} +{{- $_ := (required "Template requires either .Values.proxy.port or .Values.client.proxyUrl to set proxyUrl for client" (or .Values.clients.proxyUrl .Values.proxy.port)) -}} +{{- if .Values.clients.proxyUrl -}} +{{ printf "%s" .Values.clients.proxyUrl }} +{{- else -}} +{{ printf "http://%s.%s.svc:%d" (include "pushProxy.proxy.name" .) (include "pushprox.namespace" .) (int .Values.proxy.port) }} +{{- end -}}{{- end -}} + +# Client + +{{- define "pushProxy.client.name" -}} +{{- printf "pushprox-%s-client" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.client.serviceAccountTokenName" -}} +{{- printf "pushprox-%s-client-service-account-token" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.client.labels" -}} +k8s-app: {{ template "pushProxy.client.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +# Proxy + +{{- define "pushProxy.proxy.name" -}} +{{- printf "pushprox-%s-proxy" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.proxy.labels" -}} +k8s-app: {{ template "pushProxy.proxy.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +# ServiceMonitor + +{{- define "pushprox.serviceMonitor.name" -}} +{{- printf "%s-%s" .Release.Name (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.serviceMonitor.labels" -}} +app: {{ template "pushprox.serviceMonitor.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +{{- define "pushProxy.serviceMonitor.endpoints" -}} +{{- $proxyURL := (include "pushProxy.proxyUrl" .) -}} +{{- $useHTTPS := .Values.clients.https.enabled -}} +{{- $setHTTPSScheme := .Values.clients.https.forceHTTPSScheme -}} +{{- $insecureSkipVerify := .Values.clients.https.insecureSkipVerify -}} +{{- $useServiceAccountCredentials := .Values.clients.https.useServiceAccountCredentials -}} +{{- $serviceAccountTokenName := (include "pushProxy.client.serviceAccountTokenName" . ) -}} +{{- $metricRelabelings := list }} +{{- $endpoints := .Values.serviceMonitor.endpoints }} +{{- if .Values.proxy.enabled }} +{{- $_ := set . "proxyUrl" $proxyURL }} +{{- end }} +{{- range $endpoints }} +{{- if $.Values.proxy.enabled }} +{{- $_ := set . "proxyUrl" $proxyURL }} +{{- end }} +{{- $clusterIdRelabel := dict }} +{{- $metricRelabelings := list }} +{{- if $.Values.global.cattle.clusterId }} +{{- $_ := set $clusterIdRelabel "action" "replace" }} +{{- $_ := set $clusterIdRelabel "sourceLabels" (list "__address__") }} +{{- $_ := set $clusterIdRelabel "targetLabel" "cluster_id" }} +{{- $_ := set $clusterIdRelabel "replacement" $.Values.global.cattle.clusterId }} +{{- $metricRelabelings = append $metricRelabelings $clusterIdRelabel }} +{{- end }} +{{- $clusterNameRelabel := dict }} +{{- if $.Values.global.cattle.clusterName }} +{{- $_ := set $clusterNameRelabel "action" "replace" }} +{{- $_ := set $clusterNameRelabel "sourceLabels" (list "__address__") }} +{{- $_ := set $clusterNameRelabel "targetLabel" "cluster_name" }} +{{- $_ := set $clusterNameRelabel "replacement" $.Values.global.cattle.clusterName }} +{{- $metricRelabelings = append $metricRelabelings $clusterNameRelabel }} +{{- end }} +{{- if not (empty $metricRelabelings) }} +{{- $_ := set . "metricRelabelings" ($metricRelabelings)}} +{{- end }} +{{- if $setHTTPSScheme -}} +{{- $_ := set . "scheme" "https" }} +{{- end -}} +{{- if $useHTTPS -}} +{{- if (hasKey . "params") }} +{{- $_ := set (get . "params") "_scheme" (list "https") }} +{{- else }} +{{- $_ := set . "params" (dict "_scheme" (list "https")) }} +{{- end }} +{{- end }} +{{- if (hasKey . "tlsConfig") }} +{{- $_ := set (get . "tlsConfig") "insecureSkipVerify" $insecureSkipVerify }} +{{- else }} +{{- $_ := set . "tlsConfig" (dict "insecureSkipVerify" $insecureSkipVerify) }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.bearerTokenFile.enabled }} +{{- $_ := set . "bearerTokenFile" $.Values.clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.bearerTokenSecret.enabled }} +{{- $_ := set . "bearerTokenSecret" $serviceAccountTokenName }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.authorization.enabled }} +{{- if (hasKey . "authorization") }} +{{- $_ := set (get . "authorization") "type" $.Values.clients.https.authenticationMethod.authorization.type }} +{{- $_ := set (get . "authorization") "credentials" (dict "name" $serviceAccountTokenName "key" $.Values.clients.https.authenticationMethod.authorization.credentials.key "optional" $.Values.clients.https.authenticationMethod.authorization.credentials.optional) }} +{{- else }} +{{- $_ := set . "authorization" (dict "type" $.Values.clients.https.authenticationMethod.authorization.type) }} +{{- $_ := set . "authorization" (dict "credentials" (dict "name" $serviceAccountTokenName "key" $.Values.clients.https.authenticationMethod.authorization.credentials.key "optional" $.Values.clients.https.authenticationMethod.authorization.credentials.optional)) }} +{{- end }} +{{- end }} +{{- end }} +{{- toYaml $endpoints }} +{{- end -}} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmControllerManager/templates/pushprox-clients-rbac.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmControllerManager/templates/pushprox-clients-rbac.yaml new file mode 100644 index 00000000000..a8e27c37356 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmControllerManager/templates/pushprox-clients-rbac.yaml @@ -0,0 +1,97 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.clients }}{{- if .Values.clients.enabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ template "pushProxy.client.name" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +rules: +{{- if .Values.global.cattle.psp.enabled }} +- apiGroups: ['policy'] + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - {{ template "pushProxy.client.name" . }} +{{- end }} +{{- if and .Values.clients.https.enabled .Values.clients.https.useServiceAccountCredentials }} +- nonResourceURLs: ["/metrics"] + verbs: ["get"] +{{- if .Values.clients.rbac.additionalRules }} +{{ toYaml .Values.clients.rbac.additionalRules }} +{{- end }} +{{- end }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ template "pushProxy.client.name" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "pushProxy.client.name" . }} +subjects: + - kind: ServiceAccount + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +--- +{{- if .Values.clients.https.useServiceAccountCredentials }} +apiVersion: v1 +kind: Secret +type: kubernetes.io/service-account-token +metadata: + name: {{ template "pushProxy.client.serviceAccountTokenName" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} + annotations: + kubernetes.io/service-account.name: {{ template "pushProxy.client.name" . }} +{{- end }} +--- +{{- if .Values.global.cattle.psp.enabled }} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +spec: + privileged: false + hostNetwork: true + hostIPC: false + hostPID: false + runAsUser: + rule: 'RunAsAny' + seLinux: + rule: 'RunAsAny' +{{- if and .Values.clients.https.enabled .Values.clients.https.certDir .Values.global.seLinux.enabled .Values.clients.https.seLinuxOptions }} + seLinuxOptions: {{ .Values.clients.https.seLinuxOptions | toYaml | nindent 6 }} +{{- end }} + supplementalGroups: + rule: 'MustRunAs' + ranges: + - min: 0 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + - min: 0 + max: 65535 + readOnlyRootFilesystem: false + volumes: + - 'secret' +{{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + - 'emptyDir' + - 'hostPath' + allowedHostPaths: + - pathPrefix: {{ required "Need access to volume on host with the SSL cert files to use HTTPs" .Values.clients.https.certDir }} + readOnly: true +{{- end }} +{{- end }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmControllerManager/templates/pushprox-clients.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmControllerManager/templates/pushprox-clients.yaml new file mode 100644 index 00000000000..e8fcfb38836 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmControllerManager/templates/pushprox-clients.yaml @@ -0,0 +1,157 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.clients }}{{- if .Values.clients.enabled }} +apiVersion: apps/v1 +{{- if .Values.clients.deployment.enabled }} +kind: Deployment +{{- else }} +kind: DaemonSet +{{- end }} +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} + pushprox-exporter: "client" +spec: + {{- if .Values.clients.deployment.enabled }} + replicas: {{ .Values.clients.deployment.replicas }} + {{- end }} + selector: + matchLabels: {{ include "pushProxy.client.labels" . | nindent 6 }} + template: + metadata: + labels: {{ include "pushProxy.client.labels" . | nindent 8 }} + spec: + {{- if .Values.clients.affinity }} + affinity: {{ toYaml .Values.clients.affinity | nindent 8 }} + {{- end }} + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} +{{- if .Values.clients.nodeSelector }} +{{ toYaml .Values.clients.nodeSelector | indent 8 }} +{{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} +{{- if .Values.clients.tolerations }} +{{ toYaml .Values.clients.tolerations | indent 8 }} +{{- end }} + hostNetwork: true + dnsPolicy: ClusterFirstWithHostNet + serviceAccountName: {{ template "pushProxy.client.name" . }} + {{- if .Values.global.imagePullSecretName }} + imagePullSecrets: + - name: {{ .Values.global.imagePullSecretName }} + {{- end }} + containers: + - name: pushprox-client + image: {{ template "system_default_registry" . }}{{ .Values.clients.image.repository }}:{{ .Values.clients.image.tag }} + command: + {{- range .Values.clients.command }} + - {{ . | quote }} + {{- end }} + args: + - --fqdn=$(HOST_IP) + - --proxy-url=$(PROXY_URL) + {{- if .Values.clients.metrics.enabled }} + - --metrics-addr=$(PORT) + {{- end }} + - --allow-port={{ required "Need .Values.metricsPort to configure client to be allowed to scrape metrics at port" .Values.metricsPort}} + {{- if .Values.clients.useLocalhost }} + - --use-localhost + {{- end }} + {{- if .Values.clients.https.enabled }} + {{- if .Values.clients.https.insecureSkipVerify }} + - --insecure-skip-verify + {{- end }} + {{- if .Values.clients.https.useServiceAccountCredentials }} + - --token-path=/var/run/secrets/kubernetes.io/serviceaccount/token + {{- end }} + {{- if .Values.clients.https.certDir }} + - --tls.cert=/etc/ssl/push-proxy/push-proxy.pem + - --tls.key=/etc/ssl/push-proxy/push-proxy-key.pem + - --tls.cacert=/etc/ssl/push-proxy/push-proxy-ca-cert.pem + {{- end }} + {{- end }} + env: + - name: HOST_IP + valueFrom: + fieldRef: + fieldPath: status.hostIP + {{- if .Values.clients.metrics.enabled }} + - name: PORT + value: :{{ .Values.clients.port }} + {{- end }} + - name: PROXY_URL + value: {{ template "pushProxy.proxyUrl" . }} + securityContext: + runAsNonRoot: true + runAsUser: 1000 + {{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + volumeMounts: + - name: metrics-cert-dir + mountPath: /etc/ssl/push-proxy + {{- end }} + {{- if .Values.clients.resources }} + resources: {{ toYaml .Values.clients.resources | nindent 10 }} + {{- end }} + {{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + initContainers: + - name: copy-certs + image: {{ template "system_default_registry" . }}{{ .Values.clients.copyCertsImage.repository }}:{{ .Values.clients.copyCertsImage.tag }} + command: + - sh + - -c + - | + echo "Searching for files to copy within the source volume" + echo "cert: ${CERT_FILE_NAME}" + echo "key: ${KEY_FILE_NAME}" + echo "cacert: ${CACERT_FILE_NAME}" + + CERT_FILE_SOURCE=$(find /etc/source/ -type f -name "${CERT_FILE_NAME}" | sort -r | head -n 1) + KEY_FILE_SOURCE=$(find /etc/source/ -type f -name "${KEY_FILE_NAME}" | sort -r | head -n 1) + CACERT_FILE_SOURCE=$(find /etc/source/ -type f -name "${CACERT_FILE_NAME}" | sort -r | head -n 1) + + test -z ${CERT_FILE_SOURCE} && echo "Failed to find cert file" && exit 1 + test -z ${KEY_FILE_SOURCE} && echo "Failed to find key file" && exit 1 + test -z ${CACERT_FILE_SOURCE} && echo "Failed to find cacert file" && exit 1 + + echo "Copying cert file from $CERT_FILE_SOURCE to $CERT_FILE_TARGET" + cp $CERT_FILE_SOURCE $CERT_FILE_TARGET || exit 1 + chmod 444 $CERT_FILE_TARGET || exit 1 + + echo "Copying key file from $KEY_FILE_SOURCE to $KEY_FILE_TARGET" + cp $KEY_FILE_SOURCE $KEY_FILE_TARGET || exit 1 + chmod 444 $KEY_FILE_TARGET || exit 1 + + echo "Copying cacert file from $CACERT_FILE_SOURCE to $CACERT_FILE_TARGET" + cp $CACERT_FILE_SOURCE $CACERT_FILE_TARGET || exit 1 + chmod 444 $CACERT_FILE_TARGET || exit 1 + env: + - name: CERT_FILE_NAME + value: {{ required "Need a TLS cert file for scraping metrics endpoint over HTTPs" .Values.clients.https.certFile }} + - name: KEY_FILE_NAME + value: {{ required "Need a TLS key file for scraping metrics endpoint over HTTPs" .Values.clients.https.keyFile }} + - name: CACERT_FILE_NAME + value: {{ required "Need a TLS CA cert file for scraping metrics endpoint over HTTPs" .Values.clients.https.caCertFile }} + - name: CERT_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy.pem + - name: KEY_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy-key.pem + - name: CACERT_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy-ca-cert.pem + securityContext: + runAsNonRoot: false +{{- if and .Values.global.seLinux.enabled .Values.clients.https.seLinuxOptions }} + seLinuxOptions: {{ .Values.clients.https.seLinuxOptions | toYaml | nindent 12 }} +{{- end }} + volumeMounts: + - name: metrics-cert-dir-source + mountPath: /etc/source + readOnly: true + - name: metrics-cert-dir + mountPath: /etc/ssl/push-proxy + volumes: + - name: metrics-cert-dir-source + hostPath: + path: {{ required "Need access to volume on host with the SSL cert files to use HTTPs" .Values.clients.https.certDir }} + - name: metrics-cert-dir + emptyDir: {} + {{- end }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmControllerManager/templates/pushprox-proxy-rbac.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmControllerManager/templates/pushprox-proxy-rbac.yaml new file mode 100644 index 00000000000..eefe609058c --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmControllerManager/templates/pushprox-proxy-rbac.yaml @@ -0,0 +1,68 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if and .Values.proxy }}{{ if .Values.proxy.enabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ template "pushProxy.proxy.name" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +rules: +{{- if .Values.global.cattle.psp.enabled }} +- apiGroups: ['policy'] + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - {{ template "pushProxy.proxy.name" . }} +{{- end }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ template "pushProxy.proxy.name" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "pushProxy.proxy.name" . }} +subjects: + - kind: ServiceAccount + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +--- +{{- if .Values.global.cattle.psp.enabled }} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +spec: + privileged: false + hostNetwork: false + hostIPC: false + hostPID: false + runAsUser: + rule: 'MustRunAsNonRoot' + seLinux: + rule: 'RunAsAny' + supplementalGroups: + rule: 'MustRunAs' + ranges: + - min: 1 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + - min: 1 + max: 65535 + readOnlyRootFilesystem: false + volumes: + - 'secret' +{{- end }}{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmControllerManager/templates/pushprox-proxy.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmControllerManager/templates/pushprox-proxy.yaml new file mode 100644 index 00000000000..723bbd6c00b --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmControllerManager/templates/pushprox-proxy.yaml @@ -0,0 +1,57 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if and .Values.proxy }}{{ if .Values.proxy.enabled }} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} + pushprox-exporter: "proxy" +spec: + selector: + matchLabels: {{ include "pushProxy.proxy.labels" . | nindent 6 }} + template: + metadata: + labels: {{ include "pushProxy.proxy.labels" . | nindent 8 }} + spec: + securityContext: + runAsNonRoot: true + runAsUser: 1000 + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} +{{- if .Values.proxy.nodeSelector }} +{{ toYaml .Values.proxy.nodeSelector | indent 8 }} +{{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} +{{- if .Values.proxy.tolerations }} +{{ toYaml .Values.proxy.tolerations | indent 8 }} +{{- end }} + serviceAccountName: {{ template "pushProxy.proxy.name" . }} + {{- if .Values.global.imagePullSecretName }} + imagePullSecrets: + - name: {{ .Values.global.imagePullSecretName }} + {{- end }} + containers: + - name: pushprox-proxy + image: {{ template "system_default_registry" . }}{{ .Values.proxy.image.repository }}:{{ .Values.proxy.image.tag }} + command: + {{- range .Values.proxy.command }} + - {{ . | quote }} + {{- end }} + {{- if .Values.proxy.resources }} + resources: {{ toYaml .Values.proxy.resources | nindent 10 }} + {{- end }} +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +spec: + ports: + - name: pp-proxy + port: {{ required "Need .Values.proxy.port to configure proxy" .Values.proxy.port }} + protocol: TCP + targetPort: {{ .Values.proxy.port }} + selector: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmControllerManager/templates/pushprox-servicemonitor.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmControllerManager/templates/pushprox-servicemonitor.yaml new file mode 100644 index 00000000000..67eb2216b5a --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmControllerManager/templates/pushprox-servicemonitor.yaml @@ -0,0 +1,45 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.serviceMonitor }}{{- if .Values.serviceMonitor.enabled }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ template "pushprox.serviceMonitor.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.serviceMonitor.labels" . | nindent 4 }} +spec: + endpoints: {{include "pushProxy.serviceMonitor.endpoints" . | nindent 4 }} + jobLabel: component + podTargetLabels: + - component + - pushprox-exporter + namespaceSelector: + matchNames: + - {{ template "pushprox.namespace" . }} + selector: + matchLabels: {{ include "pushProxy.client.labels" . | nindent 6 }} +--- +{{- $selector := "" }} +{{- if not (kindIs "invalid" .Values.service) }} +{{- if not (kindIs "invalid" .Values.service.selector) }} +{{ if .Values.service.selector }} +{{- if .Values.clients.enabled }} +{{- required (printf "Cannot override .Values.service.selector=%s when .Values.clients.enabled=true" (toJson .Values.service.selector)) "" }} +{{- end }} +{{- $selector = (toYaml .Values.service.selector) }} +{{- end }} +{{- end }} +{{- end }} +apiVersion: v1 +kind: Service +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +spec: + ports: + - name: metrics + port: {{ required "Need .Values.metricsPort to configure client to listen to metrics at port" .Values.metricsPort}} + protocol: TCP + targetPort: {{ .Values.metricsPort }} + selector: {{ default (include "pushProxy.client.labels" .) $selector | nindent 4 }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmControllerManager/templates/validate-install-crd.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmControllerManager/templates/validate-install-crd.yaml new file mode 100644 index 00000000000..16abc2fa83a --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmControllerManager/templates/validate-install-crd.yaml @@ -0,0 +1,14 @@ +#{{- if gt (len (lookup "rbac.authorization.k8s.io/v1" "ClusterRole" "" "")) 0 -}} +# {{- $found := dict -}} +# {{- set $found "monitoring.coreos.com/v1/ServiceMonitor" false -}} +# {{- range .Capabilities.APIVersions -}} +# {{- if hasKey $found (toString .) -}} +# {{- set $found (toString .) true -}} +# {{- end -}} +# {{- end -}} +# {{- range $_, $exists := $found -}} +# {{- if (eq $exists false) -}} +# {{- required "Required CRDs are missing. Please install Prometheus Operator CRDs before installing this chart." "" -}} +# {{- end -}} +# {{- end -}} +#{{- end -}} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmControllerManager/templates/validate-psp-install.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmControllerManager/templates/validate-psp-install.yaml new file mode 100644 index 00000000000..a30c59d3b7c --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmControllerManager/templates/validate-psp-install.yaml @@ -0,0 +1,7 @@ +#{{- if gt (len (lookup "rbac.authorization.k8s.io/v1" "ClusterRole" "" "")) 0 -}} +#{{- if .Values.global.cattle.psp.enabled }} +#{{- if not (.Capabilities.APIVersions.Has "policy/v1beta1/PodSecurityPolicy") }} +#{{- fail "The target cluster does not have the PodSecurityPolicy API resource. Please disable PSPs in this chart before proceeding." -}} +#{{- end }} +#{{- end }} +#{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmControllerManager/values.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmControllerManager/values.yaml new file mode 100644 index 00000000000..7f38498a1c5 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmControllerManager/values.yaml @@ -0,0 +1,166 @@ +# Default values for rancher-pushprox. +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. + +# Default image containing both the proxy and the client was generated from the following Dockerfile +# https://github.com/prometheus-community/PushProx/blob/eeadbe766641699129920ccfaaaa30a85c67fe81/Dockerfile#L1-L15 + +# Configuration + +global: + cattle: + psp: + enabled: false + systemDefaultRegistry: "" + seLinux: + enabled: false + +# A list of Semver constraint strings (defined by https://github.com/Masterminds/semver) and values.yaml overrides. +# +# For each key in kubeVersionOverrides, this chart will check to see if the current Kubernetes cluster's version matches +# any of the semver constraints provided as keys on the map. +# +# On seeing a match, the default value for each values.yaml field overridden will be updated with the new value. +# +# If multiple matches are encountered (due to overlapping semver ranges), the matches will be applied in order. +# +# Notes: +# - On running a helm template, Helm generally assumes the kubeVersion is v1.20.0 +# - On running a helm install --dry-run, the correct kubeVersion should be chosen. +kubeVersionOverrides: [] +# - constraint: "< 1.21" +# values: +# metricsPort: 10252 +# clients: +# https: +# enabled: false +# insecureSkipVerify: false +# useServiceAccountCredentials: false + +namespaceOverride: "" + +# The component that is being monitored (i.e. etcd) +component: "component" + +# The port containing the metrics that need to be scraped +metricsPort: 2739 + +# Configure ServiceMonitor that monitors metrics from the metricsPort endpoint +serviceMonitor: + enabled: true + # A list of endpoints that will be added to the ServiceMonitor based on the Endpoint spec + # Source: https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#endpoint + # By default, proxyUrl and params._scheme will be overridden based on other values + endpoints: + - port: metrics + +# Configure Service that grabs scrape targets +service: + # The selector that is used to populate the Service's Endpoints object. + # The chart will error out on rendering templating if .Values.clients.enabled is set alongside this field, + # since it is expected that this service should point to the PushProx Clients Daemonset / Deployment + selector: {} + +clients: + enabled: true + # The port which the PushProx client will post PushProx metrics to + port: 9369 + # If unset, this will default to the URL for the proxy service: http://pushprox-{{component}}-proxy.{{namepsace}}.svc.cluster.local:{{proxy.port}} + # Should be modified if the clients are being deployed outside the cluster where the proxy rests, otherwise leave it null + proxyUrl: "" + # If set to true, the client will forward any requests from the host IP to 127.0.0.1 + # It will only allow proxy requests to the metricsPort specified + useLocalhost: false + # Configuration for accessing metrics via HTTPS + https: + # Does the client require https to access the metrics? + enabled: false + # Does the client require requests be sent to http or https? + forceHTTPSScheme: false + # If set to true, the client will create a service account with adequate permissions and set a flag + # on the client to use the service account token provided by it to make authorized scrape requests + useServiceAccountCredentials: false + # Configuration for authentication to metrics via https endpoint + authenticationMethod: + # Reads token from defined file in container + # This function is deprecated in the prometheus operator api and may be removed in a future version + bearerTokenFile: + enabled: false + bearerTokenFilePath: "/var/run/secrets/kubernetes.io/serviceaccount/token" + # Reads token from defined secret in namespace + # This function is deprecated in the prometheus operator api and may be removed in a future version + bearerTokenSecret: + enabled: false + # Reads token from defined secret in namespace + authorization: + enabled: false + type: "bearer" + credentials: + key: "token" + optional: false + # If set to true, the client will disable SSL security checks + insecureSkipVerify: false + # Directory on host where necessary TLS cert and key to scrape metrics can be found + certDir: "" + # Filenames for files located in .Values.clients.https.certDir that correspond to TLS settings + certFile: "" + keyFile: "" + caCertFile: "" + # seLinuxOptions to be passed into the container that copies certs. Should define a container with permissions to read the files in the certDir provided on the host. + # Required and only used if `clients.https.enabled` is set and `clients.https.certDir` is provided. + seLinuxOptions: {} + + metrics: + # Whether the client should publish PushProx client-specific metrics to .Values.clients.port + enabled: false + + rbac: + # Additional permissions to provide to the ServiceAccount bound to the client + # This can be used to provide additional permissions for the client to scrape metrics from the k8s API + # Only enabled if clients.https.enabled and clients.https.useServiceAccountCredentials are true + additionalRules: [] + + # Resource limits + resources: {} + + # Options to select all nodes to deploy client DaemonSet on + nodeSelector: {} + tolerations: [] + affinity: {} + + image: + repository: rancher/pushprox-client + tag: v0.1.4-rancher2-client + command: ["pushprox-client"] + + copyCertsImage: + repository: rancher/mirrored-library-busybox + tag: 1.31.1 + + # The default intention of rancher-pushprox clients is to scrape hostNetwork metrics across all nodes. + # This can be used to scrape internal Kubernetes components or DaemonSets of hostNetwork Pods in + # situations where a cloud provider firewall prevents Pod-To-Host communication but not Pod-To-Pod. + # However, if the underlying hostNetwork Pod that is being scraped is managed by a Deployment, + # this advanced option enables users to deploy the client as a Deployment instead of a DaemonSet. + # If a user deploys this feature and the underlying Deployment's number of replicas changes, the user will + # be responsible for upgrading this chart accordingly to the right number of replicas. + deployment: + enabled: false + replicas: 0 + +proxy: + enabled: true + # The port through which PushProx clients will communicate to the proxy + port: 8080 + + # Resource limits + resources: {} + + # Options to select a node to run a single proxy deployment on + nodeSelector: {} + tolerations: [] + + image: + repository: rancher/pushprox-proxy + tag: v0.1.4-rancher2-proxy + command: ["pushprox-proxy"] diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmEtcd/.helmignore b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmEtcd/.helmignore new file mode 100644 index 00000000000..0e8a0eb36f4 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmEtcd/.helmignore @@ -0,0 +1,23 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*.orig +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmEtcd/Chart.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmEtcd/Chart.yaml new file mode 100644 index 00000000000..389e1386b59 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmEtcd/Chart.yaml @@ -0,0 +1,15 @@ +annotations: + catalog.cattle.io/hidden: "true" + catalog.cattle.io/kube-version: '>= 1.26.0-0 < 1.31.0-0' + catalog.cattle.io/os: linux + catalog.rancher.io/certified: rancher + catalog.rancher.io/namespace: cattle-monitoring-system + catalog.rancher.io/release-name: rancher-pushprox +apiVersion: v1 +appVersion: v0.1.4-rancher2 +description: Sets up a deployment of the PushProx proxy and a DaemonSet of PushProx + clients. +kubeVersion: '>=1.26.0-0' +name: kubeAdmEtcd +type: application +version: 0.2.1 diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmEtcd/README.md b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmEtcd/README.md new file mode 100644 index 00000000000..345002f48a2 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmEtcd/README.md @@ -0,0 +1,90 @@ +# rancher-pushprox + +A Rancher chart based on Rancher [PushProx](https://github.com/rancher/PushProx) that sets up a Deployment of a PushProx proxy and a DaemonSet of PushProx clients on a Kubernetes cluster. + +Installs [rancher-pushprox](https://github.com/rancher/charts/tree/gh-pages/packages/rancher-pushprox) to create PushProx clients that can access their host's network and register with a PushProx proxy. A [Prometheus Operator](https://github.com/coreos/prometheus-operator) ServiceMonitor CR is also included that is configured to scrape the metrics from each of the clients through the proxy. + +Using an instance of this chart is suitable for the following scenarios: +- You need to scrape metrics from a port that should not be accessible outside of the host (e.g. scraping `etcd` metrics in a hardened cluster) +- You need to scrape metrics on a host that are not exposed outside of 127.0.0.1 (e.g. scraping `kube-proxy` metrics) +- You need to scrape metrics through HTTPS using certs hosted directly on `hostPath` +- You need to scrape metrics from Kubernetes components that require authorization via a service account (e.g. permissions to make request to `/metrics`) +- You need to scrape metrics without access to cacerts (i.e. enable `insecureSkipVerify`) + +The clients and proxy are created based on a Rancher fork of the [prometheus-community/PushProx](https://github.com/prometheus-community/PushProx) project. + +## Upgrading to Kubernetes v1.25+ + +Starting in Kubernetes v1.25, [Pod Security Policies](https://kubernetes.io/docs/concepts/security/pod-security-policy/) have been removed from the Kubernetes API. + +As a result, **before upgrading to Kubernetes v1.25** (or on a fresh install in a Kubernetes v1.25+ cluster), users are expected to perform an in-place upgrade of this chart with `global.cattle.psp.enabled` set to `false` if it has been previously set to `true`. +​ +> **Note:** +> In this chart release, any previous field that was associated with any PSP resources have been removed in favor of a single global field: `global.cattle.psp.enabled`. + +> **Note:** +> If you upgrade your cluster to Kubernetes v1.25+ before removing PSPs via a `helm upgrade` (even if you manually clean up resources), **it will leave the Helm release in a broken state within the cluster such that further Helm operations will not work (`helm uninstall`, `helm upgrade`, etc.).** +> +> If your charts get stuck in this state, please consult the Rancher docs on how to clean up your Helm release secrets. + +Upon setting `global.cattle.psp.enabled` to false, the chart will remove any PSP resources deployed on its behalf from the cluster. This is the default setting for this chart. + +As a replacement for PSPs, [Pod Security Admission](https://kubernetes.io/docs/concepts/security/pod-security-admission/) should be used. Please consult the Rancher docs for more details on how to configure your chart release namespaces to work with the new Pod Security Admission and apply Pod Security Standards. + +## Configuration + +The following tables list the configurable parameters of the rancher-pushprox chart and their default values. + +### General + +#### Required +| Parameter | Description | Example | +| ----- | ----------- | ------ | +| `component` | The component that is being monitored | `kube-etcd` +| `metricsPort` | The port on the host that contains the metrics you want to scrape (e.g. `http://:/metrics`) | `2379` | +| `namespaceOverride` | The namespace to install the chart | `""` + +#### Optional +| Parameter | Description | Default | +| ----- | ----------- | ------ | +| `serviceMonitor.enabled` | Deploys a [Prometheus Operator](https://github.com/coreos/prometheus-operator/blob/master/Documentation/api.md#servicemonitor) ServiceMonitor CR that is configured to scrape metrics on the hosts that the clients are deployed on via the proxy. Also deploys a Service that points to all pods with the expected client name that exposes the `metricsPort` selected | `true` | +| `serviceMonitor.endpoints` | A list of endpoints that will be added to the ServiceMonitor based on the [Endpoint spec](https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#endpoint) | `[{port: metrics}]` | +| `service.selector` | The selector that is used to populate the Service's Endpoints object. The chart will error out on rendering templating if `.Values.clients.enabled` is set alongside this field, since it is expected that this service should point to the PushProx Clients Daemonset / Deployment | `{}` | +| `clients.enabled` | Deploys a DaemonSet of clients that are each capable of scraping endpoints on the hostNetwork it is deployed on | `true` | +| `clients.port` | The port where the client will publish PushProx client-specific metrics. If deploying multiple clients onto the same node, the clients should not have conflicting ports | `9369` | +| `clients.proxyUrl` | Overrides the default proxyUrl setting of `http://pushprox-{{ .Values.component }}-proxy.{{ . Release.Namespace }}.svc.cluster.local:{{ .Values.proxy.port }}"` with the `proxyUrl` specified | `""` | +| `clients.useLocalhost` | Sets a flag on each client deployment to redirect scrapes directed to `HOST_IP` to `127.0.0.1` | `false` | +| `clients.https.enabled` | Enables scraping metrics via HTTPS using the provided TLS certs that exist on each host | `false` | +| `clients.https.forceHTTPSScheme` | Forces scraping metrics via HTTPS using the provided TLS certs that exist on each host | `false` | +| `clients.https.useServiceAccountCredentials` | If set to true, the client will create a service account with permissions to scrape `/metrics` endpoint of Kubernetes components. The client will use the service account token provided to make authorized scrape requests to the Kubernetes API | `false` | +| `clients.https.authenticationMethod.bearerTokenFile.enabled` | If set to true, the client will use service account credentials mounted at the configured path `clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath`. This requires permissions to scrape `/metrics` endpoint of Kubernetes components. This method is deprecated by the prometheus operator and may be removed in a future release | `false` | +| `clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath` | This is a volume mount on the pod with permissions to scrape `/metrics` endpoint of Kubernetes components | `"/var/run/secrets/kubernetes.io/serviceaccount/token"` | +| `clients.https.authenticationMethod.bearerTokenSecret.enabled` | If set to true, the client will use service account credentials to scrape `/metrics` endpoint of Kubernetes components. This method is deprecated by the prometheus operator and may be removed in a future release | `false` | +| `clients.https.authenticationMethod.authorization.enabled` | If set to true, the client will use service account credentials to scrape `/metrics` endpoint of Kubernetes components | `false` | +| `clients.https.authenticationMethod.authorization.type` | If set, the client will use this type of authorization in its client requests for metrics | `"bearer"` | +| `clients.https.authenticationMethod.authorization.credentials.key` | If set, the client will use this key in the secret created by `clients.https.useServiceAccountCredentials` for authorization in its client requests for metrics | `"token"` | +| `clients.https.authenticationMethod.authorization.credentials.optional` | If set to false, the client will fail if the key in the secret created by `clients.https.useServiceAccountCredentials` does not exist | `false` | +| `clients.https.insecureSkipVerify` | If set to true, the client will disable SSL security checks | `false` | +| `clients.https.certDir` | A `hostPath` where TLS certs can be found. This path is mounted as a volume on an `initContainer` which copies only the necessary files over to an EmptyDir volume used by each client. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.certFile` | The path to the TLS cert file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.keyFile` | The path to the TLS key file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.caCertFile` | The path to the TLS cacert file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.seLinuxOptions` | seLinuxOptions to be passed into the container that copies certs. Should define a container with permissions to read the files in the certDir provided on the host. Required and only used if `clients.https.enabled` is set and `clients.https.certDir` is provided. | `""` | +| `clients.metrics.enabled` | Whether the client should publish PushProx client-specific metrics. | `false` | +| `clients.rbac.additionalRules` | Additional permissions to provide to the ServiceAccount bound to the client. This can be used to provide additional permissions for the client to scrape metrics from the k8s API. Only enabled if clients.https.enabled and clients.https.useServiceAccountCredentials are true | `[]` | +| `clients.deployment.enabled` | Deploys the client as a Deployment (generally used if the underlying hostNetwork Pod that is being scraped is managed by a Deployment) | `false` | +| `clients.deployment.replicas` | The number of pods the Deployment has, it should match the number of pod the hostNetwork Deployment has. Required and only used if `client.deployment.enable` is set | `0` | +| `clients.deployment.affinity` | The affinity rules that allocate the pod to the node in which the hostNetwork Deployment's pods run. Required and only used if `client.deployment.enable` is set | `{}` | +| `clients.resources` | Set resource limits and requests for the client container | `{}` | +| `clients.nodeSelector` | Select which nodes to deploy the clients on | `{}` | +| `clients.tolerations` | Specify tolerations for clients | `[]` | +| `proxy.enabled` | Deploys the proxy that each client will register with | `true` | +| `proxy.port` | The port exposed by the proxy that each client will register with to allow metrics to be scraped from the host | `8080` | +| `proxy.resources` | Set resource limits and requests for the proxy container | `{}` | +| `proxy.nodeSelector` | Select which nodes the proxy can be deployed on | `{}` | +| `proxy.tolerations` | Specify tolerations (if necessary) to allow the proxy to be deployed on the selected node | `[]` | +| `kubeVersionOverrides` | A list of Semver constraint strings (defined by https://github.com/Masterminds/semver) and values.yaml overrides. For each key in kubeVersionOverrides, this chart will check to see if the current Kubernetes cluster's version matches any of the semver constraints provided as keys on the map. On seeing a match, the default value for each values.yaml field overridden will be updated with the new value. If multiple matches are encountered (due to overlapping semver ranges), the matches will be applied in order. | `[]` + +*Tip: The filepaths set in `clients.https.File` can include wildcard characters*. + +See [rancher-monitoring](https://github.com/rancher/charts/tree/gh-pages/packages/rancher-monitoring) for examples of how this chart can be used. diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmEtcd/templates/_helpers.tpl b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmEtcd/templates/_helpers.tpl new file mode 100644 index 00000000000..1ba50939441 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmEtcd/templates/_helpers.tpl @@ -0,0 +1,170 @@ +# Rancher + +{{- define "system_default_registry" -}} +{{- if .Values.global.cattle.systemDefaultRegistry -}} +{{- printf "%s/" .Values.global.cattle.systemDefaultRegistry -}} +{{- end -}} +{{- end -}} + +# Windows Support + +{{/* +Windows cluster will add default taint for linux nodes, +add below linux tolerations to workloads could be scheduled to those linux nodes +*/}} + +{{- define "linux-node-tolerations" -}} +- key: "cattle.io/os" + value: "linux" + effect: "NoSchedule" + operator: "Equal" +{{- end -}} + +{{- define "linux-node-selector" -}} +{{- if semverCompare "<1.14-0" .Capabilities.KubeVersion.GitVersion -}} +beta.kubernetes.io/os: linux +{{- else -}} +kubernetes.io/os: linux +{{- end -}} +{{- end -}} + +# General + +{{- define "applyKubeVersionOverrides" -}} +{{- $overrides := dict -}} +{{- range $override := .Values.kubeVersionOverrides -}} +{{- if semverCompare $override.constraint $.Capabilities.KubeVersion.Version -}} +{{- $_ := mergeOverwrite $overrides $override.values -}} +{{- end -}} +{{- end -}} +{{- $_ := mergeOverwrite .Values $overrides -}} +{{- end -}} + +{{- define "pushprox.namespace" -}} + {{- if .Values.namespaceOverride -}} + {{- .Values.namespaceOverride -}} + {{- else -}} + {{- .Release.Namespace -}} + {{- end -}} +{{- end -}} + +{{- define "pushProxy.commonLabels" -}} +release: {{ .Release.Name }} +component: {{ .Values.component | quote }} +provider: kubernetes +{{- end -}} + +{{- define "pushProxy.proxyUrl" -}} +{{- $_ := (required "Template requires either .Values.proxy.port or .Values.client.proxyUrl to set proxyUrl for client" (or .Values.clients.proxyUrl .Values.proxy.port)) -}} +{{- if .Values.clients.proxyUrl -}} +{{ printf "%s" .Values.clients.proxyUrl }} +{{- else -}} +{{ printf "http://%s.%s.svc:%d" (include "pushProxy.proxy.name" .) (include "pushprox.namespace" .) (int .Values.proxy.port) }} +{{- end -}}{{- end -}} + +# Client + +{{- define "pushProxy.client.name" -}} +{{- printf "pushprox-%s-client" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.client.serviceAccountTokenName" -}} +{{- printf "pushprox-%s-client-service-account-token" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.client.labels" -}} +k8s-app: {{ template "pushProxy.client.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +# Proxy + +{{- define "pushProxy.proxy.name" -}} +{{- printf "pushprox-%s-proxy" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.proxy.labels" -}} +k8s-app: {{ template "pushProxy.proxy.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +# ServiceMonitor + +{{- define "pushprox.serviceMonitor.name" -}} +{{- printf "%s-%s" .Release.Name (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.serviceMonitor.labels" -}} +app: {{ template "pushprox.serviceMonitor.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +{{- define "pushProxy.serviceMonitor.endpoints" -}} +{{- $proxyURL := (include "pushProxy.proxyUrl" .) -}} +{{- $useHTTPS := .Values.clients.https.enabled -}} +{{- $setHTTPSScheme := .Values.clients.https.forceHTTPSScheme -}} +{{- $insecureSkipVerify := .Values.clients.https.insecureSkipVerify -}} +{{- $useServiceAccountCredentials := .Values.clients.https.useServiceAccountCredentials -}} +{{- $serviceAccountTokenName := (include "pushProxy.client.serviceAccountTokenName" . ) -}} +{{- $metricRelabelings := list }} +{{- $endpoints := .Values.serviceMonitor.endpoints }} +{{- if .Values.proxy.enabled }} +{{- $_ := set . "proxyUrl" $proxyURL }} +{{- end }} +{{- range $endpoints }} +{{- if $.Values.proxy.enabled }} +{{- $_ := set . "proxyUrl" $proxyURL }} +{{- end }} +{{- $clusterIdRelabel := dict }} +{{- $metricRelabelings := list }} +{{- if $.Values.global.cattle.clusterId }} +{{- $_ := set $clusterIdRelabel "action" "replace" }} +{{- $_ := set $clusterIdRelabel "sourceLabels" (list "__address__") }} +{{- $_ := set $clusterIdRelabel "targetLabel" "cluster_id" }} +{{- $_ := set $clusterIdRelabel "replacement" $.Values.global.cattle.clusterId }} +{{- $metricRelabelings = append $metricRelabelings $clusterIdRelabel }} +{{- end }} +{{- $clusterNameRelabel := dict }} +{{- if $.Values.global.cattle.clusterName }} +{{- $_ := set $clusterNameRelabel "action" "replace" }} +{{- $_ := set $clusterNameRelabel "sourceLabels" (list "__address__") }} +{{- $_ := set $clusterNameRelabel "targetLabel" "cluster_name" }} +{{- $_ := set $clusterNameRelabel "replacement" $.Values.global.cattle.clusterName }} +{{- $metricRelabelings = append $metricRelabelings $clusterNameRelabel }} +{{- end }} +{{- if not (empty $metricRelabelings) }} +{{- $_ := set . "metricRelabelings" ($metricRelabelings)}} +{{- end }} +{{- if $setHTTPSScheme -}} +{{- $_ := set . "scheme" "https" }} +{{- end -}} +{{- if $useHTTPS -}} +{{- if (hasKey . "params") }} +{{- $_ := set (get . "params") "_scheme" (list "https") }} +{{- else }} +{{- $_ := set . "params" (dict "_scheme" (list "https")) }} +{{- end }} +{{- end }} +{{- if (hasKey . "tlsConfig") }} +{{- $_ := set (get . "tlsConfig") "insecureSkipVerify" $insecureSkipVerify }} +{{- else }} +{{- $_ := set . "tlsConfig" (dict "insecureSkipVerify" $insecureSkipVerify) }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.bearerTokenFile.enabled }} +{{- $_ := set . "bearerTokenFile" $.Values.clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.bearerTokenSecret.enabled }} +{{- $_ := set . "bearerTokenSecret" $serviceAccountTokenName }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.authorization.enabled }} +{{- if (hasKey . "authorization") }} +{{- $_ := set (get . "authorization") "type" $.Values.clients.https.authenticationMethod.authorization.type }} +{{- $_ := set (get . "authorization") "credentials" (dict "name" $serviceAccountTokenName "key" $.Values.clients.https.authenticationMethod.authorization.credentials.key "optional" $.Values.clients.https.authenticationMethod.authorization.credentials.optional) }} +{{- else }} +{{- $_ := set . "authorization" (dict "type" $.Values.clients.https.authenticationMethod.authorization.type) }} +{{- $_ := set . "authorization" (dict "credentials" (dict "name" $serviceAccountTokenName "key" $.Values.clients.https.authenticationMethod.authorization.credentials.key "optional" $.Values.clients.https.authenticationMethod.authorization.credentials.optional)) }} +{{- end }} +{{- end }} +{{- end }} +{{- toYaml $endpoints }} +{{- end -}} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmEtcd/templates/pushprox-clients-rbac.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmEtcd/templates/pushprox-clients-rbac.yaml new file mode 100644 index 00000000000..a8e27c37356 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmEtcd/templates/pushprox-clients-rbac.yaml @@ -0,0 +1,97 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.clients }}{{- if .Values.clients.enabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ template "pushProxy.client.name" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +rules: +{{- if .Values.global.cattle.psp.enabled }} +- apiGroups: ['policy'] + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - {{ template "pushProxy.client.name" . }} +{{- end }} +{{- if and .Values.clients.https.enabled .Values.clients.https.useServiceAccountCredentials }} +- nonResourceURLs: ["/metrics"] + verbs: ["get"] +{{- if .Values.clients.rbac.additionalRules }} +{{ toYaml .Values.clients.rbac.additionalRules }} +{{- end }} +{{- end }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ template "pushProxy.client.name" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "pushProxy.client.name" . }} +subjects: + - kind: ServiceAccount + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +--- +{{- if .Values.clients.https.useServiceAccountCredentials }} +apiVersion: v1 +kind: Secret +type: kubernetes.io/service-account-token +metadata: + name: {{ template "pushProxy.client.serviceAccountTokenName" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} + annotations: + kubernetes.io/service-account.name: {{ template "pushProxy.client.name" . }} +{{- end }} +--- +{{- if .Values.global.cattle.psp.enabled }} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +spec: + privileged: false + hostNetwork: true + hostIPC: false + hostPID: false + runAsUser: + rule: 'RunAsAny' + seLinux: + rule: 'RunAsAny' +{{- if and .Values.clients.https.enabled .Values.clients.https.certDir .Values.global.seLinux.enabled .Values.clients.https.seLinuxOptions }} + seLinuxOptions: {{ .Values.clients.https.seLinuxOptions | toYaml | nindent 6 }} +{{- end }} + supplementalGroups: + rule: 'MustRunAs' + ranges: + - min: 0 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + - min: 0 + max: 65535 + readOnlyRootFilesystem: false + volumes: + - 'secret' +{{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + - 'emptyDir' + - 'hostPath' + allowedHostPaths: + - pathPrefix: {{ required "Need access to volume on host with the SSL cert files to use HTTPs" .Values.clients.https.certDir }} + readOnly: true +{{- end }} +{{- end }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmEtcd/templates/pushprox-clients.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmEtcd/templates/pushprox-clients.yaml new file mode 100644 index 00000000000..e8fcfb38836 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmEtcd/templates/pushprox-clients.yaml @@ -0,0 +1,157 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.clients }}{{- if .Values.clients.enabled }} +apiVersion: apps/v1 +{{- if .Values.clients.deployment.enabled }} +kind: Deployment +{{- else }} +kind: DaemonSet +{{- end }} +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} + pushprox-exporter: "client" +spec: + {{- if .Values.clients.deployment.enabled }} + replicas: {{ .Values.clients.deployment.replicas }} + {{- end }} + selector: + matchLabels: {{ include "pushProxy.client.labels" . | nindent 6 }} + template: + metadata: + labels: {{ include "pushProxy.client.labels" . | nindent 8 }} + spec: + {{- if .Values.clients.affinity }} + affinity: {{ toYaml .Values.clients.affinity | nindent 8 }} + {{- end }} + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} +{{- if .Values.clients.nodeSelector }} +{{ toYaml .Values.clients.nodeSelector | indent 8 }} +{{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} +{{- if .Values.clients.tolerations }} +{{ toYaml .Values.clients.tolerations | indent 8 }} +{{- end }} + hostNetwork: true + dnsPolicy: ClusterFirstWithHostNet + serviceAccountName: {{ template "pushProxy.client.name" . }} + {{- if .Values.global.imagePullSecretName }} + imagePullSecrets: + - name: {{ .Values.global.imagePullSecretName }} + {{- end }} + containers: + - name: pushprox-client + image: {{ template "system_default_registry" . }}{{ .Values.clients.image.repository }}:{{ .Values.clients.image.tag }} + command: + {{- range .Values.clients.command }} + - {{ . | quote }} + {{- end }} + args: + - --fqdn=$(HOST_IP) + - --proxy-url=$(PROXY_URL) + {{- if .Values.clients.metrics.enabled }} + - --metrics-addr=$(PORT) + {{- end }} + - --allow-port={{ required "Need .Values.metricsPort to configure client to be allowed to scrape metrics at port" .Values.metricsPort}} + {{- if .Values.clients.useLocalhost }} + - --use-localhost + {{- end }} + {{- if .Values.clients.https.enabled }} + {{- if .Values.clients.https.insecureSkipVerify }} + - --insecure-skip-verify + {{- end }} + {{- if .Values.clients.https.useServiceAccountCredentials }} + - --token-path=/var/run/secrets/kubernetes.io/serviceaccount/token + {{- end }} + {{- if .Values.clients.https.certDir }} + - --tls.cert=/etc/ssl/push-proxy/push-proxy.pem + - --tls.key=/etc/ssl/push-proxy/push-proxy-key.pem + - --tls.cacert=/etc/ssl/push-proxy/push-proxy-ca-cert.pem + {{- end }} + {{- end }} + env: + - name: HOST_IP + valueFrom: + fieldRef: + fieldPath: status.hostIP + {{- if .Values.clients.metrics.enabled }} + - name: PORT + value: :{{ .Values.clients.port }} + {{- end }} + - name: PROXY_URL + value: {{ template "pushProxy.proxyUrl" . }} + securityContext: + runAsNonRoot: true + runAsUser: 1000 + {{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + volumeMounts: + - name: metrics-cert-dir + mountPath: /etc/ssl/push-proxy + {{- end }} + {{- if .Values.clients.resources }} + resources: {{ toYaml .Values.clients.resources | nindent 10 }} + {{- end }} + {{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + initContainers: + - name: copy-certs + image: {{ template "system_default_registry" . }}{{ .Values.clients.copyCertsImage.repository }}:{{ .Values.clients.copyCertsImage.tag }} + command: + - sh + - -c + - | + echo "Searching for files to copy within the source volume" + echo "cert: ${CERT_FILE_NAME}" + echo "key: ${KEY_FILE_NAME}" + echo "cacert: ${CACERT_FILE_NAME}" + + CERT_FILE_SOURCE=$(find /etc/source/ -type f -name "${CERT_FILE_NAME}" | sort -r | head -n 1) + KEY_FILE_SOURCE=$(find /etc/source/ -type f -name "${KEY_FILE_NAME}" | sort -r | head -n 1) + CACERT_FILE_SOURCE=$(find /etc/source/ -type f -name "${CACERT_FILE_NAME}" | sort -r | head -n 1) + + test -z ${CERT_FILE_SOURCE} && echo "Failed to find cert file" && exit 1 + test -z ${KEY_FILE_SOURCE} && echo "Failed to find key file" && exit 1 + test -z ${CACERT_FILE_SOURCE} && echo "Failed to find cacert file" && exit 1 + + echo "Copying cert file from $CERT_FILE_SOURCE to $CERT_FILE_TARGET" + cp $CERT_FILE_SOURCE $CERT_FILE_TARGET || exit 1 + chmod 444 $CERT_FILE_TARGET || exit 1 + + echo "Copying key file from $KEY_FILE_SOURCE to $KEY_FILE_TARGET" + cp $KEY_FILE_SOURCE $KEY_FILE_TARGET || exit 1 + chmod 444 $KEY_FILE_TARGET || exit 1 + + echo "Copying cacert file from $CACERT_FILE_SOURCE to $CACERT_FILE_TARGET" + cp $CACERT_FILE_SOURCE $CACERT_FILE_TARGET || exit 1 + chmod 444 $CACERT_FILE_TARGET || exit 1 + env: + - name: CERT_FILE_NAME + value: {{ required "Need a TLS cert file for scraping metrics endpoint over HTTPs" .Values.clients.https.certFile }} + - name: KEY_FILE_NAME + value: {{ required "Need a TLS key file for scraping metrics endpoint over HTTPs" .Values.clients.https.keyFile }} + - name: CACERT_FILE_NAME + value: {{ required "Need a TLS CA cert file for scraping metrics endpoint over HTTPs" .Values.clients.https.caCertFile }} + - name: CERT_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy.pem + - name: KEY_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy-key.pem + - name: CACERT_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy-ca-cert.pem + securityContext: + runAsNonRoot: false +{{- if and .Values.global.seLinux.enabled .Values.clients.https.seLinuxOptions }} + seLinuxOptions: {{ .Values.clients.https.seLinuxOptions | toYaml | nindent 12 }} +{{- end }} + volumeMounts: + - name: metrics-cert-dir-source + mountPath: /etc/source + readOnly: true + - name: metrics-cert-dir + mountPath: /etc/ssl/push-proxy + volumes: + - name: metrics-cert-dir-source + hostPath: + path: {{ required "Need access to volume on host with the SSL cert files to use HTTPs" .Values.clients.https.certDir }} + - name: metrics-cert-dir + emptyDir: {} + {{- end }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmEtcd/templates/pushprox-proxy-rbac.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmEtcd/templates/pushprox-proxy-rbac.yaml new file mode 100644 index 00000000000..eefe609058c --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmEtcd/templates/pushprox-proxy-rbac.yaml @@ -0,0 +1,68 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if and .Values.proxy }}{{ if .Values.proxy.enabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ template "pushProxy.proxy.name" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +rules: +{{- if .Values.global.cattle.psp.enabled }} +- apiGroups: ['policy'] + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - {{ template "pushProxy.proxy.name" . }} +{{- end }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ template "pushProxy.proxy.name" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "pushProxy.proxy.name" . }} +subjects: + - kind: ServiceAccount + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +--- +{{- if .Values.global.cattle.psp.enabled }} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +spec: + privileged: false + hostNetwork: false + hostIPC: false + hostPID: false + runAsUser: + rule: 'MustRunAsNonRoot' + seLinux: + rule: 'RunAsAny' + supplementalGroups: + rule: 'MustRunAs' + ranges: + - min: 1 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + - min: 1 + max: 65535 + readOnlyRootFilesystem: false + volumes: + - 'secret' +{{- end }}{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmEtcd/templates/pushprox-proxy.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmEtcd/templates/pushprox-proxy.yaml new file mode 100644 index 00000000000..723bbd6c00b --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmEtcd/templates/pushprox-proxy.yaml @@ -0,0 +1,57 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if and .Values.proxy }}{{ if .Values.proxy.enabled }} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} + pushprox-exporter: "proxy" +spec: + selector: + matchLabels: {{ include "pushProxy.proxy.labels" . | nindent 6 }} + template: + metadata: + labels: {{ include "pushProxy.proxy.labels" . | nindent 8 }} + spec: + securityContext: + runAsNonRoot: true + runAsUser: 1000 + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} +{{- if .Values.proxy.nodeSelector }} +{{ toYaml .Values.proxy.nodeSelector | indent 8 }} +{{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} +{{- if .Values.proxy.tolerations }} +{{ toYaml .Values.proxy.tolerations | indent 8 }} +{{- end }} + serviceAccountName: {{ template "pushProxy.proxy.name" . }} + {{- if .Values.global.imagePullSecretName }} + imagePullSecrets: + - name: {{ .Values.global.imagePullSecretName }} + {{- end }} + containers: + - name: pushprox-proxy + image: {{ template "system_default_registry" . }}{{ .Values.proxy.image.repository }}:{{ .Values.proxy.image.tag }} + command: + {{- range .Values.proxy.command }} + - {{ . | quote }} + {{- end }} + {{- if .Values.proxy.resources }} + resources: {{ toYaml .Values.proxy.resources | nindent 10 }} + {{- end }} +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +spec: + ports: + - name: pp-proxy + port: {{ required "Need .Values.proxy.port to configure proxy" .Values.proxy.port }} + protocol: TCP + targetPort: {{ .Values.proxy.port }} + selector: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmEtcd/templates/pushprox-servicemonitor.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmEtcd/templates/pushprox-servicemonitor.yaml new file mode 100644 index 00000000000..67eb2216b5a --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmEtcd/templates/pushprox-servicemonitor.yaml @@ -0,0 +1,45 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.serviceMonitor }}{{- if .Values.serviceMonitor.enabled }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ template "pushprox.serviceMonitor.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.serviceMonitor.labels" . | nindent 4 }} +spec: + endpoints: {{include "pushProxy.serviceMonitor.endpoints" . | nindent 4 }} + jobLabel: component + podTargetLabels: + - component + - pushprox-exporter + namespaceSelector: + matchNames: + - {{ template "pushprox.namespace" . }} + selector: + matchLabels: {{ include "pushProxy.client.labels" . | nindent 6 }} +--- +{{- $selector := "" }} +{{- if not (kindIs "invalid" .Values.service) }} +{{- if not (kindIs "invalid" .Values.service.selector) }} +{{ if .Values.service.selector }} +{{- if .Values.clients.enabled }} +{{- required (printf "Cannot override .Values.service.selector=%s when .Values.clients.enabled=true" (toJson .Values.service.selector)) "" }} +{{- end }} +{{- $selector = (toYaml .Values.service.selector) }} +{{- end }} +{{- end }} +{{- end }} +apiVersion: v1 +kind: Service +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +spec: + ports: + - name: metrics + port: {{ required "Need .Values.metricsPort to configure client to listen to metrics at port" .Values.metricsPort}} + protocol: TCP + targetPort: {{ .Values.metricsPort }} + selector: {{ default (include "pushProxy.client.labels" .) $selector | nindent 4 }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmEtcd/templates/validate-install-crd.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmEtcd/templates/validate-install-crd.yaml new file mode 100644 index 00000000000..16abc2fa83a --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmEtcd/templates/validate-install-crd.yaml @@ -0,0 +1,14 @@ +#{{- if gt (len (lookup "rbac.authorization.k8s.io/v1" "ClusterRole" "" "")) 0 -}} +# {{- $found := dict -}} +# {{- set $found "monitoring.coreos.com/v1/ServiceMonitor" false -}} +# {{- range .Capabilities.APIVersions -}} +# {{- if hasKey $found (toString .) -}} +# {{- set $found (toString .) true -}} +# {{- end -}} +# {{- end -}} +# {{- range $_, $exists := $found -}} +# {{- if (eq $exists false) -}} +# {{- required "Required CRDs are missing. Please install Prometheus Operator CRDs before installing this chart." "" -}} +# {{- end -}} +# {{- end -}} +#{{- end -}} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmEtcd/templates/validate-psp-install.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmEtcd/templates/validate-psp-install.yaml new file mode 100644 index 00000000000..a30c59d3b7c --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmEtcd/templates/validate-psp-install.yaml @@ -0,0 +1,7 @@ +#{{- if gt (len (lookup "rbac.authorization.k8s.io/v1" "ClusterRole" "" "")) 0 -}} +#{{- if .Values.global.cattle.psp.enabled }} +#{{- if not (.Capabilities.APIVersions.Has "policy/v1beta1/PodSecurityPolicy") }} +#{{- fail "The target cluster does not have the PodSecurityPolicy API resource. Please disable PSPs in this chart before proceeding." -}} +#{{- end }} +#{{- end }} +#{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmEtcd/values.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmEtcd/values.yaml new file mode 100644 index 00000000000..7f38498a1c5 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmEtcd/values.yaml @@ -0,0 +1,166 @@ +# Default values for rancher-pushprox. +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. + +# Default image containing both the proxy and the client was generated from the following Dockerfile +# https://github.com/prometheus-community/PushProx/blob/eeadbe766641699129920ccfaaaa30a85c67fe81/Dockerfile#L1-L15 + +# Configuration + +global: + cattle: + psp: + enabled: false + systemDefaultRegistry: "" + seLinux: + enabled: false + +# A list of Semver constraint strings (defined by https://github.com/Masterminds/semver) and values.yaml overrides. +# +# For each key in kubeVersionOverrides, this chart will check to see if the current Kubernetes cluster's version matches +# any of the semver constraints provided as keys on the map. +# +# On seeing a match, the default value for each values.yaml field overridden will be updated with the new value. +# +# If multiple matches are encountered (due to overlapping semver ranges), the matches will be applied in order. +# +# Notes: +# - On running a helm template, Helm generally assumes the kubeVersion is v1.20.0 +# - On running a helm install --dry-run, the correct kubeVersion should be chosen. +kubeVersionOverrides: [] +# - constraint: "< 1.21" +# values: +# metricsPort: 10252 +# clients: +# https: +# enabled: false +# insecureSkipVerify: false +# useServiceAccountCredentials: false + +namespaceOverride: "" + +# The component that is being monitored (i.e. etcd) +component: "component" + +# The port containing the metrics that need to be scraped +metricsPort: 2739 + +# Configure ServiceMonitor that monitors metrics from the metricsPort endpoint +serviceMonitor: + enabled: true + # A list of endpoints that will be added to the ServiceMonitor based on the Endpoint spec + # Source: https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#endpoint + # By default, proxyUrl and params._scheme will be overridden based on other values + endpoints: + - port: metrics + +# Configure Service that grabs scrape targets +service: + # The selector that is used to populate the Service's Endpoints object. + # The chart will error out on rendering templating if .Values.clients.enabled is set alongside this field, + # since it is expected that this service should point to the PushProx Clients Daemonset / Deployment + selector: {} + +clients: + enabled: true + # The port which the PushProx client will post PushProx metrics to + port: 9369 + # If unset, this will default to the URL for the proxy service: http://pushprox-{{component}}-proxy.{{namepsace}}.svc.cluster.local:{{proxy.port}} + # Should be modified if the clients are being deployed outside the cluster where the proxy rests, otherwise leave it null + proxyUrl: "" + # If set to true, the client will forward any requests from the host IP to 127.0.0.1 + # It will only allow proxy requests to the metricsPort specified + useLocalhost: false + # Configuration for accessing metrics via HTTPS + https: + # Does the client require https to access the metrics? + enabled: false + # Does the client require requests be sent to http or https? + forceHTTPSScheme: false + # If set to true, the client will create a service account with adequate permissions and set a flag + # on the client to use the service account token provided by it to make authorized scrape requests + useServiceAccountCredentials: false + # Configuration for authentication to metrics via https endpoint + authenticationMethod: + # Reads token from defined file in container + # This function is deprecated in the prometheus operator api and may be removed in a future version + bearerTokenFile: + enabled: false + bearerTokenFilePath: "/var/run/secrets/kubernetes.io/serviceaccount/token" + # Reads token from defined secret in namespace + # This function is deprecated in the prometheus operator api and may be removed in a future version + bearerTokenSecret: + enabled: false + # Reads token from defined secret in namespace + authorization: + enabled: false + type: "bearer" + credentials: + key: "token" + optional: false + # If set to true, the client will disable SSL security checks + insecureSkipVerify: false + # Directory on host where necessary TLS cert and key to scrape metrics can be found + certDir: "" + # Filenames for files located in .Values.clients.https.certDir that correspond to TLS settings + certFile: "" + keyFile: "" + caCertFile: "" + # seLinuxOptions to be passed into the container that copies certs. Should define a container with permissions to read the files in the certDir provided on the host. + # Required and only used if `clients.https.enabled` is set and `clients.https.certDir` is provided. + seLinuxOptions: {} + + metrics: + # Whether the client should publish PushProx client-specific metrics to .Values.clients.port + enabled: false + + rbac: + # Additional permissions to provide to the ServiceAccount bound to the client + # This can be used to provide additional permissions for the client to scrape metrics from the k8s API + # Only enabled if clients.https.enabled and clients.https.useServiceAccountCredentials are true + additionalRules: [] + + # Resource limits + resources: {} + + # Options to select all nodes to deploy client DaemonSet on + nodeSelector: {} + tolerations: [] + affinity: {} + + image: + repository: rancher/pushprox-client + tag: v0.1.4-rancher2-client + command: ["pushprox-client"] + + copyCertsImage: + repository: rancher/mirrored-library-busybox + tag: 1.31.1 + + # The default intention of rancher-pushprox clients is to scrape hostNetwork metrics across all nodes. + # This can be used to scrape internal Kubernetes components or DaemonSets of hostNetwork Pods in + # situations where a cloud provider firewall prevents Pod-To-Host communication but not Pod-To-Pod. + # However, if the underlying hostNetwork Pod that is being scraped is managed by a Deployment, + # this advanced option enables users to deploy the client as a Deployment instead of a DaemonSet. + # If a user deploys this feature and the underlying Deployment's number of replicas changes, the user will + # be responsible for upgrading this chart accordingly to the right number of replicas. + deployment: + enabled: false + replicas: 0 + +proxy: + enabled: true + # The port through which PushProx clients will communicate to the proxy + port: 8080 + + # Resource limits + resources: {} + + # Options to select a node to run a single proxy deployment on + nodeSelector: {} + tolerations: [] + + image: + repository: rancher/pushprox-proxy + tag: v0.1.4-rancher2-proxy + command: ["pushprox-proxy"] diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmProxy/.helmignore b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmProxy/.helmignore new file mode 100644 index 00000000000..0e8a0eb36f4 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmProxy/.helmignore @@ -0,0 +1,23 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*.orig +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmProxy/Chart.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmProxy/Chart.yaml new file mode 100644 index 00000000000..b755e8cd9e1 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmProxy/Chart.yaml @@ -0,0 +1,15 @@ +annotations: + catalog.cattle.io/hidden: "true" + catalog.cattle.io/kube-version: '>= 1.26.0-0 < 1.31.0-0' + catalog.cattle.io/os: linux + catalog.rancher.io/certified: rancher + catalog.rancher.io/namespace: cattle-monitoring-system + catalog.rancher.io/release-name: rancher-pushprox +apiVersion: v1 +appVersion: v0.1.4-rancher2 +description: Sets up a deployment of the PushProx proxy and a DaemonSet of PushProx + clients. +kubeVersion: '>=1.26.0-0' +name: kubeAdmProxy +type: application +version: 0.2.1 diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmProxy/README.md b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmProxy/README.md new file mode 100644 index 00000000000..345002f48a2 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmProxy/README.md @@ -0,0 +1,90 @@ +# rancher-pushprox + +A Rancher chart based on Rancher [PushProx](https://github.com/rancher/PushProx) that sets up a Deployment of a PushProx proxy and a DaemonSet of PushProx clients on a Kubernetes cluster. + +Installs [rancher-pushprox](https://github.com/rancher/charts/tree/gh-pages/packages/rancher-pushprox) to create PushProx clients that can access their host's network and register with a PushProx proxy. A [Prometheus Operator](https://github.com/coreos/prometheus-operator) ServiceMonitor CR is also included that is configured to scrape the metrics from each of the clients through the proxy. + +Using an instance of this chart is suitable for the following scenarios: +- You need to scrape metrics from a port that should not be accessible outside of the host (e.g. scraping `etcd` metrics in a hardened cluster) +- You need to scrape metrics on a host that are not exposed outside of 127.0.0.1 (e.g. scraping `kube-proxy` metrics) +- You need to scrape metrics through HTTPS using certs hosted directly on `hostPath` +- You need to scrape metrics from Kubernetes components that require authorization via a service account (e.g. permissions to make request to `/metrics`) +- You need to scrape metrics without access to cacerts (i.e. enable `insecureSkipVerify`) + +The clients and proxy are created based on a Rancher fork of the [prometheus-community/PushProx](https://github.com/prometheus-community/PushProx) project. + +## Upgrading to Kubernetes v1.25+ + +Starting in Kubernetes v1.25, [Pod Security Policies](https://kubernetes.io/docs/concepts/security/pod-security-policy/) have been removed from the Kubernetes API. + +As a result, **before upgrading to Kubernetes v1.25** (or on a fresh install in a Kubernetes v1.25+ cluster), users are expected to perform an in-place upgrade of this chart with `global.cattle.psp.enabled` set to `false` if it has been previously set to `true`. +​ +> **Note:** +> In this chart release, any previous field that was associated with any PSP resources have been removed in favor of a single global field: `global.cattle.psp.enabled`. + +> **Note:** +> If you upgrade your cluster to Kubernetes v1.25+ before removing PSPs via a `helm upgrade` (even if you manually clean up resources), **it will leave the Helm release in a broken state within the cluster such that further Helm operations will not work (`helm uninstall`, `helm upgrade`, etc.).** +> +> If your charts get stuck in this state, please consult the Rancher docs on how to clean up your Helm release secrets. + +Upon setting `global.cattle.psp.enabled` to false, the chart will remove any PSP resources deployed on its behalf from the cluster. This is the default setting for this chart. + +As a replacement for PSPs, [Pod Security Admission](https://kubernetes.io/docs/concepts/security/pod-security-admission/) should be used. Please consult the Rancher docs for more details on how to configure your chart release namespaces to work with the new Pod Security Admission and apply Pod Security Standards. + +## Configuration + +The following tables list the configurable parameters of the rancher-pushprox chart and their default values. + +### General + +#### Required +| Parameter | Description | Example | +| ----- | ----------- | ------ | +| `component` | The component that is being monitored | `kube-etcd` +| `metricsPort` | The port on the host that contains the metrics you want to scrape (e.g. `http://:/metrics`) | `2379` | +| `namespaceOverride` | The namespace to install the chart | `""` + +#### Optional +| Parameter | Description | Default | +| ----- | ----------- | ------ | +| `serviceMonitor.enabled` | Deploys a [Prometheus Operator](https://github.com/coreos/prometheus-operator/blob/master/Documentation/api.md#servicemonitor) ServiceMonitor CR that is configured to scrape metrics on the hosts that the clients are deployed on via the proxy. Also deploys a Service that points to all pods with the expected client name that exposes the `metricsPort` selected | `true` | +| `serviceMonitor.endpoints` | A list of endpoints that will be added to the ServiceMonitor based on the [Endpoint spec](https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#endpoint) | `[{port: metrics}]` | +| `service.selector` | The selector that is used to populate the Service's Endpoints object. The chart will error out on rendering templating if `.Values.clients.enabled` is set alongside this field, since it is expected that this service should point to the PushProx Clients Daemonset / Deployment | `{}` | +| `clients.enabled` | Deploys a DaemonSet of clients that are each capable of scraping endpoints on the hostNetwork it is deployed on | `true` | +| `clients.port` | The port where the client will publish PushProx client-specific metrics. If deploying multiple clients onto the same node, the clients should not have conflicting ports | `9369` | +| `clients.proxyUrl` | Overrides the default proxyUrl setting of `http://pushprox-{{ .Values.component }}-proxy.{{ . Release.Namespace }}.svc.cluster.local:{{ .Values.proxy.port }}"` with the `proxyUrl` specified | `""` | +| `clients.useLocalhost` | Sets a flag on each client deployment to redirect scrapes directed to `HOST_IP` to `127.0.0.1` | `false` | +| `clients.https.enabled` | Enables scraping metrics via HTTPS using the provided TLS certs that exist on each host | `false` | +| `clients.https.forceHTTPSScheme` | Forces scraping metrics via HTTPS using the provided TLS certs that exist on each host | `false` | +| `clients.https.useServiceAccountCredentials` | If set to true, the client will create a service account with permissions to scrape `/metrics` endpoint of Kubernetes components. The client will use the service account token provided to make authorized scrape requests to the Kubernetes API | `false` | +| `clients.https.authenticationMethod.bearerTokenFile.enabled` | If set to true, the client will use service account credentials mounted at the configured path `clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath`. This requires permissions to scrape `/metrics` endpoint of Kubernetes components. This method is deprecated by the prometheus operator and may be removed in a future release | `false` | +| `clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath` | This is a volume mount on the pod with permissions to scrape `/metrics` endpoint of Kubernetes components | `"/var/run/secrets/kubernetes.io/serviceaccount/token"` | +| `clients.https.authenticationMethod.bearerTokenSecret.enabled` | If set to true, the client will use service account credentials to scrape `/metrics` endpoint of Kubernetes components. This method is deprecated by the prometheus operator and may be removed in a future release | `false` | +| `clients.https.authenticationMethod.authorization.enabled` | If set to true, the client will use service account credentials to scrape `/metrics` endpoint of Kubernetes components | `false` | +| `clients.https.authenticationMethod.authorization.type` | If set, the client will use this type of authorization in its client requests for metrics | `"bearer"` | +| `clients.https.authenticationMethod.authorization.credentials.key` | If set, the client will use this key in the secret created by `clients.https.useServiceAccountCredentials` for authorization in its client requests for metrics | `"token"` | +| `clients.https.authenticationMethod.authorization.credentials.optional` | If set to false, the client will fail if the key in the secret created by `clients.https.useServiceAccountCredentials` does not exist | `false` | +| `clients.https.insecureSkipVerify` | If set to true, the client will disable SSL security checks | `false` | +| `clients.https.certDir` | A `hostPath` where TLS certs can be found. This path is mounted as a volume on an `initContainer` which copies only the necessary files over to an EmptyDir volume used by each client. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.certFile` | The path to the TLS cert file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.keyFile` | The path to the TLS key file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.caCertFile` | The path to the TLS cacert file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.seLinuxOptions` | seLinuxOptions to be passed into the container that copies certs. Should define a container with permissions to read the files in the certDir provided on the host. Required and only used if `clients.https.enabled` is set and `clients.https.certDir` is provided. | `""` | +| `clients.metrics.enabled` | Whether the client should publish PushProx client-specific metrics. | `false` | +| `clients.rbac.additionalRules` | Additional permissions to provide to the ServiceAccount bound to the client. This can be used to provide additional permissions for the client to scrape metrics from the k8s API. Only enabled if clients.https.enabled and clients.https.useServiceAccountCredentials are true | `[]` | +| `clients.deployment.enabled` | Deploys the client as a Deployment (generally used if the underlying hostNetwork Pod that is being scraped is managed by a Deployment) | `false` | +| `clients.deployment.replicas` | The number of pods the Deployment has, it should match the number of pod the hostNetwork Deployment has. Required and only used if `client.deployment.enable` is set | `0` | +| `clients.deployment.affinity` | The affinity rules that allocate the pod to the node in which the hostNetwork Deployment's pods run. Required and only used if `client.deployment.enable` is set | `{}` | +| `clients.resources` | Set resource limits and requests for the client container | `{}` | +| `clients.nodeSelector` | Select which nodes to deploy the clients on | `{}` | +| `clients.tolerations` | Specify tolerations for clients | `[]` | +| `proxy.enabled` | Deploys the proxy that each client will register with | `true` | +| `proxy.port` | The port exposed by the proxy that each client will register with to allow metrics to be scraped from the host | `8080` | +| `proxy.resources` | Set resource limits and requests for the proxy container | `{}` | +| `proxy.nodeSelector` | Select which nodes the proxy can be deployed on | `{}` | +| `proxy.tolerations` | Specify tolerations (if necessary) to allow the proxy to be deployed on the selected node | `[]` | +| `kubeVersionOverrides` | A list of Semver constraint strings (defined by https://github.com/Masterminds/semver) and values.yaml overrides. For each key in kubeVersionOverrides, this chart will check to see if the current Kubernetes cluster's version matches any of the semver constraints provided as keys on the map. On seeing a match, the default value for each values.yaml field overridden will be updated with the new value. If multiple matches are encountered (due to overlapping semver ranges), the matches will be applied in order. | `[]` + +*Tip: The filepaths set in `clients.https.File` can include wildcard characters*. + +See [rancher-monitoring](https://github.com/rancher/charts/tree/gh-pages/packages/rancher-monitoring) for examples of how this chart can be used. diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmProxy/templates/_helpers.tpl b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmProxy/templates/_helpers.tpl new file mode 100644 index 00000000000..1ba50939441 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmProxy/templates/_helpers.tpl @@ -0,0 +1,170 @@ +# Rancher + +{{- define "system_default_registry" -}} +{{- if .Values.global.cattle.systemDefaultRegistry -}} +{{- printf "%s/" .Values.global.cattle.systemDefaultRegistry -}} +{{- end -}} +{{- end -}} + +# Windows Support + +{{/* +Windows cluster will add default taint for linux nodes, +add below linux tolerations to workloads could be scheduled to those linux nodes +*/}} + +{{- define "linux-node-tolerations" -}} +- key: "cattle.io/os" + value: "linux" + effect: "NoSchedule" + operator: "Equal" +{{- end -}} + +{{- define "linux-node-selector" -}} +{{- if semverCompare "<1.14-0" .Capabilities.KubeVersion.GitVersion -}} +beta.kubernetes.io/os: linux +{{- else -}} +kubernetes.io/os: linux +{{- end -}} +{{- end -}} + +# General + +{{- define "applyKubeVersionOverrides" -}} +{{- $overrides := dict -}} +{{- range $override := .Values.kubeVersionOverrides -}} +{{- if semverCompare $override.constraint $.Capabilities.KubeVersion.Version -}} +{{- $_ := mergeOverwrite $overrides $override.values -}} +{{- end -}} +{{- end -}} +{{- $_ := mergeOverwrite .Values $overrides -}} +{{- end -}} + +{{- define "pushprox.namespace" -}} + {{- if .Values.namespaceOverride -}} + {{- .Values.namespaceOverride -}} + {{- else -}} + {{- .Release.Namespace -}} + {{- end -}} +{{- end -}} + +{{- define "pushProxy.commonLabels" -}} +release: {{ .Release.Name }} +component: {{ .Values.component | quote }} +provider: kubernetes +{{- end -}} + +{{- define "pushProxy.proxyUrl" -}} +{{- $_ := (required "Template requires either .Values.proxy.port or .Values.client.proxyUrl to set proxyUrl for client" (or .Values.clients.proxyUrl .Values.proxy.port)) -}} +{{- if .Values.clients.proxyUrl -}} +{{ printf "%s" .Values.clients.proxyUrl }} +{{- else -}} +{{ printf "http://%s.%s.svc:%d" (include "pushProxy.proxy.name" .) (include "pushprox.namespace" .) (int .Values.proxy.port) }} +{{- end -}}{{- end -}} + +# Client + +{{- define "pushProxy.client.name" -}} +{{- printf "pushprox-%s-client" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.client.serviceAccountTokenName" -}} +{{- printf "pushprox-%s-client-service-account-token" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.client.labels" -}} +k8s-app: {{ template "pushProxy.client.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +# Proxy + +{{- define "pushProxy.proxy.name" -}} +{{- printf "pushprox-%s-proxy" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.proxy.labels" -}} +k8s-app: {{ template "pushProxy.proxy.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +# ServiceMonitor + +{{- define "pushprox.serviceMonitor.name" -}} +{{- printf "%s-%s" .Release.Name (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.serviceMonitor.labels" -}} +app: {{ template "pushprox.serviceMonitor.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +{{- define "pushProxy.serviceMonitor.endpoints" -}} +{{- $proxyURL := (include "pushProxy.proxyUrl" .) -}} +{{- $useHTTPS := .Values.clients.https.enabled -}} +{{- $setHTTPSScheme := .Values.clients.https.forceHTTPSScheme -}} +{{- $insecureSkipVerify := .Values.clients.https.insecureSkipVerify -}} +{{- $useServiceAccountCredentials := .Values.clients.https.useServiceAccountCredentials -}} +{{- $serviceAccountTokenName := (include "pushProxy.client.serviceAccountTokenName" . ) -}} +{{- $metricRelabelings := list }} +{{- $endpoints := .Values.serviceMonitor.endpoints }} +{{- if .Values.proxy.enabled }} +{{- $_ := set . "proxyUrl" $proxyURL }} +{{- end }} +{{- range $endpoints }} +{{- if $.Values.proxy.enabled }} +{{- $_ := set . "proxyUrl" $proxyURL }} +{{- end }} +{{- $clusterIdRelabel := dict }} +{{- $metricRelabelings := list }} +{{- if $.Values.global.cattle.clusterId }} +{{- $_ := set $clusterIdRelabel "action" "replace" }} +{{- $_ := set $clusterIdRelabel "sourceLabels" (list "__address__") }} +{{- $_ := set $clusterIdRelabel "targetLabel" "cluster_id" }} +{{- $_ := set $clusterIdRelabel "replacement" $.Values.global.cattle.clusterId }} +{{- $metricRelabelings = append $metricRelabelings $clusterIdRelabel }} +{{- end }} +{{- $clusterNameRelabel := dict }} +{{- if $.Values.global.cattle.clusterName }} +{{- $_ := set $clusterNameRelabel "action" "replace" }} +{{- $_ := set $clusterNameRelabel "sourceLabels" (list "__address__") }} +{{- $_ := set $clusterNameRelabel "targetLabel" "cluster_name" }} +{{- $_ := set $clusterNameRelabel "replacement" $.Values.global.cattle.clusterName }} +{{- $metricRelabelings = append $metricRelabelings $clusterNameRelabel }} +{{- end }} +{{- if not (empty $metricRelabelings) }} +{{- $_ := set . "metricRelabelings" ($metricRelabelings)}} +{{- end }} +{{- if $setHTTPSScheme -}} +{{- $_ := set . "scheme" "https" }} +{{- end -}} +{{- if $useHTTPS -}} +{{- if (hasKey . "params") }} +{{- $_ := set (get . "params") "_scheme" (list "https") }} +{{- else }} +{{- $_ := set . "params" (dict "_scheme" (list "https")) }} +{{- end }} +{{- end }} +{{- if (hasKey . "tlsConfig") }} +{{- $_ := set (get . "tlsConfig") "insecureSkipVerify" $insecureSkipVerify }} +{{- else }} +{{- $_ := set . "tlsConfig" (dict "insecureSkipVerify" $insecureSkipVerify) }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.bearerTokenFile.enabled }} +{{- $_ := set . "bearerTokenFile" $.Values.clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.bearerTokenSecret.enabled }} +{{- $_ := set . "bearerTokenSecret" $serviceAccountTokenName }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.authorization.enabled }} +{{- if (hasKey . "authorization") }} +{{- $_ := set (get . "authorization") "type" $.Values.clients.https.authenticationMethod.authorization.type }} +{{- $_ := set (get . "authorization") "credentials" (dict "name" $serviceAccountTokenName "key" $.Values.clients.https.authenticationMethod.authorization.credentials.key "optional" $.Values.clients.https.authenticationMethod.authorization.credentials.optional) }} +{{- else }} +{{- $_ := set . "authorization" (dict "type" $.Values.clients.https.authenticationMethod.authorization.type) }} +{{- $_ := set . "authorization" (dict "credentials" (dict "name" $serviceAccountTokenName "key" $.Values.clients.https.authenticationMethod.authorization.credentials.key "optional" $.Values.clients.https.authenticationMethod.authorization.credentials.optional)) }} +{{- end }} +{{- end }} +{{- end }} +{{- toYaml $endpoints }} +{{- end -}} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmProxy/templates/pushprox-clients-rbac.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmProxy/templates/pushprox-clients-rbac.yaml new file mode 100644 index 00000000000..a8e27c37356 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmProxy/templates/pushprox-clients-rbac.yaml @@ -0,0 +1,97 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.clients }}{{- if .Values.clients.enabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ template "pushProxy.client.name" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +rules: +{{- if .Values.global.cattle.psp.enabled }} +- apiGroups: ['policy'] + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - {{ template "pushProxy.client.name" . }} +{{- end }} +{{- if and .Values.clients.https.enabled .Values.clients.https.useServiceAccountCredentials }} +- nonResourceURLs: ["/metrics"] + verbs: ["get"] +{{- if .Values.clients.rbac.additionalRules }} +{{ toYaml .Values.clients.rbac.additionalRules }} +{{- end }} +{{- end }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ template "pushProxy.client.name" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "pushProxy.client.name" . }} +subjects: + - kind: ServiceAccount + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +--- +{{- if .Values.clients.https.useServiceAccountCredentials }} +apiVersion: v1 +kind: Secret +type: kubernetes.io/service-account-token +metadata: + name: {{ template "pushProxy.client.serviceAccountTokenName" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} + annotations: + kubernetes.io/service-account.name: {{ template "pushProxy.client.name" . }} +{{- end }} +--- +{{- if .Values.global.cattle.psp.enabled }} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +spec: + privileged: false + hostNetwork: true + hostIPC: false + hostPID: false + runAsUser: + rule: 'RunAsAny' + seLinux: + rule: 'RunAsAny' +{{- if and .Values.clients.https.enabled .Values.clients.https.certDir .Values.global.seLinux.enabled .Values.clients.https.seLinuxOptions }} + seLinuxOptions: {{ .Values.clients.https.seLinuxOptions | toYaml | nindent 6 }} +{{- end }} + supplementalGroups: + rule: 'MustRunAs' + ranges: + - min: 0 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + - min: 0 + max: 65535 + readOnlyRootFilesystem: false + volumes: + - 'secret' +{{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + - 'emptyDir' + - 'hostPath' + allowedHostPaths: + - pathPrefix: {{ required "Need access to volume on host with the SSL cert files to use HTTPs" .Values.clients.https.certDir }} + readOnly: true +{{- end }} +{{- end }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmProxy/templates/pushprox-clients.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmProxy/templates/pushprox-clients.yaml new file mode 100644 index 00000000000..e8fcfb38836 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmProxy/templates/pushprox-clients.yaml @@ -0,0 +1,157 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.clients }}{{- if .Values.clients.enabled }} +apiVersion: apps/v1 +{{- if .Values.clients.deployment.enabled }} +kind: Deployment +{{- else }} +kind: DaemonSet +{{- end }} +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} + pushprox-exporter: "client" +spec: + {{- if .Values.clients.deployment.enabled }} + replicas: {{ .Values.clients.deployment.replicas }} + {{- end }} + selector: + matchLabels: {{ include "pushProxy.client.labels" . | nindent 6 }} + template: + metadata: + labels: {{ include "pushProxy.client.labels" . | nindent 8 }} + spec: + {{- if .Values.clients.affinity }} + affinity: {{ toYaml .Values.clients.affinity | nindent 8 }} + {{- end }} + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} +{{- if .Values.clients.nodeSelector }} +{{ toYaml .Values.clients.nodeSelector | indent 8 }} +{{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} +{{- if .Values.clients.tolerations }} +{{ toYaml .Values.clients.tolerations | indent 8 }} +{{- end }} + hostNetwork: true + dnsPolicy: ClusterFirstWithHostNet + serviceAccountName: {{ template "pushProxy.client.name" . }} + {{- if .Values.global.imagePullSecretName }} + imagePullSecrets: + - name: {{ .Values.global.imagePullSecretName }} + {{- end }} + containers: + - name: pushprox-client + image: {{ template "system_default_registry" . }}{{ .Values.clients.image.repository }}:{{ .Values.clients.image.tag }} + command: + {{- range .Values.clients.command }} + - {{ . | quote }} + {{- end }} + args: + - --fqdn=$(HOST_IP) + - --proxy-url=$(PROXY_URL) + {{- if .Values.clients.metrics.enabled }} + - --metrics-addr=$(PORT) + {{- end }} + - --allow-port={{ required "Need .Values.metricsPort to configure client to be allowed to scrape metrics at port" .Values.metricsPort}} + {{- if .Values.clients.useLocalhost }} + - --use-localhost + {{- end }} + {{- if .Values.clients.https.enabled }} + {{- if .Values.clients.https.insecureSkipVerify }} + - --insecure-skip-verify + {{- end }} + {{- if .Values.clients.https.useServiceAccountCredentials }} + - --token-path=/var/run/secrets/kubernetes.io/serviceaccount/token + {{- end }} + {{- if .Values.clients.https.certDir }} + - --tls.cert=/etc/ssl/push-proxy/push-proxy.pem + - --tls.key=/etc/ssl/push-proxy/push-proxy-key.pem + - --tls.cacert=/etc/ssl/push-proxy/push-proxy-ca-cert.pem + {{- end }} + {{- end }} + env: + - name: HOST_IP + valueFrom: + fieldRef: + fieldPath: status.hostIP + {{- if .Values.clients.metrics.enabled }} + - name: PORT + value: :{{ .Values.clients.port }} + {{- end }} + - name: PROXY_URL + value: {{ template "pushProxy.proxyUrl" . }} + securityContext: + runAsNonRoot: true + runAsUser: 1000 + {{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + volumeMounts: + - name: metrics-cert-dir + mountPath: /etc/ssl/push-proxy + {{- end }} + {{- if .Values.clients.resources }} + resources: {{ toYaml .Values.clients.resources | nindent 10 }} + {{- end }} + {{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + initContainers: + - name: copy-certs + image: {{ template "system_default_registry" . }}{{ .Values.clients.copyCertsImage.repository }}:{{ .Values.clients.copyCertsImage.tag }} + command: + - sh + - -c + - | + echo "Searching for files to copy within the source volume" + echo "cert: ${CERT_FILE_NAME}" + echo "key: ${KEY_FILE_NAME}" + echo "cacert: ${CACERT_FILE_NAME}" + + CERT_FILE_SOURCE=$(find /etc/source/ -type f -name "${CERT_FILE_NAME}" | sort -r | head -n 1) + KEY_FILE_SOURCE=$(find /etc/source/ -type f -name "${KEY_FILE_NAME}" | sort -r | head -n 1) + CACERT_FILE_SOURCE=$(find /etc/source/ -type f -name "${CACERT_FILE_NAME}" | sort -r | head -n 1) + + test -z ${CERT_FILE_SOURCE} && echo "Failed to find cert file" && exit 1 + test -z ${KEY_FILE_SOURCE} && echo "Failed to find key file" && exit 1 + test -z ${CACERT_FILE_SOURCE} && echo "Failed to find cacert file" && exit 1 + + echo "Copying cert file from $CERT_FILE_SOURCE to $CERT_FILE_TARGET" + cp $CERT_FILE_SOURCE $CERT_FILE_TARGET || exit 1 + chmod 444 $CERT_FILE_TARGET || exit 1 + + echo "Copying key file from $KEY_FILE_SOURCE to $KEY_FILE_TARGET" + cp $KEY_FILE_SOURCE $KEY_FILE_TARGET || exit 1 + chmod 444 $KEY_FILE_TARGET || exit 1 + + echo "Copying cacert file from $CACERT_FILE_SOURCE to $CACERT_FILE_TARGET" + cp $CACERT_FILE_SOURCE $CACERT_FILE_TARGET || exit 1 + chmod 444 $CACERT_FILE_TARGET || exit 1 + env: + - name: CERT_FILE_NAME + value: {{ required "Need a TLS cert file for scraping metrics endpoint over HTTPs" .Values.clients.https.certFile }} + - name: KEY_FILE_NAME + value: {{ required "Need a TLS key file for scraping metrics endpoint over HTTPs" .Values.clients.https.keyFile }} + - name: CACERT_FILE_NAME + value: {{ required "Need a TLS CA cert file for scraping metrics endpoint over HTTPs" .Values.clients.https.caCertFile }} + - name: CERT_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy.pem + - name: KEY_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy-key.pem + - name: CACERT_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy-ca-cert.pem + securityContext: + runAsNonRoot: false +{{- if and .Values.global.seLinux.enabled .Values.clients.https.seLinuxOptions }} + seLinuxOptions: {{ .Values.clients.https.seLinuxOptions | toYaml | nindent 12 }} +{{- end }} + volumeMounts: + - name: metrics-cert-dir-source + mountPath: /etc/source + readOnly: true + - name: metrics-cert-dir + mountPath: /etc/ssl/push-proxy + volumes: + - name: metrics-cert-dir-source + hostPath: + path: {{ required "Need access to volume on host with the SSL cert files to use HTTPs" .Values.clients.https.certDir }} + - name: metrics-cert-dir + emptyDir: {} + {{- end }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmProxy/templates/pushprox-proxy-rbac.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmProxy/templates/pushprox-proxy-rbac.yaml new file mode 100644 index 00000000000..eefe609058c --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmProxy/templates/pushprox-proxy-rbac.yaml @@ -0,0 +1,68 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if and .Values.proxy }}{{ if .Values.proxy.enabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ template "pushProxy.proxy.name" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +rules: +{{- if .Values.global.cattle.psp.enabled }} +- apiGroups: ['policy'] + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - {{ template "pushProxy.proxy.name" . }} +{{- end }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ template "pushProxy.proxy.name" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "pushProxy.proxy.name" . }} +subjects: + - kind: ServiceAccount + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +--- +{{- if .Values.global.cattle.psp.enabled }} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +spec: + privileged: false + hostNetwork: false + hostIPC: false + hostPID: false + runAsUser: + rule: 'MustRunAsNonRoot' + seLinux: + rule: 'RunAsAny' + supplementalGroups: + rule: 'MustRunAs' + ranges: + - min: 1 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + - min: 1 + max: 65535 + readOnlyRootFilesystem: false + volumes: + - 'secret' +{{- end }}{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmProxy/templates/pushprox-proxy.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmProxy/templates/pushprox-proxy.yaml new file mode 100644 index 00000000000..723bbd6c00b --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmProxy/templates/pushprox-proxy.yaml @@ -0,0 +1,57 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if and .Values.proxy }}{{ if .Values.proxy.enabled }} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} + pushprox-exporter: "proxy" +spec: + selector: + matchLabels: {{ include "pushProxy.proxy.labels" . | nindent 6 }} + template: + metadata: + labels: {{ include "pushProxy.proxy.labels" . | nindent 8 }} + spec: + securityContext: + runAsNonRoot: true + runAsUser: 1000 + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} +{{- if .Values.proxy.nodeSelector }} +{{ toYaml .Values.proxy.nodeSelector | indent 8 }} +{{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} +{{- if .Values.proxy.tolerations }} +{{ toYaml .Values.proxy.tolerations | indent 8 }} +{{- end }} + serviceAccountName: {{ template "pushProxy.proxy.name" . }} + {{- if .Values.global.imagePullSecretName }} + imagePullSecrets: + - name: {{ .Values.global.imagePullSecretName }} + {{- end }} + containers: + - name: pushprox-proxy + image: {{ template "system_default_registry" . }}{{ .Values.proxy.image.repository }}:{{ .Values.proxy.image.tag }} + command: + {{- range .Values.proxy.command }} + - {{ . | quote }} + {{- end }} + {{- if .Values.proxy.resources }} + resources: {{ toYaml .Values.proxy.resources | nindent 10 }} + {{- end }} +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +spec: + ports: + - name: pp-proxy + port: {{ required "Need .Values.proxy.port to configure proxy" .Values.proxy.port }} + protocol: TCP + targetPort: {{ .Values.proxy.port }} + selector: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmProxy/templates/pushprox-servicemonitor.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmProxy/templates/pushprox-servicemonitor.yaml new file mode 100644 index 00000000000..67eb2216b5a --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmProxy/templates/pushprox-servicemonitor.yaml @@ -0,0 +1,45 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.serviceMonitor }}{{- if .Values.serviceMonitor.enabled }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ template "pushprox.serviceMonitor.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.serviceMonitor.labels" . | nindent 4 }} +spec: + endpoints: {{include "pushProxy.serviceMonitor.endpoints" . | nindent 4 }} + jobLabel: component + podTargetLabels: + - component + - pushprox-exporter + namespaceSelector: + matchNames: + - {{ template "pushprox.namespace" . }} + selector: + matchLabels: {{ include "pushProxy.client.labels" . | nindent 6 }} +--- +{{- $selector := "" }} +{{- if not (kindIs "invalid" .Values.service) }} +{{- if not (kindIs "invalid" .Values.service.selector) }} +{{ if .Values.service.selector }} +{{- if .Values.clients.enabled }} +{{- required (printf "Cannot override .Values.service.selector=%s when .Values.clients.enabled=true" (toJson .Values.service.selector)) "" }} +{{- end }} +{{- $selector = (toYaml .Values.service.selector) }} +{{- end }} +{{- end }} +{{- end }} +apiVersion: v1 +kind: Service +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +spec: + ports: + - name: metrics + port: {{ required "Need .Values.metricsPort to configure client to listen to metrics at port" .Values.metricsPort}} + protocol: TCP + targetPort: {{ .Values.metricsPort }} + selector: {{ default (include "pushProxy.client.labels" .) $selector | nindent 4 }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmProxy/templates/validate-install-crd.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmProxy/templates/validate-install-crd.yaml new file mode 100644 index 00000000000..16abc2fa83a --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmProxy/templates/validate-install-crd.yaml @@ -0,0 +1,14 @@ +#{{- if gt (len (lookup "rbac.authorization.k8s.io/v1" "ClusterRole" "" "")) 0 -}} +# {{- $found := dict -}} +# {{- set $found "monitoring.coreos.com/v1/ServiceMonitor" false -}} +# {{- range .Capabilities.APIVersions -}} +# {{- if hasKey $found (toString .) -}} +# {{- set $found (toString .) true -}} +# {{- end -}} +# {{- end -}} +# {{- range $_, $exists := $found -}} +# {{- if (eq $exists false) -}} +# {{- required "Required CRDs are missing. Please install Prometheus Operator CRDs before installing this chart." "" -}} +# {{- end -}} +# {{- end -}} +#{{- end -}} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmProxy/templates/validate-psp-install.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmProxy/templates/validate-psp-install.yaml new file mode 100644 index 00000000000..a30c59d3b7c --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmProxy/templates/validate-psp-install.yaml @@ -0,0 +1,7 @@ +#{{- if gt (len (lookup "rbac.authorization.k8s.io/v1" "ClusterRole" "" "")) 0 -}} +#{{- if .Values.global.cattle.psp.enabled }} +#{{- if not (.Capabilities.APIVersions.Has "policy/v1beta1/PodSecurityPolicy") }} +#{{- fail "The target cluster does not have the PodSecurityPolicy API resource. Please disable PSPs in this chart before proceeding." -}} +#{{- end }} +#{{- end }} +#{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmProxy/values.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmProxy/values.yaml new file mode 100644 index 00000000000..7f38498a1c5 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmProxy/values.yaml @@ -0,0 +1,166 @@ +# Default values for rancher-pushprox. +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. + +# Default image containing both the proxy and the client was generated from the following Dockerfile +# https://github.com/prometheus-community/PushProx/blob/eeadbe766641699129920ccfaaaa30a85c67fe81/Dockerfile#L1-L15 + +# Configuration + +global: + cattle: + psp: + enabled: false + systemDefaultRegistry: "" + seLinux: + enabled: false + +# A list of Semver constraint strings (defined by https://github.com/Masterminds/semver) and values.yaml overrides. +# +# For each key in kubeVersionOverrides, this chart will check to see if the current Kubernetes cluster's version matches +# any of the semver constraints provided as keys on the map. +# +# On seeing a match, the default value for each values.yaml field overridden will be updated with the new value. +# +# If multiple matches are encountered (due to overlapping semver ranges), the matches will be applied in order. +# +# Notes: +# - On running a helm template, Helm generally assumes the kubeVersion is v1.20.0 +# - On running a helm install --dry-run, the correct kubeVersion should be chosen. +kubeVersionOverrides: [] +# - constraint: "< 1.21" +# values: +# metricsPort: 10252 +# clients: +# https: +# enabled: false +# insecureSkipVerify: false +# useServiceAccountCredentials: false + +namespaceOverride: "" + +# The component that is being monitored (i.e. etcd) +component: "component" + +# The port containing the metrics that need to be scraped +metricsPort: 2739 + +# Configure ServiceMonitor that monitors metrics from the metricsPort endpoint +serviceMonitor: + enabled: true + # A list of endpoints that will be added to the ServiceMonitor based on the Endpoint spec + # Source: https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#endpoint + # By default, proxyUrl and params._scheme will be overridden based on other values + endpoints: + - port: metrics + +# Configure Service that grabs scrape targets +service: + # The selector that is used to populate the Service's Endpoints object. + # The chart will error out on rendering templating if .Values.clients.enabled is set alongside this field, + # since it is expected that this service should point to the PushProx Clients Daemonset / Deployment + selector: {} + +clients: + enabled: true + # The port which the PushProx client will post PushProx metrics to + port: 9369 + # If unset, this will default to the URL for the proxy service: http://pushprox-{{component}}-proxy.{{namepsace}}.svc.cluster.local:{{proxy.port}} + # Should be modified if the clients are being deployed outside the cluster where the proxy rests, otherwise leave it null + proxyUrl: "" + # If set to true, the client will forward any requests from the host IP to 127.0.0.1 + # It will only allow proxy requests to the metricsPort specified + useLocalhost: false + # Configuration for accessing metrics via HTTPS + https: + # Does the client require https to access the metrics? + enabled: false + # Does the client require requests be sent to http or https? + forceHTTPSScheme: false + # If set to true, the client will create a service account with adequate permissions and set a flag + # on the client to use the service account token provided by it to make authorized scrape requests + useServiceAccountCredentials: false + # Configuration for authentication to metrics via https endpoint + authenticationMethod: + # Reads token from defined file in container + # This function is deprecated in the prometheus operator api and may be removed in a future version + bearerTokenFile: + enabled: false + bearerTokenFilePath: "/var/run/secrets/kubernetes.io/serviceaccount/token" + # Reads token from defined secret in namespace + # This function is deprecated in the prometheus operator api and may be removed in a future version + bearerTokenSecret: + enabled: false + # Reads token from defined secret in namespace + authorization: + enabled: false + type: "bearer" + credentials: + key: "token" + optional: false + # If set to true, the client will disable SSL security checks + insecureSkipVerify: false + # Directory on host where necessary TLS cert and key to scrape metrics can be found + certDir: "" + # Filenames for files located in .Values.clients.https.certDir that correspond to TLS settings + certFile: "" + keyFile: "" + caCertFile: "" + # seLinuxOptions to be passed into the container that copies certs. Should define a container with permissions to read the files in the certDir provided on the host. + # Required and only used if `clients.https.enabled` is set and `clients.https.certDir` is provided. + seLinuxOptions: {} + + metrics: + # Whether the client should publish PushProx client-specific metrics to .Values.clients.port + enabled: false + + rbac: + # Additional permissions to provide to the ServiceAccount bound to the client + # This can be used to provide additional permissions for the client to scrape metrics from the k8s API + # Only enabled if clients.https.enabled and clients.https.useServiceAccountCredentials are true + additionalRules: [] + + # Resource limits + resources: {} + + # Options to select all nodes to deploy client DaemonSet on + nodeSelector: {} + tolerations: [] + affinity: {} + + image: + repository: rancher/pushprox-client + tag: v0.1.4-rancher2-client + command: ["pushprox-client"] + + copyCertsImage: + repository: rancher/mirrored-library-busybox + tag: 1.31.1 + + # The default intention of rancher-pushprox clients is to scrape hostNetwork metrics across all nodes. + # This can be used to scrape internal Kubernetes components or DaemonSets of hostNetwork Pods in + # situations where a cloud provider firewall prevents Pod-To-Host communication but not Pod-To-Pod. + # However, if the underlying hostNetwork Pod that is being scraped is managed by a Deployment, + # this advanced option enables users to deploy the client as a Deployment instead of a DaemonSet. + # If a user deploys this feature and the underlying Deployment's number of replicas changes, the user will + # be responsible for upgrading this chart accordingly to the right number of replicas. + deployment: + enabled: false + replicas: 0 + +proxy: + enabled: true + # The port through which PushProx clients will communicate to the proxy + port: 8080 + + # Resource limits + resources: {} + + # Options to select a node to run a single proxy deployment on + nodeSelector: {} + tolerations: [] + + image: + repository: rancher/pushprox-proxy + tag: v0.1.4-rancher2-proxy + command: ["pushprox-proxy"] diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmScheduler/.helmignore b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmScheduler/.helmignore new file mode 100644 index 00000000000..0e8a0eb36f4 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmScheduler/.helmignore @@ -0,0 +1,23 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*.orig +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmScheduler/Chart.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmScheduler/Chart.yaml new file mode 100644 index 00000000000..f7c201833a6 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmScheduler/Chart.yaml @@ -0,0 +1,15 @@ +annotations: + catalog.cattle.io/hidden: "true" + catalog.cattle.io/kube-version: '>= 1.26.0-0 < 1.31.0-0' + catalog.cattle.io/os: linux + catalog.rancher.io/certified: rancher + catalog.rancher.io/namespace: cattle-monitoring-system + catalog.rancher.io/release-name: rancher-pushprox +apiVersion: v1 +appVersion: v0.1.4-rancher2 +description: Sets up a deployment of the PushProx proxy and a DaemonSet of PushProx + clients. +kubeVersion: '>=1.26.0-0' +name: kubeAdmScheduler +type: application +version: 0.2.1 diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmScheduler/README.md b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmScheduler/README.md new file mode 100644 index 00000000000..345002f48a2 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmScheduler/README.md @@ -0,0 +1,90 @@ +# rancher-pushprox + +A Rancher chart based on Rancher [PushProx](https://github.com/rancher/PushProx) that sets up a Deployment of a PushProx proxy and a DaemonSet of PushProx clients on a Kubernetes cluster. + +Installs [rancher-pushprox](https://github.com/rancher/charts/tree/gh-pages/packages/rancher-pushprox) to create PushProx clients that can access their host's network and register with a PushProx proxy. A [Prometheus Operator](https://github.com/coreos/prometheus-operator) ServiceMonitor CR is also included that is configured to scrape the metrics from each of the clients through the proxy. + +Using an instance of this chart is suitable for the following scenarios: +- You need to scrape metrics from a port that should not be accessible outside of the host (e.g. scraping `etcd` metrics in a hardened cluster) +- You need to scrape metrics on a host that are not exposed outside of 127.0.0.1 (e.g. scraping `kube-proxy` metrics) +- You need to scrape metrics through HTTPS using certs hosted directly on `hostPath` +- You need to scrape metrics from Kubernetes components that require authorization via a service account (e.g. permissions to make request to `/metrics`) +- You need to scrape metrics without access to cacerts (i.e. enable `insecureSkipVerify`) + +The clients and proxy are created based on a Rancher fork of the [prometheus-community/PushProx](https://github.com/prometheus-community/PushProx) project. + +## Upgrading to Kubernetes v1.25+ + +Starting in Kubernetes v1.25, [Pod Security Policies](https://kubernetes.io/docs/concepts/security/pod-security-policy/) have been removed from the Kubernetes API. + +As a result, **before upgrading to Kubernetes v1.25** (or on a fresh install in a Kubernetes v1.25+ cluster), users are expected to perform an in-place upgrade of this chart with `global.cattle.psp.enabled` set to `false` if it has been previously set to `true`. +​ +> **Note:** +> In this chart release, any previous field that was associated with any PSP resources have been removed in favor of a single global field: `global.cattle.psp.enabled`. + +> **Note:** +> If you upgrade your cluster to Kubernetes v1.25+ before removing PSPs via a `helm upgrade` (even if you manually clean up resources), **it will leave the Helm release in a broken state within the cluster such that further Helm operations will not work (`helm uninstall`, `helm upgrade`, etc.).** +> +> If your charts get stuck in this state, please consult the Rancher docs on how to clean up your Helm release secrets. + +Upon setting `global.cattle.psp.enabled` to false, the chart will remove any PSP resources deployed on its behalf from the cluster. This is the default setting for this chart. + +As a replacement for PSPs, [Pod Security Admission](https://kubernetes.io/docs/concepts/security/pod-security-admission/) should be used. Please consult the Rancher docs for more details on how to configure your chart release namespaces to work with the new Pod Security Admission and apply Pod Security Standards. + +## Configuration + +The following tables list the configurable parameters of the rancher-pushprox chart and their default values. + +### General + +#### Required +| Parameter | Description | Example | +| ----- | ----------- | ------ | +| `component` | The component that is being monitored | `kube-etcd` +| `metricsPort` | The port on the host that contains the metrics you want to scrape (e.g. `http://:/metrics`) | `2379` | +| `namespaceOverride` | The namespace to install the chart | `""` + +#### Optional +| Parameter | Description | Default | +| ----- | ----------- | ------ | +| `serviceMonitor.enabled` | Deploys a [Prometheus Operator](https://github.com/coreos/prometheus-operator/blob/master/Documentation/api.md#servicemonitor) ServiceMonitor CR that is configured to scrape metrics on the hosts that the clients are deployed on via the proxy. Also deploys a Service that points to all pods with the expected client name that exposes the `metricsPort` selected | `true` | +| `serviceMonitor.endpoints` | A list of endpoints that will be added to the ServiceMonitor based on the [Endpoint spec](https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#endpoint) | `[{port: metrics}]` | +| `service.selector` | The selector that is used to populate the Service's Endpoints object. The chart will error out on rendering templating if `.Values.clients.enabled` is set alongside this field, since it is expected that this service should point to the PushProx Clients Daemonset / Deployment | `{}` | +| `clients.enabled` | Deploys a DaemonSet of clients that are each capable of scraping endpoints on the hostNetwork it is deployed on | `true` | +| `clients.port` | The port where the client will publish PushProx client-specific metrics. If deploying multiple clients onto the same node, the clients should not have conflicting ports | `9369` | +| `clients.proxyUrl` | Overrides the default proxyUrl setting of `http://pushprox-{{ .Values.component }}-proxy.{{ . Release.Namespace }}.svc.cluster.local:{{ .Values.proxy.port }}"` with the `proxyUrl` specified | `""` | +| `clients.useLocalhost` | Sets a flag on each client deployment to redirect scrapes directed to `HOST_IP` to `127.0.0.1` | `false` | +| `clients.https.enabled` | Enables scraping metrics via HTTPS using the provided TLS certs that exist on each host | `false` | +| `clients.https.forceHTTPSScheme` | Forces scraping metrics via HTTPS using the provided TLS certs that exist on each host | `false` | +| `clients.https.useServiceAccountCredentials` | If set to true, the client will create a service account with permissions to scrape `/metrics` endpoint of Kubernetes components. The client will use the service account token provided to make authorized scrape requests to the Kubernetes API | `false` | +| `clients.https.authenticationMethod.bearerTokenFile.enabled` | If set to true, the client will use service account credentials mounted at the configured path `clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath`. This requires permissions to scrape `/metrics` endpoint of Kubernetes components. This method is deprecated by the prometheus operator and may be removed in a future release | `false` | +| `clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath` | This is a volume mount on the pod with permissions to scrape `/metrics` endpoint of Kubernetes components | `"/var/run/secrets/kubernetes.io/serviceaccount/token"` | +| `clients.https.authenticationMethod.bearerTokenSecret.enabled` | If set to true, the client will use service account credentials to scrape `/metrics` endpoint of Kubernetes components. This method is deprecated by the prometheus operator and may be removed in a future release | `false` | +| `clients.https.authenticationMethod.authorization.enabled` | If set to true, the client will use service account credentials to scrape `/metrics` endpoint of Kubernetes components | `false` | +| `clients.https.authenticationMethod.authorization.type` | If set, the client will use this type of authorization in its client requests for metrics | `"bearer"` | +| `clients.https.authenticationMethod.authorization.credentials.key` | If set, the client will use this key in the secret created by `clients.https.useServiceAccountCredentials` for authorization in its client requests for metrics | `"token"` | +| `clients.https.authenticationMethod.authorization.credentials.optional` | If set to false, the client will fail if the key in the secret created by `clients.https.useServiceAccountCredentials` does not exist | `false` | +| `clients.https.insecureSkipVerify` | If set to true, the client will disable SSL security checks | `false` | +| `clients.https.certDir` | A `hostPath` where TLS certs can be found. This path is mounted as a volume on an `initContainer` which copies only the necessary files over to an EmptyDir volume used by each client. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.certFile` | The path to the TLS cert file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.keyFile` | The path to the TLS key file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.caCertFile` | The path to the TLS cacert file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.seLinuxOptions` | seLinuxOptions to be passed into the container that copies certs. Should define a container with permissions to read the files in the certDir provided on the host. Required and only used if `clients.https.enabled` is set and `clients.https.certDir` is provided. | `""` | +| `clients.metrics.enabled` | Whether the client should publish PushProx client-specific metrics. | `false` | +| `clients.rbac.additionalRules` | Additional permissions to provide to the ServiceAccount bound to the client. This can be used to provide additional permissions for the client to scrape metrics from the k8s API. Only enabled if clients.https.enabled and clients.https.useServiceAccountCredentials are true | `[]` | +| `clients.deployment.enabled` | Deploys the client as a Deployment (generally used if the underlying hostNetwork Pod that is being scraped is managed by a Deployment) | `false` | +| `clients.deployment.replicas` | The number of pods the Deployment has, it should match the number of pod the hostNetwork Deployment has. Required and only used if `client.deployment.enable` is set | `0` | +| `clients.deployment.affinity` | The affinity rules that allocate the pod to the node in which the hostNetwork Deployment's pods run. Required and only used if `client.deployment.enable` is set | `{}` | +| `clients.resources` | Set resource limits and requests for the client container | `{}` | +| `clients.nodeSelector` | Select which nodes to deploy the clients on | `{}` | +| `clients.tolerations` | Specify tolerations for clients | `[]` | +| `proxy.enabled` | Deploys the proxy that each client will register with | `true` | +| `proxy.port` | The port exposed by the proxy that each client will register with to allow metrics to be scraped from the host | `8080` | +| `proxy.resources` | Set resource limits and requests for the proxy container | `{}` | +| `proxy.nodeSelector` | Select which nodes the proxy can be deployed on | `{}` | +| `proxy.tolerations` | Specify tolerations (if necessary) to allow the proxy to be deployed on the selected node | `[]` | +| `kubeVersionOverrides` | A list of Semver constraint strings (defined by https://github.com/Masterminds/semver) and values.yaml overrides. For each key in kubeVersionOverrides, this chart will check to see if the current Kubernetes cluster's version matches any of the semver constraints provided as keys on the map. On seeing a match, the default value for each values.yaml field overridden will be updated with the new value. If multiple matches are encountered (due to overlapping semver ranges), the matches will be applied in order. | `[]` + +*Tip: The filepaths set in `clients.https.File` can include wildcard characters*. + +See [rancher-monitoring](https://github.com/rancher/charts/tree/gh-pages/packages/rancher-monitoring) for examples of how this chart can be used. diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmScheduler/templates/_helpers.tpl b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmScheduler/templates/_helpers.tpl new file mode 100644 index 00000000000..1ba50939441 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmScheduler/templates/_helpers.tpl @@ -0,0 +1,170 @@ +# Rancher + +{{- define "system_default_registry" -}} +{{- if .Values.global.cattle.systemDefaultRegistry -}} +{{- printf "%s/" .Values.global.cattle.systemDefaultRegistry -}} +{{- end -}} +{{- end -}} + +# Windows Support + +{{/* +Windows cluster will add default taint for linux nodes, +add below linux tolerations to workloads could be scheduled to those linux nodes +*/}} + +{{- define "linux-node-tolerations" -}} +- key: "cattle.io/os" + value: "linux" + effect: "NoSchedule" + operator: "Equal" +{{- end -}} + +{{- define "linux-node-selector" -}} +{{- if semverCompare "<1.14-0" .Capabilities.KubeVersion.GitVersion -}} +beta.kubernetes.io/os: linux +{{- else -}} +kubernetes.io/os: linux +{{- end -}} +{{- end -}} + +# General + +{{- define "applyKubeVersionOverrides" -}} +{{- $overrides := dict -}} +{{- range $override := .Values.kubeVersionOverrides -}} +{{- if semverCompare $override.constraint $.Capabilities.KubeVersion.Version -}} +{{- $_ := mergeOverwrite $overrides $override.values -}} +{{- end -}} +{{- end -}} +{{- $_ := mergeOverwrite .Values $overrides -}} +{{- end -}} + +{{- define "pushprox.namespace" -}} + {{- if .Values.namespaceOverride -}} + {{- .Values.namespaceOverride -}} + {{- else -}} + {{- .Release.Namespace -}} + {{- end -}} +{{- end -}} + +{{- define "pushProxy.commonLabels" -}} +release: {{ .Release.Name }} +component: {{ .Values.component | quote }} +provider: kubernetes +{{- end -}} + +{{- define "pushProxy.proxyUrl" -}} +{{- $_ := (required "Template requires either .Values.proxy.port or .Values.client.proxyUrl to set proxyUrl for client" (or .Values.clients.proxyUrl .Values.proxy.port)) -}} +{{- if .Values.clients.proxyUrl -}} +{{ printf "%s" .Values.clients.proxyUrl }} +{{- else -}} +{{ printf "http://%s.%s.svc:%d" (include "pushProxy.proxy.name" .) (include "pushprox.namespace" .) (int .Values.proxy.port) }} +{{- end -}}{{- end -}} + +# Client + +{{- define "pushProxy.client.name" -}} +{{- printf "pushprox-%s-client" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.client.serviceAccountTokenName" -}} +{{- printf "pushprox-%s-client-service-account-token" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.client.labels" -}} +k8s-app: {{ template "pushProxy.client.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +# Proxy + +{{- define "pushProxy.proxy.name" -}} +{{- printf "pushprox-%s-proxy" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.proxy.labels" -}} +k8s-app: {{ template "pushProxy.proxy.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +# ServiceMonitor + +{{- define "pushprox.serviceMonitor.name" -}} +{{- printf "%s-%s" .Release.Name (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.serviceMonitor.labels" -}} +app: {{ template "pushprox.serviceMonitor.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +{{- define "pushProxy.serviceMonitor.endpoints" -}} +{{- $proxyURL := (include "pushProxy.proxyUrl" .) -}} +{{- $useHTTPS := .Values.clients.https.enabled -}} +{{- $setHTTPSScheme := .Values.clients.https.forceHTTPSScheme -}} +{{- $insecureSkipVerify := .Values.clients.https.insecureSkipVerify -}} +{{- $useServiceAccountCredentials := .Values.clients.https.useServiceAccountCredentials -}} +{{- $serviceAccountTokenName := (include "pushProxy.client.serviceAccountTokenName" . ) -}} +{{- $metricRelabelings := list }} +{{- $endpoints := .Values.serviceMonitor.endpoints }} +{{- if .Values.proxy.enabled }} +{{- $_ := set . "proxyUrl" $proxyURL }} +{{- end }} +{{- range $endpoints }} +{{- if $.Values.proxy.enabled }} +{{- $_ := set . "proxyUrl" $proxyURL }} +{{- end }} +{{- $clusterIdRelabel := dict }} +{{- $metricRelabelings := list }} +{{- if $.Values.global.cattle.clusterId }} +{{- $_ := set $clusterIdRelabel "action" "replace" }} +{{- $_ := set $clusterIdRelabel "sourceLabels" (list "__address__") }} +{{- $_ := set $clusterIdRelabel "targetLabel" "cluster_id" }} +{{- $_ := set $clusterIdRelabel "replacement" $.Values.global.cattle.clusterId }} +{{- $metricRelabelings = append $metricRelabelings $clusterIdRelabel }} +{{- end }} +{{- $clusterNameRelabel := dict }} +{{- if $.Values.global.cattle.clusterName }} +{{- $_ := set $clusterNameRelabel "action" "replace" }} +{{- $_ := set $clusterNameRelabel "sourceLabels" (list "__address__") }} +{{- $_ := set $clusterNameRelabel "targetLabel" "cluster_name" }} +{{- $_ := set $clusterNameRelabel "replacement" $.Values.global.cattle.clusterName }} +{{- $metricRelabelings = append $metricRelabelings $clusterNameRelabel }} +{{- end }} +{{- if not (empty $metricRelabelings) }} +{{- $_ := set . "metricRelabelings" ($metricRelabelings)}} +{{- end }} +{{- if $setHTTPSScheme -}} +{{- $_ := set . "scheme" "https" }} +{{- end -}} +{{- if $useHTTPS -}} +{{- if (hasKey . "params") }} +{{- $_ := set (get . "params") "_scheme" (list "https") }} +{{- else }} +{{- $_ := set . "params" (dict "_scheme" (list "https")) }} +{{- end }} +{{- end }} +{{- if (hasKey . "tlsConfig") }} +{{- $_ := set (get . "tlsConfig") "insecureSkipVerify" $insecureSkipVerify }} +{{- else }} +{{- $_ := set . "tlsConfig" (dict "insecureSkipVerify" $insecureSkipVerify) }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.bearerTokenFile.enabled }} +{{- $_ := set . "bearerTokenFile" $.Values.clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.bearerTokenSecret.enabled }} +{{- $_ := set . "bearerTokenSecret" $serviceAccountTokenName }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.authorization.enabled }} +{{- if (hasKey . "authorization") }} +{{- $_ := set (get . "authorization") "type" $.Values.clients.https.authenticationMethod.authorization.type }} +{{- $_ := set (get . "authorization") "credentials" (dict "name" $serviceAccountTokenName "key" $.Values.clients.https.authenticationMethod.authorization.credentials.key "optional" $.Values.clients.https.authenticationMethod.authorization.credentials.optional) }} +{{- else }} +{{- $_ := set . "authorization" (dict "type" $.Values.clients.https.authenticationMethod.authorization.type) }} +{{- $_ := set . "authorization" (dict "credentials" (dict "name" $serviceAccountTokenName "key" $.Values.clients.https.authenticationMethod.authorization.credentials.key "optional" $.Values.clients.https.authenticationMethod.authorization.credentials.optional)) }} +{{- end }} +{{- end }} +{{- end }} +{{- toYaml $endpoints }} +{{- end -}} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmScheduler/templates/pushprox-clients-rbac.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmScheduler/templates/pushprox-clients-rbac.yaml new file mode 100644 index 00000000000..a8e27c37356 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmScheduler/templates/pushprox-clients-rbac.yaml @@ -0,0 +1,97 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.clients }}{{- if .Values.clients.enabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ template "pushProxy.client.name" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +rules: +{{- if .Values.global.cattle.psp.enabled }} +- apiGroups: ['policy'] + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - {{ template "pushProxy.client.name" . }} +{{- end }} +{{- if and .Values.clients.https.enabled .Values.clients.https.useServiceAccountCredentials }} +- nonResourceURLs: ["/metrics"] + verbs: ["get"] +{{- if .Values.clients.rbac.additionalRules }} +{{ toYaml .Values.clients.rbac.additionalRules }} +{{- end }} +{{- end }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ template "pushProxy.client.name" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "pushProxy.client.name" . }} +subjects: + - kind: ServiceAccount + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +--- +{{- if .Values.clients.https.useServiceAccountCredentials }} +apiVersion: v1 +kind: Secret +type: kubernetes.io/service-account-token +metadata: + name: {{ template "pushProxy.client.serviceAccountTokenName" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} + annotations: + kubernetes.io/service-account.name: {{ template "pushProxy.client.name" . }} +{{- end }} +--- +{{- if .Values.global.cattle.psp.enabled }} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +spec: + privileged: false + hostNetwork: true + hostIPC: false + hostPID: false + runAsUser: + rule: 'RunAsAny' + seLinux: + rule: 'RunAsAny' +{{- if and .Values.clients.https.enabled .Values.clients.https.certDir .Values.global.seLinux.enabled .Values.clients.https.seLinuxOptions }} + seLinuxOptions: {{ .Values.clients.https.seLinuxOptions | toYaml | nindent 6 }} +{{- end }} + supplementalGroups: + rule: 'MustRunAs' + ranges: + - min: 0 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + - min: 0 + max: 65535 + readOnlyRootFilesystem: false + volumes: + - 'secret' +{{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + - 'emptyDir' + - 'hostPath' + allowedHostPaths: + - pathPrefix: {{ required "Need access to volume on host with the SSL cert files to use HTTPs" .Values.clients.https.certDir }} + readOnly: true +{{- end }} +{{- end }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmScheduler/templates/pushprox-clients.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmScheduler/templates/pushprox-clients.yaml new file mode 100644 index 00000000000..e8fcfb38836 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmScheduler/templates/pushprox-clients.yaml @@ -0,0 +1,157 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.clients }}{{- if .Values.clients.enabled }} +apiVersion: apps/v1 +{{- if .Values.clients.deployment.enabled }} +kind: Deployment +{{- else }} +kind: DaemonSet +{{- end }} +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} + pushprox-exporter: "client" +spec: + {{- if .Values.clients.deployment.enabled }} + replicas: {{ .Values.clients.deployment.replicas }} + {{- end }} + selector: + matchLabels: {{ include "pushProxy.client.labels" . | nindent 6 }} + template: + metadata: + labels: {{ include "pushProxy.client.labels" . | nindent 8 }} + spec: + {{- if .Values.clients.affinity }} + affinity: {{ toYaml .Values.clients.affinity | nindent 8 }} + {{- end }} + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} +{{- if .Values.clients.nodeSelector }} +{{ toYaml .Values.clients.nodeSelector | indent 8 }} +{{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} +{{- if .Values.clients.tolerations }} +{{ toYaml .Values.clients.tolerations | indent 8 }} +{{- end }} + hostNetwork: true + dnsPolicy: ClusterFirstWithHostNet + serviceAccountName: {{ template "pushProxy.client.name" . }} + {{- if .Values.global.imagePullSecretName }} + imagePullSecrets: + - name: {{ .Values.global.imagePullSecretName }} + {{- end }} + containers: + - name: pushprox-client + image: {{ template "system_default_registry" . }}{{ .Values.clients.image.repository }}:{{ .Values.clients.image.tag }} + command: + {{- range .Values.clients.command }} + - {{ . | quote }} + {{- end }} + args: + - --fqdn=$(HOST_IP) + - --proxy-url=$(PROXY_URL) + {{- if .Values.clients.metrics.enabled }} + - --metrics-addr=$(PORT) + {{- end }} + - --allow-port={{ required "Need .Values.metricsPort to configure client to be allowed to scrape metrics at port" .Values.metricsPort}} + {{- if .Values.clients.useLocalhost }} + - --use-localhost + {{- end }} + {{- if .Values.clients.https.enabled }} + {{- if .Values.clients.https.insecureSkipVerify }} + - --insecure-skip-verify + {{- end }} + {{- if .Values.clients.https.useServiceAccountCredentials }} + - --token-path=/var/run/secrets/kubernetes.io/serviceaccount/token + {{- end }} + {{- if .Values.clients.https.certDir }} + - --tls.cert=/etc/ssl/push-proxy/push-proxy.pem + - --tls.key=/etc/ssl/push-proxy/push-proxy-key.pem + - --tls.cacert=/etc/ssl/push-proxy/push-proxy-ca-cert.pem + {{- end }} + {{- end }} + env: + - name: HOST_IP + valueFrom: + fieldRef: + fieldPath: status.hostIP + {{- if .Values.clients.metrics.enabled }} + - name: PORT + value: :{{ .Values.clients.port }} + {{- end }} + - name: PROXY_URL + value: {{ template "pushProxy.proxyUrl" . }} + securityContext: + runAsNonRoot: true + runAsUser: 1000 + {{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + volumeMounts: + - name: metrics-cert-dir + mountPath: /etc/ssl/push-proxy + {{- end }} + {{- if .Values.clients.resources }} + resources: {{ toYaml .Values.clients.resources | nindent 10 }} + {{- end }} + {{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + initContainers: + - name: copy-certs + image: {{ template "system_default_registry" . }}{{ .Values.clients.copyCertsImage.repository }}:{{ .Values.clients.copyCertsImage.tag }} + command: + - sh + - -c + - | + echo "Searching for files to copy within the source volume" + echo "cert: ${CERT_FILE_NAME}" + echo "key: ${KEY_FILE_NAME}" + echo "cacert: ${CACERT_FILE_NAME}" + + CERT_FILE_SOURCE=$(find /etc/source/ -type f -name "${CERT_FILE_NAME}" | sort -r | head -n 1) + KEY_FILE_SOURCE=$(find /etc/source/ -type f -name "${KEY_FILE_NAME}" | sort -r | head -n 1) + CACERT_FILE_SOURCE=$(find /etc/source/ -type f -name "${CACERT_FILE_NAME}" | sort -r | head -n 1) + + test -z ${CERT_FILE_SOURCE} && echo "Failed to find cert file" && exit 1 + test -z ${KEY_FILE_SOURCE} && echo "Failed to find key file" && exit 1 + test -z ${CACERT_FILE_SOURCE} && echo "Failed to find cacert file" && exit 1 + + echo "Copying cert file from $CERT_FILE_SOURCE to $CERT_FILE_TARGET" + cp $CERT_FILE_SOURCE $CERT_FILE_TARGET || exit 1 + chmod 444 $CERT_FILE_TARGET || exit 1 + + echo "Copying key file from $KEY_FILE_SOURCE to $KEY_FILE_TARGET" + cp $KEY_FILE_SOURCE $KEY_FILE_TARGET || exit 1 + chmod 444 $KEY_FILE_TARGET || exit 1 + + echo "Copying cacert file from $CACERT_FILE_SOURCE to $CACERT_FILE_TARGET" + cp $CACERT_FILE_SOURCE $CACERT_FILE_TARGET || exit 1 + chmod 444 $CACERT_FILE_TARGET || exit 1 + env: + - name: CERT_FILE_NAME + value: {{ required "Need a TLS cert file for scraping metrics endpoint over HTTPs" .Values.clients.https.certFile }} + - name: KEY_FILE_NAME + value: {{ required "Need a TLS key file for scraping metrics endpoint over HTTPs" .Values.clients.https.keyFile }} + - name: CACERT_FILE_NAME + value: {{ required "Need a TLS CA cert file for scraping metrics endpoint over HTTPs" .Values.clients.https.caCertFile }} + - name: CERT_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy.pem + - name: KEY_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy-key.pem + - name: CACERT_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy-ca-cert.pem + securityContext: + runAsNonRoot: false +{{- if and .Values.global.seLinux.enabled .Values.clients.https.seLinuxOptions }} + seLinuxOptions: {{ .Values.clients.https.seLinuxOptions | toYaml | nindent 12 }} +{{- end }} + volumeMounts: + - name: metrics-cert-dir-source + mountPath: /etc/source + readOnly: true + - name: metrics-cert-dir + mountPath: /etc/ssl/push-proxy + volumes: + - name: metrics-cert-dir-source + hostPath: + path: {{ required "Need access to volume on host with the SSL cert files to use HTTPs" .Values.clients.https.certDir }} + - name: metrics-cert-dir + emptyDir: {} + {{- end }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmScheduler/templates/pushprox-proxy-rbac.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmScheduler/templates/pushprox-proxy-rbac.yaml new file mode 100644 index 00000000000..eefe609058c --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmScheduler/templates/pushprox-proxy-rbac.yaml @@ -0,0 +1,68 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if and .Values.proxy }}{{ if .Values.proxy.enabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ template "pushProxy.proxy.name" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +rules: +{{- if .Values.global.cattle.psp.enabled }} +- apiGroups: ['policy'] + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - {{ template "pushProxy.proxy.name" . }} +{{- end }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ template "pushProxy.proxy.name" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "pushProxy.proxy.name" . }} +subjects: + - kind: ServiceAccount + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +--- +{{- if .Values.global.cattle.psp.enabled }} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +spec: + privileged: false + hostNetwork: false + hostIPC: false + hostPID: false + runAsUser: + rule: 'MustRunAsNonRoot' + seLinux: + rule: 'RunAsAny' + supplementalGroups: + rule: 'MustRunAs' + ranges: + - min: 1 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + - min: 1 + max: 65535 + readOnlyRootFilesystem: false + volumes: + - 'secret' +{{- end }}{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmScheduler/templates/pushprox-proxy.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmScheduler/templates/pushprox-proxy.yaml new file mode 100644 index 00000000000..723bbd6c00b --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmScheduler/templates/pushprox-proxy.yaml @@ -0,0 +1,57 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if and .Values.proxy }}{{ if .Values.proxy.enabled }} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} + pushprox-exporter: "proxy" +spec: + selector: + matchLabels: {{ include "pushProxy.proxy.labels" . | nindent 6 }} + template: + metadata: + labels: {{ include "pushProxy.proxy.labels" . | nindent 8 }} + spec: + securityContext: + runAsNonRoot: true + runAsUser: 1000 + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} +{{- if .Values.proxy.nodeSelector }} +{{ toYaml .Values.proxy.nodeSelector | indent 8 }} +{{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} +{{- if .Values.proxy.tolerations }} +{{ toYaml .Values.proxy.tolerations | indent 8 }} +{{- end }} + serviceAccountName: {{ template "pushProxy.proxy.name" . }} + {{- if .Values.global.imagePullSecretName }} + imagePullSecrets: + - name: {{ .Values.global.imagePullSecretName }} + {{- end }} + containers: + - name: pushprox-proxy + image: {{ template "system_default_registry" . }}{{ .Values.proxy.image.repository }}:{{ .Values.proxy.image.tag }} + command: + {{- range .Values.proxy.command }} + - {{ . | quote }} + {{- end }} + {{- if .Values.proxy.resources }} + resources: {{ toYaml .Values.proxy.resources | nindent 10 }} + {{- end }} +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +spec: + ports: + - name: pp-proxy + port: {{ required "Need .Values.proxy.port to configure proxy" .Values.proxy.port }} + protocol: TCP + targetPort: {{ .Values.proxy.port }} + selector: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmScheduler/templates/pushprox-servicemonitor.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmScheduler/templates/pushprox-servicemonitor.yaml new file mode 100644 index 00000000000..67eb2216b5a --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmScheduler/templates/pushprox-servicemonitor.yaml @@ -0,0 +1,45 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.serviceMonitor }}{{- if .Values.serviceMonitor.enabled }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ template "pushprox.serviceMonitor.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.serviceMonitor.labels" . | nindent 4 }} +spec: + endpoints: {{include "pushProxy.serviceMonitor.endpoints" . | nindent 4 }} + jobLabel: component + podTargetLabels: + - component + - pushprox-exporter + namespaceSelector: + matchNames: + - {{ template "pushprox.namespace" . }} + selector: + matchLabels: {{ include "pushProxy.client.labels" . | nindent 6 }} +--- +{{- $selector := "" }} +{{- if not (kindIs "invalid" .Values.service) }} +{{- if not (kindIs "invalid" .Values.service.selector) }} +{{ if .Values.service.selector }} +{{- if .Values.clients.enabled }} +{{- required (printf "Cannot override .Values.service.selector=%s when .Values.clients.enabled=true" (toJson .Values.service.selector)) "" }} +{{- end }} +{{- $selector = (toYaml .Values.service.selector) }} +{{- end }} +{{- end }} +{{- end }} +apiVersion: v1 +kind: Service +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +spec: + ports: + - name: metrics + port: {{ required "Need .Values.metricsPort to configure client to listen to metrics at port" .Values.metricsPort}} + protocol: TCP + targetPort: {{ .Values.metricsPort }} + selector: {{ default (include "pushProxy.client.labels" .) $selector | nindent 4 }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmScheduler/templates/validate-install-crd.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmScheduler/templates/validate-install-crd.yaml new file mode 100644 index 00000000000..16abc2fa83a --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmScheduler/templates/validate-install-crd.yaml @@ -0,0 +1,14 @@ +#{{- if gt (len (lookup "rbac.authorization.k8s.io/v1" "ClusterRole" "" "")) 0 -}} +# {{- $found := dict -}} +# {{- set $found "monitoring.coreos.com/v1/ServiceMonitor" false -}} +# {{- range .Capabilities.APIVersions -}} +# {{- if hasKey $found (toString .) -}} +# {{- set $found (toString .) true -}} +# {{- end -}} +# {{- end -}} +# {{- range $_, $exists := $found -}} +# {{- if (eq $exists false) -}} +# {{- required "Required CRDs are missing. Please install Prometheus Operator CRDs before installing this chart." "" -}} +# {{- end -}} +# {{- end -}} +#{{- end -}} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmScheduler/templates/validate-psp-install.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmScheduler/templates/validate-psp-install.yaml new file mode 100644 index 00000000000..a30c59d3b7c --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmScheduler/templates/validate-psp-install.yaml @@ -0,0 +1,7 @@ +#{{- if gt (len (lookup "rbac.authorization.k8s.io/v1" "ClusterRole" "" "")) 0 -}} +#{{- if .Values.global.cattle.psp.enabled }} +#{{- if not (.Capabilities.APIVersions.Has "policy/v1beta1/PodSecurityPolicy") }} +#{{- fail "The target cluster does not have the PodSecurityPolicy API resource. Please disable PSPs in this chart before proceeding." -}} +#{{- end }} +#{{- end }} +#{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmScheduler/values.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmScheduler/values.yaml new file mode 100644 index 00000000000..7f38498a1c5 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/kubeAdmScheduler/values.yaml @@ -0,0 +1,166 @@ +# Default values for rancher-pushprox. +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. + +# Default image containing both the proxy and the client was generated from the following Dockerfile +# https://github.com/prometheus-community/PushProx/blob/eeadbe766641699129920ccfaaaa30a85c67fe81/Dockerfile#L1-L15 + +# Configuration + +global: + cattle: + psp: + enabled: false + systemDefaultRegistry: "" + seLinux: + enabled: false + +# A list of Semver constraint strings (defined by https://github.com/Masterminds/semver) and values.yaml overrides. +# +# For each key in kubeVersionOverrides, this chart will check to see if the current Kubernetes cluster's version matches +# any of the semver constraints provided as keys on the map. +# +# On seeing a match, the default value for each values.yaml field overridden will be updated with the new value. +# +# If multiple matches are encountered (due to overlapping semver ranges), the matches will be applied in order. +# +# Notes: +# - On running a helm template, Helm generally assumes the kubeVersion is v1.20.0 +# - On running a helm install --dry-run, the correct kubeVersion should be chosen. +kubeVersionOverrides: [] +# - constraint: "< 1.21" +# values: +# metricsPort: 10252 +# clients: +# https: +# enabled: false +# insecureSkipVerify: false +# useServiceAccountCredentials: false + +namespaceOverride: "" + +# The component that is being monitored (i.e. etcd) +component: "component" + +# The port containing the metrics that need to be scraped +metricsPort: 2739 + +# Configure ServiceMonitor that monitors metrics from the metricsPort endpoint +serviceMonitor: + enabled: true + # A list of endpoints that will be added to the ServiceMonitor based on the Endpoint spec + # Source: https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#endpoint + # By default, proxyUrl and params._scheme will be overridden based on other values + endpoints: + - port: metrics + +# Configure Service that grabs scrape targets +service: + # The selector that is used to populate the Service's Endpoints object. + # The chart will error out on rendering templating if .Values.clients.enabled is set alongside this field, + # since it is expected that this service should point to the PushProx Clients Daemonset / Deployment + selector: {} + +clients: + enabled: true + # The port which the PushProx client will post PushProx metrics to + port: 9369 + # If unset, this will default to the URL for the proxy service: http://pushprox-{{component}}-proxy.{{namepsace}}.svc.cluster.local:{{proxy.port}} + # Should be modified if the clients are being deployed outside the cluster where the proxy rests, otherwise leave it null + proxyUrl: "" + # If set to true, the client will forward any requests from the host IP to 127.0.0.1 + # It will only allow proxy requests to the metricsPort specified + useLocalhost: false + # Configuration for accessing metrics via HTTPS + https: + # Does the client require https to access the metrics? + enabled: false + # Does the client require requests be sent to http or https? + forceHTTPSScheme: false + # If set to true, the client will create a service account with adequate permissions and set a flag + # on the client to use the service account token provided by it to make authorized scrape requests + useServiceAccountCredentials: false + # Configuration for authentication to metrics via https endpoint + authenticationMethod: + # Reads token from defined file in container + # This function is deprecated in the prometheus operator api and may be removed in a future version + bearerTokenFile: + enabled: false + bearerTokenFilePath: "/var/run/secrets/kubernetes.io/serviceaccount/token" + # Reads token from defined secret in namespace + # This function is deprecated in the prometheus operator api and may be removed in a future version + bearerTokenSecret: + enabled: false + # Reads token from defined secret in namespace + authorization: + enabled: false + type: "bearer" + credentials: + key: "token" + optional: false + # If set to true, the client will disable SSL security checks + insecureSkipVerify: false + # Directory on host where necessary TLS cert and key to scrape metrics can be found + certDir: "" + # Filenames for files located in .Values.clients.https.certDir that correspond to TLS settings + certFile: "" + keyFile: "" + caCertFile: "" + # seLinuxOptions to be passed into the container that copies certs. Should define a container with permissions to read the files in the certDir provided on the host. + # Required and only used if `clients.https.enabled` is set and `clients.https.certDir` is provided. + seLinuxOptions: {} + + metrics: + # Whether the client should publish PushProx client-specific metrics to .Values.clients.port + enabled: false + + rbac: + # Additional permissions to provide to the ServiceAccount bound to the client + # This can be used to provide additional permissions for the client to scrape metrics from the k8s API + # Only enabled if clients.https.enabled and clients.https.useServiceAccountCredentials are true + additionalRules: [] + + # Resource limits + resources: {} + + # Options to select all nodes to deploy client DaemonSet on + nodeSelector: {} + tolerations: [] + affinity: {} + + image: + repository: rancher/pushprox-client + tag: v0.1.4-rancher2-client + command: ["pushprox-client"] + + copyCertsImage: + repository: rancher/mirrored-library-busybox + tag: 1.31.1 + + # The default intention of rancher-pushprox clients is to scrape hostNetwork metrics across all nodes. + # This can be used to scrape internal Kubernetes components or DaemonSets of hostNetwork Pods in + # situations where a cloud provider firewall prevents Pod-To-Host communication but not Pod-To-Pod. + # However, if the underlying hostNetwork Pod that is being scraped is managed by a Deployment, + # this advanced option enables users to deploy the client as a Deployment instead of a DaemonSet. + # If a user deploys this feature and the underlying Deployment's number of replicas changes, the user will + # be responsible for upgrading this chart accordingly to the right number of replicas. + deployment: + enabled: false + replicas: 0 + +proxy: + enabled: true + # The port through which PushProx clients will communicate to the proxy + port: 8080 + + # Resource limits + resources: {} + + # Options to select a node to run a single proxy deployment on + nodeSelector: {} + tolerations: [] + + image: + repository: rancher/pushprox-proxy + tag: v0.1.4-rancher2-proxy + command: ["pushprox-proxy"] diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-adapter/.helmignore b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-adapter/.helmignore new file mode 100644 index 00000000000..f0c13194444 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-adapter/.helmignore @@ -0,0 +1,21 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*~ +# Various IDEs +.project +.idea/ +*.tmproj diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-adapter/Chart.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-adapter/Chart.yaml new file mode 100644 index 00000000000..d067725a175 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-adapter/Chart.yaml @@ -0,0 +1,28 @@ +annotations: + catalog.cattle.io/hidden: "true" + catalog.cattle.io/kube-version: '>= 1.26.0-0 < 1.31.0-0' + catalog.cattle.io/os: linux + catalog.rancher.io/certified: rancher + catalog.rancher.io/namespace: cattle-monitoring-system + catalog.rancher.io/release-name: rancher-prometheus-adapter +apiVersion: v1 +appVersion: v0.10.0 +description: A Helm chart for k8s prometheus adapter +home: https://github.com/kubernetes-sigs/prometheus-adapter +keywords: +- hpa +- metrics +- prometheus +- adapter +kubeVersion: '>=1.26.0-0' +maintainers: +- email: mattias.gees@jetstack.io + name: mattiasgees +- name: steven-sheehy +- email: hfernandez@mesosphere.com + name: hectorj2f +name: prometheus-adapter +sources: +- https://github.com/kubernetes/charts +- https://github.com/kubernetes-sigs/prometheus-adapter +version: 4.2.0 diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-adapter/README.md b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-adapter/README.md new file mode 100644 index 00000000000..d77bb0c9202 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-adapter/README.md @@ -0,0 +1,160 @@ +# Prometheus Adapter + +Installs the [Prometheus Adapter](https://github.com/kubernetes-sigs/prometheus-adapter) for the Custom Metrics API. Custom metrics are used in Kubernetes by [Horizontal Pod Autoscalers](https://kubernetes.io/docs/tasks/run-application/horizontal-pod-autoscale/) to scale workloads based upon your own metric pulled from an external metrics provider like Prometheus. This chart complements the [metrics-server](https://github.com/helm/charts/tree/master/stable/metrics-server) chart that provides resource only metrics. + +## Prerequisites + +Kubernetes 1.14+ + +## Get Helm Repositories Info + +```console +helm repo add prometheus-community https://prometheus-community.github.io/helm-charts +helm repo update +``` + +_See [`helm repo`](https://helm.sh/docs/helm/helm_repo/) for command documentation._ + +## Install Helm Chart + +```console +helm install [RELEASE_NAME] prometheus-community/prometheus-adapter +``` + +_See [configuration](#configuration) below._ + +_See [helm install](https://helm.sh/docs/helm/helm_install/) for command documentation._ + +## Uninstall Helm Chart + +```console +helm uninstall [RELEASE_NAME] +``` + +This removes all the Kubernetes components associated with the chart and deletes the release. + +_See [helm uninstall](https://helm.sh/docs/helm/helm_uninstall/) for command documentation._ + +## Upgrading Helm Chart + +```console +helm upgrade [RELEASE_NAME] [CHART] --install +``` + +_See [helm upgrade](https://helm.sh/docs/helm/helm_upgrade/) for command documentation._ + +### To 4.2.0 + +Readiness and liveness probes are now fully configurable through values `readinessProbe` and `livenessProbe`. The previous values have been kept as defaults. + +### To 4.0.0 + +Previously, security context of the container was set directly in the deployment template. This release makes it configurable through the new configuration variable `securityContext` whilst keeping the previously set values as defaults. Furthermore, previous variable `runAsUser` is now set in `securityContext` and is not used any longer. Please, use `securityContext.runAsUser` instead. In the same security context, `seccompProfile` has been enabled and set to type `RuntimeDefault`. + +### To 3.0.0 + +Due to a change in deployment labels, the upgrade requires `helm upgrade --force` in order to re-create the deployment. + +## Configuration + +See [Customizing the Chart Before Installing](https://helm.sh/docs/intro/using_helm/#customizing-the-chart-before-installing). To see all configurable options with detailed comments, visit the chart's [values.yaml](./values.yaml), or run these configuration commands: + +```console +helm show values prometheus-community/prometheus-adapter +``` + +### Prometheus Service Endpoint + +To use the chart, ensure the `prometheus.url` and `prometheus.port` are configured with the correct Prometheus service endpoint. If Prometheus is exposed under HTTPS the host's CA Bundle must be exposed to the container using `extraVolumes` and `extraVolumeMounts`. + +### Adapter Rules + +Additionally, the chart comes with a set of default rules out of the box but they may pull in too many metrics or not map them correctly for your needs. Therefore, it is recommended to populate `rules.custom` with a list of rules (see the [config document](https://github.com/kubernetes-sigs/prometheus-adapter/blob/master/docs/config.md) for the proper format). + +### Horizontal Pod Autoscaler Metrics + +Finally, to configure your Horizontal Pod Autoscaler to use the custom metric, see the custom metrics section of the [HPA walkthrough](https://kubernetes.io/docs/tasks/run-application/horizontal-pod-autoscale-walkthrough/#autoscaling-on-multiple-metrics-and-custom-metrics). + +The Prometheus Adapter can serve three different [metrics APIs](https://kubernetes.io/docs/tasks/run-application/horizontal-pod-autoscale/#support-for-metrics-apis): + +### Custom Metrics + +Enabling this option will cause custom metrics to be served at `/apis/custom.metrics.k8s.io/v1beta1`. Enabled by default when `rules.default` is true, but can be customized by populating `rules.custom`: + +```yaml +rules: + custom: + - seriesQuery: '{__name__=~"^some_metric_count$"}' + resources: + template: <<.Resource>> + name: + matches: "" + as: "my_custom_metric" + metricsQuery: sum(<<.Series>>{<<.LabelMatchers>>}) by (<<.GroupBy>>) +``` + +### External Metrics + +Enabling this option will cause external metrics to be served at `/apis/external.metrics.k8s.io/v1beta1`. Can be enabled by populating `rules.external`: + +```yaml +rules: + external: + - seriesQuery: '{__name__=~"^some_metric_count$"}' + resources: + template: <<.Resource>> + name: + matches: "" + as: "my_external_metric" + metricsQuery: sum(<<.Series>>{<<.LabelMatchers>>}) by (<<.GroupBy>>) +``` + +### Resource Metrics + +Enabling this option will cause resource metrics to be served at `/apis/metrics.k8s.io/v1beta1`. Resource metrics will allow pod CPU and Memory metrics to be used in [Horizontal Pod Autoscalers](https://kubernetes.io/docs/tasks/run-application/horizontal-pod-autoscale/) as well as the `kubectl top` command. Can be enabled by populating `rules.resource`: + +```yaml +rules: + resource: + cpu: + containerQuery: | + sum by (<<.GroupBy>>) ( + rate(container_cpu_usage_seconds_total{container!="",<<.LabelMatchers>>}[3m]) + ) + nodeQuery: | + sum by (<<.GroupBy>>) ( + rate(node_cpu_seconds_total{mode!="idle",mode!="iowait",mode!="steal",<<.LabelMatchers>>}[3m]) + ) + resources: + overrides: + node: + resource: node + namespace: + resource: namespace + pod: + resource: pod + containerLabel: container + memory: + containerQuery: | + sum by (<<.GroupBy>>) ( + avg_over_time(container_memory_working_set_bytes{container!="",<<.LabelMatchers>>}[3m]) + ) + nodeQuery: | + sum by (<<.GroupBy>>) ( + avg_over_time(node_memory_MemTotal_bytes{<<.LabelMatchers>>}[3m]) + - + avg_over_time(node_memory_MemAvailable_bytes{<<.LabelMatchers>>}[3m]) + ) + resources: + overrides: + node: + resource: node + namespace: + resource: namespace + pod: + resource: pod + containerLabel: container + window: 3m +``` + +**NOTE:** Setting a value for `rules.resource` will also deploy the resource metrics API service, providing the same functionality as [metrics-server](https://github.com/helm/charts/tree/master/stable/metrics-server). As such it is not possible to deploy them both in the same cluster. diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-adapter/templates/NOTES.txt b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-adapter/templates/NOTES.txt new file mode 100644 index 00000000000..b7b9b99322f --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-adapter/templates/NOTES.txt @@ -0,0 +1,9 @@ +{{ template "k8s-prometheus-adapter.fullname" . }} has been deployed. +In a few minutes you should be able to list metrics using the following command(s): +{{ if .Values.rules.resource }} + kubectl get --raw /apis/metrics.k8s.io/v1beta1 +{{- end }} + kubectl get --raw /apis/custom.metrics.k8s.io/v1beta1 +{{ if .Values.rules.external }} + kubectl get --raw /apis/external.metrics.k8s.io/v1beta1 +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-adapter/templates/_helpers.tpl b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-adapter/templates/_helpers.tpl new file mode 100644 index 00000000000..edbb829b2b7 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-adapter/templates/_helpers.tpl @@ -0,0 +1,113 @@ +# Rancher +{{- define "system_default_registry" -}} +{{- if .Values.global.cattle.systemDefaultRegistry -}} +{{- printf "%s/" .Values.global.cattle.systemDefaultRegistry -}} +{{- end -}} +{{- end -}} + +# Windows Support + +{{/* +Windows cluster will add default taint for linux nodes, +add below linux tolerations to workloads could be scheduled to those linux nodes +*/}} + +{{- define "linux-node-tolerations" -}} +- key: "cattle.io/os" + value: "linux" + effect: "NoSchedule" + operator: "Equal" +{{- end -}} + +{{- define "linux-node-selector" -}} +{{- if semverCompare "<1.14-0" .Capabilities.KubeVersion.GitVersion -}} +beta.kubernetes.io/os: linux +{{- else -}} +kubernetes.io/os: linux +{{- end -}} +{{- end -}} + +{{/* vim: set filetype=mustache: */}} +{{/* +Expand the name of the chart. +*/}} +{{- define "k8s-prometheus-adapter.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "k8s-prometheus-adapter.fullname" -}} +{{- if .Values.fullnameOverride -}} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- $name := default .Chart.Name .Values.nameOverride -}} +{{- if contains $name .Release.Name -}} +{{- .Release.Name | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}} +{{- end -}} +{{- end -}} +{{- end -}} + +{{/* +Allow the release namespace to be overridden for multi-namespace deployments in combined charts +*/}} +{{- define "k8s-prometheus-adapter.namespace" -}} +{{- default .Release.Namespace .Values.namespaceOverride -}} +{{- end -}} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "k8s-prometheus-adapter.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{/* +Generate basic labels +*/}} +{{- define "k8s-prometheus-adapter.labels" }} +helm.sh/chart: {{ include "k8s-prometheus-adapter.chart" . }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +app.kubernetes.io/component: metrics +app.kubernetes.io/part-of: {{ template "k8s-prometheus-adapter.name" . }} +{{- include "k8s-prometheus-adapter.selectorLabels" . }} +{{- if .Chart.AppVersion }} +app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} +{{- end }} +{{- if .Values.customLabels }} +{{ toYaml .Values.customLabels }} +{{- end }} +{{- end }} + +{{/* +Selector labels +*/}} +{{- define "k8s-prometheus-adapter.selectorLabels" }} +app.kubernetes.io/name: {{ include "k8s-prometheus-adapter.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end }} + +{{/* +Create the name of the service account to use +*/}} +{{- define "k8s-prometheus-adapter.serviceAccountName" -}} +{{- if .Values.serviceAccount.create -}} + {{ default (include "k8s-prometheus-adapter.fullname" .) .Values.serviceAccount.name }} +{{- else -}} + {{ default "default" .Values.serviceAccount.name }} +{{- end -}} +{{- end -}} + +{{/* Get Policy API Version */}} +{{- define "k8s-prometheus-adapter.pdb.apiVersion" -}} +{{- if and (.Capabilities.APIVersions.Has "policy/v1") (semverCompare ">= 1.21-0" .Capabilities.KubeVersion.Version) -}} + {{- print "policy/v1" -}} +{{- else -}} + {{- print "policy/v1beta1" -}} +{{- end -}} +{{- end -}} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-adapter/templates/certmanager.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-adapter/templates/certmanager.yaml new file mode 100644 index 00000000000..4e32c964c6f --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-adapter/templates/certmanager.yaml @@ -0,0 +1,76 @@ +{{- if .Values.certManager.enabled -}} +--- +# Create a selfsigned Issuer, in order to create a root CA certificate for +# signing webhook serving certificates +apiVersion: cert-manager.io/v1 +kind: Issuer +metadata: + name: {{ template "k8s-prometheus-adapter.fullname" . }}-self-signed-issuer + namespace: {{ include "k8s-prometheus-adapter.namespace" . }} + {{- if .Values.customAnnotations }} + annotations: + {{- toYaml .Values.customAnnotations | nindent 4 }} + {{- end }} + labels: + {{- include "k8s-prometheus-adapter.labels" . | indent 4 }} +spec: + selfSigned: {} +--- +# Generate a CA Certificate used to sign certificates for the webhook +apiVersion: cert-manager.io/v1 +kind: Certificate +metadata: + name: {{ template "k8s-prometheus-adapter.fullname" . }}-root-cert + namespace: {{ include "k8s-prometheus-adapter.namespace" . }} + {{- if .Values.customAnnotations }} + annotations: + {{- toYaml .Values.customAnnotations | nindent 4 }} + {{- end }} + labels: + {{- include "k8s-prometheus-adapter.labels" . | indent 4 }} +spec: + secretName: {{ template "k8s-prometheus-adapter.fullname" . }}-root-cert + duration: {{ .Values.certManager.caCertDuration }} + issuerRef: + name: {{ template "k8s-prometheus-adapter.fullname" . }}-self-signed-issuer + commonName: "ca.webhook.prometheus-adapter" + isCA: true +--- +# Create an Issuer that uses the above generated CA certificate to issue certs +apiVersion: cert-manager.io/v1 +kind: Issuer +metadata: + name: {{ template "k8s-prometheus-adapter.fullname" . }}-root-issuer + namespace: {{ include "k8s-prometheus-adapter.namespace" . }} + {{- if .Values.customAnnotations }} + annotations: + {{- toYaml .Values.customAnnotations | nindent 4 }} + {{- end }} + labels: + {{- include "k8s-prometheus-adapter.labels" . | indent 4 }} +spec: + ca: + secretName: {{ template "k8s-prometheus-adapter.fullname" . }}-root-cert +--- +# Finally, generate a serving certificate for the apiservices to use +apiVersion: cert-manager.io/v1 +kind: Certificate +metadata: + name: {{ template "k8s-prometheus-adapter.fullname" . }}-cert + namespace: {{ include "k8s-prometheus-adapter.namespace" . }} + {{- if .Values.customAnnotations }} + annotations: + {{- toYaml .Values.customAnnotations | nindent 4 }} + {{- end }} + labels: + {{- include "k8s-prometheus-adapter.labels" . | indent 4 }} +spec: + secretName: {{ template "k8s-prometheus-adapter.fullname" . }} + duration: {{ .Values.certManager.certDuration }} + issuerRef: + name: {{ template "k8s-prometheus-adapter.fullname" . }}-root-issuer + dnsNames: + - {{ template "k8s-prometheus-adapter.fullname" . }} + - {{ template "k8s-prometheus-adapter.fullname" . }}.{{ include "k8s-prometheus-adapter.namespace" . }} + - {{ template "k8s-prometheus-adapter.fullname" . }}.{{ include "k8s-prometheus-adapter.namespace" . }}.svc +{{- end -}} \ No newline at end of file diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-adapter/templates/cluster-role-binding-auth-delegator.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-adapter/templates/cluster-role-binding-auth-delegator.yaml new file mode 100644 index 00000000000..6701e6ba08d --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-adapter/templates/cluster-role-binding-auth-delegator.yaml @@ -0,0 +1,20 @@ +{{- if .Values.rbac.create -}} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + {{- if .Values.customAnnotations }} + annotations: + {{- toYaml .Values.customAnnotations | nindent 4 }} + {{- end }} + labels: + {{- include "k8s-prometheus-adapter.labels" . | indent 4 }} + name: {{ template "k8s-prometheus-adapter.name" . }}-system-auth-delegator +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: system:auth-delegator +subjects: +- kind: ServiceAccount + name: {{ template "k8s-prometheus-adapter.serviceAccountName" . }} + namespace: {{ include "k8s-prometheus-adapter.namespace" . | quote }} +{{- end -}} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-adapter/templates/cluster-role-binding-resource-reader.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-adapter/templates/cluster-role-binding-resource-reader.yaml new file mode 100644 index 00000000000..67efd2aa2fe --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-adapter/templates/cluster-role-binding-resource-reader.yaml @@ -0,0 +1,20 @@ +{{- if .Values.rbac.create -}} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + {{- if .Values.customAnnotations }} + annotations: + {{- toYaml .Values.customAnnotations | nindent 4 }} + {{- end }} + labels: + {{- include "k8s-prometheus-adapter.labels" . | indent 4 }} + name: {{ template "k8s-prometheus-adapter.name" . }}-resource-reader +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "k8s-prometheus-adapter.name" . }}-resource-reader +subjects: +- kind: ServiceAccount + name: {{ template "k8s-prometheus-adapter.serviceAccountName" . }} + namespace: {{ include "k8s-prometheus-adapter.namespace" . | quote }} +{{- end -}} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-adapter/templates/cluster-role-resource-reader.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-adapter/templates/cluster-role-resource-reader.yaml new file mode 100644 index 00000000000..2c690a03cc5 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-adapter/templates/cluster-role-resource-reader.yaml @@ -0,0 +1,24 @@ +{{- if .Values.rbac.create -}} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + {{- if .Values.customAnnotations }} + annotations: + {{- toYaml .Values.customAnnotations | nindent 4 }} + {{- end }} + labels: + {{- include "k8s-prometheus-adapter.labels" . | indent 4 }} + name: {{ template "k8s-prometheus-adapter.name" . }}-resource-reader +rules: +- apiGroups: + - "" + resources: + - namespaces + - pods + - services + - configmaps + verbs: + - get + - list + - watch +{{- end -}} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-adapter/templates/configmap.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-adapter/templates/configmap.yaml new file mode 100644 index 00000000000..17f415d9701 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-adapter/templates/configmap.yaml @@ -0,0 +1,97 @@ +{{- if not .Values.rules.existing -}} +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ template "k8s-prometheus-adapter.fullname" . }} + namespace: {{ include "k8s-prometheus-adapter.namespace" . }} + {{- if .Values.customAnnotations }} + annotations: + {{- toYaml .Values.customAnnotations | nindent 4 }} + {{- end }} + labels: + {{- include "k8s-prometheus-adapter.labels" . | indent 4 }} +data: + config.yaml: | +{{- if or .Values.rules.default .Values.rules.custom }} + rules: +{{- if .Values.rules.default }} + - seriesQuery: '{__name__=~"^container_.*",container!="POD",namespace!="",pod!=""}' + seriesFilters: [] + resources: + overrides: + namespace: + resource: namespace + pod: + resource: pod + name: + matches: ^container_(.*)_seconds_total$ + as: "" + metricsQuery: sum(rate(<<.Series>>{<<.LabelMatchers>>,container!="POD"}[5m])) + by (<<.GroupBy>>) + - seriesQuery: '{__name__=~"^container_.*",container!="POD",namespace!="",pod!=""}' + seriesFilters: + - isNot: ^container_.*_seconds_total$ + resources: + overrides: + namespace: + resource: namespace + pod: + resource: pod + name: + matches: ^container_(.*)_total$ + as: "" + metricsQuery: sum(rate(<<.Series>>{<<.LabelMatchers>>,container!="POD"}[5m])) + by (<<.GroupBy>>) + - seriesQuery: '{__name__=~"^container_.*",container!="POD",namespace!="",pod!=""}' + seriesFilters: + - isNot: ^container_.*_total$ + resources: + overrides: + namespace: + resource: namespace + pod: + resource: pod + name: + matches: ^container_(.*)$ + as: "" + metricsQuery: sum(<<.Series>>{<<.LabelMatchers>>,container!="POD"}) by (<<.GroupBy>>) + - seriesQuery: '{namespace!="",__name__!~"^container_.*"}' + seriesFilters: + - isNot: .*_total$ + resources: + template: <<.Resource>> + name: + matches: "" + as: "" + metricsQuery: sum(<<.Series>>{<<.LabelMatchers>>}) by (<<.GroupBy>>) + - seriesQuery: '{namespace!="",__name__!~"^container_.*"}' + seriesFilters: + - isNot: .*_seconds_total + resources: + template: <<.Resource>> + name: + matches: ^(.*)_total$ + as: "" + metricsQuery: sum(rate(<<.Series>>{<<.LabelMatchers>>}[5m])) by (<<.GroupBy>>) + - seriesQuery: '{namespace!="",__name__!~"^container_.*"}' + seriesFilters: [] + resources: + template: <<.Resource>> + name: + matches: ^(.*)_seconds_total$ + as: "" + metricsQuery: sum(rate(<<.Series>>{<<.LabelMatchers>>}[5m])) by (<<.GroupBy>>) +{{- end -}} +{{- if .Values.rules.custom }} +{{ toYaml .Values.rules.custom | indent 4 }} +{{- end -}} +{{- end -}} +{{- if .Values.rules.external }} + externalRules: +{{ toYaml .Values.rules.external | indent 4 }} +{{- end -}} +{{- if .Values.rules.resource }} + resourceRules: +{{ toYaml .Values.rules.resource | indent 6 }} +{{- end -}} +{{- end -}} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-adapter/templates/custom-metrics-apiservice.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-adapter/templates/custom-metrics-apiservice.yaml new file mode 100644 index 00000000000..8b7b4e555ef --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-adapter/templates/custom-metrics-apiservice.yaml @@ -0,0 +1,34 @@ +{{- if or .Values.rules.default .Values.rules.custom }} +{{- if .Capabilities.APIVersions.Has "apiregistration.k8s.io/v1" }} +apiVersion: apiregistration.k8s.io/v1 +{{- else }} +apiVersion: apiregistration.k8s.io/v1beta1 +{{- end }} +kind: APIService +metadata: +{{- if or .Values.certManager.enabled .Values.customAnnotations }} + annotations: + certmanager.k8s.io/inject-ca-from: {{ printf "%s/%s-root-cert" (include "k8s-prometheus-adapter.namespace" .) (include "k8s-prometheus-adapter.fullname" .) | quote }} + cert-manager.io/inject-ca-from: {{ printf "%s/%s-root-cert" (include "k8s-prometheus-adapter.namespace" .) (include "k8s-prometheus-adapter.fullname" .) | quote }} + {{- if .Values.customAnnotations }} + {{- toYaml .Values.customAnnotations | nindent 4 }} + {{- end }} +{{- end }} + labels: + {{- include "k8s-prometheus-adapter.labels" . | indent 4 }} + name: v1beta1.custom.metrics.k8s.io +spec: + service: + name: {{ template "k8s-prometheus-adapter.fullname" . }} + namespace: {{ include "k8s-prometheus-adapter.namespace" . | quote }} + {{- if .Values.tls.enable }} + caBundle: {{ b64enc .Values.tls.ca }} + {{- end }} + group: custom.metrics.k8s.io + version: v1beta1 + {{- if not (or .Values.tls.enable .Values.certManager.enabled) }} + insecureSkipTLSVerify: true + {{- end }} + groupPriorityMinimum: 100 + versionPriority: 100 +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-adapter/templates/custom-metrics-cluster-role-binding-hpa.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-adapter/templates/custom-metrics-cluster-role-binding-hpa.yaml new file mode 100644 index 00000000000..0cc69208363 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-adapter/templates/custom-metrics-cluster-role-binding-hpa.yaml @@ -0,0 +1,24 @@ +{{- /* +This if must be aligned with custom-metrics-cluster-role.yaml +as otherwise this binding will point to not existing role. +*/ -}} +{{- if and .Values.rbac.create (or .Values.rules.default .Values.rules.custom) -}} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + {{- if .Values.customAnnotations }} + annotations: + {{- toYaml .Values.customAnnotations | nindent 4 }} + {{- end }} + labels: + {{- include "k8s-prometheus-adapter.labels" . | indent 4 }} + name: {{ template "k8s-prometheus-adapter.name" . }}-hpa-controller +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "k8s-prometheus-adapter.name" . }}-server-resources +subjects: +- kind: ServiceAccount + name: {{ template "k8s-prometheus-adapter.serviceAccountName" . }} + namespace: {{ include "k8s-prometheus-adapter.namespace" . | quote }} +{{- end -}} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-adapter/templates/custom-metrics-cluster-role.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-adapter/templates/custom-metrics-cluster-role.yaml new file mode 100644 index 00000000000..4aa15ffe991 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-adapter/templates/custom-metrics-cluster-role.yaml @@ -0,0 +1,17 @@ +{{- if and .Values.rbac.create (or .Values.rules.default .Values.rules.custom) -}} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + {{- if .Values.customAnnotations }} + annotations: + {{- toYaml .Values.customAnnotations | nindent 4 }} + {{- end }} + labels: + {{- include "k8s-prometheus-adapter.labels" . | indent 4 }} + name: {{ template "k8s-prometheus-adapter.name" . }}-server-resources +rules: +- apiGroups: + - custom.metrics.k8s.io + resources: ["*"] + verbs: ["*"] +{{- end -}} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-adapter/templates/deployment.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-adapter/templates/deployment.yaml new file mode 100644 index 00000000000..a7ea3310a02 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-adapter/templates/deployment.yaml @@ -0,0 +1,143 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + {{- if or .Values.customAnnotations .Values.deploymentAnnotations }} + annotations: + {{- with .Values.customAnnotations }} + {{- toYaml . | nindent 4 }} + {{- end }} + {{- with .Values.deploymentAnnotations }} + {{- toYaml . | nindent 4 }} + {{- end }} + {{- end }} + labels: + {{- include "k8s-prometheus-adapter.labels" . | indent 4 }} + name: {{ template "k8s-prometheus-adapter.fullname" . }} + namespace: {{ include "k8s-prometheus-adapter.namespace" . }} +spec: + replicas: {{ .Values.replicas }} + strategy: {{ toYaml .Values.strategy | nindent 4 }} + selector: + matchLabels: + {{- include "k8s-prometheus-adapter.selectorLabels" . | indent 6 }} + template: + metadata: + labels: + {{- include "k8s-prometheus-adapter.labels" . | indent 8 }} + {{- with .Values.podLabels }} + {{- toYaml . | trim | nindent 8 }} + {{- end }} + name: {{ template "k8s-prometheus-adapter.name" . }} + annotations: + checksum/config: {{ include (print $.Template.BasePath "/configmap.yaml") . | sha256sum }} + {{- with .Values.podAnnotations }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.customAnnotations }} + {{- toYaml . | nindent 8 }} + {{- end }} + spec: + serviceAccountName: {{ template "k8s-prometheus-adapter.serviceAccountName" . }} + {{- if .Values.hostNetwork.enabled }} + hostNetwork: true + {{- end }} + {{- if .Values.dnsPolicy }} + dnsPolicy: {{ .Values.dnsPolicy }} + {{- end}} + containers: + - name: {{ .Chart.Name }} + image: "{{ template "system_default_registry" . }}{{ .Values.image.repository }}:{{ .Values.image.tag }}" + imagePullPolicy: {{ .Values.image.pullPolicy }} + {{- with .Values.env }} + env: + {{- toYaml . | nindent 8 }} + {{- end }} + args: + - /adapter + - --secure-port={{ .Values.listenPort }} + {{- if or .Values.tls.enable .Values.certManager.enabled }} + - --tls-cert-file=/var/run/serving-cert/tls.crt + - --tls-private-key-file=/var/run/serving-cert/tls.key + {{- end }} + - --cert-dir=/tmp/cert + - --prometheus-url={{ tpl .Values.prometheus.url . }}{{ if .Values.prometheus.port }}:{{ .Values.prometheus.port }}{{end}}{{ .Values.prometheus.path }} + - --metrics-relist-interval={{ .Values.metricsRelistInterval }} + - --v={{ .Values.logLevel }} + - --config=/etc/adapter/config.yaml + {{- if .Values.extraArguments }} + {{- toYaml .Values.extraArguments | trim | nindent 8 }} + {{- end }} + ports: + - containerPort: {{ .Values.listenPort }} + name: https + {{- with .Values.livenessProbe }} + livenessProbe: + {{- toYaml . | nindent 10 }} + {{- end }} + {{- with .Values.readinessProbe }} + readinessProbe: + {{- toYaml . | nindent 10 }} + {{- end }} + {{- if .Values.resources }} + resources: + {{- toYaml .Values.resources | nindent 10 }} + {{- end }} + {{- with .Values.dnsConfig }} + dnsConfig: + {{ toYaml . | indent 8 }} + {{- end }} + {{- with .Values.securityContext }} + securityContext: + {{- toYaml . | nindent 10 }} + {{- end }} + volumeMounts: + {{- if .Values.extraVolumeMounts }} + {{ toYaml .Values.extraVolumeMounts | trim | nindent 8 }} + {{ end }} + - mountPath: /etc/adapter/ + name: config + readOnly: true + - mountPath: /tmp + name: tmp + {{- if or .Values.tls.enable .Values.certManager.enabled }} + - mountPath: /var/run/serving-cert + name: volume-serving-cert + readOnly: true + {{- end }} + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} +{{- if .Values.nodeSelector }} +{{- toYaml .Values.nodeSelector | nindent 8 }} +{{- end }} + affinity: + {{- toYaml .Values.affinity | nindent 8 }} + topologySpreadConstraints: + {{- toYaml .Values.topologySpreadConstraints | nindent 8 }} + priorityClassName: {{ .Values.priorityClassName }} + {{- if .Values.podSecurityContext }} + securityContext: + {{- toYaml .Values.podSecurityContext | nindent 8 }} + {{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} +{{- if .Values.tolerations }} +{{- toYaml .Values.tolerations | nindent 8 }} +{{- end }} + {{- if .Values.image.pullSecrets }} + imagePullSecrets: + {{- range .Values.image.pullSecrets }} + - name: {{ . }} + {{- end }} + {{- end }} + volumes: + {{- if .Values.extraVolumes }} + {{ toYaml .Values.extraVolumes | trim | nindent 6 }} + {{ end }} + - name: config + configMap: + name: {{ .Values.rules.existing | default (include "k8s-prometheus-adapter.fullname" . ) }} + - name: tmp + emptyDir: {} + {{- if or .Values.tls.enable .Values.certManager.enabled }} + - name: volume-serving-cert + secret: + secretName: {{ template "k8s-prometheus-adapter.fullname" . }} + {{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-adapter/templates/external-metrics-apiservice.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-adapter/templates/external-metrics-apiservice.yaml new file mode 100644 index 00000000000..21339af1284 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-adapter/templates/external-metrics-apiservice.yaml @@ -0,0 +1,34 @@ +{{- if .Values.rules.external }} +{{- if .Capabilities.APIVersions.Has "apiregistration.k8s.io/v1" }} +apiVersion: apiregistration.k8s.io/v1 +{{- else }} +apiVersion: apiregistration.k8s.io/v1beta1 +{{- end }} +kind: APIService +metadata: +{{- if or .Values.certManager.enabled .Values.customAnnotations }} + annotations: + certmanager.k8s.io/inject-ca-from: {{ printf "%s/%s-root-cert" (include "k8s-prometheus-adapter.namespace" .) (include "k8s-prometheus-adapter.fullname" .) | quote }} + cert-manager.io/inject-ca-from: {{ printf "%s/%s-root-cert" (include "k8s-prometheus-adapter.namespace" .) (include "k8s-prometheus-adapter.fullname" .) | quote }} + {{- if .Values.customAnnotations }} + {{- toYaml .Values.customAnnotations | nindent 4 }} + {{- end }} +{{- end }} + labels: + {{- include "k8s-prometheus-adapter.labels" . | indent 4 }} + name: v1beta1.external.metrics.k8s.io +spec: + service: + name: {{ template "k8s-prometheus-adapter.fullname" . }} + namespace: {{ include "k8s-prometheus-adapter.namespace" . | quote }} + {{- if .Values.tls.enable }} + caBundle: {{ b64enc .Values.tls.ca }} + {{- end }} + group: external.metrics.k8s.io + version: v1beta1 + {{- if not (or .Values.tls.enable .Values.certManager.enabled) }} + insecureSkipTLSVerify: true + {{- end }} + groupPriorityMinimum: 100 + versionPriority: 100 +{{- end -}} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-adapter/templates/external-metrics-cluster-role-binding-hpa.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-adapter/templates/external-metrics-cluster-role-binding-hpa.yaml new file mode 100644 index 00000000000..05547bd323a --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-adapter/templates/external-metrics-cluster-role-binding-hpa.yaml @@ -0,0 +1,20 @@ +{{- if and .Values.rbac.create .Values.rules.external -}} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + {{- if .Values.customAnnotations }} + annotations: + {{- toYaml .Values.customAnnotations | nindent 4 }} + {{- end }} + labels: + {{- include "k8s-prometheus-adapter.labels" . | indent 4 }} + name: {{ template "k8s-prometheus-adapter.name" . }}-hpa-controller-external-metrics +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "k8s-prometheus-adapter.name" . }}-external-metrics +subjects: +- kind: ServiceAccount + name: horizontal-pod-autoscaler + namespace: kube-system +{{- end -}} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-adapter/templates/external-metrics-cluster-role.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-adapter/templates/external-metrics-cluster-role.yaml new file mode 100644 index 00000000000..212ea78b25a --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-adapter/templates/external-metrics-cluster-role.yaml @@ -0,0 +1,21 @@ +{{- if and .Values.rbac.create .Values.rules.external -}} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + {{- if .Values.customAnnotations }} + annotations: + {{- toYaml .Values.customAnnotations | nindent 4 }} + {{- end }} + labels: + {{- include "k8s-prometheus-adapter.labels" . | indent 4 }} + name: {{ template "k8s-prometheus-adapter.name" . }}-external-metrics +rules: +- apiGroups: + - "external.metrics.k8s.io" + resources: + - "*" + verbs: + - list + - get + - watch +{{- end -}} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-adapter/templates/pdb.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-adapter/templates/pdb.yaml new file mode 100644 index 00000000000..205761a9f16 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-adapter/templates/pdb.yaml @@ -0,0 +1,23 @@ +{{- if .Values.podDisruptionBudget.enabled }} +apiVersion: {{ include "k8s-prometheus-adapter.pdb.apiVersion" . }} +kind: PodDisruptionBudget +metadata: + name: {{ template "k8s-prometheus-adapter.fullname" . }} + namespace: {{ include "k8s-prometheus-adapter.namespace" . }} + {{- if .Values.customAnnotations }} + annotations: + {{- toYaml .Values.customAnnotations | nindent 4 }} + {{- end }} + labels: + {{- include "k8s-prometheus-adapter.labels" . | indent 4 }} +spec: + {{- if .Values.podDisruptionBudget.minAvailable }} + minAvailable: {{ .Values.podDisruptionBudget.minAvailable }} + {{- end }} + {{- if .Values.podDisruptionBudget.maxUnavailable }} + maxUnavailable: {{ .Values.podDisruptionBudget.maxUnavailable }} + {{- end }} + selector: + matchLabels: + {{- include "k8s-prometheus-adapter.selectorLabels" . | indent 6 }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-adapter/templates/psp.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-adapter/templates/psp.yaml new file mode 100644 index 00000000000..fded5a74912 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-adapter/templates/psp.yaml @@ -0,0 +1,66 @@ +{{- if and (or .Values.global.cattle.psp.enabled .Values.psp.create) (.Capabilities.APIVersions.Has "policy/v1beta1/PodSecurityPolicy") }} +--- +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ template "k8s-prometheus-adapter.fullname" . }} + {{- if .Values.customAnnotations }} + annotations: + {{- toYaml .Values.customAnnotations | nindent 4 }} + {{- end }} + labels: + {{- include "k8s-prometheus-adapter.labels" . | indent 4 }} +spec: + {{- if .Values.hostNetwork.enabled }} + hostNetwork: true + hostPorts: + - min: {{ .Values.listenPort }} + max: {{ .Values.listenPort }} + {{- end }} + fsGroup: + rule: RunAsAny + runAsGroup: + rule: RunAsAny + runAsUser: + rule: MustRunAs + ranges: + - min: 1024 + max: 65535 + seLinux: + rule: RunAsAny + supplementalGroups: + rule: RunAsAny + volumes: + - secret + - emptyDir + - configMap +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + {{- include "k8s-prometheus-adapter.labels" . | indent 4 }} + name: {{ template "k8s-prometheus-adapter.name" . }}-psp +rules: +- apiGroups: + - 'policy' + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - {{ template "k8s-prometheus-adapter.fullname" . }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + labels: + {{- include "k8s-prometheus-adapter.labels" . | indent 4 }} + name: {{ template "k8s-prometheus-adapter.name" . }}-psp +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "k8s-prometheus-adapter.name" . }}-psp +subjects: +- kind: ServiceAccount + name: {{ template "k8s-prometheus-adapter.serviceAccountName" . }} + namespace: {{ include "k8s-prometheus-adapter.namespace" . | quote }} +{{- end -}} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-adapter/templates/resource-metrics-apiservice.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-adapter/templates/resource-metrics-apiservice.yaml new file mode 100644 index 00000000000..0cc9fff6a25 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-adapter/templates/resource-metrics-apiservice.yaml @@ -0,0 +1,34 @@ +{{- if .Values.rules.resource}} +{{- if .Capabilities.APIVersions.Has "apiregistration.k8s.io/v1" }} +apiVersion: apiregistration.k8s.io/v1 +{{- else }} +apiVersion: apiregistration.k8s.io/v1beta1 +{{- end }} +kind: APIService +metadata: +{{- if or .Values.certManager.enabled .Values.customAnnotations }} + annotations: + certmanager.k8s.io/inject-ca-from: {{ printf "%s/%s-root-cert" (include "k8s-prometheus-adapter.namespace" .) (include "k8s-prometheus-adapter.fullname" .) | quote }} + cert-manager.io/inject-ca-from: {{ printf "%s/%s-root-cert" (include "k8s-prometheus-adapter.namespace" .) (include "k8s-prometheus-adapter.fullname" .) | quote }} + {{- if .Values.customAnnotations }} + {{- toYaml .Values.customAnnotations | nindent 4 }} + {{- end }} +{{- end }} + labels: + {{- include "k8s-prometheus-adapter.labels" . | indent 4 }} + name: v1beta1.metrics.k8s.io +spec: + service: + name: {{ template "k8s-prometheus-adapter.fullname" . }} + namespace: {{ include "k8s-prometheus-adapter.namespace" . | quote }} + {{- if .Values.tls.enable }} + caBundle: {{ b64enc .Values.tls.ca }} + {{- end }} + group: metrics.k8s.io + version: v1beta1 + {{- if not (or .Values.tls.enable .Values.certManager.enabled) }} + insecureSkipTLSVerify: true + {{- end }} + groupPriorityMinimum: 100 + versionPriority: 100 +{{- end -}} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-adapter/templates/resource-metrics-cluster-role-binding.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-adapter/templates/resource-metrics-cluster-role-binding.yaml new file mode 100644 index 00000000000..3c247e48d28 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-adapter/templates/resource-metrics-cluster-role-binding.yaml @@ -0,0 +1,20 @@ +{{- if and .Values.rbac.create .Values.rules.resource -}} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + {{- if .Values.customAnnotations }} + annotations: + {{- toYaml .Values.customAnnotations | nindent 4 }} + {{- end }} + labels: + {{- include "k8s-prometheus-adapter.labels" . | indent 4 }} + name: {{ template "k8s-prometheus-adapter.name" . }}-hpa-controller-metrics +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "k8s-prometheus-adapter.name" . }}-metrics +subjects: +- kind: ServiceAccount + name: {{ template "k8s-prometheus-adapter.serviceAccountName" . }} + namespace: {{ include "k8s-prometheus-adapter.namespace" . | quote }} +{{- end -}} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-adapter/templates/resource-metrics-cluster-role.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-adapter/templates/resource-metrics-cluster-role.yaml new file mode 100644 index 00000000000..73d8953046f --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-adapter/templates/resource-metrics-cluster-role.yaml @@ -0,0 +1,23 @@ +{{- if and .Values.rbac.create .Values.rules.resource -}} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + {{- if .Values.customAnnotations }} + annotations: + {{- toYaml .Values.customAnnotations | nindent 4 }} + {{- end }} + labels: + {{- include "k8s-prometheus-adapter.labels" . | indent 4 }} + name: {{ template "k8s-prometheus-adapter.name" . }}-metrics +rules: +- apiGroups: + - "" + resources: + - pods + - nodes + - nodes/stats + verbs: + - get + - list + - watch +{{- end -}} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-adapter/templates/role-binding-auth-reader.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-adapter/templates/role-binding-auth-reader.yaml new file mode 100644 index 00000000000..d3c77c1c65d --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-adapter/templates/role-binding-auth-reader.yaml @@ -0,0 +1,21 @@ +{{- if .Values.rbac.create -}} +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + {{- if .Values.customAnnotations }} + annotations: + {{- toYaml .Values.customAnnotations | nindent 4 }} + {{- end }} + labels: + {{- include "k8s-prometheus-adapter.labels" . | indent 4 }} + name: {{ template "k8s-prometheus-adapter.name" . }}-auth-reader + namespace: kube-system +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: extension-apiserver-authentication-reader +subjects: +- kind: ServiceAccount + name: {{ template "k8s-prometheus-adapter.serviceAccountName" . }} + namespace: {{ include "k8s-prometheus-adapter.namespace" . | quote }} +{{- end -}} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-adapter/templates/secret.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-adapter/templates/secret.yaml new file mode 100644 index 00000000000..3e7e8887bd2 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-adapter/templates/secret.yaml @@ -0,0 +1,17 @@ +{{- if .Values.tls.enable -}} +apiVersion: v1 +kind: Secret +metadata: + {{- if .Values.customAnnotations }} + annotations: + {{- toYaml .Values.customAnnotations | nindent 4 }} + {{- end }} + labels: + {{- include "k8s-prometheus-adapter.labels" . | indent 4 }} + name: {{ template "k8s-prometheus-adapter.fullname" . }} + namespace: {{ include "k8s-prometheus-adapter.namespace" . }} +type: kubernetes.io/tls +data: + tls.crt: {{ b64enc .Values.tls.certificate }} + tls.key: {{ b64enc .Values.tls.key }} +{{- end -}} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-adapter/templates/service.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-adapter/templates/service.yaml new file mode 100644 index 00000000000..ddac37cfa1d --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-adapter/templates/service.yaml @@ -0,0 +1,27 @@ +apiVersion: v1 +kind: Service +metadata: + {{- if or .Values.service.annotations .Values.customAnnotations }} + annotations: + {{- if .Values.service.annotations }} + {{ toYaml .Values.service.annotations | indent 4 }} + {{- end }} + {{- if .Values.customAnnotations }} + {{- toYaml .Values.customAnnotations | nindent 4 }} + {{- end }} + {{- end }} + labels: + {{- include "k8s-prometheus-adapter.labels" . | indent 4 }} + name: {{ template "k8s-prometheus-adapter.fullname" . }} + namespace: {{ include "k8s-prometheus-adapter.namespace" . }} +spec: + ports: + - port: {{ .Values.service.port }} + protocol: TCP + targetPort: https + selector: + {{- include "k8s-prometheus-adapter.selectorLabels" . | indent 4 }} + type: {{ .Values.service.type }} + {{- if .Values.service.clusterIP }} + clusterIP: {{ .Values.service.clusterIP }} + {{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-adapter/templates/serviceaccount.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-adapter/templates/serviceaccount.yaml new file mode 100644 index 00000000000..30a169ae0e9 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-adapter/templates/serviceaccount.yaml @@ -0,0 +1,18 @@ +{{- if .Values.serviceAccount.create -}} +apiVersion: v1 +kind: ServiceAccount +metadata: + labels: + {{- include "k8s-prometheus-adapter.labels" . | indent 4 }} + name: {{ template "k8s-prometheus-adapter.serviceAccountName" . }} + namespace: {{ include "k8s-prometheus-adapter.namespace" . }} +{{- if or .Values.serviceAccount.annotations .Values.customAnnotations }} + annotations: + {{- if .Values.serviceAccount.annotations }} + {{- toYaml .Values.serviceAccount.annotations | nindent 4 }} + {{- end }} + {{- if .Values.customAnnotations }} + {{- toYaml .Values.customAnnotations | nindent 4 }} + {{- end }} +{{- end }} +{{- end -}} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-adapter/values.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-adapter/values.yaml new file mode 100644 index 00000000000..a1445a23f13 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-adapter/values.yaml @@ -0,0 +1,277 @@ +# Default values for k8s-prometheus-adapter.. +global: + cattle: + psp: + enabled: false + systemDefaultRegistry: "" + +affinity: {} + +topologySpreadConstraints: [] + +image: + repository: rancher/mirrored-prometheus-adapter-prometheus-adapter + tag: v0.12.0 + pullPolicy: IfNotPresent + +logLevel: 4 + +metricsRelistInterval: 1m + +listenPort: 6443 + +nodeSelector: {} + +priorityClassName: "" + +## Override the release namespace (for multi-namespace deployments in combined charts) +namespaceOverride: "" + +## Additional annotations to add to all resources +customAnnotations: {} + # role: custom-metrics + +## Additional labels to add to all resources +customLabels: {} + # monitoring: prometheus-adapter + +# Url to access prometheus +prometheus: + # Value is templated + url: http://prometheus.default.svc + port: 9090 + path: "" + +replicas: 1 + +# k8s 1.21 needs fsGroup to be set for non root deployments +# ref: https://github.com/kubernetes/kubernetes/issues/70679 +podSecurityContext: + fsGroup: 10001 + +# SecurityContext of the container +# ref. https://kubernetes.io/docs/tasks/configure-pod-container/security-context +securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: ["all"] + readOnlyRootFilesystem: true + runAsNonRoot: true + runAsUser: 10001 + seccompProfile: + type: RuntimeDefault + +rbac: + # Specifies whether RBAC resources should be created + create: true + +psp: + # Specifies whether PSP resources should be created + create: false + +serviceAccount: + # Specifies whether a service account should be created + create: true + # The name of the service account to use. + # If not set and create is true, a name is generated using the fullname template + name: + # ServiceAccount annotations. + # Use case: AWS EKS IAM roles for service accounts + # ref: https://docs.aws.amazon.com/eks/latest/userguide/specify-service-account-role.html + annotations: {} + +# Custom DNS configuration to be added to prometheus-adapter pods +dnsConfig: {} + # nameservers: + # - 1.2.3.4 + # searches: + # - ns1.svc.cluster-domain.example + # - my.dns.search.suffix + # options: + # - name: ndots + # value: "2" + # - name: edns0 + +resources: {} + # requests: + # cpu: 100m + # memory: 128Mi + # limits: + # cpu: 100m + # memory: 128Mi + +# Configure liveness probe +# https://kubernetes.io/docs/reference/kubernetes-api/workload-resources/pod-v1/#Probe +livenessProbe: + httpGet: + path: /healthz + port: https + scheme: HTTPS + initialDelaySeconds: 30 + timeoutSeconds: 5 + +# Configure readiness probe +readinessProbe: + httpGet: + path: /healthz + port: https + scheme: HTTPS + initialDelaySeconds: 30 + timeoutSeconds: 5 + +rules: + default: true + + custom: [] + # - seriesQuery: '{__name__=~"^some_metric_count$"}' + # resources: + # template: <<.Resource>> + # name: + # matches: "" + # as: "my_custom_metric" + # metricsQuery: sum(<<.Series>>{<<.LabelMatchers>>}) by (<<.GroupBy>>) + + # Mounts a configMap with pre-generated rules for use. Overrides the + # default, custom, external and resource entries + existing: + + external: [] + # - seriesQuery: '{__name__=~"^some_metric_count$"}' + # resources: + # template: <<.Resource>> + # name: + # matches: "" + # as: "my_external_metric" + # metricsQuery: sum(<<.Series>>{<<.LabelMatchers>>}) by (<<.GroupBy>>) + + # resource: + # cpu: + # containerQuery: | + # sum by (<<.GroupBy>>) ( + # rate(container_cpu_usage_seconds_total{container!="",<<.LabelMatchers>>}[3m]) + # ) + # nodeQuery: | + # sum by (<<.GroupBy>>) ( + # rate(node_cpu_seconds_total{mode!="idle",mode!="iowait",mode!="steal",<<.LabelMatchers>>}[3m]) + # ) + # resources: + # overrides: + # node: + # resource: node + # namespace: + # resource: namespace + # pod: + # resource: pod + # containerLabel: container + # memory: + # containerQuery: | + # sum by (<<.GroupBy>>) ( + # avg_over_time(container_memory_working_set_bytes{container!="",<<.LabelMatchers>>}[3m]) + # ) + # nodeQuery: | + # sum by (<<.GroupBy>>) ( + # avg_over_time(node_memory_MemTotal_bytes{<<.LabelMatchers>>}[3m]) + # - + # avg_over_time(node_memory_MemAvailable_bytes{<<.LabelMatchers>>}[3m]) + # ) + # resources: + # overrides: + # node: + # resource: node + # namespace: + # resource: namespace + # pod: + # resource: pod + # containerLabel: container + # window: 3m + +service: + annotations: {} + port: 443 + type: ClusterIP + # clusterIP: 1.2.3.4 + +tls: + enable: false + ca: |- + # Public CA file that signed the APIService + key: |- + # Private key of the APIService + certificate: |- + # Public key of the APIService + +# Set environment variables from secrets, configmaps or by setting them as name/value +env: [] + # - name: TMP_DIR + # value: /tmp + # - name: PASSWORD + # valueFrom: + # secretKeyRef: + # name: mysecret + # key: password + # optional: false + +# Any extra arguments +extraArguments: [] + # - --tls-private-key-file=/etc/tls/tls.key + # - --tls-cert-file=/etc/tls/tls.crt + +# Any extra volumes +extraVolumes: [] + # - name: example-name + # hostPath: + # path: /path/on/host + # type: DirectoryOrCreate + # - name: ssl-certs + # hostPath: + # path: /etc/ssl/certs/ca-bundle.crt + # type: File + +# Any extra volume mounts +extraVolumeMounts: [] + # - name: example-name + # mountPath: /path/in/container + # - name: ssl-certs + # mountPath: /etc/ssl/certs/ca-certificates.crt + # readOnly: true + +tolerations: [] + +# Labels added to the pod +podLabels: {} + +# Annotations added to the pod +podAnnotations: {} + +# Annotations added to the deployment +deploymentAnnotations: {} + +hostNetwork: + # Specifies if prometheus-adapter should be started in hostNetwork mode. + # + # You would require this enabled if you use alternate overlay networking for pods and + # API server unable to communicate with metrics-server. As an example, this is required + # if you use Weave network on EKS. See also dnsPolicy + enabled: false + +# When hostNetwork is enabled, you probably want to set this to ClusterFirstWithHostNet +# dnsPolicy: ClusterFirstWithHostNet + +# Deployment strategy type +strategy: + type: RollingUpdate + rollingUpdate: + maxUnavailable: 25% + maxSurge: 25% + +podDisruptionBudget: + # Specifies if PodDisruptionBudget should be enabled + # When enabled, minAvailable or maxUnavailable should also be defined. + enabled: false + minAvailable: + maxUnavailable: 1 + +certManager: + enabled: false + caCertDuration: 43800h + certDuration: 8760h diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-node-exporter/.helmignore b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-node-exporter/.helmignore new file mode 100644 index 00000000000..f0c13194444 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-node-exporter/.helmignore @@ -0,0 +1,21 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*~ +# Various IDEs +.project +.idea/ +*.tmproj diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-node-exporter/Chart.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-node-exporter/Chart.yaml new file mode 100644 index 00000000000..9130cbcc91d --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-node-exporter/Chart.yaml @@ -0,0 +1,25 @@ +annotations: + artifacthub.io/license: Apache-2.0 + artifacthub.io/links: |- + - name: Chart Source + url: https://github.com/prometheus-community/helm-charts +apiVersion: v2 +appVersion: 1.7.0 +description: A Helm chart for prometheus node-exporter +home: https://github.com/prometheus/node_exporter/ +keywords: +- node-exporter +- prometheus +- exporter +maintainers: +- email: gianrubio@gmail.com + name: gianrubio +- email: zanhsieh@gmail.com + name: zanhsieh +- email: rootsandtrees@posteo.de + name: zeritti +name: prometheus-node-exporter +sources: +- https://github.com/prometheus/node_exporter/ +type: application +version: 4.30.3 diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-node-exporter/README.md b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-node-exporter/README.md new file mode 100644 index 00000000000..149b9822670 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-node-exporter/README.md @@ -0,0 +1,97 @@ + +# Prometheus Node Exporter + +Prometheus exporter for hardware and OS metrics exposed by *NIX kernels, written in Go with pluggable metric collectors. + +This chart bootstraps a Prometheus [Node Exporter](http://github.com/prometheus/node_exporter) daemonset on a [Kubernetes](http://kubernetes.io) cluster using the [Helm](https://helm.sh) package manager. + +## Get Repository Info + +```console +helm repo add prometheus-community https://prometheus-community.github.io/helm-charts +helm repo update +``` + +_See [helm repo](https://helm.sh/docs/helm/helm_repo/) for command documentation._ + +## Install Chart + +```console +helm install [RELEASE_NAME] prometheus-community/prometheus-node-exporter +``` + +_See [configuration](#configuring) below._ + +_See [helm install](https://helm.sh/docs/helm/helm_install/) for command documentation._ + +## Uninstall Chart + +```console +helm uninstall [RELEASE_NAME] +``` + +This removes all the Kubernetes components associated with the chart and deletes the release. + +_See [helm uninstall](https://helm.sh/docs/helm/helm_uninstall/) for command documentation._ + +## Upgrading Chart + +```console +helm upgrade [RELEASE_NAME] prometheus-community/prometheus-node-exporter --install +``` + +_See [helm upgrade](https://helm.sh/docs/helm/helm_upgrade/) for command documentation._ + +### 3.x to 4.x + +Starting from version 4.0.0, the `node exporter` chart is using the [Kubernetes recommended labels](https://kubernetes.io/docs/concepts/overview/working-with-objects/common-labels/). Therefore you have to delete the daemonset before you upgrade. + +```console +kubectl delete daemonset -l app=prometheus-node-exporter +helm upgrade -i prometheus-node-exporter prometheus-community/prometheus-node-exporter +``` + +If you use your own custom [ServiceMonitor](https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#servicemonitor) or [PodMonitor](https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#podmonitor), please ensure to upgrade their `selector` fields accordingly to the new labels. + +### From 2.x to 3.x + +Change the following: + +```yaml +hostRootFsMount: true +``` + +to: + +```yaml +hostRootFsMount: + enabled: true + mountPropagation: HostToContainer +``` + +## Configuring + +See [Customizing the Chart Before Installing](https://helm.sh/docs/intro/using_helm/#customizing-the-chart-before-installing). To see all configurable options with detailed comments, visit the chart's [values.yaml](./values.yaml), or run these configuration commands: + +```console +helm show values prometheus-community/prometheus-node-exporter +``` + +### kube-rbac-proxy + +You can enable `prometheus-node-exporter` endpoint protection using `kube-rbac-proxy`. By setting `kubeRBACProxy.enabled: true`, this chart will deploy a RBAC proxy container protecting the node-exporter endpoint. +To authorize access, authenticate your requests (via a `ServiceAccount` for example) with a `ClusterRole` attached such as: + +```yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: prometheus-node-exporter-read +rules: + - apiGroups: [ "" ] + resources: ["services/node-exporter-prometheus-node-exporter"] + verbs: + - get +``` + +See [kube-rbac-proxy examples](https://github.com/brancz/kube-rbac-proxy/tree/master/examples/resource-attributes) for more details. diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-node-exporter/templates/NOTES.txt b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-node-exporter/templates/NOTES.txt new file mode 100644 index 00000000000..8c5391f1f77 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-node-exporter/templates/NOTES.txt @@ -0,0 +1,29 @@ +1. Get the application URL by running these commands: +{{- if contains "NodePort" .Values.service.type }} + export NODE_PORT=$(kubectl get --namespace {{ template "prometheus-node-exporter.namespace" . }} -o jsonpath="{.spec.ports[0].nodePort}" services {{ template "prometheus-node-exporter.fullname" . }}) + export NODE_IP=$(kubectl get nodes --namespace {{ template "prometheus-node-exporter.namespace" . }} -o jsonpath="{.items[0].status.addresses[0].address}") + echo http://$NODE_IP:$NODE_PORT +{{- else if contains "LoadBalancer" .Values.service.type }} + NOTE: It may take a few minutes for the LoadBalancer IP to be available. + You can watch the status of by running 'kubectl get svc -w {{ template "prometheus-node-exporter.fullname" . }}' + export SERVICE_IP=$(kubectl get svc --namespace {{ template "prometheus-node-exporter.namespace" . }} {{ template "prometheus-node-exporter.fullname" . }} -o jsonpath='{.status.loadBalancer.ingress[0].ip}') + echo http://$SERVICE_IP:{{ .Values.service.port }} +{{- else if contains "ClusterIP" .Values.service.type }} + export POD_NAME=$(kubectl get pods --namespace {{ template "prometheus-node-exporter.namespace" . }} -l "app.kubernetes.io/name={{ template "prometheus-node-exporter.name" . }},app.kubernetes.io/instance={{ .Release.Name }}" -o jsonpath="{.items[0].metadata.name}") + echo "Visit http://127.0.0.1:{{ .Values.service.port }} to use your application" + kubectl port-forward --namespace {{ template "prometheus-node-exporter.namespace" . }} $POD_NAME {{ .Values.service.port }} +{{- end }} + +{{- if .Values.kubeRBACProxy.enabled}} + +kube-rbac-proxy endpoint protections is enabled: +- Metrics endpoints is now HTTPS +- Ensure that the client authenticates the requests (e.g. via service account) with the following role permissions: +``` +rules: + - apiGroups: [ "" ] + resources: ["services/{{ template "prometheus-node-exporter.fullname" . }}"] + verbs: + - get +``` +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-node-exporter/templates/_helpers.tpl b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-node-exporter/templates/_helpers.tpl new file mode 100644 index 00000000000..72a6db45a15 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-node-exporter/templates/_helpers.tpl @@ -0,0 +1,236 @@ +# Rancher +{{- define "system_default_registry" -}} +{{- if .Values.global.cattle.systemDefaultRegistry -}} +{{- printf "%s/" .Values.global.cattle.systemDefaultRegistry -}} +{{- end -}} +{{- end -}} + +# Windows Support + +{{/* +Windows cluster will add default taint for linux nodes, +add below linux tolerations to workloads could be scheduled to those linux nodes +*/}} + +{{- define "linux-node-tolerations" -}} +- key: "cattle.io/os" + value: "linux" + effect: "NoSchedule" + operator: "Equal" +{{- end -}} + +{{- define "linux-node-selector" -}} +{{- if semverCompare "<1.14-0" .Capabilities.KubeVersion.GitVersion -}} +beta.kubernetes.io/os: linux +{{- else -}} +kubernetes.io/os: linux +{{- end -}} +{{- end -}} + +{{/* vim: set filetype=mustache: */}} +{{/* +Expand the name of the chart. +*/}} +{{- define "prometheus-node-exporter.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "prometheus-node-exporter.fullname" -}} +{{- if .Values.fullnameOverride }} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- $name := default .Chart.Name .Values.nameOverride }} +{{- if contains $name .Release.Name }} +{{- .Release.Name | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }} +{{- end }} +{{- end }} +{{- end }} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "prometheus-node-exporter.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Common labels +*/}} +{{- define "prometheus-node-exporter.labels" -}} +helm.sh/chart: {{ include "prometheus-node-exporter.chart" . }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +app.kubernetes.io/component: metrics +app.kubernetes.io/part-of: {{ include "prometheus-node-exporter.name" . }} +{{ include "prometheus-node-exporter.selectorLabels" . }} +{{- with .Chart.AppVersion }} +app.kubernetes.io/version: {{ . | quote }} +{{- end }} +{{- with .Values.podLabels }} +{{ toYaml . }} +{{- end }} +{{- if .Values.releaseLabel }} +release: {{ .Release.Name }} +{{- end }} +{{- end }} + +{{/* +Selector labels +*/}} +{{- define "prometheus-node-exporter.selectorLabels" -}} +app.kubernetes.io/name: {{ include "prometheus-node-exporter.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end }} + + +{{/* +Create the name of the service account to use +*/}} +{{- define "prometheus-node-exporter.serviceAccountName" -}} +{{- if .Values.serviceAccount.create }} +{{- default (include "prometheus-node-exporter.fullname" .) .Values.serviceAccount.name }} +{{- else }} +{{- default "default" .Values.serviceAccount.name }} +{{- end }} +{{- end }} + +{{/* +The image to use +*/}} +{{- define "prometheus-node-exporter.image" -}} +{{- $temp_registry := (include "system_default_registry" .) }} +{{- if .Values.image.sha }} +{{- fail "image.sha forbidden. Use image.digest instead" }} +{{- else if .Values.image.digest }} +{{- if $temp_registry }} +{{- printf "%s%s:%s@%s" $temp_registry .Values.image.repository (default (printf "v%s" .Chart.AppVersion) .Values.image.tag) .Values.image.digest }} +{{- else if .Values.global.imageRegistry }} +{{- printf "%s/%s:%s@%s" .Values.global.imageRegistry .Values.image.repository (default (printf "v%s" .Chart.AppVersion) .Values.image.tag) .Values.image.digest }} +{{- else }} +{{- printf "%s/%s:%s@%s" .Values.image.registry .Values.image.repository (default (printf "v%s" .Chart.AppVersion) .Values.image.tag) .Values.image.digest }} +{{- end }} +{{- else }} +{{- if $temp_registry }} +{{- printf "%s%s:%s" $temp_registry .Values.image.repository (default (printf "v%s" .Chart.AppVersion) .Values.image.tag) }} +{{- else if .Values.global.imageRegistry }} +{{- printf "%s/%s:%s" .Values.global.imageRegistry .Values.image.repository (default (printf "v%s" .Chart.AppVersion) .Values.image.tag) }} +{{- else }} +{{- printf "%s/%s:%s" .Values.image.registry .Values.image.repository (default (printf "v%s" .Chart.AppVersion) .Values.image.tag) }} +{{- end }} +{{- end }} +{{- end }} + +{{/* +Allow the release namespace to be overridden for multi-namespace deployments in combined charts +*/}} +{{- define "prometheus-node-exporter.namespace" -}} +{{- if .Values.namespaceOverride }} +{{- .Values.namespaceOverride }} +{{- else }} +{{- .Release.Namespace }} +{{- end }} +{{- end }} + +{{/* +Create the namespace name of the service monitor +*/}} +{{- define "prometheus-node-exporter.monitor-namespace" -}} +{{- if .Values.namespaceOverride }} +{{- .Values.namespaceOverride }} +{{- else }} +{{- if .Values.prometheus.monitor.namespace }} +{{- .Values.prometheus.monitor.namespace }} +{{- else }} +{{- .Release.Namespace }} +{{- end }} +{{- end }} +{{- end }} + +{{/* Sets default scrape limits for servicemonitor */}} +{{- define "servicemonitor.scrapeLimits" -}} +{{- with .sampleLimit }} +sampleLimit: {{ . }} +{{- end }} +{{- with .targetLimit }} +targetLimit: {{ . }} +{{- end }} +{{- with .labelLimit }} +labelLimit: {{ . }} +{{- end }} +{{- with .labelNameLengthLimit }} +labelNameLengthLimit: {{ . }} +{{- end }} +{{- with .labelValueLengthLimit }} +labelValueLengthLimit: {{ . }} +{{- end }} +{{- end }} + +{{/* +Formats imagePullSecrets. Input is (dict "Values" .Values "imagePullSecrets" .{specific imagePullSecrets}) +*/}} +{{- define "prometheus-node-exporter.imagePullSecrets" -}} +{{- range (concat .Values.global.imagePullSecrets .imagePullSecrets) }} + {{- if eq (typeOf .) "map[string]interface {}" }} +- {{ toYaml . | trim }} + {{- else }} +- name: {{ . }} + {{- end }} +{{- end }} +{{- end -}} + +{{/* +Create the namespace name of the pod monitor +*/}} +{{- define "prometheus-node-exporter.podmonitor-namespace" -}} +{{- if .Values.namespaceOverride }} +{{- .Values.namespaceOverride }} +{{- else }} +{{- if .Values.prometheus.podMonitor.namespace }} +{{- .Values.prometheus.podMonitor.namespace }} +{{- else }} +{{- .Release.Namespace }} +{{- end }} +{{- end }} +{{- end }} + +{{/* Sets default scrape limits for podmonitor */}} +{{- define "podmonitor.scrapeLimits" -}} +{{- with .sampleLimit }} +sampleLimit: {{ . }} +{{- end }} +{{- with .targetLimit }} +targetLimit: {{ . }} +{{- end }} +{{- with .labelLimit }} +labelLimit: {{ . }} +{{- end }} +{{- with .labelNameLengthLimit }} +labelNameLengthLimit: {{ . }} +{{- end }} +{{- with .labelValueLengthLimit }} +labelValueLengthLimit: {{ . }} +{{- end }} +{{- end }} + +{{/* Sets sidecar volumeMounts */}} +{{- define "prometheus-node-exporter.sidecarVolumeMounts" -}} +{{- range $_, $mount := $.Values.sidecarVolumeMount }} +- name: {{ $mount.name }} + mountPath: {{ $mount.mountPath }} + readOnly: {{ $mount.readOnly }} +{{- end }} +{{- range $_, $mount := $.Values.sidecarHostVolumeMounts }} +- name: {{ $mount.name }} + mountPath: {{ $mount.mountPath }} + readOnly: {{ $mount.readOnly }} +{{- if $mount.mountPropagation }} + mountPropagation: {{ $mount.mountPropagation }} +{{- end }} +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-node-exporter/templates/clusterrole.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-node-exporter/templates/clusterrole.yaml new file mode 100644 index 00000000000..c256dba73db --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-node-exporter/templates/clusterrole.yaml @@ -0,0 +1,19 @@ +{{- if and (eq .Values.rbac.create true) (eq .Values.kubeRBACProxy.enabled true) -}} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ include "prometheus-node-exporter.fullname" . }} + labels: + {{- include "prometheus-node-exporter.labels" . | nindent 4 }} +rules: + {{- if $.Values.kubeRBACProxy.enabled }} + - apiGroups: [ "authentication.k8s.io" ] + resources: + - tokenreviews + verbs: [ "create" ] + - apiGroups: [ "authorization.k8s.io" ] + resources: + - subjectaccessreviews + verbs: [ "create" ] + {{- end }} +{{- end -}} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-node-exporter/templates/clusterrolebinding.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-node-exporter/templates/clusterrolebinding.yaml new file mode 100644 index 00000000000..653305ad9e7 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-node-exporter/templates/clusterrolebinding.yaml @@ -0,0 +1,20 @@ +{{- if and (eq .Values.rbac.create true) (eq .Values.kubeRBACProxy.enabled true) -}} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + labels: + {{- include "prometheus-node-exporter.labels" . | nindent 4 }} + name: {{ template "prometheus-node-exporter.fullname" . }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole +{{- if .Values.rbac.useExistingRole }} + name: {{ .Values.rbac.useExistingRole }} +{{- else }} + name: {{ template "prometheus-node-exporter.fullname" . }} +{{- end }} +subjects: +- kind: ServiceAccount + name: {{ template "prometheus-node-exporter.serviceAccountName" . }} + namespace: {{ template "prometheus-node-exporter.namespace" . }} +{{- end -}} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-node-exporter/templates/daemonset.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-node-exporter/templates/daemonset.yaml new file mode 100644 index 00000000000..48d274f1b18 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-node-exporter/templates/daemonset.yaml @@ -0,0 +1,309 @@ +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: {{ include "prometheus-node-exporter.fullname" . }} + namespace: {{ include "prometheus-node-exporter.namespace" . }} + labels: + {{- include "prometheus-node-exporter.labels" . | nindent 4 }} + {{- with .Values.daemonsetAnnotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + selector: + matchLabels: + {{- include "prometheus-node-exporter.selectorLabels" . | nindent 6 }} + revisionHistoryLimit: {{ .Values.revisionHistoryLimit }} + {{- with .Values.updateStrategy }} + updateStrategy: + {{- toYaml . | nindent 4 }} + {{- end }} + template: + metadata: + {{- with .Values.podAnnotations }} + annotations: + {{- toYaml . | nindent 8 }} + {{- end }} + labels: + {{- include "prometheus-node-exporter.labels" . | nindent 8 }} + spec: + automountServiceAccountToken: {{ ternary true false (or .Values.serviceAccount.automountServiceAccountToken .Values.kubeRBACProxy.enabled) }} + {{- with .Values.securityContext }} + securityContext: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.priorityClassName }} + priorityClassName: {{ . }} + {{- end }} + {{- with .Values.extraInitContainers }} + initContainers: + {{- toYaml . | nindent 8 }} + {{- end }} + serviceAccountName: {{ include "prometheus-node-exporter.serviceAccountName" . }} + {{- with .Values.terminationGracePeriodSeconds }} + terminationGracePeriodSeconds: {{ . }} + {{- end }} + containers: + {{- $servicePort := ternary .Values.kubeRBACProxy.port .Values.service.port .Values.kubeRBACProxy.enabled }} + - name: node-exporter + image: {{ include "prometheus-node-exporter.image" . }} + imagePullPolicy: {{ .Values.image.pullPolicy }} + args: + - --path.procfs=/host/proc + - --path.sysfs=/host/sys + {{- if .Values.hostRootFsMount.enabled }} + - --path.rootfs=/host/root + {{- if semverCompare ">=1.4.0-0" (coalesce .Values.version .Values.image.tag .Chart.AppVersion) }} + - --path.udev.data=/host/root/run/udev/data + {{- end }} + {{- end }} + - --web.listen-address=[$(HOST_IP)]:{{ $servicePort }} + {{- with .Values.extraArgs }} + {{- toYaml . | nindent 12 }} + {{- end }} + {{- with .Values.containerSecurityContext }} + securityContext: + {{- toYaml . | nindent 12 }} + {{- end }} + env: + - name: HOST_IP + {{- if .Values.kubeRBACProxy.enabled }} + value: 127.0.0.1 + {{- else if .Values.service.listenOnAllInterfaces }} + value: 0.0.0.0 + {{- else }} + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: status.hostIP + {{- end }} + {{- range $key, $value := .Values.env }} + - name: {{ $key }} + value: {{ $value | quote }} + {{- end }} + {{- if eq .Values.kubeRBACProxy.enabled false }} + ports: + - name: {{ .Values.service.portName }} + containerPort: {{ .Values.service.port }} + protocol: TCP + {{- end }} + livenessProbe: + failureThreshold: {{ .Values.livenessProbe.failureThreshold }} + httpGet: + {{- if .Values.kubeRBACProxy.enabled }} + host: 127.0.0.1 + {{- end }} + httpHeaders: + {{- range $_, $header := .Values.livenessProbe.httpGet.httpHeaders }} + - name: {{ $header.name }} + value: {{ $header.value }} + {{- end }} + path: / + port: {{ $servicePort }} + scheme: {{ upper .Values.livenessProbe.httpGet.scheme }} + initialDelaySeconds: {{ .Values.livenessProbe.initialDelaySeconds }} + periodSeconds: {{ .Values.livenessProbe.periodSeconds }} + successThreshold: {{ .Values.livenessProbe.successThreshold }} + timeoutSeconds: {{ .Values.livenessProbe.timeoutSeconds }} + readinessProbe: + failureThreshold: {{ .Values.readinessProbe.failureThreshold }} + httpGet: + {{- if .Values.kubeRBACProxy.enabled }} + host: 127.0.0.1 + {{- end }} + httpHeaders: + {{- range $_, $header := .Values.readinessProbe.httpGet.httpHeaders }} + - name: {{ $header.name }} + value: {{ $header.value }} + {{- end }} + path: / + port: {{ $servicePort }} + scheme: {{ upper .Values.readinessProbe.httpGet.scheme }} + initialDelaySeconds: {{ .Values.readinessProbe.initialDelaySeconds }} + periodSeconds: {{ .Values.readinessProbe.periodSeconds }} + successThreshold: {{ .Values.readinessProbe.successThreshold }} + timeoutSeconds: {{ .Values.readinessProbe.timeoutSeconds }} + {{- with .Values.resources }} + resources: + {{- toYaml . | nindent 12 }} + {{- end }} + {{- if .Values.terminationMessageParams.enabled }} + {{- with .Values.terminationMessageParams }} + terminationMessagePath: {{ .terminationMessagePath }} + terminationMessagePolicy: {{ .terminationMessagePolicy }} + {{- end }} + {{- end }} + volumeMounts: + - name: proc + mountPath: /host/proc + {{- with .Values.hostProcFsMount.mountPropagation }} + mountPropagation: {{ . }} + {{- end }} + readOnly: true + - name: sys + mountPath: /host/sys + {{- with .Values.hostSysFsMount.mountPropagation }} + mountPropagation: {{ . }} + {{- end }} + readOnly: true + {{- if .Values.hostRootFsMount.enabled }} + - name: root + mountPath: /host/root + {{- with .Values.hostRootFsMount.mountPropagation }} + mountPropagation: {{ . }} + {{- end }} + readOnly: true + {{- end }} + {{- range $_, $mount := .Values.extraHostVolumeMounts }} + - name: {{ $mount.name }} + mountPath: {{ $mount.mountPath }} + readOnly: {{ $mount.readOnly }} + {{- with $mount.mountPropagation }} + mountPropagation: {{ . }} + {{- end }} + {{- end }} + {{- range $_, $mount := .Values.sidecarVolumeMount }} + - name: {{ $mount.name }} + mountPath: {{ $mount.mountPath }} + readOnly: true + {{- end }} + {{- range $_, $mount := .Values.configmaps }} + - name: {{ $mount.name }} + mountPath: {{ $mount.mountPath }} + {{- end }} + {{- range $_, $mount := .Values.secrets }} + - name: {{ .name }} + mountPath: {{ .mountPath }} + {{- end }} + {{- range .Values.sidecars }} + {{- $overwrites := dict "volumeMounts" (concat (include "prometheus-node-exporter.sidecarVolumeMounts" $ | fromYamlArray) (.volumeMounts | default list) | default list) }} + {{- $defaults := dict "image" (include "prometheus-node-exporter.image" $) "securityContext" $.Values.containerSecurityContext "imagePullPolicy" $.Values.image.pullPolicy }} + - {{- toYaml (merge $overwrites . $defaults) | nindent 10 }} + {{- end }} + {{- if .Values.kubeRBACProxy.enabled }} + - name: kube-rbac-proxy + args: + {{- if .Values.kubeRBACProxy.extraArgs }} + {{- .Values.kubeRBACProxy.extraArgs | toYaml | nindent 12 }} + {{- end }} + - --secure-listen-address=:{{ .Values.service.port}} + - --upstream=http://127.0.0.1:{{ $servicePort }}/ + - --proxy-endpoints-port=8888 + - --config-file=/etc/kube-rbac-proxy-config/config-file.yaml + volumeMounts: + - name: kube-rbac-proxy-config + mountPath: /etc/kube-rbac-proxy-config + imagePullPolicy: {{ .Values.kubeRBACProxy.image.pullPolicy }} + {{- $base_registry := (include "monitoring_registry" .) }} + {{- if .Values.kubeRBACProxy.image.sha }} + image: "{{ $base_registry | default .Values.kubeRBACProxy.image.registry}}/{{ .Values.kubeRBACProxy.image.repository }}:{{ .Values.kubeRBACProxy.image.tag }}@sha256:{{ .Values.kubeRBACProxy.image.sha }}" + {{- else }} + image: "{{ $base_registry | default .Values.kubeRBACProxy.image.registry}}/{{ .Values.kubeRBACProxy.image.repository }}:{{ .Values.kubeRBACProxy.image.tag }}" + {{- end }} + ports: + - containerPort: {{ .Values.service.port}} + name: {{ .Values.kubeRBACProxy.portName }} + {{- if .Values.kubeRBACProxy.enableHostPort }} + hostPort: {{ .Values.service.port }} + {{- end }} + - containerPort: 8888 + name: "http-healthz" + readinessProbe: + httpGet: + scheme: HTTPS + port: 8888 + path: healthz + initialDelaySeconds: 5 + timeoutSeconds: 5 + {{- if .Values.kubeRBACProxy.resources }} + resources: + {{- toYaml .Values.kubeRBACProxy.resources | nindent 12 }} + {{- end }} + {{- if .Values.terminationMessageParams.enabled }} + {{- with .Values.terminationMessageParams }} + terminationMessagePath: {{ .terminationMessagePath }} + terminationMessagePolicy: {{ .terminationMessagePolicy }} + {{- end }} + {{- end }} + {{- with .Values.kubeRBACProxy.env }} + env: + {{- range $key, $value := $.Values.kubeRBACProxy.env }} + - name: {{ $key }} + value: {{ $value | quote }} + {{- end }} + {{- end }} + {{- if .Values.kubeRBACProxy.containerSecurityContext }} + securityContext: + {{ toYaml .Values.kubeRBACProxy.containerSecurityContext | nindent 12 }} + {{- end }} + {{- end }} + {{- if or .Values.imagePullSecrets .Values.global.imagePullSecrets }} + imagePullSecrets: + {{- include "prometheus-node-exporter.imagePullSecrets" (dict "Values" .Values "imagePullSecrets" .Values.imagePullSecrets) | indent 8 }} + {{- end }} + hostNetwork: {{ .Values.hostNetwork }} + hostPID: {{ .Values.hostPID }} + {{- with .Values.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.dnsConfig }} + dnsConfig: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.restartPolicy }} + restartPolicy: {{ . }} + {{- end }} + {{- with .Values.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} + volumes: + - name: proc + hostPath: + path: /proc + - name: sys + hostPath: + path: /sys + {{- if .Values.hostRootFsMount.enabled }} + - name: root + hostPath: + path: / + {{- end }} + {{- range $_, $mount := .Values.extraHostVolumeMounts }} + - name: {{ $mount.name }} + hostPath: + path: {{ $mount.hostPath }} + {{- with $mount.type }} + type: {{ . }} + {{- end }} + {{- end }} + {{- range $_, $mount := .Values.sidecarVolumeMount }} + - name: {{ $mount.name }} + emptyDir: + medium: Memory + {{- end }} + {{- range $_, $mount := .Values.sidecarHostVolumeMounts }} + - name: {{ $mount.name }} + hostPath: + path: {{ $mount.hostPath }} + {{- end }} + {{- range $_, $mount := .Values.configmaps }} + - name: {{ $mount.name }} + configMap: + name: {{ $mount.name }} + {{- end }} + {{- range $_, $mount := .Values.secrets }} + - name: {{ $mount.name }} + secret: + secretName: {{ $mount.name }} + {{- end }} + {{- if .Values.kubeRBACProxy.enabled }} + - name: kube-rbac-proxy-config + configMap: + name: {{ template "prometheus-node-exporter.fullname" . }}-rbac-config + {{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-node-exporter/templates/endpoints.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-node-exporter/templates/endpoints.yaml new file mode 100644 index 00000000000..56b695203a4 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-node-exporter/templates/endpoints.yaml @@ -0,0 +1,18 @@ +{{- if .Values.endpoints }} +apiVersion: v1 +kind: Endpoints +metadata: + name: {{ include "prometheus-node-exporter.fullname" . }} + namespace: {{ include "prometheus-node-exporter.namespace" . }} + labels: + {{- include "prometheus-node-exporter.labels" . | nindent 4 }} +subsets: + - addresses: + {{- range .Values.endpoints }} + - ip: {{ . }} + {{- end }} + ports: + - name: {{ .Values.service.portName }} + port: {{ .Values.service.port }} + protocol: TCP +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-node-exporter/templates/extra-manifests.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-node-exporter/templates/extra-manifests.yaml new file mode 100644 index 00000000000..2b21b710621 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-node-exporter/templates/extra-manifests.yaml @@ -0,0 +1,4 @@ +{{ range .Values.extraManifests }} +--- +{{ tpl . $ }} +{{ end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-node-exporter/templates/networkpolicy.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-node-exporter/templates/networkpolicy.yaml new file mode 100644 index 00000000000..825722729d3 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-node-exporter/templates/networkpolicy.yaml @@ -0,0 +1,23 @@ +{{- if .Values.networkPolicy.enabled }} +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: {{ include "prometheus-node-exporter.fullname" . }} + namespace: {{ include "prometheus-node-exporter.namespace" . }} + labels: + {{- include "prometheus-node-exporter.labels" $ | nindent 4 }} + {{- with .Values.service.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + ingress: + - ports: + - port: {{ .Values.service.port }} + policyTypes: + - Egress + - Ingress + podSelector: + matchLabels: + {{- include "prometheus-node-exporter.selectorLabels" . | nindent 6 }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-node-exporter/templates/podmonitor.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-node-exporter/templates/podmonitor.yaml new file mode 100644 index 00000000000..f88da6a34e6 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-node-exporter/templates/podmonitor.yaml @@ -0,0 +1,91 @@ +{{- if .Values.prometheus.podMonitor.enabled }} +apiVersion: {{ .Values.prometheus.podMonitor.apiVersion | default "monitoring.coreos.com/v1" }} +kind: PodMonitor +metadata: + name: {{ include "prometheus-node-exporter.fullname" . }} + namespace: {{ include "prometheus-node-exporter.podmonitor-namespace" . }} + labels: + {{- include "prometheus-node-exporter.labels" . | nindent 4 }} + {{- with .Values.prometheus.podMonitor.additionalLabels }} + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + jobLabel: {{ default "app.kubernetes.io/name" .Values.prometheus.podMonitor.jobLabel }} + {{- include "podmonitor.scrapeLimits" .Values.prometheus.podMonitor | nindent 2 }} + selector: + matchLabels: + {{- with .Values.prometheus.podMonitor.selectorOverride }} + {{- toYaml . | nindent 6 }} + {{- else }} + {{- include "prometheus-node-exporter.selectorLabels" . | nindent 6 }} + {{- end }} + namespaceSelector: + matchNames: + - {{ include "prometheus-node-exporter.namespace" . }} + {{- with .Values.prometheus.podMonitor.attachMetadata }} + attachMetadata: + {{- toYaml . | nindent 4 }} + {{- end }} + {{- with .Values.prometheus.podMonitor.podTargetLabels }} + podTargetLabels: + {{- toYaml . | nindent 4 }} + {{- end }} + podMetricsEndpoints: + - port: {{ .Values.service.portName }} + {{- with .Values.prometheus.podMonitor.scheme }} + scheme: {{ . }} + {{- end }} + {{- with .Values.prometheus.podMonitor.path }} + path: {{ . }} + {{- end }} + {{- with .Values.prometheus.podMonitor.basicAuth }} + basicAuth: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.prometheus.podMonitor.bearerTokenSecret }} + bearerTokenSecret: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.prometheus.podMonitor.tlsConfig }} + tlsConfig: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.prometheus.podMonitor.authorization }} + authorization: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.prometheus.podMonitor.oauth2 }} + oauth2: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.prometheus.podMonitor.proxyUrl }} + proxyUrl: {{ . }} + {{- end }} + {{- with .Values.prometheus.podMonitor.interval }} + interval: {{ . }} + {{- end }} + {{- with .Values.prometheus.podMonitor.honorTimestamps }} + honorTimestamps: {{ . }} + {{- end }} + {{- with .Values.prometheus.podMonitor.honorLabels }} + honorLabels: {{ . }} + {{- end }} + {{- with .Values.prometheus.podMonitor.scrapeTimeout }} + scrapeTimeout: {{ . }} + {{- end }} + {{- with .Values.prometheus.podMonitor.relabelings }} + relabelings: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.prometheus.podMonitor.metricRelabelings }} + metricRelabelings: + {{- toYaml . | nindent 8 }} + {{- end }} + enableHttp2: {{ default false .Values.prometheus.podMonitor.enableHttp2 }} + filterRunning: {{ default true .Values.prometheus.podMonitor.filterRunning }} + followRedirects: {{ default false .Values.prometheus.podMonitor.followRedirects }} + {{- with .Values.prometheus.podMonitor.params }} + params: + {{- toYaml . | nindent 8 }} + {{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-node-exporter/templates/psp-clusterrole.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-node-exporter/templates/psp-clusterrole.yaml new file mode 100644 index 00000000000..ee5bbba4a54 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-node-exporter/templates/psp-clusterrole.yaml @@ -0,0 +1,14 @@ +{{- if and (or .Values.global.cattle.psp.enable (and .Values.rbac.create .Values.rbac.pspEnabled)) (.Capabilities.APIVersions.Has "policy/v1beta1/PodSecurityPolicy") }} +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: psp-{{ include "prometheus-node-exporter.fullname" . }} + labels: + {{- include "prometheus-node-exporter.labels" . | nindent 4 }} +rules: +- apiGroups: ['extensions'] + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - {{ include "prometheus-node-exporter.fullname" . }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-node-exporter/templates/psp-clusterrolebinding.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-node-exporter/templates/psp-clusterrolebinding.yaml new file mode 100644 index 00000000000..160f2bbf7ac --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-node-exporter/templates/psp-clusterrolebinding.yaml @@ -0,0 +1,16 @@ +{{- if and (or .Values.global.cattle.psp.enable (and .Values.rbac.create .Values.rbac.pspEnabled)) (.Capabilities.APIVersions.Has "policy/v1beta1/PodSecurityPolicy") }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: psp-{{ include "prometheus-node-exporter.fullname" . }} + labels: + {{- include "prometheus-node-exporter.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: psp-{{ include "prometheus-node-exporter.fullname" . }} +subjects: + - kind: ServiceAccount + name: {{ include "prometheus-node-exporter.fullname" . }} + namespace: {{ include "prometheus-node-exporter.namespace" . }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-node-exporter/templates/psp.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-node-exporter/templates/psp.yaml new file mode 100644 index 00000000000..f3b52e1120e --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-node-exporter/templates/psp.yaml @@ -0,0 +1,49 @@ +{{- if and (or .Values.global.cattle.psp.enable (and .Values.rbac.create .Values.rbac.pspEnabled)) (.Capabilities.APIVersions.Has "policy/v1beta1/PodSecurityPolicy") }} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ include "prometheus-node-exporter.fullname" . }} + namespace: {{ include "prometheus-node-exporter.namespace" . }} + labels: + {{- include "prometheus-node-exporter.labels" . | nindent 4 }} + {{- with .Values.rbac.pspAnnotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + privileged: false + # Allow core volume types. + volumes: + - 'configMap' + - 'emptyDir' + - 'projected' + - 'secret' + - 'downwardAPI' + - 'persistentVolumeClaim' + - 'hostPath' + hostNetwork: true + hostIPC: false + hostPID: true + hostPorts: + - min: 0 + max: 65535 + runAsUser: + # Permits the container to run with root privileges as well. + rule: 'RunAsAny' + seLinux: + # This policy assumes the nodes are using AppArmor rather than SELinux. + rule: 'RunAsAny' + supplementalGroups: + rule: 'MustRunAs' + ranges: + # Allow adding the root group. + - min: 0 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + # Allow adding the root group. + - min: 0 + max: 65535 + readOnlyRootFilesystem: false +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-node-exporter/templates/rbac-configmap.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-node-exporter/templates/rbac-configmap.yaml new file mode 100644 index 00000000000..814e1103375 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-node-exporter/templates/rbac-configmap.yaml @@ -0,0 +1,16 @@ +{{- if .Values.kubeRBACProxy.enabled}} +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ template "prometheus-node-exporter.fullname" . }}-rbac-config + namespace: {{ include "prometheus-node-exporter.namespace" . }} +data: + config-file.yaml: |+ + authorization: + resourceAttributes: + namespace: {{ template "prometheus-node-exporter.namespace" . }} + apiVersion: v1 + resource: services + subresource: {{ template "prometheus-node-exporter.fullname" . }} + name: {{ template "prometheus-node-exporter.fullname" . }} +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-node-exporter/templates/service.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-node-exporter/templates/service.yaml new file mode 100644 index 00000000000..a065e46e390 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-node-exporter/templates/service.yaml @@ -0,0 +1,29 @@ +{{- if .Values.service.enabled }} +apiVersion: v1 +kind: Service +metadata: + name: {{ include "prometheus-node-exporter.fullname" . }} + namespace: {{ include "prometheus-node-exporter.namespace" . }} + labels: + {{- include "prometheus-node-exporter.labels" $ | nindent 4 }} + {{- with .Values.service.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: +{{- if .Values.service.ipDualStack.enabled }} + ipFamilies: {{ toYaml .Values.service.ipDualStack.ipFamilies | nindent 4 }} + ipFamilyPolicy: {{ .Values.service.ipDualStack.ipFamilyPolicy }} +{{- end }} + type: {{ .Values.service.type }} + ports: + - port: {{ .Values.service.port }} + {{- if ( and (eq .Values.service.type "NodePort" ) (not (empty .Values.service.nodePort)) ) }} + nodePort: {{ .Values.service.nodePort }} + {{- end }} + targetPort: {{ .Values.service.targetPort }} + protocol: TCP + name: {{ .Values.service.portName }} + selector: + {{- include "prometheus-node-exporter.selectorLabels" . | nindent 4 }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-node-exporter/templates/serviceaccount.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-node-exporter/templates/serviceaccount.yaml new file mode 100644 index 00000000000..5c3348c09ba --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-node-exporter/templates/serviceaccount.yaml @@ -0,0 +1,17 @@ +{{- if and .Values.rbac.create .Values.serviceAccount.create -}} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ include "prometheus-node-exporter.serviceAccountName" . }} + namespace: {{ include "prometheus-node-exporter.namespace" . }} + labels: + {{- include "prometheus-node-exporter.labels" . | nindent 4 }} + {{- with .Values.serviceAccount.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +{{- if or .Values.serviceAccount.imagePullSecrets .Values.global.imagePullSecrets }} +imagePullSecrets: + {{- include "prometheus-node-exporter.imagePullSecrets" (dict "Values" .Values "imagePullSecrets" .Values.serviceAccount.imagePullSecrets) | indent 2 }} +{{- end }} +{{- end -}} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-node-exporter/templates/servicemonitor.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-node-exporter/templates/servicemonitor.yaml new file mode 100644 index 00000000000..6d6e4404738 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-node-exporter/templates/servicemonitor.yaml @@ -0,0 +1,71 @@ +{{- if .Values.prometheus.monitor.enabled }} +apiVersion: {{ .Values.prometheus.monitor.apiVersion | default "monitoring.coreos.com/v1" }} +kind: ServiceMonitor +metadata: + name: {{ include "prometheus-node-exporter.fullname" . }} + namespace: {{ include "prometheus-node-exporter.monitor-namespace" . }} + labels: + {{- include "prometheus-node-exporter.labels" . | nindent 4 }} + {{- with .Values.prometheus.monitor.additionalLabels }} + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + jobLabel: {{ default "app.kubernetes.io/name" .Values.prometheus.monitor.jobLabel }} + {{- include "servicemonitor.scrapeLimits" .Values.prometheus.monitor | nindent 2 }} + {{- with .Values.prometheus.monitor.podTargetLabels }} + podTargetLabels: + {{- toYaml . | nindent 4 }} + {{- end }} + selector: + matchLabels: + {{- with .Values.prometheus.monitor.selectorOverride }} + {{- toYaml . | nindent 6 }} + {{- else }} + {{- include "prometheus-node-exporter.selectorLabels" . | nindent 6 }} + {{- end }} + {{- with .Values.prometheus.monitor.attachMetadata }} + attachMetadata: + {{- toYaml . | nindent 4 }} + {{- end }} + endpoints: + - port: {{ .Values.service.portName }} + scheme: {{ .Values.prometheus.monitor.scheme }} + {{- with .Values.prometheus.monitor.basicAuth }} + basicAuth: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.prometheus.monitor.bearerTokenFile }} + bearerTokenFile: {{ . }} + {{- end }} + {{- with .Values.prometheus.monitor.tlsConfig }} + tlsConfig: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.prometheus.monitor.proxyUrl }} + proxyUrl: {{ . }} + {{- end }} + {{- with .Values.prometheus.monitor.interval }} + interval: {{ . }} + {{- end }} + {{- with .Values.prometheus.monitor.scrapeTimeout }} + scrapeTimeout: {{ . }} + {{- end }} + {{- with .Values.prometheus.monitor.relabelings }} + relabelings: + {{- toYaml . | nindent 8 }} + {{- end }} + metricRelabelings: + {{- with .Values.prometheus.monitor.metricRelabelings }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{ if .Values.global.cattle.clusterId }} + - sourceLabels: [__address__] + targetLabel: cluster_id + replacement: {{ .Values.global.cattle.clusterId }} + {{- end }} + {{ if .Values.global.cattle.clusterName }} + - sourceLabels: [__address__] + targetLabel: cluster_name + replacement: {{ .Values.global.cattle.clusterName }} + {{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-node-exporter/templates/verticalpodautoscaler.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-node-exporter/templates/verticalpodautoscaler.yaml new file mode 100644 index 00000000000..2c2705f8724 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-node-exporter/templates/verticalpodautoscaler.yaml @@ -0,0 +1,40 @@ +{{- if and (.Capabilities.APIVersions.Has "autoscaling.k8s.io/v1") (.Values.verticalPodAutoscaler.enabled) }} +apiVersion: autoscaling.k8s.io/v1 +kind: VerticalPodAutoscaler +metadata: + name: {{ include "prometheus-node-exporter.fullname" . }} + namespace: {{ include "prometheus-node-exporter.namespace" . }} + labels: + {{- include "prometheus-node-exporter.labels" . | nindent 4 }} +spec: + {{- with .Values.verticalPodAutoscaler.recommenders }} + recommenders: + {{- toYaml . | nindent 4 }} + {{- end }} + resourcePolicy: + containerPolicies: + - containerName: node-exporter + {{- with .Values.verticalPodAutoscaler.controlledResources }} + controlledResources: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.verticalPodAutoscaler.controlledValues }} + controlledValues: {{ . }} + {{- end }} + {{- with .Values.verticalPodAutoscaler.maxAllowed }} + maxAllowed: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.verticalPodAutoscaler.minAllowed }} + minAllowed: + {{- toYaml . | nindent 8 }} + {{- end }} + targetRef: + apiVersion: apps/v1 + kind: DaemonSet + name: {{ include "prometheus-node-exporter.fullname" . }} + {{- with .Values.verticalPodAutoscaler.updatePolicy }} + updatePolicy: + {{- toYaml . | nindent 4 }} + {{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-node-exporter/values.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-node-exporter/values.yaml new file mode 100644 index 00000000000..b9f2f7ab87b --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/prometheus-node-exporter/values.yaml @@ -0,0 +1,530 @@ +# Default values for prometheus-node-exporter. +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. + +image: + registry: docker.io + repository: rancher/mirrored-prometheus-node-exporter + # Overrides the image tag whose default is {{ printf "v%s" .Chart.AppVersion }} + tag: v1.7.0 + pullPolicy: IfNotPresent + digest: "" + +imagePullSecrets: [] +# - name: "image-pull-secret" +nameOverride: "" +fullnameOverride: "" + +# Number of old history to retain to allow rollback +# Default Kubernetes value is set to 10 +revisionHistoryLimit: 10 + +global: + cattle: + psp: + enable: true + systemDefaultRegistry: "" + + # To help compatibility with other charts which use global.imagePullSecrets. + # Allow either an array of {name: pullSecret} maps (k8s-style), or an array of strings (more common helm-style). + # global: + # imagePullSecrets: + # - name: pullSecret1 + # - name: pullSecret2 + # or + # global: + # imagePullSecrets: + # - pullSecret1 + # - pullSecret2 + imagePullSecrets: [] + # + # Allow parent charts to override registry hostname + imageRegistry: "docker.io" + +# Configure kube-rbac-proxy. When enabled, creates a kube-rbac-proxy to protect the node-exporter http endpoint. +# The requests are served through the same service but requests are HTTPS. +kubeRBACProxy: + enabled: false + ## Set environment variables as name/value pairs + env: {} + # VARIABLE: value + image: + registry: docker.io + repository: rancher/mirrored-kube-rbac-proxy + tag: v0.15.0 + sha: "" + pullPolicy: IfNotPresent + + # List of additional cli arguments to configure kube-rbac-proxy + # for example: --tls-cipher-suites, --log-file, etc. + # all the possible args can be found here: https://github.com/brancz/kube-rbac-proxy#usage + extraArgs: [] + + ## Specify security settings for a Container + ## Allows overrides and additional options compared to (Pod) securityContext + ## Ref: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/#set-the-security-context-for-a-container + containerSecurityContext: {} + + # Specify the port used for the Node exporter container (upstream port) + port: 8100 + # Specify the name of the container port + portName: http + # Configure a hostPort. If true, hostPort will be enabled in the container and set to service.port. + enableHostPort: false + + resources: {} + # We usually recommend not to specify default resources and to leave this as a conscious + # choice for the user. This also increases chances charts run on environments with little + # resources, such as Minikube. If you do want to specify resources, uncomment the following + # lines, adjust them as necessary, and remove the curly braces after 'resources:'. + # limits: + # cpu: 100m + # memory: 64Mi + # requests: + # cpu: 10m + # memory: 32Mi + +service: + enabled: true + type: ClusterIP + port: 9796 + targetPort: 9796 + nodePort: + portName: metrics + listenOnAllInterfaces: true + annotations: + prometheus.io/scrape: "true" + ipDualStack: + enabled: false + ipFamilies: ["IPv6", "IPv4"] + ipFamilyPolicy: "PreferDualStack" + +# Set a NetworkPolicy with: +# ingress only on service.port +# no egress permitted +networkPolicy: + enabled: false + +# Additional environment variables that will be passed to the daemonset +env: {} +## env: +## VARIABLE: value + +prometheus: + monitor: + enabled: false + additionalLabels: {} + namespace: "" + + jobLabel: "" + + # List of pod labels to add to node exporter metrics + # https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#servicemonitor + podTargetLabels: [] + + scheme: http + basicAuth: {} + bearerTokenFile: + tlsConfig: {} + + ## proxyUrl: URL of a proxy that should be used for scraping. + ## + proxyUrl: "" + + ## Override serviceMonitor selector + ## + selectorOverride: {} + + ## Attach node metadata to discovered targets. Requires Prometheus v2.35.0 and above. + ## + attachMetadata: + node: false + + relabelings: [] + metricRelabelings: [] + interval: "" + scrapeTimeout: 10s + ## prometheus.monitor.apiVersion ApiVersion for the serviceMonitor Resource(defaults to "monitoring.coreos.com/v1") + apiVersion: "" + + ## SampleLimit defines per-scrape limit on number of scraped samples that will be accepted. + ## + sampleLimit: 0 + + ## TargetLimit defines a limit on the number of scraped targets that will be accepted. + ## + targetLimit: 0 + + ## Per-scrape limit on number of labels that will be accepted for a sample. Only valid in Prometheus versions 2.27.0 and newer. + ## + labelLimit: 0 + + ## Per-scrape limit on length of labels name that will be accepted for a sample. Only valid in Prometheus versions 2.27.0 and newer. + ## + labelNameLengthLimit: 0 + + ## Per-scrape limit on length of labels value that will be accepted for a sample. Only valid in Prometheus versions 2.27.0 and newer. + ## + labelValueLengthLimit: 0 + + # PodMonitor defines monitoring for a set of pods. + # ref. https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#monitoring.coreos.com/v1.PodMonitor + # Using a PodMonitor may be preferred in some environments where there is very large number + # of Node Exporter endpoints (1000+) behind a single service. + # The PodMonitor is disabled by default. When switching from ServiceMonitor to PodMonitor, + # the time series resulting from the configuration through PodMonitor may have different labels. + # For instance, there will not be the service label any longer which might + # affect PromQL queries selecting that label. + podMonitor: + enabled: false + # Namespace in which to deploy the pod monitor. Defaults to the release namespace. + namespace: "" + # Additional labels, e.g. setting a label for pod monitor selector as set in prometheus + additionalLabels: {} + # release: kube-prometheus-stack + # PodTargetLabels transfers labels of the Kubernetes Pod onto the target. + podTargetLabels: [] + # apiVersion defaults to monitoring.coreos.com/v1. + apiVersion: "" + # Override pod selector to select pod objects. + selectorOverride: {} + # Attach node metadata to discovered targets. Requires Prometheus v2.35.0 and above. + attachMetadata: + node: false + # The label to use to retrieve the job name from. Defaults to label app.kubernetes.io/name. + jobLabel: "" + + # Scheme/protocol to use for scraping. + scheme: "http" + # Path to scrape metrics at. + path: "/metrics" + + # BasicAuth allow an endpoint to authenticate over basic authentication. + # More info: https://prometheus.io/docs/operating/configuration/#endpoint + basicAuth: {} + # Secret to mount to read bearer token for scraping targets. + # The secret needs to be in the same namespace as the pod monitor and accessible by the Prometheus Operator. + # https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.24/#secretkeyselector-v1-core + bearerTokenSecret: {} + # TLS configuration to use when scraping the endpoint. + tlsConfig: {} + # Authorization section for this endpoint. + # https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#monitoring.coreos.com/v1.SafeAuthorization + authorization: {} + # OAuth2 for the URL. Only valid in Prometheus versions 2.27.0 and newer. + # https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#monitoring.coreos.com/v1.OAuth2 + oauth2: {} + + # ProxyURL eg http://proxyserver:2195. Directs scrapes through proxy to this endpoint. + proxyUrl: "" + # Interval at which endpoints should be scraped. If not specified Prometheus’ global scrape interval is used. + interval: "" + # Timeout after which the scrape is ended. If not specified, the Prometheus global scrape interval is used. + scrapeTimeout: "" + # HonorTimestamps controls whether Prometheus respects the timestamps present in scraped data. + honorTimestamps: true + # HonorLabels chooses the metric’s labels on collisions with target labels. + honorLabels: true + # Whether to enable HTTP2. Default false. + enableHttp2: "" + # Drop pods that are not running. (Failed, Succeeded). + # Enabled by default. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle/#pod-phase + filterRunning: "" + # FollowRedirects configures whether scrape requests follow HTTP 3xx redirects. Default false. + followRedirects: "" + # Optional HTTP URL parameters + params: {} + + # RelabelConfigs to apply to samples before scraping. Prometheus Operator automatically adds + # relabelings for a few standard Kubernetes fields. The original scrape job’s name + # is available via the __tmp_prometheus_job_name label. + # More info: https://prometheus.io/docs/prometheus/latest/configuration/configuration/#relabel_config + relabelings: [] + # MetricRelabelConfigs to apply to samples before ingestion. + metricRelabelings: [] + + # SampleLimit defines per-scrape limit on number of scraped samples that will be accepted. + sampleLimit: 0 + # TargetLimit defines a limit on the number of scraped targets that will be accepted. + targetLimit: 0 + # Per-scrape limit on number of labels that will be accepted for a sample. + # Only valid in Prometheus versions 2.27.0 and newer. + labelLimit: 0 + # Per-scrape limit on length of labels name that will be accepted for a sample. + # Only valid in Prometheus versions 2.27.0 and newer. + labelNameLengthLimit: 0 + # Per-scrape limit on length of labels value that will be accepted for a sample. + # Only valid in Prometheus versions 2.27.0 and newer. + labelValueLengthLimit: 0 + +## Customize the updateStrategy if set +updateStrategy: + type: RollingUpdate + rollingUpdate: + maxUnavailable: 1 + +resources: {} + # We usually recommend not to specify default resources and to leave this as a conscious + # choice for the user. This also increases chances charts run on environments with little + # resources, such as Minikube. If you do want to specify resources, uncomment the following + # lines, adjust them as necessary, and remove the curly braces after 'resources:'. + # limits: + # cpu: 200m + # memory: 50Mi + # requests: + # cpu: 100m + # memory: 30Mi + +# Specify the container restart policy passed to the Node Export container +# Possible Values: Always (default)|OnFailure|Never +restartPolicy: null + +serviceAccount: + # Specifies whether a ServiceAccount should be created + create: true + # The name of the ServiceAccount to use. + # If not set and create is true, a name is generated using the fullname template + name: + annotations: {} + imagePullSecrets: [] + automountServiceAccountToken: false + +securityContext: + fsGroup: 65534 + runAsGroup: 65534 + runAsNonRoot: true + runAsUser: 65534 + +containerSecurityContext: + readOnlyRootFilesystem: true + # capabilities: + # add: + # - SYS_TIME + +rbac: + ## If true, create & use RBAC resources + ## + create: true + pspAnnotations: {} + +# for deployments that have node_exporter deployed outside of the cluster, list +# their addresses here +endpoints: [] + +# Expose the service to the host network +hostNetwork: true + +# Share the host process ID namespace +hostPID: true + +# Mount the node's root file system (/) at /host/root in the container +hostRootFsMount: + enabled: true + # Defines how new mounts in existing mounts on the node or in the container + # are propagated to the container or node, respectively. Possible values are + # None, HostToContainer, and Bidirectional. If this field is omitted, then + # None is used. More information on: + # https://kubernetes.io/docs/concepts/storage/volumes/#mount-propagation + mountPropagation: HostToContainer + +# Mount the node's proc file system (/proc) at /host/proc in the container +hostProcFsMount: + # Possible values are None, HostToContainer, and Bidirectional + mountPropagation: "" + +# Mount the node's sys file system (/sys) at /host/sys in the container +hostSysFsMount: + # Possible values are None, HostToContainer, and Bidirectional + mountPropagation: "" + +## Assign a group of affinity scheduling rules +## +affinity: {} +# nodeAffinity: +# requiredDuringSchedulingIgnoredDuringExecution: +# nodeSelectorTerms: +# - matchFields: +# - key: metadata.name +# operator: In +# values: +# - target-host-name + +# Annotations to be added to node exporter pods +podAnnotations: + # Fix for very slow GKE cluster upgrades + cluster-autoscaler.kubernetes.io/safe-to-evict: "true" + +# Extra labels to be added to node exporter pods +podLabels: {} + +# Annotations to be added to node exporter daemonset +daemonsetAnnotations: {} + +## set to true to add the release label so scraping of the servicemonitor with kube-prometheus-stack works out of the box +releaseLabel: false + +# Custom DNS configuration to be added to prometheus-node-exporter pods +dnsConfig: {} +# nameservers: +# - 1.2.3.4 +# searches: +# - ns1.svc.cluster-domain.example +# - my.dns.search.suffix +# options: +# - name: ndots +# value: "2" +# - name: edns0 + +## Assign a nodeSelector if operating a hybrid cluster +## +nodeSelector: + kubernetes.io/os: linux + # kubernetes.io/arch: amd64 + +# Specify grace period for graceful termination of pods. Defaults to 30 if null or not specified +terminationGracePeriodSeconds: null + +tolerations: + - effect: NoSchedule + operator: Exists + - effect: NoExecute + operator: Exists + +# Enable or disable container termination message settings +# https://kubernetes.io/docs/tasks/debug/debug-application/determine-reason-pod-failure/ +terminationMessageParams: + enabled: false + # If enabled, specify the path for termination messages + terminationMessagePath: /dev/termination-log + # If enabled, specify the policy for termination messages + terminationMessagePolicy: File + + +## Assign a PriorityClassName to pods if set +# priorityClassName: "" + +## Additional container arguments +## +extraArgs: [] +# - --collector.diskstats.ignored-devices=^(ram|loop|fd|(h|s|v)d[a-z]|nvme\\d+n\\d+p)\\d+$ +# - --collector.textfile.directory=/run/prometheus + +## Additional mounts from the host to node-exporter container +## +extraHostVolumeMounts: [] +# - name: +# hostPath: +# https://kubernetes.io/docs/concepts/storage/volumes/#hostpath-volume-types +# type: "" (Default)|DirectoryOrCreate|Directory|FileOrCreate|File|Socket|CharDevice|BlockDevice +# mountPath: +# readOnly: true|false +# mountPropagation: None|HostToContainer|Bidirectional + +## Additional configmaps to be mounted. +## +configmaps: [] +# - name: +# mountPath: +secrets: [] +# - name: +# mountPath: +## Override the deployment namespace +## +namespaceOverride: "" + +## Additional containers for export metrics to text file; fields image,imagePullPolicy,securityContext take default value from main container +## +sidecars: [] +# - name: nvidia-dcgm-exporter +# image: nvidia/dcgm-exporter:1.4.3 +# volumeMounts: +# - name: tmp +# mountPath: /tmp + +## Volume for sidecar containers +## +sidecarVolumeMount: [] +# - name: collector-textfiles +# mountPath: /run/prometheus +# readOnly: false + +## Additional mounts from the host to sidecar containers +## +sidecarHostVolumeMounts: [] +# - name: +# hostPath: +# mountPath: +# readOnly: true|false +# mountPropagation: None|HostToContainer|Bidirectional + +## Additional InitContainers to initialize the pod +## +extraInitContainers: [] + +## Liveness probe +## +livenessProbe: + failureThreshold: 3 + httpGet: + httpHeaders: [] + scheme: http + initialDelaySeconds: 0 + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 1 + +## Readiness probe +## +readinessProbe: + failureThreshold: 3 + httpGet: + httpHeaders: [] + scheme: http + initialDelaySeconds: 0 + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 1 + +# Enable vertical pod autoscaler support for prometheus-node-exporter +verticalPodAutoscaler: + enabled: false + + # Recommender responsible for generating recommendation for the object. + # List should be empty (then the default recommender will generate the recommendation) + # or contain exactly one recommender. + # recommenders: + # - name: custom-recommender-performance + + # List of resources that the vertical pod autoscaler can control. Defaults to cpu and memory + controlledResources: [] + # Specifies which resource values should be controlled: RequestsOnly or RequestsAndLimits. + # controlledValues: RequestsAndLimits + + # Define the max allowed resources for the pod + maxAllowed: {} + # cpu: 200m + # memory: 100Mi + # Define the min allowed resources for the pod + minAllowed: {} + # cpu: 200m + # memory: 100Mi + + # updatePolicy: + # Specifies minimal number of replicas which need to be alive for VPA Updater to attempt pod eviction + # minReplicas: 1 + # Specifies whether recommended updates are applied when a Pod is started and whether recommended updates + # are applied during the life of a Pod. Possible values are "Off", "Initial", "Recreate", and "Auto". + # updateMode: Auto + +# Extra manifests to deploy as an array +extraManifests: [] + # - | + # apiVersion: v1 + # kind: ConfigMap + # metadata: + # name: prometheus-extra + # data: + # extra-data: "value" + +# Override version of app, required if image.tag is defined and does not follow semver +version: "" diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2ControllerManager/.helmignore b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2ControllerManager/.helmignore new file mode 100644 index 00000000000..0e8a0eb36f4 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2ControllerManager/.helmignore @@ -0,0 +1,23 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*.orig +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2ControllerManager/Chart.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2ControllerManager/Chart.yaml new file mode 100644 index 00000000000..c50eeb0fafb --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2ControllerManager/Chart.yaml @@ -0,0 +1,15 @@ +annotations: + catalog.cattle.io/hidden: "true" + catalog.cattle.io/kube-version: '>= 1.26.0-0 < 1.31.0-0' + catalog.cattle.io/os: linux + catalog.rancher.io/certified: rancher + catalog.rancher.io/namespace: cattle-monitoring-system + catalog.rancher.io/release-name: rancher-pushprox +apiVersion: v1 +appVersion: v0.1.4-rancher2 +description: Sets up a deployment of the PushProx proxy and a DaemonSet of PushProx + clients. +kubeVersion: '>=1.26.0-0' +name: rke2ControllerManager +type: application +version: 0.2.1 diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2ControllerManager/README.md b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2ControllerManager/README.md new file mode 100644 index 00000000000..345002f48a2 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2ControllerManager/README.md @@ -0,0 +1,90 @@ +# rancher-pushprox + +A Rancher chart based on Rancher [PushProx](https://github.com/rancher/PushProx) that sets up a Deployment of a PushProx proxy and a DaemonSet of PushProx clients on a Kubernetes cluster. + +Installs [rancher-pushprox](https://github.com/rancher/charts/tree/gh-pages/packages/rancher-pushprox) to create PushProx clients that can access their host's network and register with a PushProx proxy. A [Prometheus Operator](https://github.com/coreos/prometheus-operator) ServiceMonitor CR is also included that is configured to scrape the metrics from each of the clients through the proxy. + +Using an instance of this chart is suitable for the following scenarios: +- You need to scrape metrics from a port that should not be accessible outside of the host (e.g. scraping `etcd` metrics in a hardened cluster) +- You need to scrape metrics on a host that are not exposed outside of 127.0.0.1 (e.g. scraping `kube-proxy` metrics) +- You need to scrape metrics through HTTPS using certs hosted directly on `hostPath` +- You need to scrape metrics from Kubernetes components that require authorization via a service account (e.g. permissions to make request to `/metrics`) +- You need to scrape metrics without access to cacerts (i.e. enable `insecureSkipVerify`) + +The clients and proxy are created based on a Rancher fork of the [prometheus-community/PushProx](https://github.com/prometheus-community/PushProx) project. + +## Upgrading to Kubernetes v1.25+ + +Starting in Kubernetes v1.25, [Pod Security Policies](https://kubernetes.io/docs/concepts/security/pod-security-policy/) have been removed from the Kubernetes API. + +As a result, **before upgrading to Kubernetes v1.25** (or on a fresh install in a Kubernetes v1.25+ cluster), users are expected to perform an in-place upgrade of this chart with `global.cattle.psp.enabled` set to `false` if it has been previously set to `true`. +​ +> **Note:** +> In this chart release, any previous field that was associated with any PSP resources have been removed in favor of a single global field: `global.cattle.psp.enabled`. + +> **Note:** +> If you upgrade your cluster to Kubernetes v1.25+ before removing PSPs via a `helm upgrade` (even if you manually clean up resources), **it will leave the Helm release in a broken state within the cluster such that further Helm operations will not work (`helm uninstall`, `helm upgrade`, etc.).** +> +> If your charts get stuck in this state, please consult the Rancher docs on how to clean up your Helm release secrets. + +Upon setting `global.cattle.psp.enabled` to false, the chart will remove any PSP resources deployed on its behalf from the cluster. This is the default setting for this chart. + +As a replacement for PSPs, [Pod Security Admission](https://kubernetes.io/docs/concepts/security/pod-security-admission/) should be used. Please consult the Rancher docs for more details on how to configure your chart release namespaces to work with the new Pod Security Admission and apply Pod Security Standards. + +## Configuration + +The following tables list the configurable parameters of the rancher-pushprox chart and their default values. + +### General + +#### Required +| Parameter | Description | Example | +| ----- | ----------- | ------ | +| `component` | The component that is being monitored | `kube-etcd` +| `metricsPort` | The port on the host that contains the metrics you want to scrape (e.g. `http://:/metrics`) | `2379` | +| `namespaceOverride` | The namespace to install the chart | `""` + +#### Optional +| Parameter | Description | Default | +| ----- | ----------- | ------ | +| `serviceMonitor.enabled` | Deploys a [Prometheus Operator](https://github.com/coreos/prometheus-operator/blob/master/Documentation/api.md#servicemonitor) ServiceMonitor CR that is configured to scrape metrics on the hosts that the clients are deployed on via the proxy. Also deploys a Service that points to all pods with the expected client name that exposes the `metricsPort` selected | `true` | +| `serviceMonitor.endpoints` | A list of endpoints that will be added to the ServiceMonitor based on the [Endpoint spec](https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#endpoint) | `[{port: metrics}]` | +| `service.selector` | The selector that is used to populate the Service's Endpoints object. The chart will error out on rendering templating if `.Values.clients.enabled` is set alongside this field, since it is expected that this service should point to the PushProx Clients Daemonset / Deployment | `{}` | +| `clients.enabled` | Deploys a DaemonSet of clients that are each capable of scraping endpoints on the hostNetwork it is deployed on | `true` | +| `clients.port` | The port where the client will publish PushProx client-specific metrics. If deploying multiple clients onto the same node, the clients should not have conflicting ports | `9369` | +| `clients.proxyUrl` | Overrides the default proxyUrl setting of `http://pushprox-{{ .Values.component }}-proxy.{{ . Release.Namespace }}.svc.cluster.local:{{ .Values.proxy.port }}"` with the `proxyUrl` specified | `""` | +| `clients.useLocalhost` | Sets a flag on each client deployment to redirect scrapes directed to `HOST_IP` to `127.0.0.1` | `false` | +| `clients.https.enabled` | Enables scraping metrics via HTTPS using the provided TLS certs that exist on each host | `false` | +| `clients.https.forceHTTPSScheme` | Forces scraping metrics via HTTPS using the provided TLS certs that exist on each host | `false` | +| `clients.https.useServiceAccountCredentials` | If set to true, the client will create a service account with permissions to scrape `/metrics` endpoint of Kubernetes components. The client will use the service account token provided to make authorized scrape requests to the Kubernetes API | `false` | +| `clients.https.authenticationMethod.bearerTokenFile.enabled` | If set to true, the client will use service account credentials mounted at the configured path `clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath`. This requires permissions to scrape `/metrics` endpoint of Kubernetes components. This method is deprecated by the prometheus operator and may be removed in a future release | `false` | +| `clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath` | This is a volume mount on the pod with permissions to scrape `/metrics` endpoint of Kubernetes components | `"/var/run/secrets/kubernetes.io/serviceaccount/token"` | +| `clients.https.authenticationMethod.bearerTokenSecret.enabled` | If set to true, the client will use service account credentials to scrape `/metrics` endpoint of Kubernetes components. This method is deprecated by the prometheus operator and may be removed in a future release | `false` | +| `clients.https.authenticationMethod.authorization.enabled` | If set to true, the client will use service account credentials to scrape `/metrics` endpoint of Kubernetes components | `false` | +| `clients.https.authenticationMethod.authorization.type` | If set, the client will use this type of authorization in its client requests for metrics | `"bearer"` | +| `clients.https.authenticationMethod.authorization.credentials.key` | If set, the client will use this key in the secret created by `clients.https.useServiceAccountCredentials` for authorization in its client requests for metrics | `"token"` | +| `clients.https.authenticationMethod.authorization.credentials.optional` | If set to false, the client will fail if the key in the secret created by `clients.https.useServiceAccountCredentials` does not exist | `false` | +| `clients.https.insecureSkipVerify` | If set to true, the client will disable SSL security checks | `false` | +| `clients.https.certDir` | A `hostPath` where TLS certs can be found. This path is mounted as a volume on an `initContainer` which copies only the necessary files over to an EmptyDir volume used by each client. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.certFile` | The path to the TLS cert file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.keyFile` | The path to the TLS key file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.caCertFile` | The path to the TLS cacert file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.seLinuxOptions` | seLinuxOptions to be passed into the container that copies certs. Should define a container with permissions to read the files in the certDir provided on the host. Required and only used if `clients.https.enabled` is set and `clients.https.certDir` is provided. | `""` | +| `clients.metrics.enabled` | Whether the client should publish PushProx client-specific metrics. | `false` | +| `clients.rbac.additionalRules` | Additional permissions to provide to the ServiceAccount bound to the client. This can be used to provide additional permissions for the client to scrape metrics from the k8s API. Only enabled if clients.https.enabled and clients.https.useServiceAccountCredentials are true | `[]` | +| `clients.deployment.enabled` | Deploys the client as a Deployment (generally used if the underlying hostNetwork Pod that is being scraped is managed by a Deployment) | `false` | +| `clients.deployment.replicas` | The number of pods the Deployment has, it should match the number of pod the hostNetwork Deployment has. Required and only used if `client.deployment.enable` is set | `0` | +| `clients.deployment.affinity` | The affinity rules that allocate the pod to the node in which the hostNetwork Deployment's pods run. Required and only used if `client.deployment.enable` is set | `{}` | +| `clients.resources` | Set resource limits and requests for the client container | `{}` | +| `clients.nodeSelector` | Select which nodes to deploy the clients on | `{}` | +| `clients.tolerations` | Specify tolerations for clients | `[]` | +| `proxy.enabled` | Deploys the proxy that each client will register with | `true` | +| `proxy.port` | The port exposed by the proxy that each client will register with to allow metrics to be scraped from the host | `8080` | +| `proxy.resources` | Set resource limits and requests for the proxy container | `{}` | +| `proxy.nodeSelector` | Select which nodes the proxy can be deployed on | `{}` | +| `proxy.tolerations` | Specify tolerations (if necessary) to allow the proxy to be deployed on the selected node | `[]` | +| `kubeVersionOverrides` | A list of Semver constraint strings (defined by https://github.com/Masterminds/semver) and values.yaml overrides. For each key in kubeVersionOverrides, this chart will check to see if the current Kubernetes cluster's version matches any of the semver constraints provided as keys on the map. On seeing a match, the default value for each values.yaml field overridden will be updated with the new value. If multiple matches are encountered (due to overlapping semver ranges), the matches will be applied in order. | `[]` + +*Tip: The filepaths set in `clients.https.File` can include wildcard characters*. + +See [rancher-monitoring](https://github.com/rancher/charts/tree/gh-pages/packages/rancher-monitoring) for examples of how this chart can be used. diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2ControllerManager/templates/_helpers.tpl b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2ControllerManager/templates/_helpers.tpl new file mode 100644 index 00000000000..1ba50939441 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2ControllerManager/templates/_helpers.tpl @@ -0,0 +1,170 @@ +# Rancher + +{{- define "system_default_registry" -}} +{{- if .Values.global.cattle.systemDefaultRegistry -}} +{{- printf "%s/" .Values.global.cattle.systemDefaultRegistry -}} +{{- end -}} +{{- end -}} + +# Windows Support + +{{/* +Windows cluster will add default taint for linux nodes, +add below linux tolerations to workloads could be scheduled to those linux nodes +*/}} + +{{- define "linux-node-tolerations" -}} +- key: "cattle.io/os" + value: "linux" + effect: "NoSchedule" + operator: "Equal" +{{- end -}} + +{{- define "linux-node-selector" -}} +{{- if semverCompare "<1.14-0" .Capabilities.KubeVersion.GitVersion -}} +beta.kubernetes.io/os: linux +{{- else -}} +kubernetes.io/os: linux +{{- end -}} +{{- end -}} + +# General + +{{- define "applyKubeVersionOverrides" -}} +{{- $overrides := dict -}} +{{- range $override := .Values.kubeVersionOverrides -}} +{{- if semverCompare $override.constraint $.Capabilities.KubeVersion.Version -}} +{{- $_ := mergeOverwrite $overrides $override.values -}} +{{- end -}} +{{- end -}} +{{- $_ := mergeOverwrite .Values $overrides -}} +{{- end -}} + +{{- define "pushprox.namespace" -}} + {{- if .Values.namespaceOverride -}} + {{- .Values.namespaceOverride -}} + {{- else -}} + {{- .Release.Namespace -}} + {{- end -}} +{{- end -}} + +{{- define "pushProxy.commonLabels" -}} +release: {{ .Release.Name }} +component: {{ .Values.component | quote }} +provider: kubernetes +{{- end -}} + +{{- define "pushProxy.proxyUrl" -}} +{{- $_ := (required "Template requires either .Values.proxy.port or .Values.client.proxyUrl to set proxyUrl for client" (or .Values.clients.proxyUrl .Values.proxy.port)) -}} +{{- if .Values.clients.proxyUrl -}} +{{ printf "%s" .Values.clients.proxyUrl }} +{{- else -}} +{{ printf "http://%s.%s.svc:%d" (include "pushProxy.proxy.name" .) (include "pushprox.namespace" .) (int .Values.proxy.port) }} +{{- end -}}{{- end -}} + +# Client + +{{- define "pushProxy.client.name" -}} +{{- printf "pushprox-%s-client" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.client.serviceAccountTokenName" -}} +{{- printf "pushprox-%s-client-service-account-token" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.client.labels" -}} +k8s-app: {{ template "pushProxy.client.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +# Proxy + +{{- define "pushProxy.proxy.name" -}} +{{- printf "pushprox-%s-proxy" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.proxy.labels" -}} +k8s-app: {{ template "pushProxy.proxy.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +# ServiceMonitor + +{{- define "pushprox.serviceMonitor.name" -}} +{{- printf "%s-%s" .Release.Name (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.serviceMonitor.labels" -}} +app: {{ template "pushprox.serviceMonitor.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +{{- define "pushProxy.serviceMonitor.endpoints" -}} +{{- $proxyURL := (include "pushProxy.proxyUrl" .) -}} +{{- $useHTTPS := .Values.clients.https.enabled -}} +{{- $setHTTPSScheme := .Values.clients.https.forceHTTPSScheme -}} +{{- $insecureSkipVerify := .Values.clients.https.insecureSkipVerify -}} +{{- $useServiceAccountCredentials := .Values.clients.https.useServiceAccountCredentials -}} +{{- $serviceAccountTokenName := (include "pushProxy.client.serviceAccountTokenName" . ) -}} +{{- $metricRelabelings := list }} +{{- $endpoints := .Values.serviceMonitor.endpoints }} +{{- if .Values.proxy.enabled }} +{{- $_ := set . "proxyUrl" $proxyURL }} +{{- end }} +{{- range $endpoints }} +{{- if $.Values.proxy.enabled }} +{{- $_ := set . "proxyUrl" $proxyURL }} +{{- end }} +{{- $clusterIdRelabel := dict }} +{{- $metricRelabelings := list }} +{{- if $.Values.global.cattle.clusterId }} +{{- $_ := set $clusterIdRelabel "action" "replace" }} +{{- $_ := set $clusterIdRelabel "sourceLabels" (list "__address__") }} +{{- $_ := set $clusterIdRelabel "targetLabel" "cluster_id" }} +{{- $_ := set $clusterIdRelabel "replacement" $.Values.global.cattle.clusterId }} +{{- $metricRelabelings = append $metricRelabelings $clusterIdRelabel }} +{{- end }} +{{- $clusterNameRelabel := dict }} +{{- if $.Values.global.cattle.clusterName }} +{{- $_ := set $clusterNameRelabel "action" "replace" }} +{{- $_ := set $clusterNameRelabel "sourceLabels" (list "__address__") }} +{{- $_ := set $clusterNameRelabel "targetLabel" "cluster_name" }} +{{- $_ := set $clusterNameRelabel "replacement" $.Values.global.cattle.clusterName }} +{{- $metricRelabelings = append $metricRelabelings $clusterNameRelabel }} +{{- end }} +{{- if not (empty $metricRelabelings) }} +{{- $_ := set . "metricRelabelings" ($metricRelabelings)}} +{{- end }} +{{- if $setHTTPSScheme -}} +{{- $_ := set . "scheme" "https" }} +{{- end -}} +{{- if $useHTTPS -}} +{{- if (hasKey . "params") }} +{{- $_ := set (get . "params") "_scheme" (list "https") }} +{{- else }} +{{- $_ := set . "params" (dict "_scheme" (list "https")) }} +{{- end }} +{{- end }} +{{- if (hasKey . "tlsConfig") }} +{{- $_ := set (get . "tlsConfig") "insecureSkipVerify" $insecureSkipVerify }} +{{- else }} +{{- $_ := set . "tlsConfig" (dict "insecureSkipVerify" $insecureSkipVerify) }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.bearerTokenFile.enabled }} +{{- $_ := set . "bearerTokenFile" $.Values.clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.bearerTokenSecret.enabled }} +{{- $_ := set . "bearerTokenSecret" $serviceAccountTokenName }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.authorization.enabled }} +{{- if (hasKey . "authorization") }} +{{- $_ := set (get . "authorization") "type" $.Values.clients.https.authenticationMethod.authorization.type }} +{{- $_ := set (get . "authorization") "credentials" (dict "name" $serviceAccountTokenName "key" $.Values.clients.https.authenticationMethod.authorization.credentials.key "optional" $.Values.clients.https.authenticationMethod.authorization.credentials.optional) }} +{{- else }} +{{- $_ := set . "authorization" (dict "type" $.Values.clients.https.authenticationMethod.authorization.type) }} +{{- $_ := set . "authorization" (dict "credentials" (dict "name" $serviceAccountTokenName "key" $.Values.clients.https.authenticationMethod.authorization.credentials.key "optional" $.Values.clients.https.authenticationMethod.authorization.credentials.optional)) }} +{{- end }} +{{- end }} +{{- end }} +{{- toYaml $endpoints }} +{{- end -}} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2ControllerManager/templates/pushprox-clients-rbac.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2ControllerManager/templates/pushprox-clients-rbac.yaml new file mode 100644 index 00000000000..a8e27c37356 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2ControllerManager/templates/pushprox-clients-rbac.yaml @@ -0,0 +1,97 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.clients }}{{- if .Values.clients.enabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ template "pushProxy.client.name" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +rules: +{{- if .Values.global.cattle.psp.enabled }} +- apiGroups: ['policy'] + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - {{ template "pushProxy.client.name" . }} +{{- end }} +{{- if and .Values.clients.https.enabled .Values.clients.https.useServiceAccountCredentials }} +- nonResourceURLs: ["/metrics"] + verbs: ["get"] +{{- if .Values.clients.rbac.additionalRules }} +{{ toYaml .Values.clients.rbac.additionalRules }} +{{- end }} +{{- end }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ template "pushProxy.client.name" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "pushProxy.client.name" . }} +subjects: + - kind: ServiceAccount + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +--- +{{- if .Values.clients.https.useServiceAccountCredentials }} +apiVersion: v1 +kind: Secret +type: kubernetes.io/service-account-token +metadata: + name: {{ template "pushProxy.client.serviceAccountTokenName" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} + annotations: + kubernetes.io/service-account.name: {{ template "pushProxy.client.name" . }} +{{- end }} +--- +{{- if .Values.global.cattle.psp.enabled }} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +spec: + privileged: false + hostNetwork: true + hostIPC: false + hostPID: false + runAsUser: + rule: 'RunAsAny' + seLinux: + rule: 'RunAsAny' +{{- if and .Values.clients.https.enabled .Values.clients.https.certDir .Values.global.seLinux.enabled .Values.clients.https.seLinuxOptions }} + seLinuxOptions: {{ .Values.clients.https.seLinuxOptions | toYaml | nindent 6 }} +{{- end }} + supplementalGroups: + rule: 'MustRunAs' + ranges: + - min: 0 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + - min: 0 + max: 65535 + readOnlyRootFilesystem: false + volumes: + - 'secret' +{{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + - 'emptyDir' + - 'hostPath' + allowedHostPaths: + - pathPrefix: {{ required "Need access to volume on host with the SSL cert files to use HTTPs" .Values.clients.https.certDir }} + readOnly: true +{{- end }} +{{- end }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2ControllerManager/templates/pushprox-clients.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2ControllerManager/templates/pushprox-clients.yaml new file mode 100644 index 00000000000..e8fcfb38836 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2ControllerManager/templates/pushprox-clients.yaml @@ -0,0 +1,157 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.clients }}{{- if .Values.clients.enabled }} +apiVersion: apps/v1 +{{- if .Values.clients.deployment.enabled }} +kind: Deployment +{{- else }} +kind: DaemonSet +{{- end }} +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} + pushprox-exporter: "client" +spec: + {{- if .Values.clients.deployment.enabled }} + replicas: {{ .Values.clients.deployment.replicas }} + {{- end }} + selector: + matchLabels: {{ include "pushProxy.client.labels" . | nindent 6 }} + template: + metadata: + labels: {{ include "pushProxy.client.labels" . | nindent 8 }} + spec: + {{- if .Values.clients.affinity }} + affinity: {{ toYaml .Values.clients.affinity | nindent 8 }} + {{- end }} + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} +{{- if .Values.clients.nodeSelector }} +{{ toYaml .Values.clients.nodeSelector | indent 8 }} +{{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} +{{- if .Values.clients.tolerations }} +{{ toYaml .Values.clients.tolerations | indent 8 }} +{{- end }} + hostNetwork: true + dnsPolicy: ClusterFirstWithHostNet + serviceAccountName: {{ template "pushProxy.client.name" . }} + {{- if .Values.global.imagePullSecretName }} + imagePullSecrets: + - name: {{ .Values.global.imagePullSecretName }} + {{- end }} + containers: + - name: pushprox-client + image: {{ template "system_default_registry" . }}{{ .Values.clients.image.repository }}:{{ .Values.clients.image.tag }} + command: + {{- range .Values.clients.command }} + - {{ . | quote }} + {{- end }} + args: + - --fqdn=$(HOST_IP) + - --proxy-url=$(PROXY_URL) + {{- if .Values.clients.metrics.enabled }} + - --metrics-addr=$(PORT) + {{- end }} + - --allow-port={{ required "Need .Values.metricsPort to configure client to be allowed to scrape metrics at port" .Values.metricsPort}} + {{- if .Values.clients.useLocalhost }} + - --use-localhost + {{- end }} + {{- if .Values.clients.https.enabled }} + {{- if .Values.clients.https.insecureSkipVerify }} + - --insecure-skip-verify + {{- end }} + {{- if .Values.clients.https.useServiceAccountCredentials }} + - --token-path=/var/run/secrets/kubernetes.io/serviceaccount/token + {{- end }} + {{- if .Values.clients.https.certDir }} + - --tls.cert=/etc/ssl/push-proxy/push-proxy.pem + - --tls.key=/etc/ssl/push-proxy/push-proxy-key.pem + - --tls.cacert=/etc/ssl/push-proxy/push-proxy-ca-cert.pem + {{- end }} + {{- end }} + env: + - name: HOST_IP + valueFrom: + fieldRef: + fieldPath: status.hostIP + {{- if .Values.clients.metrics.enabled }} + - name: PORT + value: :{{ .Values.clients.port }} + {{- end }} + - name: PROXY_URL + value: {{ template "pushProxy.proxyUrl" . }} + securityContext: + runAsNonRoot: true + runAsUser: 1000 + {{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + volumeMounts: + - name: metrics-cert-dir + mountPath: /etc/ssl/push-proxy + {{- end }} + {{- if .Values.clients.resources }} + resources: {{ toYaml .Values.clients.resources | nindent 10 }} + {{- end }} + {{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + initContainers: + - name: copy-certs + image: {{ template "system_default_registry" . }}{{ .Values.clients.copyCertsImage.repository }}:{{ .Values.clients.copyCertsImage.tag }} + command: + - sh + - -c + - | + echo "Searching for files to copy within the source volume" + echo "cert: ${CERT_FILE_NAME}" + echo "key: ${KEY_FILE_NAME}" + echo "cacert: ${CACERT_FILE_NAME}" + + CERT_FILE_SOURCE=$(find /etc/source/ -type f -name "${CERT_FILE_NAME}" | sort -r | head -n 1) + KEY_FILE_SOURCE=$(find /etc/source/ -type f -name "${KEY_FILE_NAME}" | sort -r | head -n 1) + CACERT_FILE_SOURCE=$(find /etc/source/ -type f -name "${CACERT_FILE_NAME}" | sort -r | head -n 1) + + test -z ${CERT_FILE_SOURCE} && echo "Failed to find cert file" && exit 1 + test -z ${KEY_FILE_SOURCE} && echo "Failed to find key file" && exit 1 + test -z ${CACERT_FILE_SOURCE} && echo "Failed to find cacert file" && exit 1 + + echo "Copying cert file from $CERT_FILE_SOURCE to $CERT_FILE_TARGET" + cp $CERT_FILE_SOURCE $CERT_FILE_TARGET || exit 1 + chmod 444 $CERT_FILE_TARGET || exit 1 + + echo "Copying key file from $KEY_FILE_SOURCE to $KEY_FILE_TARGET" + cp $KEY_FILE_SOURCE $KEY_FILE_TARGET || exit 1 + chmod 444 $KEY_FILE_TARGET || exit 1 + + echo "Copying cacert file from $CACERT_FILE_SOURCE to $CACERT_FILE_TARGET" + cp $CACERT_FILE_SOURCE $CACERT_FILE_TARGET || exit 1 + chmod 444 $CACERT_FILE_TARGET || exit 1 + env: + - name: CERT_FILE_NAME + value: {{ required "Need a TLS cert file for scraping metrics endpoint over HTTPs" .Values.clients.https.certFile }} + - name: KEY_FILE_NAME + value: {{ required "Need a TLS key file for scraping metrics endpoint over HTTPs" .Values.clients.https.keyFile }} + - name: CACERT_FILE_NAME + value: {{ required "Need a TLS CA cert file for scraping metrics endpoint over HTTPs" .Values.clients.https.caCertFile }} + - name: CERT_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy.pem + - name: KEY_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy-key.pem + - name: CACERT_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy-ca-cert.pem + securityContext: + runAsNonRoot: false +{{- if and .Values.global.seLinux.enabled .Values.clients.https.seLinuxOptions }} + seLinuxOptions: {{ .Values.clients.https.seLinuxOptions | toYaml | nindent 12 }} +{{- end }} + volumeMounts: + - name: metrics-cert-dir-source + mountPath: /etc/source + readOnly: true + - name: metrics-cert-dir + mountPath: /etc/ssl/push-proxy + volumes: + - name: metrics-cert-dir-source + hostPath: + path: {{ required "Need access to volume on host with the SSL cert files to use HTTPs" .Values.clients.https.certDir }} + - name: metrics-cert-dir + emptyDir: {} + {{- end }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2ControllerManager/templates/pushprox-proxy-rbac.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2ControllerManager/templates/pushprox-proxy-rbac.yaml new file mode 100644 index 00000000000..eefe609058c --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2ControllerManager/templates/pushprox-proxy-rbac.yaml @@ -0,0 +1,68 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if and .Values.proxy }}{{ if .Values.proxy.enabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ template "pushProxy.proxy.name" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +rules: +{{- if .Values.global.cattle.psp.enabled }} +- apiGroups: ['policy'] + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - {{ template "pushProxy.proxy.name" . }} +{{- end }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ template "pushProxy.proxy.name" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "pushProxy.proxy.name" . }} +subjects: + - kind: ServiceAccount + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +--- +{{- if .Values.global.cattle.psp.enabled }} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +spec: + privileged: false + hostNetwork: false + hostIPC: false + hostPID: false + runAsUser: + rule: 'MustRunAsNonRoot' + seLinux: + rule: 'RunAsAny' + supplementalGroups: + rule: 'MustRunAs' + ranges: + - min: 1 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + - min: 1 + max: 65535 + readOnlyRootFilesystem: false + volumes: + - 'secret' +{{- end }}{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2ControllerManager/templates/pushprox-proxy.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2ControllerManager/templates/pushprox-proxy.yaml new file mode 100644 index 00000000000..723bbd6c00b --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2ControllerManager/templates/pushprox-proxy.yaml @@ -0,0 +1,57 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if and .Values.proxy }}{{ if .Values.proxy.enabled }} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} + pushprox-exporter: "proxy" +spec: + selector: + matchLabels: {{ include "pushProxy.proxy.labels" . | nindent 6 }} + template: + metadata: + labels: {{ include "pushProxy.proxy.labels" . | nindent 8 }} + spec: + securityContext: + runAsNonRoot: true + runAsUser: 1000 + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} +{{- if .Values.proxy.nodeSelector }} +{{ toYaml .Values.proxy.nodeSelector | indent 8 }} +{{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} +{{- if .Values.proxy.tolerations }} +{{ toYaml .Values.proxy.tolerations | indent 8 }} +{{- end }} + serviceAccountName: {{ template "pushProxy.proxy.name" . }} + {{- if .Values.global.imagePullSecretName }} + imagePullSecrets: + - name: {{ .Values.global.imagePullSecretName }} + {{- end }} + containers: + - name: pushprox-proxy + image: {{ template "system_default_registry" . }}{{ .Values.proxy.image.repository }}:{{ .Values.proxy.image.tag }} + command: + {{- range .Values.proxy.command }} + - {{ . | quote }} + {{- end }} + {{- if .Values.proxy.resources }} + resources: {{ toYaml .Values.proxy.resources | nindent 10 }} + {{- end }} +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +spec: + ports: + - name: pp-proxy + port: {{ required "Need .Values.proxy.port to configure proxy" .Values.proxy.port }} + protocol: TCP + targetPort: {{ .Values.proxy.port }} + selector: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2ControllerManager/templates/pushprox-servicemonitor.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2ControllerManager/templates/pushprox-servicemonitor.yaml new file mode 100644 index 00000000000..67eb2216b5a --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2ControllerManager/templates/pushprox-servicemonitor.yaml @@ -0,0 +1,45 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.serviceMonitor }}{{- if .Values.serviceMonitor.enabled }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ template "pushprox.serviceMonitor.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.serviceMonitor.labels" . | nindent 4 }} +spec: + endpoints: {{include "pushProxy.serviceMonitor.endpoints" . | nindent 4 }} + jobLabel: component + podTargetLabels: + - component + - pushprox-exporter + namespaceSelector: + matchNames: + - {{ template "pushprox.namespace" . }} + selector: + matchLabels: {{ include "pushProxy.client.labels" . | nindent 6 }} +--- +{{- $selector := "" }} +{{- if not (kindIs "invalid" .Values.service) }} +{{- if not (kindIs "invalid" .Values.service.selector) }} +{{ if .Values.service.selector }} +{{- if .Values.clients.enabled }} +{{- required (printf "Cannot override .Values.service.selector=%s when .Values.clients.enabled=true" (toJson .Values.service.selector)) "" }} +{{- end }} +{{- $selector = (toYaml .Values.service.selector) }} +{{- end }} +{{- end }} +{{- end }} +apiVersion: v1 +kind: Service +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +spec: + ports: + - name: metrics + port: {{ required "Need .Values.metricsPort to configure client to listen to metrics at port" .Values.metricsPort}} + protocol: TCP + targetPort: {{ .Values.metricsPort }} + selector: {{ default (include "pushProxy.client.labels" .) $selector | nindent 4 }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2ControllerManager/templates/validate-install-crd.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2ControllerManager/templates/validate-install-crd.yaml new file mode 100644 index 00000000000..16abc2fa83a --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2ControllerManager/templates/validate-install-crd.yaml @@ -0,0 +1,14 @@ +#{{- if gt (len (lookup "rbac.authorization.k8s.io/v1" "ClusterRole" "" "")) 0 -}} +# {{- $found := dict -}} +# {{- set $found "monitoring.coreos.com/v1/ServiceMonitor" false -}} +# {{- range .Capabilities.APIVersions -}} +# {{- if hasKey $found (toString .) -}} +# {{- set $found (toString .) true -}} +# {{- end -}} +# {{- end -}} +# {{- range $_, $exists := $found -}} +# {{- if (eq $exists false) -}} +# {{- required "Required CRDs are missing. Please install Prometheus Operator CRDs before installing this chart." "" -}} +# {{- end -}} +# {{- end -}} +#{{- end -}} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2ControllerManager/templates/validate-psp-install.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2ControllerManager/templates/validate-psp-install.yaml new file mode 100644 index 00000000000..a30c59d3b7c --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2ControllerManager/templates/validate-psp-install.yaml @@ -0,0 +1,7 @@ +#{{- if gt (len (lookup "rbac.authorization.k8s.io/v1" "ClusterRole" "" "")) 0 -}} +#{{- if .Values.global.cattle.psp.enabled }} +#{{- if not (.Capabilities.APIVersions.Has "policy/v1beta1/PodSecurityPolicy") }} +#{{- fail "The target cluster does not have the PodSecurityPolicy API resource. Please disable PSPs in this chart before proceeding." -}} +#{{- end }} +#{{- end }} +#{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2ControllerManager/values.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2ControllerManager/values.yaml new file mode 100644 index 00000000000..7f38498a1c5 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2ControllerManager/values.yaml @@ -0,0 +1,166 @@ +# Default values for rancher-pushprox. +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. + +# Default image containing both the proxy and the client was generated from the following Dockerfile +# https://github.com/prometheus-community/PushProx/blob/eeadbe766641699129920ccfaaaa30a85c67fe81/Dockerfile#L1-L15 + +# Configuration + +global: + cattle: + psp: + enabled: false + systemDefaultRegistry: "" + seLinux: + enabled: false + +# A list of Semver constraint strings (defined by https://github.com/Masterminds/semver) and values.yaml overrides. +# +# For each key in kubeVersionOverrides, this chart will check to see if the current Kubernetes cluster's version matches +# any of the semver constraints provided as keys on the map. +# +# On seeing a match, the default value for each values.yaml field overridden will be updated with the new value. +# +# If multiple matches are encountered (due to overlapping semver ranges), the matches will be applied in order. +# +# Notes: +# - On running a helm template, Helm generally assumes the kubeVersion is v1.20.0 +# - On running a helm install --dry-run, the correct kubeVersion should be chosen. +kubeVersionOverrides: [] +# - constraint: "< 1.21" +# values: +# metricsPort: 10252 +# clients: +# https: +# enabled: false +# insecureSkipVerify: false +# useServiceAccountCredentials: false + +namespaceOverride: "" + +# The component that is being monitored (i.e. etcd) +component: "component" + +# The port containing the metrics that need to be scraped +metricsPort: 2739 + +# Configure ServiceMonitor that monitors metrics from the metricsPort endpoint +serviceMonitor: + enabled: true + # A list of endpoints that will be added to the ServiceMonitor based on the Endpoint spec + # Source: https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#endpoint + # By default, proxyUrl and params._scheme will be overridden based on other values + endpoints: + - port: metrics + +# Configure Service that grabs scrape targets +service: + # The selector that is used to populate the Service's Endpoints object. + # The chart will error out on rendering templating if .Values.clients.enabled is set alongside this field, + # since it is expected that this service should point to the PushProx Clients Daemonset / Deployment + selector: {} + +clients: + enabled: true + # The port which the PushProx client will post PushProx metrics to + port: 9369 + # If unset, this will default to the URL for the proxy service: http://pushprox-{{component}}-proxy.{{namepsace}}.svc.cluster.local:{{proxy.port}} + # Should be modified if the clients are being deployed outside the cluster where the proxy rests, otherwise leave it null + proxyUrl: "" + # If set to true, the client will forward any requests from the host IP to 127.0.0.1 + # It will only allow proxy requests to the metricsPort specified + useLocalhost: false + # Configuration for accessing metrics via HTTPS + https: + # Does the client require https to access the metrics? + enabled: false + # Does the client require requests be sent to http or https? + forceHTTPSScheme: false + # If set to true, the client will create a service account with adequate permissions and set a flag + # on the client to use the service account token provided by it to make authorized scrape requests + useServiceAccountCredentials: false + # Configuration for authentication to metrics via https endpoint + authenticationMethod: + # Reads token from defined file in container + # This function is deprecated in the prometheus operator api and may be removed in a future version + bearerTokenFile: + enabled: false + bearerTokenFilePath: "/var/run/secrets/kubernetes.io/serviceaccount/token" + # Reads token from defined secret in namespace + # This function is deprecated in the prometheus operator api and may be removed in a future version + bearerTokenSecret: + enabled: false + # Reads token from defined secret in namespace + authorization: + enabled: false + type: "bearer" + credentials: + key: "token" + optional: false + # If set to true, the client will disable SSL security checks + insecureSkipVerify: false + # Directory on host where necessary TLS cert and key to scrape metrics can be found + certDir: "" + # Filenames for files located in .Values.clients.https.certDir that correspond to TLS settings + certFile: "" + keyFile: "" + caCertFile: "" + # seLinuxOptions to be passed into the container that copies certs. Should define a container with permissions to read the files in the certDir provided on the host. + # Required and only used if `clients.https.enabled` is set and `clients.https.certDir` is provided. + seLinuxOptions: {} + + metrics: + # Whether the client should publish PushProx client-specific metrics to .Values.clients.port + enabled: false + + rbac: + # Additional permissions to provide to the ServiceAccount bound to the client + # This can be used to provide additional permissions for the client to scrape metrics from the k8s API + # Only enabled if clients.https.enabled and clients.https.useServiceAccountCredentials are true + additionalRules: [] + + # Resource limits + resources: {} + + # Options to select all nodes to deploy client DaemonSet on + nodeSelector: {} + tolerations: [] + affinity: {} + + image: + repository: rancher/pushprox-client + tag: v0.1.4-rancher2-client + command: ["pushprox-client"] + + copyCertsImage: + repository: rancher/mirrored-library-busybox + tag: 1.31.1 + + # The default intention of rancher-pushprox clients is to scrape hostNetwork metrics across all nodes. + # This can be used to scrape internal Kubernetes components or DaemonSets of hostNetwork Pods in + # situations where a cloud provider firewall prevents Pod-To-Host communication but not Pod-To-Pod. + # However, if the underlying hostNetwork Pod that is being scraped is managed by a Deployment, + # this advanced option enables users to deploy the client as a Deployment instead of a DaemonSet. + # If a user deploys this feature and the underlying Deployment's number of replicas changes, the user will + # be responsible for upgrading this chart accordingly to the right number of replicas. + deployment: + enabled: false + replicas: 0 + +proxy: + enabled: true + # The port through which PushProx clients will communicate to the proxy + port: 8080 + + # Resource limits + resources: {} + + # Options to select a node to run a single proxy deployment on + nodeSelector: {} + tolerations: [] + + image: + repository: rancher/pushprox-proxy + tag: v0.1.4-rancher2-proxy + command: ["pushprox-proxy"] diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2Etcd/.helmignore b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2Etcd/.helmignore new file mode 100644 index 00000000000..0e8a0eb36f4 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2Etcd/.helmignore @@ -0,0 +1,23 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*.orig +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2Etcd/Chart.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2Etcd/Chart.yaml new file mode 100644 index 00000000000..266f100ff9c --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2Etcd/Chart.yaml @@ -0,0 +1,15 @@ +annotations: + catalog.cattle.io/hidden: "true" + catalog.cattle.io/kube-version: '>= 1.26.0-0 < 1.31.0-0' + catalog.cattle.io/os: linux + catalog.rancher.io/certified: rancher + catalog.rancher.io/namespace: cattle-monitoring-system + catalog.rancher.io/release-name: rancher-pushprox +apiVersion: v1 +appVersion: v0.1.4-rancher2 +description: Sets up a deployment of the PushProx proxy and a DaemonSet of PushProx + clients. +kubeVersion: '>=1.26.0-0' +name: rke2Etcd +type: application +version: 0.2.1 diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2Etcd/README.md b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2Etcd/README.md new file mode 100644 index 00000000000..345002f48a2 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2Etcd/README.md @@ -0,0 +1,90 @@ +# rancher-pushprox + +A Rancher chart based on Rancher [PushProx](https://github.com/rancher/PushProx) that sets up a Deployment of a PushProx proxy and a DaemonSet of PushProx clients on a Kubernetes cluster. + +Installs [rancher-pushprox](https://github.com/rancher/charts/tree/gh-pages/packages/rancher-pushprox) to create PushProx clients that can access their host's network and register with a PushProx proxy. A [Prometheus Operator](https://github.com/coreos/prometheus-operator) ServiceMonitor CR is also included that is configured to scrape the metrics from each of the clients through the proxy. + +Using an instance of this chart is suitable for the following scenarios: +- You need to scrape metrics from a port that should not be accessible outside of the host (e.g. scraping `etcd` metrics in a hardened cluster) +- You need to scrape metrics on a host that are not exposed outside of 127.0.0.1 (e.g. scraping `kube-proxy` metrics) +- You need to scrape metrics through HTTPS using certs hosted directly on `hostPath` +- You need to scrape metrics from Kubernetes components that require authorization via a service account (e.g. permissions to make request to `/metrics`) +- You need to scrape metrics without access to cacerts (i.e. enable `insecureSkipVerify`) + +The clients and proxy are created based on a Rancher fork of the [prometheus-community/PushProx](https://github.com/prometheus-community/PushProx) project. + +## Upgrading to Kubernetes v1.25+ + +Starting in Kubernetes v1.25, [Pod Security Policies](https://kubernetes.io/docs/concepts/security/pod-security-policy/) have been removed from the Kubernetes API. + +As a result, **before upgrading to Kubernetes v1.25** (or on a fresh install in a Kubernetes v1.25+ cluster), users are expected to perform an in-place upgrade of this chart with `global.cattle.psp.enabled` set to `false` if it has been previously set to `true`. +​ +> **Note:** +> In this chart release, any previous field that was associated with any PSP resources have been removed in favor of a single global field: `global.cattle.psp.enabled`. + +> **Note:** +> If you upgrade your cluster to Kubernetes v1.25+ before removing PSPs via a `helm upgrade` (even if you manually clean up resources), **it will leave the Helm release in a broken state within the cluster such that further Helm operations will not work (`helm uninstall`, `helm upgrade`, etc.).** +> +> If your charts get stuck in this state, please consult the Rancher docs on how to clean up your Helm release secrets. + +Upon setting `global.cattle.psp.enabled` to false, the chart will remove any PSP resources deployed on its behalf from the cluster. This is the default setting for this chart. + +As a replacement for PSPs, [Pod Security Admission](https://kubernetes.io/docs/concepts/security/pod-security-admission/) should be used. Please consult the Rancher docs for more details on how to configure your chart release namespaces to work with the new Pod Security Admission and apply Pod Security Standards. + +## Configuration + +The following tables list the configurable parameters of the rancher-pushprox chart and their default values. + +### General + +#### Required +| Parameter | Description | Example | +| ----- | ----------- | ------ | +| `component` | The component that is being monitored | `kube-etcd` +| `metricsPort` | The port on the host that contains the metrics you want to scrape (e.g. `http://:/metrics`) | `2379` | +| `namespaceOverride` | The namespace to install the chart | `""` + +#### Optional +| Parameter | Description | Default | +| ----- | ----------- | ------ | +| `serviceMonitor.enabled` | Deploys a [Prometheus Operator](https://github.com/coreos/prometheus-operator/blob/master/Documentation/api.md#servicemonitor) ServiceMonitor CR that is configured to scrape metrics on the hosts that the clients are deployed on via the proxy. Also deploys a Service that points to all pods with the expected client name that exposes the `metricsPort` selected | `true` | +| `serviceMonitor.endpoints` | A list of endpoints that will be added to the ServiceMonitor based on the [Endpoint spec](https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#endpoint) | `[{port: metrics}]` | +| `service.selector` | The selector that is used to populate the Service's Endpoints object. The chart will error out on rendering templating if `.Values.clients.enabled` is set alongside this field, since it is expected that this service should point to the PushProx Clients Daemonset / Deployment | `{}` | +| `clients.enabled` | Deploys a DaemonSet of clients that are each capable of scraping endpoints on the hostNetwork it is deployed on | `true` | +| `clients.port` | The port where the client will publish PushProx client-specific metrics. If deploying multiple clients onto the same node, the clients should not have conflicting ports | `9369` | +| `clients.proxyUrl` | Overrides the default proxyUrl setting of `http://pushprox-{{ .Values.component }}-proxy.{{ . Release.Namespace }}.svc.cluster.local:{{ .Values.proxy.port }}"` with the `proxyUrl` specified | `""` | +| `clients.useLocalhost` | Sets a flag on each client deployment to redirect scrapes directed to `HOST_IP` to `127.0.0.1` | `false` | +| `clients.https.enabled` | Enables scraping metrics via HTTPS using the provided TLS certs that exist on each host | `false` | +| `clients.https.forceHTTPSScheme` | Forces scraping metrics via HTTPS using the provided TLS certs that exist on each host | `false` | +| `clients.https.useServiceAccountCredentials` | If set to true, the client will create a service account with permissions to scrape `/metrics` endpoint of Kubernetes components. The client will use the service account token provided to make authorized scrape requests to the Kubernetes API | `false` | +| `clients.https.authenticationMethod.bearerTokenFile.enabled` | If set to true, the client will use service account credentials mounted at the configured path `clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath`. This requires permissions to scrape `/metrics` endpoint of Kubernetes components. This method is deprecated by the prometheus operator and may be removed in a future release | `false` | +| `clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath` | This is a volume mount on the pod with permissions to scrape `/metrics` endpoint of Kubernetes components | `"/var/run/secrets/kubernetes.io/serviceaccount/token"` | +| `clients.https.authenticationMethod.bearerTokenSecret.enabled` | If set to true, the client will use service account credentials to scrape `/metrics` endpoint of Kubernetes components. This method is deprecated by the prometheus operator and may be removed in a future release | `false` | +| `clients.https.authenticationMethod.authorization.enabled` | If set to true, the client will use service account credentials to scrape `/metrics` endpoint of Kubernetes components | `false` | +| `clients.https.authenticationMethod.authorization.type` | If set, the client will use this type of authorization in its client requests for metrics | `"bearer"` | +| `clients.https.authenticationMethod.authorization.credentials.key` | If set, the client will use this key in the secret created by `clients.https.useServiceAccountCredentials` for authorization in its client requests for metrics | `"token"` | +| `clients.https.authenticationMethod.authorization.credentials.optional` | If set to false, the client will fail if the key in the secret created by `clients.https.useServiceAccountCredentials` does not exist | `false` | +| `clients.https.insecureSkipVerify` | If set to true, the client will disable SSL security checks | `false` | +| `clients.https.certDir` | A `hostPath` where TLS certs can be found. This path is mounted as a volume on an `initContainer` which copies only the necessary files over to an EmptyDir volume used by each client. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.certFile` | The path to the TLS cert file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.keyFile` | The path to the TLS key file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.caCertFile` | The path to the TLS cacert file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.seLinuxOptions` | seLinuxOptions to be passed into the container that copies certs. Should define a container with permissions to read the files in the certDir provided on the host. Required and only used if `clients.https.enabled` is set and `clients.https.certDir` is provided. | `""` | +| `clients.metrics.enabled` | Whether the client should publish PushProx client-specific metrics. | `false` | +| `clients.rbac.additionalRules` | Additional permissions to provide to the ServiceAccount bound to the client. This can be used to provide additional permissions for the client to scrape metrics from the k8s API. Only enabled if clients.https.enabled and clients.https.useServiceAccountCredentials are true | `[]` | +| `clients.deployment.enabled` | Deploys the client as a Deployment (generally used if the underlying hostNetwork Pod that is being scraped is managed by a Deployment) | `false` | +| `clients.deployment.replicas` | The number of pods the Deployment has, it should match the number of pod the hostNetwork Deployment has. Required and only used if `client.deployment.enable` is set | `0` | +| `clients.deployment.affinity` | The affinity rules that allocate the pod to the node in which the hostNetwork Deployment's pods run. Required and only used if `client.deployment.enable` is set | `{}` | +| `clients.resources` | Set resource limits and requests for the client container | `{}` | +| `clients.nodeSelector` | Select which nodes to deploy the clients on | `{}` | +| `clients.tolerations` | Specify tolerations for clients | `[]` | +| `proxy.enabled` | Deploys the proxy that each client will register with | `true` | +| `proxy.port` | The port exposed by the proxy that each client will register with to allow metrics to be scraped from the host | `8080` | +| `proxy.resources` | Set resource limits and requests for the proxy container | `{}` | +| `proxy.nodeSelector` | Select which nodes the proxy can be deployed on | `{}` | +| `proxy.tolerations` | Specify tolerations (if necessary) to allow the proxy to be deployed on the selected node | `[]` | +| `kubeVersionOverrides` | A list of Semver constraint strings (defined by https://github.com/Masterminds/semver) and values.yaml overrides. For each key in kubeVersionOverrides, this chart will check to see if the current Kubernetes cluster's version matches any of the semver constraints provided as keys on the map. On seeing a match, the default value for each values.yaml field overridden will be updated with the new value. If multiple matches are encountered (due to overlapping semver ranges), the matches will be applied in order. | `[]` + +*Tip: The filepaths set in `clients.https.File` can include wildcard characters*. + +See [rancher-monitoring](https://github.com/rancher/charts/tree/gh-pages/packages/rancher-monitoring) for examples of how this chart can be used. diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2Etcd/templates/_helpers.tpl b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2Etcd/templates/_helpers.tpl new file mode 100644 index 00000000000..1ba50939441 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2Etcd/templates/_helpers.tpl @@ -0,0 +1,170 @@ +# Rancher + +{{- define "system_default_registry" -}} +{{- if .Values.global.cattle.systemDefaultRegistry -}} +{{- printf "%s/" .Values.global.cattle.systemDefaultRegistry -}} +{{- end -}} +{{- end -}} + +# Windows Support + +{{/* +Windows cluster will add default taint for linux nodes, +add below linux tolerations to workloads could be scheduled to those linux nodes +*/}} + +{{- define "linux-node-tolerations" -}} +- key: "cattle.io/os" + value: "linux" + effect: "NoSchedule" + operator: "Equal" +{{- end -}} + +{{- define "linux-node-selector" -}} +{{- if semverCompare "<1.14-0" .Capabilities.KubeVersion.GitVersion -}} +beta.kubernetes.io/os: linux +{{- else -}} +kubernetes.io/os: linux +{{- end -}} +{{- end -}} + +# General + +{{- define "applyKubeVersionOverrides" -}} +{{- $overrides := dict -}} +{{- range $override := .Values.kubeVersionOverrides -}} +{{- if semverCompare $override.constraint $.Capabilities.KubeVersion.Version -}} +{{- $_ := mergeOverwrite $overrides $override.values -}} +{{- end -}} +{{- end -}} +{{- $_ := mergeOverwrite .Values $overrides -}} +{{- end -}} + +{{- define "pushprox.namespace" -}} + {{- if .Values.namespaceOverride -}} + {{- .Values.namespaceOverride -}} + {{- else -}} + {{- .Release.Namespace -}} + {{- end -}} +{{- end -}} + +{{- define "pushProxy.commonLabels" -}} +release: {{ .Release.Name }} +component: {{ .Values.component | quote }} +provider: kubernetes +{{- end -}} + +{{- define "pushProxy.proxyUrl" -}} +{{- $_ := (required "Template requires either .Values.proxy.port or .Values.client.proxyUrl to set proxyUrl for client" (or .Values.clients.proxyUrl .Values.proxy.port)) -}} +{{- if .Values.clients.proxyUrl -}} +{{ printf "%s" .Values.clients.proxyUrl }} +{{- else -}} +{{ printf "http://%s.%s.svc:%d" (include "pushProxy.proxy.name" .) (include "pushprox.namespace" .) (int .Values.proxy.port) }} +{{- end -}}{{- end -}} + +# Client + +{{- define "pushProxy.client.name" -}} +{{- printf "pushprox-%s-client" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.client.serviceAccountTokenName" -}} +{{- printf "pushprox-%s-client-service-account-token" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.client.labels" -}} +k8s-app: {{ template "pushProxy.client.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +# Proxy + +{{- define "pushProxy.proxy.name" -}} +{{- printf "pushprox-%s-proxy" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.proxy.labels" -}} +k8s-app: {{ template "pushProxy.proxy.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +# ServiceMonitor + +{{- define "pushprox.serviceMonitor.name" -}} +{{- printf "%s-%s" .Release.Name (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.serviceMonitor.labels" -}} +app: {{ template "pushprox.serviceMonitor.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +{{- define "pushProxy.serviceMonitor.endpoints" -}} +{{- $proxyURL := (include "pushProxy.proxyUrl" .) -}} +{{- $useHTTPS := .Values.clients.https.enabled -}} +{{- $setHTTPSScheme := .Values.clients.https.forceHTTPSScheme -}} +{{- $insecureSkipVerify := .Values.clients.https.insecureSkipVerify -}} +{{- $useServiceAccountCredentials := .Values.clients.https.useServiceAccountCredentials -}} +{{- $serviceAccountTokenName := (include "pushProxy.client.serviceAccountTokenName" . ) -}} +{{- $metricRelabelings := list }} +{{- $endpoints := .Values.serviceMonitor.endpoints }} +{{- if .Values.proxy.enabled }} +{{- $_ := set . "proxyUrl" $proxyURL }} +{{- end }} +{{- range $endpoints }} +{{- if $.Values.proxy.enabled }} +{{- $_ := set . "proxyUrl" $proxyURL }} +{{- end }} +{{- $clusterIdRelabel := dict }} +{{- $metricRelabelings := list }} +{{- if $.Values.global.cattle.clusterId }} +{{- $_ := set $clusterIdRelabel "action" "replace" }} +{{- $_ := set $clusterIdRelabel "sourceLabels" (list "__address__") }} +{{- $_ := set $clusterIdRelabel "targetLabel" "cluster_id" }} +{{- $_ := set $clusterIdRelabel "replacement" $.Values.global.cattle.clusterId }} +{{- $metricRelabelings = append $metricRelabelings $clusterIdRelabel }} +{{- end }} +{{- $clusterNameRelabel := dict }} +{{- if $.Values.global.cattle.clusterName }} +{{- $_ := set $clusterNameRelabel "action" "replace" }} +{{- $_ := set $clusterNameRelabel "sourceLabels" (list "__address__") }} +{{- $_ := set $clusterNameRelabel "targetLabel" "cluster_name" }} +{{- $_ := set $clusterNameRelabel "replacement" $.Values.global.cattle.clusterName }} +{{- $metricRelabelings = append $metricRelabelings $clusterNameRelabel }} +{{- end }} +{{- if not (empty $metricRelabelings) }} +{{- $_ := set . "metricRelabelings" ($metricRelabelings)}} +{{- end }} +{{- if $setHTTPSScheme -}} +{{- $_ := set . "scheme" "https" }} +{{- end -}} +{{- if $useHTTPS -}} +{{- if (hasKey . "params") }} +{{- $_ := set (get . "params") "_scheme" (list "https") }} +{{- else }} +{{- $_ := set . "params" (dict "_scheme" (list "https")) }} +{{- end }} +{{- end }} +{{- if (hasKey . "tlsConfig") }} +{{- $_ := set (get . "tlsConfig") "insecureSkipVerify" $insecureSkipVerify }} +{{- else }} +{{- $_ := set . "tlsConfig" (dict "insecureSkipVerify" $insecureSkipVerify) }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.bearerTokenFile.enabled }} +{{- $_ := set . "bearerTokenFile" $.Values.clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.bearerTokenSecret.enabled }} +{{- $_ := set . "bearerTokenSecret" $serviceAccountTokenName }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.authorization.enabled }} +{{- if (hasKey . "authorization") }} +{{- $_ := set (get . "authorization") "type" $.Values.clients.https.authenticationMethod.authorization.type }} +{{- $_ := set (get . "authorization") "credentials" (dict "name" $serviceAccountTokenName "key" $.Values.clients.https.authenticationMethod.authorization.credentials.key "optional" $.Values.clients.https.authenticationMethod.authorization.credentials.optional) }} +{{- else }} +{{- $_ := set . "authorization" (dict "type" $.Values.clients.https.authenticationMethod.authorization.type) }} +{{- $_ := set . "authorization" (dict "credentials" (dict "name" $serviceAccountTokenName "key" $.Values.clients.https.authenticationMethod.authorization.credentials.key "optional" $.Values.clients.https.authenticationMethod.authorization.credentials.optional)) }} +{{- end }} +{{- end }} +{{- end }} +{{- toYaml $endpoints }} +{{- end -}} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2Etcd/templates/pushprox-clients-rbac.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2Etcd/templates/pushprox-clients-rbac.yaml new file mode 100644 index 00000000000..a8e27c37356 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2Etcd/templates/pushprox-clients-rbac.yaml @@ -0,0 +1,97 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.clients }}{{- if .Values.clients.enabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ template "pushProxy.client.name" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +rules: +{{- if .Values.global.cattle.psp.enabled }} +- apiGroups: ['policy'] + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - {{ template "pushProxy.client.name" . }} +{{- end }} +{{- if and .Values.clients.https.enabled .Values.clients.https.useServiceAccountCredentials }} +- nonResourceURLs: ["/metrics"] + verbs: ["get"] +{{- if .Values.clients.rbac.additionalRules }} +{{ toYaml .Values.clients.rbac.additionalRules }} +{{- end }} +{{- end }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ template "pushProxy.client.name" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "pushProxy.client.name" . }} +subjects: + - kind: ServiceAccount + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +--- +{{- if .Values.clients.https.useServiceAccountCredentials }} +apiVersion: v1 +kind: Secret +type: kubernetes.io/service-account-token +metadata: + name: {{ template "pushProxy.client.serviceAccountTokenName" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} + annotations: + kubernetes.io/service-account.name: {{ template "pushProxy.client.name" . }} +{{- end }} +--- +{{- if .Values.global.cattle.psp.enabled }} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +spec: + privileged: false + hostNetwork: true + hostIPC: false + hostPID: false + runAsUser: + rule: 'RunAsAny' + seLinux: + rule: 'RunAsAny' +{{- if and .Values.clients.https.enabled .Values.clients.https.certDir .Values.global.seLinux.enabled .Values.clients.https.seLinuxOptions }} + seLinuxOptions: {{ .Values.clients.https.seLinuxOptions | toYaml | nindent 6 }} +{{- end }} + supplementalGroups: + rule: 'MustRunAs' + ranges: + - min: 0 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + - min: 0 + max: 65535 + readOnlyRootFilesystem: false + volumes: + - 'secret' +{{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + - 'emptyDir' + - 'hostPath' + allowedHostPaths: + - pathPrefix: {{ required "Need access to volume on host with the SSL cert files to use HTTPs" .Values.clients.https.certDir }} + readOnly: true +{{- end }} +{{- end }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2Etcd/templates/pushprox-clients.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2Etcd/templates/pushprox-clients.yaml new file mode 100644 index 00000000000..e8fcfb38836 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2Etcd/templates/pushprox-clients.yaml @@ -0,0 +1,157 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.clients }}{{- if .Values.clients.enabled }} +apiVersion: apps/v1 +{{- if .Values.clients.deployment.enabled }} +kind: Deployment +{{- else }} +kind: DaemonSet +{{- end }} +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} + pushprox-exporter: "client" +spec: + {{- if .Values.clients.deployment.enabled }} + replicas: {{ .Values.clients.deployment.replicas }} + {{- end }} + selector: + matchLabels: {{ include "pushProxy.client.labels" . | nindent 6 }} + template: + metadata: + labels: {{ include "pushProxy.client.labels" . | nindent 8 }} + spec: + {{- if .Values.clients.affinity }} + affinity: {{ toYaml .Values.clients.affinity | nindent 8 }} + {{- end }} + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} +{{- if .Values.clients.nodeSelector }} +{{ toYaml .Values.clients.nodeSelector | indent 8 }} +{{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} +{{- if .Values.clients.tolerations }} +{{ toYaml .Values.clients.tolerations | indent 8 }} +{{- end }} + hostNetwork: true + dnsPolicy: ClusterFirstWithHostNet + serviceAccountName: {{ template "pushProxy.client.name" . }} + {{- if .Values.global.imagePullSecretName }} + imagePullSecrets: + - name: {{ .Values.global.imagePullSecretName }} + {{- end }} + containers: + - name: pushprox-client + image: {{ template "system_default_registry" . }}{{ .Values.clients.image.repository }}:{{ .Values.clients.image.tag }} + command: + {{- range .Values.clients.command }} + - {{ . | quote }} + {{- end }} + args: + - --fqdn=$(HOST_IP) + - --proxy-url=$(PROXY_URL) + {{- if .Values.clients.metrics.enabled }} + - --metrics-addr=$(PORT) + {{- end }} + - --allow-port={{ required "Need .Values.metricsPort to configure client to be allowed to scrape metrics at port" .Values.metricsPort}} + {{- if .Values.clients.useLocalhost }} + - --use-localhost + {{- end }} + {{- if .Values.clients.https.enabled }} + {{- if .Values.clients.https.insecureSkipVerify }} + - --insecure-skip-verify + {{- end }} + {{- if .Values.clients.https.useServiceAccountCredentials }} + - --token-path=/var/run/secrets/kubernetes.io/serviceaccount/token + {{- end }} + {{- if .Values.clients.https.certDir }} + - --tls.cert=/etc/ssl/push-proxy/push-proxy.pem + - --tls.key=/etc/ssl/push-proxy/push-proxy-key.pem + - --tls.cacert=/etc/ssl/push-proxy/push-proxy-ca-cert.pem + {{- end }} + {{- end }} + env: + - name: HOST_IP + valueFrom: + fieldRef: + fieldPath: status.hostIP + {{- if .Values.clients.metrics.enabled }} + - name: PORT + value: :{{ .Values.clients.port }} + {{- end }} + - name: PROXY_URL + value: {{ template "pushProxy.proxyUrl" . }} + securityContext: + runAsNonRoot: true + runAsUser: 1000 + {{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + volumeMounts: + - name: metrics-cert-dir + mountPath: /etc/ssl/push-proxy + {{- end }} + {{- if .Values.clients.resources }} + resources: {{ toYaml .Values.clients.resources | nindent 10 }} + {{- end }} + {{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + initContainers: + - name: copy-certs + image: {{ template "system_default_registry" . }}{{ .Values.clients.copyCertsImage.repository }}:{{ .Values.clients.copyCertsImage.tag }} + command: + - sh + - -c + - | + echo "Searching for files to copy within the source volume" + echo "cert: ${CERT_FILE_NAME}" + echo "key: ${KEY_FILE_NAME}" + echo "cacert: ${CACERT_FILE_NAME}" + + CERT_FILE_SOURCE=$(find /etc/source/ -type f -name "${CERT_FILE_NAME}" | sort -r | head -n 1) + KEY_FILE_SOURCE=$(find /etc/source/ -type f -name "${KEY_FILE_NAME}" | sort -r | head -n 1) + CACERT_FILE_SOURCE=$(find /etc/source/ -type f -name "${CACERT_FILE_NAME}" | sort -r | head -n 1) + + test -z ${CERT_FILE_SOURCE} && echo "Failed to find cert file" && exit 1 + test -z ${KEY_FILE_SOURCE} && echo "Failed to find key file" && exit 1 + test -z ${CACERT_FILE_SOURCE} && echo "Failed to find cacert file" && exit 1 + + echo "Copying cert file from $CERT_FILE_SOURCE to $CERT_FILE_TARGET" + cp $CERT_FILE_SOURCE $CERT_FILE_TARGET || exit 1 + chmod 444 $CERT_FILE_TARGET || exit 1 + + echo "Copying key file from $KEY_FILE_SOURCE to $KEY_FILE_TARGET" + cp $KEY_FILE_SOURCE $KEY_FILE_TARGET || exit 1 + chmod 444 $KEY_FILE_TARGET || exit 1 + + echo "Copying cacert file from $CACERT_FILE_SOURCE to $CACERT_FILE_TARGET" + cp $CACERT_FILE_SOURCE $CACERT_FILE_TARGET || exit 1 + chmod 444 $CACERT_FILE_TARGET || exit 1 + env: + - name: CERT_FILE_NAME + value: {{ required "Need a TLS cert file for scraping metrics endpoint over HTTPs" .Values.clients.https.certFile }} + - name: KEY_FILE_NAME + value: {{ required "Need a TLS key file for scraping metrics endpoint over HTTPs" .Values.clients.https.keyFile }} + - name: CACERT_FILE_NAME + value: {{ required "Need a TLS CA cert file for scraping metrics endpoint over HTTPs" .Values.clients.https.caCertFile }} + - name: CERT_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy.pem + - name: KEY_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy-key.pem + - name: CACERT_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy-ca-cert.pem + securityContext: + runAsNonRoot: false +{{- if and .Values.global.seLinux.enabled .Values.clients.https.seLinuxOptions }} + seLinuxOptions: {{ .Values.clients.https.seLinuxOptions | toYaml | nindent 12 }} +{{- end }} + volumeMounts: + - name: metrics-cert-dir-source + mountPath: /etc/source + readOnly: true + - name: metrics-cert-dir + mountPath: /etc/ssl/push-proxy + volumes: + - name: metrics-cert-dir-source + hostPath: + path: {{ required "Need access to volume on host with the SSL cert files to use HTTPs" .Values.clients.https.certDir }} + - name: metrics-cert-dir + emptyDir: {} + {{- end }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2Etcd/templates/pushprox-proxy-rbac.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2Etcd/templates/pushprox-proxy-rbac.yaml new file mode 100644 index 00000000000..eefe609058c --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2Etcd/templates/pushprox-proxy-rbac.yaml @@ -0,0 +1,68 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if and .Values.proxy }}{{ if .Values.proxy.enabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ template "pushProxy.proxy.name" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +rules: +{{- if .Values.global.cattle.psp.enabled }} +- apiGroups: ['policy'] + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - {{ template "pushProxy.proxy.name" . }} +{{- end }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ template "pushProxy.proxy.name" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "pushProxy.proxy.name" . }} +subjects: + - kind: ServiceAccount + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +--- +{{- if .Values.global.cattle.psp.enabled }} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +spec: + privileged: false + hostNetwork: false + hostIPC: false + hostPID: false + runAsUser: + rule: 'MustRunAsNonRoot' + seLinux: + rule: 'RunAsAny' + supplementalGroups: + rule: 'MustRunAs' + ranges: + - min: 1 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + - min: 1 + max: 65535 + readOnlyRootFilesystem: false + volumes: + - 'secret' +{{- end }}{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2Etcd/templates/pushprox-proxy.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2Etcd/templates/pushprox-proxy.yaml new file mode 100644 index 00000000000..723bbd6c00b --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2Etcd/templates/pushprox-proxy.yaml @@ -0,0 +1,57 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if and .Values.proxy }}{{ if .Values.proxy.enabled }} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} + pushprox-exporter: "proxy" +spec: + selector: + matchLabels: {{ include "pushProxy.proxy.labels" . | nindent 6 }} + template: + metadata: + labels: {{ include "pushProxy.proxy.labels" . | nindent 8 }} + spec: + securityContext: + runAsNonRoot: true + runAsUser: 1000 + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} +{{- if .Values.proxy.nodeSelector }} +{{ toYaml .Values.proxy.nodeSelector | indent 8 }} +{{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} +{{- if .Values.proxy.tolerations }} +{{ toYaml .Values.proxy.tolerations | indent 8 }} +{{- end }} + serviceAccountName: {{ template "pushProxy.proxy.name" . }} + {{- if .Values.global.imagePullSecretName }} + imagePullSecrets: + - name: {{ .Values.global.imagePullSecretName }} + {{- end }} + containers: + - name: pushprox-proxy + image: {{ template "system_default_registry" . }}{{ .Values.proxy.image.repository }}:{{ .Values.proxy.image.tag }} + command: + {{- range .Values.proxy.command }} + - {{ . | quote }} + {{- end }} + {{- if .Values.proxy.resources }} + resources: {{ toYaml .Values.proxy.resources | nindent 10 }} + {{- end }} +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +spec: + ports: + - name: pp-proxy + port: {{ required "Need .Values.proxy.port to configure proxy" .Values.proxy.port }} + protocol: TCP + targetPort: {{ .Values.proxy.port }} + selector: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2Etcd/templates/pushprox-servicemonitor.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2Etcd/templates/pushprox-servicemonitor.yaml new file mode 100644 index 00000000000..67eb2216b5a --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2Etcd/templates/pushprox-servicemonitor.yaml @@ -0,0 +1,45 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.serviceMonitor }}{{- if .Values.serviceMonitor.enabled }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ template "pushprox.serviceMonitor.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.serviceMonitor.labels" . | nindent 4 }} +spec: + endpoints: {{include "pushProxy.serviceMonitor.endpoints" . | nindent 4 }} + jobLabel: component + podTargetLabels: + - component + - pushprox-exporter + namespaceSelector: + matchNames: + - {{ template "pushprox.namespace" . }} + selector: + matchLabels: {{ include "pushProxy.client.labels" . | nindent 6 }} +--- +{{- $selector := "" }} +{{- if not (kindIs "invalid" .Values.service) }} +{{- if not (kindIs "invalid" .Values.service.selector) }} +{{ if .Values.service.selector }} +{{- if .Values.clients.enabled }} +{{- required (printf "Cannot override .Values.service.selector=%s when .Values.clients.enabled=true" (toJson .Values.service.selector)) "" }} +{{- end }} +{{- $selector = (toYaml .Values.service.selector) }} +{{- end }} +{{- end }} +{{- end }} +apiVersion: v1 +kind: Service +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +spec: + ports: + - name: metrics + port: {{ required "Need .Values.metricsPort to configure client to listen to metrics at port" .Values.metricsPort}} + protocol: TCP + targetPort: {{ .Values.metricsPort }} + selector: {{ default (include "pushProxy.client.labels" .) $selector | nindent 4 }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2Etcd/templates/validate-install-crd.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2Etcd/templates/validate-install-crd.yaml new file mode 100644 index 00000000000..16abc2fa83a --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2Etcd/templates/validate-install-crd.yaml @@ -0,0 +1,14 @@ +#{{- if gt (len (lookup "rbac.authorization.k8s.io/v1" "ClusterRole" "" "")) 0 -}} +# {{- $found := dict -}} +# {{- set $found "monitoring.coreos.com/v1/ServiceMonitor" false -}} +# {{- range .Capabilities.APIVersions -}} +# {{- if hasKey $found (toString .) -}} +# {{- set $found (toString .) true -}} +# {{- end -}} +# {{- end -}} +# {{- range $_, $exists := $found -}} +# {{- if (eq $exists false) -}} +# {{- required "Required CRDs are missing. Please install Prometheus Operator CRDs before installing this chart." "" -}} +# {{- end -}} +# {{- end -}} +#{{- end -}} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2Etcd/templates/validate-psp-install.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2Etcd/templates/validate-psp-install.yaml new file mode 100644 index 00000000000..a30c59d3b7c --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2Etcd/templates/validate-psp-install.yaml @@ -0,0 +1,7 @@ +#{{- if gt (len (lookup "rbac.authorization.k8s.io/v1" "ClusterRole" "" "")) 0 -}} +#{{- if .Values.global.cattle.psp.enabled }} +#{{- if not (.Capabilities.APIVersions.Has "policy/v1beta1/PodSecurityPolicy") }} +#{{- fail "The target cluster does not have the PodSecurityPolicy API resource. Please disable PSPs in this chart before proceeding." -}} +#{{- end }} +#{{- end }} +#{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2Etcd/values.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2Etcd/values.yaml new file mode 100644 index 00000000000..7f38498a1c5 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2Etcd/values.yaml @@ -0,0 +1,166 @@ +# Default values for rancher-pushprox. +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. + +# Default image containing both the proxy and the client was generated from the following Dockerfile +# https://github.com/prometheus-community/PushProx/blob/eeadbe766641699129920ccfaaaa30a85c67fe81/Dockerfile#L1-L15 + +# Configuration + +global: + cattle: + psp: + enabled: false + systemDefaultRegistry: "" + seLinux: + enabled: false + +# A list of Semver constraint strings (defined by https://github.com/Masterminds/semver) and values.yaml overrides. +# +# For each key in kubeVersionOverrides, this chart will check to see if the current Kubernetes cluster's version matches +# any of the semver constraints provided as keys on the map. +# +# On seeing a match, the default value for each values.yaml field overridden will be updated with the new value. +# +# If multiple matches are encountered (due to overlapping semver ranges), the matches will be applied in order. +# +# Notes: +# - On running a helm template, Helm generally assumes the kubeVersion is v1.20.0 +# - On running a helm install --dry-run, the correct kubeVersion should be chosen. +kubeVersionOverrides: [] +# - constraint: "< 1.21" +# values: +# metricsPort: 10252 +# clients: +# https: +# enabled: false +# insecureSkipVerify: false +# useServiceAccountCredentials: false + +namespaceOverride: "" + +# The component that is being monitored (i.e. etcd) +component: "component" + +# The port containing the metrics that need to be scraped +metricsPort: 2739 + +# Configure ServiceMonitor that monitors metrics from the metricsPort endpoint +serviceMonitor: + enabled: true + # A list of endpoints that will be added to the ServiceMonitor based on the Endpoint spec + # Source: https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#endpoint + # By default, proxyUrl and params._scheme will be overridden based on other values + endpoints: + - port: metrics + +# Configure Service that grabs scrape targets +service: + # The selector that is used to populate the Service's Endpoints object. + # The chart will error out on rendering templating if .Values.clients.enabled is set alongside this field, + # since it is expected that this service should point to the PushProx Clients Daemonset / Deployment + selector: {} + +clients: + enabled: true + # The port which the PushProx client will post PushProx metrics to + port: 9369 + # If unset, this will default to the URL for the proxy service: http://pushprox-{{component}}-proxy.{{namepsace}}.svc.cluster.local:{{proxy.port}} + # Should be modified if the clients are being deployed outside the cluster where the proxy rests, otherwise leave it null + proxyUrl: "" + # If set to true, the client will forward any requests from the host IP to 127.0.0.1 + # It will only allow proxy requests to the metricsPort specified + useLocalhost: false + # Configuration for accessing metrics via HTTPS + https: + # Does the client require https to access the metrics? + enabled: false + # Does the client require requests be sent to http or https? + forceHTTPSScheme: false + # If set to true, the client will create a service account with adequate permissions and set a flag + # on the client to use the service account token provided by it to make authorized scrape requests + useServiceAccountCredentials: false + # Configuration for authentication to metrics via https endpoint + authenticationMethod: + # Reads token from defined file in container + # This function is deprecated in the prometheus operator api and may be removed in a future version + bearerTokenFile: + enabled: false + bearerTokenFilePath: "/var/run/secrets/kubernetes.io/serviceaccount/token" + # Reads token from defined secret in namespace + # This function is deprecated in the prometheus operator api and may be removed in a future version + bearerTokenSecret: + enabled: false + # Reads token from defined secret in namespace + authorization: + enabled: false + type: "bearer" + credentials: + key: "token" + optional: false + # If set to true, the client will disable SSL security checks + insecureSkipVerify: false + # Directory on host where necessary TLS cert and key to scrape metrics can be found + certDir: "" + # Filenames for files located in .Values.clients.https.certDir that correspond to TLS settings + certFile: "" + keyFile: "" + caCertFile: "" + # seLinuxOptions to be passed into the container that copies certs. Should define a container with permissions to read the files in the certDir provided on the host. + # Required and only used if `clients.https.enabled` is set and `clients.https.certDir` is provided. + seLinuxOptions: {} + + metrics: + # Whether the client should publish PushProx client-specific metrics to .Values.clients.port + enabled: false + + rbac: + # Additional permissions to provide to the ServiceAccount bound to the client + # This can be used to provide additional permissions for the client to scrape metrics from the k8s API + # Only enabled if clients.https.enabled and clients.https.useServiceAccountCredentials are true + additionalRules: [] + + # Resource limits + resources: {} + + # Options to select all nodes to deploy client DaemonSet on + nodeSelector: {} + tolerations: [] + affinity: {} + + image: + repository: rancher/pushprox-client + tag: v0.1.4-rancher2-client + command: ["pushprox-client"] + + copyCertsImage: + repository: rancher/mirrored-library-busybox + tag: 1.31.1 + + # The default intention of rancher-pushprox clients is to scrape hostNetwork metrics across all nodes. + # This can be used to scrape internal Kubernetes components or DaemonSets of hostNetwork Pods in + # situations where a cloud provider firewall prevents Pod-To-Host communication but not Pod-To-Pod. + # However, if the underlying hostNetwork Pod that is being scraped is managed by a Deployment, + # this advanced option enables users to deploy the client as a Deployment instead of a DaemonSet. + # If a user deploys this feature and the underlying Deployment's number of replicas changes, the user will + # be responsible for upgrading this chart accordingly to the right number of replicas. + deployment: + enabled: false + replicas: 0 + +proxy: + enabled: true + # The port through which PushProx clients will communicate to the proxy + port: 8080 + + # Resource limits + resources: {} + + # Options to select a node to run a single proxy deployment on + nodeSelector: {} + tolerations: [] + + image: + repository: rancher/pushprox-proxy + tag: v0.1.4-rancher2-proxy + command: ["pushprox-proxy"] diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2IngressNginx/.helmignore b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2IngressNginx/.helmignore new file mode 100644 index 00000000000..0e8a0eb36f4 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2IngressNginx/.helmignore @@ -0,0 +1,23 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*.orig +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2IngressNginx/Chart.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2IngressNginx/Chart.yaml new file mode 100644 index 00000000000..8978b710276 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2IngressNginx/Chart.yaml @@ -0,0 +1,15 @@ +annotations: + catalog.cattle.io/hidden: "true" + catalog.cattle.io/kube-version: '>= 1.26.0-0 < 1.31.0-0' + catalog.cattle.io/os: linux + catalog.rancher.io/certified: rancher + catalog.rancher.io/namespace: cattle-monitoring-system + catalog.rancher.io/release-name: rancher-pushprox +apiVersion: v1 +appVersion: v0.1.4-rancher2 +description: Sets up a deployment of the PushProx proxy and a DaemonSet of PushProx + clients. +kubeVersion: '>=1.26.0-0' +name: rke2IngressNginx +type: application +version: 0.2.1 diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2IngressNginx/README.md b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2IngressNginx/README.md new file mode 100644 index 00000000000..345002f48a2 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2IngressNginx/README.md @@ -0,0 +1,90 @@ +# rancher-pushprox + +A Rancher chart based on Rancher [PushProx](https://github.com/rancher/PushProx) that sets up a Deployment of a PushProx proxy and a DaemonSet of PushProx clients on a Kubernetes cluster. + +Installs [rancher-pushprox](https://github.com/rancher/charts/tree/gh-pages/packages/rancher-pushprox) to create PushProx clients that can access their host's network and register with a PushProx proxy. A [Prometheus Operator](https://github.com/coreos/prometheus-operator) ServiceMonitor CR is also included that is configured to scrape the metrics from each of the clients through the proxy. + +Using an instance of this chart is suitable for the following scenarios: +- You need to scrape metrics from a port that should not be accessible outside of the host (e.g. scraping `etcd` metrics in a hardened cluster) +- You need to scrape metrics on a host that are not exposed outside of 127.0.0.1 (e.g. scraping `kube-proxy` metrics) +- You need to scrape metrics through HTTPS using certs hosted directly on `hostPath` +- You need to scrape metrics from Kubernetes components that require authorization via a service account (e.g. permissions to make request to `/metrics`) +- You need to scrape metrics without access to cacerts (i.e. enable `insecureSkipVerify`) + +The clients and proxy are created based on a Rancher fork of the [prometheus-community/PushProx](https://github.com/prometheus-community/PushProx) project. + +## Upgrading to Kubernetes v1.25+ + +Starting in Kubernetes v1.25, [Pod Security Policies](https://kubernetes.io/docs/concepts/security/pod-security-policy/) have been removed from the Kubernetes API. + +As a result, **before upgrading to Kubernetes v1.25** (or on a fresh install in a Kubernetes v1.25+ cluster), users are expected to perform an in-place upgrade of this chart with `global.cattle.psp.enabled` set to `false` if it has been previously set to `true`. +​ +> **Note:** +> In this chart release, any previous field that was associated with any PSP resources have been removed in favor of a single global field: `global.cattle.psp.enabled`. + +> **Note:** +> If you upgrade your cluster to Kubernetes v1.25+ before removing PSPs via a `helm upgrade` (even if you manually clean up resources), **it will leave the Helm release in a broken state within the cluster such that further Helm operations will not work (`helm uninstall`, `helm upgrade`, etc.).** +> +> If your charts get stuck in this state, please consult the Rancher docs on how to clean up your Helm release secrets. + +Upon setting `global.cattle.psp.enabled` to false, the chart will remove any PSP resources deployed on its behalf from the cluster. This is the default setting for this chart. + +As a replacement for PSPs, [Pod Security Admission](https://kubernetes.io/docs/concepts/security/pod-security-admission/) should be used. Please consult the Rancher docs for more details on how to configure your chart release namespaces to work with the new Pod Security Admission and apply Pod Security Standards. + +## Configuration + +The following tables list the configurable parameters of the rancher-pushprox chart and their default values. + +### General + +#### Required +| Parameter | Description | Example | +| ----- | ----------- | ------ | +| `component` | The component that is being monitored | `kube-etcd` +| `metricsPort` | The port on the host that contains the metrics you want to scrape (e.g. `http://:/metrics`) | `2379` | +| `namespaceOverride` | The namespace to install the chart | `""` + +#### Optional +| Parameter | Description | Default | +| ----- | ----------- | ------ | +| `serviceMonitor.enabled` | Deploys a [Prometheus Operator](https://github.com/coreos/prometheus-operator/blob/master/Documentation/api.md#servicemonitor) ServiceMonitor CR that is configured to scrape metrics on the hosts that the clients are deployed on via the proxy. Also deploys a Service that points to all pods with the expected client name that exposes the `metricsPort` selected | `true` | +| `serviceMonitor.endpoints` | A list of endpoints that will be added to the ServiceMonitor based on the [Endpoint spec](https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#endpoint) | `[{port: metrics}]` | +| `service.selector` | The selector that is used to populate the Service's Endpoints object. The chart will error out on rendering templating if `.Values.clients.enabled` is set alongside this field, since it is expected that this service should point to the PushProx Clients Daemonset / Deployment | `{}` | +| `clients.enabled` | Deploys a DaemonSet of clients that are each capable of scraping endpoints on the hostNetwork it is deployed on | `true` | +| `clients.port` | The port where the client will publish PushProx client-specific metrics. If deploying multiple clients onto the same node, the clients should not have conflicting ports | `9369` | +| `clients.proxyUrl` | Overrides the default proxyUrl setting of `http://pushprox-{{ .Values.component }}-proxy.{{ . Release.Namespace }}.svc.cluster.local:{{ .Values.proxy.port }}"` with the `proxyUrl` specified | `""` | +| `clients.useLocalhost` | Sets a flag on each client deployment to redirect scrapes directed to `HOST_IP` to `127.0.0.1` | `false` | +| `clients.https.enabled` | Enables scraping metrics via HTTPS using the provided TLS certs that exist on each host | `false` | +| `clients.https.forceHTTPSScheme` | Forces scraping metrics via HTTPS using the provided TLS certs that exist on each host | `false` | +| `clients.https.useServiceAccountCredentials` | If set to true, the client will create a service account with permissions to scrape `/metrics` endpoint of Kubernetes components. The client will use the service account token provided to make authorized scrape requests to the Kubernetes API | `false` | +| `clients.https.authenticationMethod.bearerTokenFile.enabled` | If set to true, the client will use service account credentials mounted at the configured path `clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath`. This requires permissions to scrape `/metrics` endpoint of Kubernetes components. This method is deprecated by the prometheus operator and may be removed in a future release | `false` | +| `clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath` | This is a volume mount on the pod with permissions to scrape `/metrics` endpoint of Kubernetes components | `"/var/run/secrets/kubernetes.io/serviceaccount/token"` | +| `clients.https.authenticationMethod.bearerTokenSecret.enabled` | If set to true, the client will use service account credentials to scrape `/metrics` endpoint of Kubernetes components. This method is deprecated by the prometheus operator and may be removed in a future release | `false` | +| `clients.https.authenticationMethod.authorization.enabled` | If set to true, the client will use service account credentials to scrape `/metrics` endpoint of Kubernetes components | `false` | +| `clients.https.authenticationMethod.authorization.type` | If set, the client will use this type of authorization in its client requests for metrics | `"bearer"` | +| `clients.https.authenticationMethod.authorization.credentials.key` | If set, the client will use this key in the secret created by `clients.https.useServiceAccountCredentials` for authorization in its client requests for metrics | `"token"` | +| `clients.https.authenticationMethod.authorization.credentials.optional` | If set to false, the client will fail if the key in the secret created by `clients.https.useServiceAccountCredentials` does not exist | `false` | +| `clients.https.insecureSkipVerify` | If set to true, the client will disable SSL security checks | `false` | +| `clients.https.certDir` | A `hostPath` where TLS certs can be found. This path is mounted as a volume on an `initContainer` which copies only the necessary files over to an EmptyDir volume used by each client. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.certFile` | The path to the TLS cert file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.keyFile` | The path to the TLS key file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.caCertFile` | The path to the TLS cacert file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.seLinuxOptions` | seLinuxOptions to be passed into the container that copies certs. Should define a container with permissions to read the files in the certDir provided on the host. Required and only used if `clients.https.enabled` is set and `clients.https.certDir` is provided. | `""` | +| `clients.metrics.enabled` | Whether the client should publish PushProx client-specific metrics. | `false` | +| `clients.rbac.additionalRules` | Additional permissions to provide to the ServiceAccount bound to the client. This can be used to provide additional permissions for the client to scrape metrics from the k8s API. Only enabled if clients.https.enabled and clients.https.useServiceAccountCredentials are true | `[]` | +| `clients.deployment.enabled` | Deploys the client as a Deployment (generally used if the underlying hostNetwork Pod that is being scraped is managed by a Deployment) | `false` | +| `clients.deployment.replicas` | The number of pods the Deployment has, it should match the number of pod the hostNetwork Deployment has. Required and only used if `client.deployment.enable` is set | `0` | +| `clients.deployment.affinity` | The affinity rules that allocate the pod to the node in which the hostNetwork Deployment's pods run. Required and only used if `client.deployment.enable` is set | `{}` | +| `clients.resources` | Set resource limits and requests for the client container | `{}` | +| `clients.nodeSelector` | Select which nodes to deploy the clients on | `{}` | +| `clients.tolerations` | Specify tolerations for clients | `[]` | +| `proxy.enabled` | Deploys the proxy that each client will register with | `true` | +| `proxy.port` | The port exposed by the proxy that each client will register with to allow metrics to be scraped from the host | `8080` | +| `proxy.resources` | Set resource limits and requests for the proxy container | `{}` | +| `proxy.nodeSelector` | Select which nodes the proxy can be deployed on | `{}` | +| `proxy.tolerations` | Specify tolerations (if necessary) to allow the proxy to be deployed on the selected node | `[]` | +| `kubeVersionOverrides` | A list of Semver constraint strings (defined by https://github.com/Masterminds/semver) and values.yaml overrides. For each key in kubeVersionOverrides, this chart will check to see if the current Kubernetes cluster's version matches any of the semver constraints provided as keys on the map. On seeing a match, the default value for each values.yaml field overridden will be updated with the new value. If multiple matches are encountered (due to overlapping semver ranges), the matches will be applied in order. | `[]` + +*Tip: The filepaths set in `clients.https.File` can include wildcard characters*. + +See [rancher-monitoring](https://github.com/rancher/charts/tree/gh-pages/packages/rancher-monitoring) for examples of how this chart can be used. diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2IngressNginx/templates/_helpers.tpl b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2IngressNginx/templates/_helpers.tpl new file mode 100644 index 00000000000..1ba50939441 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2IngressNginx/templates/_helpers.tpl @@ -0,0 +1,170 @@ +# Rancher + +{{- define "system_default_registry" -}} +{{- if .Values.global.cattle.systemDefaultRegistry -}} +{{- printf "%s/" .Values.global.cattle.systemDefaultRegistry -}} +{{- end -}} +{{- end -}} + +# Windows Support + +{{/* +Windows cluster will add default taint for linux nodes, +add below linux tolerations to workloads could be scheduled to those linux nodes +*/}} + +{{- define "linux-node-tolerations" -}} +- key: "cattle.io/os" + value: "linux" + effect: "NoSchedule" + operator: "Equal" +{{- end -}} + +{{- define "linux-node-selector" -}} +{{- if semverCompare "<1.14-0" .Capabilities.KubeVersion.GitVersion -}} +beta.kubernetes.io/os: linux +{{- else -}} +kubernetes.io/os: linux +{{- end -}} +{{- end -}} + +# General + +{{- define "applyKubeVersionOverrides" -}} +{{- $overrides := dict -}} +{{- range $override := .Values.kubeVersionOverrides -}} +{{- if semverCompare $override.constraint $.Capabilities.KubeVersion.Version -}} +{{- $_ := mergeOverwrite $overrides $override.values -}} +{{- end -}} +{{- end -}} +{{- $_ := mergeOverwrite .Values $overrides -}} +{{- end -}} + +{{- define "pushprox.namespace" -}} + {{- if .Values.namespaceOverride -}} + {{- .Values.namespaceOverride -}} + {{- else -}} + {{- .Release.Namespace -}} + {{- end -}} +{{- end -}} + +{{- define "pushProxy.commonLabels" -}} +release: {{ .Release.Name }} +component: {{ .Values.component | quote }} +provider: kubernetes +{{- end -}} + +{{- define "pushProxy.proxyUrl" -}} +{{- $_ := (required "Template requires either .Values.proxy.port or .Values.client.proxyUrl to set proxyUrl for client" (or .Values.clients.proxyUrl .Values.proxy.port)) -}} +{{- if .Values.clients.proxyUrl -}} +{{ printf "%s" .Values.clients.proxyUrl }} +{{- else -}} +{{ printf "http://%s.%s.svc:%d" (include "pushProxy.proxy.name" .) (include "pushprox.namespace" .) (int .Values.proxy.port) }} +{{- end -}}{{- end -}} + +# Client + +{{- define "pushProxy.client.name" -}} +{{- printf "pushprox-%s-client" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.client.serviceAccountTokenName" -}} +{{- printf "pushprox-%s-client-service-account-token" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.client.labels" -}} +k8s-app: {{ template "pushProxy.client.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +# Proxy + +{{- define "pushProxy.proxy.name" -}} +{{- printf "pushprox-%s-proxy" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.proxy.labels" -}} +k8s-app: {{ template "pushProxy.proxy.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +# ServiceMonitor + +{{- define "pushprox.serviceMonitor.name" -}} +{{- printf "%s-%s" .Release.Name (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.serviceMonitor.labels" -}} +app: {{ template "pushprox.serviceMonitor.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +{{- define "pushProxy.serviceMonitor.endpoints" -}} +{{- $proxyURL := (include "pushProxy.proxyUrl" .) -}} +{{- $useHTTPS := .Values.clients.https.enabled -}} +{{- $setHTTPSScheme := .Values.clients.https.forceHTTPSScheme -}} +{{- $insecureSkipVerify := .Values.clients.https.insecureSkipVerify -}} +{{- $useServiceAccountCredentials := .Values.clients.https.useServiceAccountCredentials -}} +{{- $serviceAccountTokenName := (include "pushProxy.client.serviceAccountTokenName" . ) -}} +{{- $metricRelabelings := list }} +{{- $endpoints := .Values.serviceMonitor.endpoints }} +{{- if .Values.proxy.enabled }} +{{- $_ := set . "proxyUrl" $proxyURL }} +{{- end }} +{{- range $endpoints }} +{{- if $.Values.proxy.enabled }} +{{- $_ := set . "proxyUrl" $proxyURL }} +{{- end }} +{{- $clusterIdRelabel := dict }} +{{- $metricRelabelings := list }} +{{- if $.Values.global.cattle.clusterId }} +{{- $_ := set $clusterIdRelabel "action" "replace" }} +{{- $_ := set $clusterIdRelabel "sourceLabels" (list "__address__") }} +{{- $_ := set $clusterIdRelabel "targetLabel" "cluster_id" }} +{{- $_ := set $clusterIdRelabel "replacement" $.Values.global.cattle.clusterId }} +{{- $metricRelabelings = append $metricRelabelings $clusterIdRelabel }} +{{- end }} +{{- $clusterNameRelabel := dict }} +{{- if $.Values.global.cattle.clusterName }} +{{- $_ := set $clusterNameRelabel "action" "replace" }} +{{- $_ := set $clusterNameRelabel "sourceLabels" (list "__address__") }} +{{- $_ := set $clusterNameRelabel "targetLabel" "cluster_name" }} +{{- $_ := set $clusterNameRelabel "replacement" $.Values.global.cattle.clusterName }} +{{- $metricRelabelings = append $metricRelabelings $clusterNameRelabel }} +{{- end }} +{{- if not (empty $metricRelabelings) }} +{{- $_ := set . "metricRelabelings" ($metricRelabelings)}} +{{- end }} +{{- if $setHTTPSScheme -}} +{{- $_ := set . "scheme" "https" }} +{{- end -}} +{{- if $useHTTPS -}} +{{- if (hasKey . "params") }} +{{- $_ := set (get . "params") "_scheme" (list "https") }} +{{- else }} +{{- $_ := set . "params" (dict "_scheme" (list "https")) }} +{{- end }} +{{- end }} +{{- if (hasKey . "tlsConfig") }} +{{- $_ := set (get . "tlsConfig") "insecureSkipVerify" $insecureSkipVerify }} +{{- else }} +{{- $_ := set . "tlsConfig" (dict "insecureSkipVerify" $insecureSkipVerify) }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.bearerTokenFile.enabled }} +{{- $_ := set . "bearerTokenFile" $.Values.clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.bearerTokenSecret.enabled }} +{{- $_ := set . "bearerTokenSecret" $serviceAccountTokenName }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.authorization.enabled }} +{{- if (hasKey . "authorization") }} +{{- $_ := set (get . "authorization") "type" $.Values.clients.https.authenticationMethod.authorization.type }} +{{- $_ := set (get . "authorization") "credentials" (dict "name" $serviceAccountTokenName "key" $.Values.clients.https.authenticationMethod.authorization.credentials.key "optional" $.Values.clients.https.authenticationMethod.authorization.credentials.optional) }} +{{- else }} +{{- $_ := set . "authorization" (dict "type" $.Values.clients.https.authenticationMethod.authorization.type) }} +{{- $_ := set . "authorization" (dict "credentials" (dict "name" $serviceAccountTokenName "key" $.Values.clients.https.authenticationMethod.authorization.credentials.key "optional" $.Values.clients.https.authenticationMethod.authorization.credentials.optional)) }} +{{- end }} +{{- end }} +{{- end }} +{{- toYaml $endpoints }} +{{- end -}} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2IngressNginx/templates/pushprox-clients-rbac.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2IngressNginx/templates/pushprox-clients-rbac.yaml new file mode 100644 index 00000000000..a8e27c37356 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2IngressNginx/templates/pushprox-clients-rbac.yaml @@ -0,0 +1,97 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.clients }}{{- if .Values.clients.enabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ template "pushProxy.client.name" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +rules: +{{- if .Values.global.cattle.psp.enabled }} +- apiGroups: ['policy'] + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - {{ template "pushProxy.client.name" . }} +{{- end }} +{{- if and .Values.clients.https.enabled .Values.clients.https.useServiceAccountCredentials }} +- nonResourceURLs: ["/metrics"] + verbs: ["get"] +{{- if .Values.clients.rbac.additionalRules }} +{{ toYaml .Values.clients.rbac.additionalRules }} +{{- end }} +{{- end }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ template "pushProxy.client.name" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "pushProxy.client.name" . }} +subjects: + - kind: ServiceAccount + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +--- +{{- if .Values.clients.https.useServiceAccountCredentials }} +apiVersion: v1 +kind: Secret +type: kubernetes.io/service-account-token +metadata: + name: {{ template "pushProxy.client.serviceAccountTokenName" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} + annotations: + kubernetes.io/service-account.name: {{ template "pushProxy.client.name" . }} +{{- end }} +--- +{{- if .Values.global.cattle.psp.enabled }} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +spec: + privileged: false + hostNetwork: true + hostIPC: false + hostPID: false + runAsUser: + rule: 'RunAsAny' + seLinux: + rule: 'RunAsAny' +{{- if and .Values.clients.https.enabled .Values.clients.https.certDir .Values.global.seLinux.enabled .Values.clients.https.seLinuxOptions }} + seLinuxOptions: {{ .Values.clients.https.seLinuxOptions | toYaml | nindent 6 }} +{{- end }} + supplementalGroups: + rule: 'MustRunAs' + ranges: + - min: 0 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + - min: 0 + max: 65535 + readOnlyRootFilesystem: false + volumes: + - 'secret' +{{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + - 'emptyDir' + - 'hostPath' + allowedHostPaths: + - pathPrefix: {{ required "Need access to volume on host with the SSL cert files to use HTTPs" .Values.clients.https.certDir }} + readOnly: true +{{- end }} +{{- end }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2IngressNginx/templates/pushprox-clients.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2IngressNginx/templates/pushprox-clients.yaml new file mode 100644 index 00000000000..e8fcfb38836 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2IngressNginx/templates/pushprox-clients.yaml @@ -0,0 +1,157 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.clients }}{{- if .Values.clients.enabled }} +apiVersion: apps/v1 +{{- if .Values.clients.deployment.enabled }} +kind: Deployment +{{- else }} +kind: DaemonSet +{{- end }} +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} + pushprox-exporter: "client" +spec: + {{- if .Values.clients.deployment.enabled }} + replicas: {{ .Values.clients.deployment.replicas }} + {{- end }} + selector: + matchLabels: {{ include "pushProxy.client.labels" . | nindent 6 }} + template: + metadata: + labels: {{ include "pushProxy.client.labels" . | nindent 8 }} + spec: + {{- if .Values.clients.affinity }} + affinity: {{ toYaml .Values.clients.affinity | nindent 8 }} + {{- end }} + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} +{{- if .Values.clients.nodeSelector }} +{{ toYaml .Values.clients.nodeSelector | indent 8 }} +{{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} +{{- if .Values.clients.tolerations }} +{{ toYaml .Values.clients.tolerations | indent 8 }} +{{- end }} + hostNetwork: true + dnsPolicy: ClusterFirstWithHostNet + serviceAccountName: {{ template "pushProxy.client.name" . }} + {{- if .Values.global.imagePullSecretName }} + imagePullSecrets: + - name: {{ .Values.global.imagePullSecretName }} + {{- end }} + containers: + - name: pushprox-client + image: {{ template "system_default_registry" . }}{{ .Values.clients.image.repository }}:{{ .Values.clients.image.tag }} + command: + {{- range .Values.clients.command }} + - {{ . | quote }} + {{- end }} + args: + - --fqdn=$(HOST_IP) + - --proxy-url=$(PROXY_URL) + {{- if .Values.clients.metrics.enabled }} + - --metrics-addr=$(PORT) + {{- end }} + - --allow-port={{ required "Need .Values.metricsPort to configure client to be allowed to scrape metrics at port" .Values.metricsPort}} + {{- if .Values.clients.useLocalhost }} + - --use-localhost + {{- end }} + {{- if .Values.clients.https.enabled }} + {{- if .Values.clients.https.insecureSkipVerify }} + - --insecure-skip-verify + {{- end }} + {{- if .Values.clients.https.useServiceAccountCredentials }} + - --token-path=/var/run/secrets/kubernetes.io/serviceaccount/token + {{- end }} + {{- if .Values.clients.https.certDir }} + - --tls.cert=/etc/ssl/push-proxy/push-proxy.pem + - --tls.key=/etc/ssl/push-proxy/push-proxy-key.pem + - --tls.cacert=/etc/ssl/push-proxy/push-proxy-ca-cert.pem + {{- end }} + {{- end }} + env: + - name: HOST_IP + valueFrom: + fieldRef: + fieldPath: status.hostIP + {{- if .Values.clients.metrics.enabled }} + - name: PORT + value: :{{ .Values.clients.port }} + {{- end }} + - name: PROXY_URL + value: {{ template "pushProxy.proxyUrl" . }} + securityContext: + runAsNonRoot: true + runAsUser: 1000 + {{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + volumeMounts: + - name: metrics-cert-dir + mountPath: /etc/ssl/push-proxy + {{- end }} + {{- if .Values.clients.resources }} + resources: {{ toYaml .Values.clients.resources | nindent 10 }} + {{- end }} + {{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + initContainers: + - name: copy-certs + image: {{ template "system_default_registry" . }}{{ .Values.clients.copyCertsImage.repository }}:{{ .Values.clients.copyCertsImage.tag }} + command: + - sh + - -c + - | + echo "Searching for files to copy within the source volume" + echo "cert: ${CERT_FILE_NAME}" + echo "key: ${KEY_FILE_NAME}" + echo "cacert: ${CACERT_FILE_NAME}" + + CERT_FILE_SOURCE=$(find /etc/source/ -type f -name "${CERT_FILE_NAME}" | sort -r | head -n 1) + KEY_FILE_SOURCE=$(find /etc/source/ -type f -name "${KEY_FILE_NAME}" | sort -r | head -n 1) + CACERT_FILE_SOURCE=$(find /etc/source/ -type f -name "${CACERT_FILE_NAME}" | sort -r | head -n 1) + + test -z ${CERT_FILE_SOURCE} && echo "Failed to find cert file" && exit 1 + test -z ${KEY_FILE_SOURCE} && echo "Failed to find key file" && exit 1 + test -z ${CACERT_FILE_SOURCE} && echo "Failed to find cacert file" && exit 1 + + echo "Copying cert file from $CERT_FILE_SOURCE to $CERT_FILE_TARGET" + cp $CERT_FILE_SOURCE $CERT_FILE_TARGET || exit 1 + chmod 444 $CERT_FILE_TARGET || exit 1 + + echo "Copying key file from $KEY_FILE_SOURCE to $KEY_FILE_TARGET" + cp $KEY_FILE_SOURCE $KEY_FILE_TARGET || exit 1 + chmod 444 $KEY_FILE_TARGET || exit 1 + + echo "Copying cacert file from $CACERT_FILE_SOURCE to $CACERT_FILE_TARGET" + cp $CACERT_FILE_SOURCE $CACERT_FILE_TARGET || exit 1 + chmod 444 $CACERT_FILE_TARGET || exit 1 + env: + - name: CERT_FILE_NAME + value: {{ required "Need a TLS cert file for scraping metrics endpoint over HTTPs" .Values.clients.https.certFile }} + - name: KEY_FILE_NAME + value: {{ required "Need a TLS key file for scraping metrics endpoint over HTTPs" .Values.clients.https.keyFile }} + - name: CACERT_FILE_NAME + value: {{ required "Need a TLS CA cert file for scraping metrics endpoint over HTTPs" .Values.clients.https.caCertFile }} + - name: CERT_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy.pem + - name: KEY_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy-key.pem + - name: CACERT_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy-ca-cert.pem + securityContext: + runAsNonRoot: false +{{- if and .Values.global.seLinux.enabled .Values.clients.https.seLinuxOptions }} + seLinuxOptions: {{ .Values.clients.https.seLinuxOptions | toYaml | nindent 12 }} +{{- end }} + volumeMounts: + - name: metrics-cert-dir-source + mountPath: /etc/source + readOnly: true + - name: metrics-cert-dir + mountPath: /etc/ssl/push-proxy + volumes: + - name: metrics-cert-dir-source + hostPath: + path: {{ required "Need access to volume on host with the SSL cert files to use HTTPs" .Values.clients.https.certDir }} + - name: metrics-cert-dir + emptyDir: {} + {{- end }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2IngressNginx/templates/pushprox-proxy-rbac.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2IngressNginx/templates/pushprox-proxy-rbac.yaml new file mode 100644 index 00000000000..eefe609058c --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2IngressNginx/templates/pushprox-proxy-rbac.yaml @@ -0,0 +1,68 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if and .Values.proxy }}{{ if .Values.proxy.enabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ template "pushProxy.proxy.name" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +rules: +{{- if .Values.global.cattle.psp.enabled }} +- apiGroups: ['policy'] + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - {{ template "pushProxy.proxy.name" . }} +{{- end }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ template "pushProxy.proxy.name" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "pushProxy.proxy.name" . }} +subjects: + - kind: ServiceAccount + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +--- +{{- if .Values.global.cattle.psp.enabled }} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +spec: + privileged: false + hostNetwork: false + hostIPC: false + hostPID: false + runAsUser: + rule: 'MustRunAsNonRoot' + seLinux: + rule: 'RunAsAny' + supplementalGroups: + rule: 'MustRunAs' + ranges: + - min: 1 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + - min: 1 + max: 65535 + readOnlyRootFilesystem: false + volumes: + - 'secret' +{{- end }}{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2IngressNginx/templates/pushprox-proxy.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2IngressNginx/templates/pushprox-proxy.yaml new file mode 100644 index 00000000000..723bbd6c00b --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2IngressNginx/templates/pushprox-proxy.yaml @@ -0,0 +1,57 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if and .Values.proxy }}{{ if .Values.proxy.enabled }} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} + pushprox-exporter: "proxy" +spec: + selector: + matchLabels: {{ include "pushProxy.proxy.labels" . | nindent 6 }} + template: + metadata: + labels: {{ include "pushProxy.proxy.labels" . | nindent 8 }} + spec: + securityContext: + runAsNonRoot: true + runAsUser: 1000 + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} +{{- if .Values.proxy.nodeSelector }} +{{ toYaml .Values.proxy.nodeSelector | indent 8 }} +{{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} +{{- if .Values.proxy.tolerations }} +{{ toYaml .Values.proxy.tolerations | indent 8 }} +{{- end }} + serviceAccountName: {{ template "pushProxy.proxy.name" . }} + {{- if .Values.global.imagePullSecretName }} + imagePullSecrets: + - name: {{ .Values.global.imagePullSecretName }} + {{- end }} + containers: + - name: pushprox-proxy + image: {{ template "system_default_registry" . }}{{ .Values.proxy.image.repository }}:{{ .Values.proxy.image.tag }} + command: + {{- range .Values.proxy.command }} + - {{ . | quote }} + {{- end }} + {{- if .Values.proxy.resources }} + resources: {{ toYaml .Values.proxy.resources | nindent 10 }} + {{- end }} +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +spec: + ports: + - name: pp-proxy + port: {{ required "Need .Values.proxy.port to configure proxy" .Values.proxy.port }} + protocol: TCP + targetPort: {{ .Values.proxy.port }} + selector: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2IngressNginx/templates/pushprox-servicemonitor.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2IngressNginx/templates/pushprox-servicemonitor.yaml new file mode 100644 index 00000000000..67eb2216b5a --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2IngressNginx/templates/pushprox-servicemonitor.yaml @@ -0,0 +1,45 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.serviceMonitor }}{{- if .Values.serviceMonitor.enabled }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ template "pushprox.serviceMonitor.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.serviceMonitor.labels" . | nindent 4 }} +spec: + endpoints: {{include "pushProxy.serviceMonitor.endpoints" . | nindent 4 }} + jobLabel: component + podTargetLabels: + - component + - pushprox-exporter + namespaceSelector: + matchNames: + - {{ template "pushprox.namespace" . }} + selector: + matchLabels: {{ include "pushProxy.client.labels" . | nindent 6 }} +--- +{{- $selector := "" }} +{{- if not (kindIs "invalid" .Values.service) }} +{{- if not (kindIs "invalid" .Values.service.selector) }} +{{ if .Values.service.selector }} +{{- if .Values.clients.enabled }} +{{- required (printf "Cannot override .Values.service.selector=%s when .Values.clients.enabled=true" (toJson .Values.service.selector)) "" }} +{{- end }} +{{- $selector = (toYaml .Values.service.selector) }} +{{- end }} +{{- end }} +{{- end }} +apiVersion: v1 +kind: Service +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +spec: + ports: + - name: metrics + port: {{ required "Need .Values.metricsPort to configure client to listen to metrics at port" .Values.metricsPort}} + protocol: TCP + targetPort: {{ .Values.metricsPort }} + selector: {{ default (include "pushProxy.client.labels" .) $selector | nindent 4 }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2IngressNginx/templates/validate-install-crd.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2IngressNginx/templates/validate-install-crd.yaml new file mode 100644 index 00000000000..16abc2fa83a --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2IngressNginx/templates/validate-install-crd.yaml @@ -0,0 +1,14 @@ +#{{- if gt (len (lookup "rbac.authorization.k8s.io/v1" "ClusterRole" "" "")) 0 -}} +# {{- $found := dict -}} +# {{- set $found "monitoring.coreos.com/v1/ServiceMonitor" false -}} +# {{- range .Capabilities.APIVersions -}} +# {{- if hasKey $found (toString .) -}} +# {{- set $found (toString .) true -}} +# {{- end -}} +# {{- end -}} +# {{- range $_, $exists := $found -}} +# {{- if (eq $exists false) -}} +# {{- required "Required CRDs are missing. Please install Prometheus Operator CRDs before installing this chart." "" -}} +# {{- end -}} +# {{- end -}} +#{{- end -}} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2IngressNginx/templates/validate-psp-install.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2IngressNginx/templates/validate-psp-install.yaml new file mode 100644 index 00000000000..a30c59d3b7c --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2IngressNginx/templates/validate-psp-install.yaml @@ -0,0 +1,7 @@ +#{{- if gt (len (lookup "rbac.authorization.k8s.io/v1" "ClusterRole" "" "")) 0 -}} +#{{- if .Values.global.cattle.psp.enabled }} +#{{- if not (.Capabilities.APIVersions.Has "policy/v1beta1/PodSecurityPolicy") }} +#{{- fail "The target cluster does not have the PodSecurityPolicy API resource. Please disable PSPs in this chart before proceeding." -}} +#{{- end }} +#{{- end }} +#{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2IngressNginx/values.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2IngressNginx/values.yaml new file mode 100644 index 00000000000..7f38498a1c5 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2IngressNginx/values.yaml @@ -0,0 +1,166 @@ +# Default values for rancher-pushprox. +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. + +# Default image containing both the proxy and the client was generated from the following Dockerfile +# https://github.com/prometheus-community/PushProx/blob/eeadbe766641699129920ccfaaaa30a85c67fe81/Dockerfile#L1-L15 + +# Configuration + +global: + cattle: + psp: + enabled: false + systemDefaultRegistry: "" + seLinux: + enabled: false + +# A list of Semver constraint strings (defined by https://github.com/Masterminds/semver) and values.yaml overrides. +# +# For each key in kubeVersionOverrides, this chart will check to see if the current Kubernetes cluster's version matches +# any of the semver constraints provided as keys on the map. +# +# On seeing a match, the default value for each values.yaml field overridden will be updated with the new value. +# +# If multiple matches are encountered (due to overlapping semver ranges), the matches will be applied in order. +# +# Notes: +# - On running a helm template, Helm generally assumes the kubeVersion is v1.20.0 +# - On running a helm install --dry-run, the correct kubeVersion should be chosen. +kubeVersionOverrides: [] +# - constraint: "< 1.21" +# values: +# metricsPort: 10252 +# clients: +# https: +# enabled: false +# insecureSkipVerify: false +# useServiceAccountCredentials: false + +namespaceOverride: "" + +# The component that is being monitored (i.e. etcd) +component: "component" + +# The port containing the metrics that need to be scraped +metricsPort: 2739 + +# Configure ServiceMonitor that monitors metrics from the metricsPort endpoint +serviceMonitor: + enabled: true + # A list of endpoints that will be added to the ServiceMonitor based on the Endpoint spec + # Source: https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#endpoint + # By default, proxyUrl and params._scheme will be overridden based on other values + endpoints: + - port: metrics + +# Configure Service that grabs scrape targets +service: + # The selector that is used to populate the Service's Endpoints object. + # The chart will error out on rendering templating if .Values.clients.enabled is set alongside this field, + # since it is expected that this service should point to the PushProx Clients Daemonset / Deployment + selector: {} + +clients: + enabled: true + # The port which the PushProx client will post PushProx metrics to + port: 9369 + # If unset, this will default to the URL for the proxy service: http://pushprox-{{component}}-proxy.{{namepsace}}.svc.cluster.local:{{proxy.port}} + # Should be modified if the clients are being deployed outside the cluster where the proxy rests, otherwise leave it null + proxyUrl: "" + # If set to true, the client will forward any requests from the host IP to 127.0.0.1 + # It will only allow proxy requests to the metricsPort specified + useLocalhost: false + # Configuration for accessing metrics via HTTPS + https: + # Does the client require https to access the metrics? + enabled: false + # Does the client require requests be sent to http or https? + forceHTTPSScheme: false + # If set to true, the client will create a service account with adequate permissions and set a flag + # on the client to use the service account token provided by it to make authorized scrape requests + useServiceAccountCredentials: false + # Configuration for authentication to metrics via https endpoint + authenticationMethod: + # Reads token from defined file in container + # This function is deprecated in the prometheus operator api and may be removed in a future version + bearerTokenFile: + enabled: false + bearerTokenFilePath: "/var/run/secrets/kubernetes.io/serviceaccount/token" + # Reads token from defined secret in namespace + # This function is deprecated in the prometheus operator api and may be removed in a future version + bearerTokenSecret: + enabled: false + # Reads token from defined secret in namespace + authorization: + enabled: false + type: "bearer" + credentials: + key: "token" + optional: false + # If set to true, the client will disable SSL security checks + insecureSkipVerify: false + # Directory on host where necessary TLS cert and key to scrape metrics can be found + certDir: "" + # Filenames for files located in .Values.clients.https.certDir that correspond to TLS settings + certFile: "" + keyFile: "" + caCertFile: "" + # seLinuxOptions to be passed into the container that copies certs. Should define a container with permissions to read the files in the certDir provided on the host. + # Required and only used if `clients.https.enabled` is set and `clients.https.certDir` is provided. + seLinuxOptions: {} + + metrics: + # Whether the client should publish PushProx client-specific metrics to .Values.clients.port + enabled: false + + rbac: + # Additional permissions to provide to the ServiceAccount bound to the client + # This can be used to provide additional permissions for the client to scrape metrics from the k8s API + # Only enabled if clients.https.enabled and clients.https.useServiceAccountCredentials are true + additionalRules: [] + + # Resource limits + resources: {} + + # Options to select all nodes to deploy client DaemonSet on + nodeSelector: {} + tolerations: [] + affinity: {} + + image: + repository: rancher/pushprox-client + tag: v0.1.4-rancher2-client + command: ["pushprox-client"] + + copyCertsImage: + repository: rancher/mirrored-library-busybox + tag: 1.31.1 + + # The default intention of rancher-pushprox clients is to scrape hostNetwork metrics across all nodes. + # This can be used to scrape internal Kubernetes components or DaemonSets of hostNetwork Pods in + # situations where a cloud provider firewall prevents Pod-To-Host communication but not Pod-To-Pod. + # However, if the underlying hostNetwork Pod that is being scraped is managed by a Deployment, + # this advanced option enables users to deploy the client as a Deployment instead of a DaemonSet. + # If a user deploys this feature and the underlying Deployment's number of replicas changes, the user will + # be responsible for upgrading this chart accordingly to the right number of replicas. + deployment: + enabled: false + replicas: 0 + +proxy: + enabled: true + # The port through which PushProx clients will communicate to the proxy + port: 8080 + + # Resource limits + resources: {} + + # Options to select a node to run a single proxy deployment on + nodeSelector: {} + tolerations: [] + + image: + repository: rancher/pushprox-proxy + tag: v0.1.4-rancher2-proxy + command: ["pushprox-proxy"] diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2Proxy/.helmignore b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2Proxy/.helmignore new file mode 100644 index 00000000000..0e8a0eb36f4 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2Proxy/.helmignore @@ -0,0 +1,23 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*.orig +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2Proxy/Chart.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2Proxy/Chart.yaml new file mode 100644 index 00000000000..a8d19e93263 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2Proxy/Chart.yaml @@ -0,0 +1,15 @@ +annotations: + catalog.cattle.io/hidden: "true" + catalog.cattle.io/kube-version: '>= 1.26.0-0 < 1.31.0-0' + catalog.cattle.io/os: linux + catalog.rancher.io/certified: rancher + catalog.rancher.io/namespace: cattle-monitoring-system + catalog.rancher.io/release-name: rancher-pushprox +apiVersion: v1 +appVersion: v0.1.4-rancher2 +description: Sets up a deployment of the PushProx proxy and a DaemonSet of PushProx + clients. +kubeVersion: '>=1.26.0-0' +name: rke2Proxy +type: application +version: 0.2.1 diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2Proxy/README.md b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2Proxy/README.md new file mode 100644 index 00000000000..345002f48a2 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2Proxy/README.md @@ -0,0 +1,90 @@ +# rancher-pushprox + +A Rancher chart based on Rancher [PushProx](https://github.com/rancher/PushProx) that sets up a Deployment of a PushProx proxy and a DaemonSet of PushProx clients on a Kubernetes cluster. + +Installs [rancher-pushprox](https://github.com/rancher/charts/tree/gh-pages/packages/rancher-pushprox) to create PushProx clients that can access their host's network and register with a PushProx proxy. A [Prometheus Operator](https://github.com/coreos/prometheus-operator) ServiceMonitor CR is also included that is configured to scrape the metrics from each of the clients through the proxy. + +Using an instance of this chart is suitable for the following scenarios: +- You need to scrape metrics from a port that should not be accessible outside of the host (e.g. scraping `etcd` metrics in a hardened cluster) +- You need to scrape metrics on a host that are not exposed outside of 127.0.0.1 (e.g. scraping `kube-proxy` metrics) +- You need to scrape metrics through HTTPS using certs hosted directly on `hostPath` +- You need to scrape metrics from Kubernetes components that require authorization via a service account (e.g. permissions to make request to `/metrics`) +- You need to scrape metrics without access to cacerts (i.e. enable `insecureSkipVerify`) + +The clients and proxy are created based on a Rancher fork of the [prometheus-community/PushProx](https://github.com/prometheus-community/PushProx) project. + +## Upgrading to Kubernetes v1.25+ + +Starting in Kubernetes v1.25, [Pod Security Policies](https://kubernetes.io/docs/concepts/security/pod-security-policy/) have been removed from the Kubernetes API. + +As a result, **before upgrading to Kubernetes v1.25** (or on a fresh install in a Kubernetes v1.25+ cluster), users are expected to perform an in-place upgrade of this chart with `global.cattle.psp.enabled` set to `false` if it has been previously set to `true`. +​ +> **Note:** +> In this chart release, any previous field that was associated with any PSP resources have been removed in favor of a single global field: `global.cattle.psp.enabled`. + +> **Note:** +> If you upgrade your cluster to Kubernetes v1.25+ before removing PSPs via a `helm upgrade` (even if you manually clean up resources), **it will leave the Helm release in a broken state within the cluster such that further Helm operations will not work (`helm uninstall`, `helm upgrade`, etc.).** +> +> If your charts get stuck in this state, please consult the Rancher docs on how to clean up your Helm release secrets. + +Upon setting `global.cattle.psp.enabled` to false, the chart will remove any PSP resources deployed on its behalf from the cluster. This is the default setting for this chart. + +As a replacement for PSPs, [Pod Security Admission](https://kubernetes.io/docs/concepts/security/pod-security-admission/) should be used. Please consult the Rancher docs for more details on how to configure your chart release namespaces to work with the new Pod Security Admission and apply Pod Security Standards. + +## Configuration + +The following tables list the configurable parameters of the rancher-pushprox chart and their default values. + +### General + +#### Required +| Parameter | Description | Example | +| ----- | ----------- | ------ | +| `component` | The component that is being monitored | `kube-etcd` +| `metricsPort` | The port on the host that contains the metrics you want to scrape (e.g. `http://:/metrics`) | `2379` | +| `namespaceOverride` | The namespace to install the chart | `""` + +#### Optional +| Parameter | Description | Default | +| ----- | ----------- | ------ | +| `serviceMonitor.enabled` | Deploys a [Prometheus Operator](https://github.com/coreos/prometheus-operator/blob/master/Documentation/api.md#servicemonitor) ServiceMonitor CR that is configured to scrape metrics on the hosts that the clients are deployed on via the proxy. Also deploys a Service that points to all pods with the expected client name that exposes the `metricsPort` selected | `true` | +| `serviceMonitor.endpoints` | A list of endpoints that will be added to the ServiceMonitor based on the [Endpoint spec](https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#endpoint) | `[{port: metrics}]` | +| `service.selector` | The selector that is used to populate the Service's Endpoints object. The chart will error out on rendering templating if `.Values.clients.enabled` is set alongside this field, since it is expected that this service should point to the PushProx Clients Daemonset / Deployment | `{}` | +| `clients.enabled` | Deploys a DaemonSet of clients that are each capable of scraping endpoints on the hostNetwork it is deployed on | `true` | +| `clients.port` | The port where the client will publish PushProx client-specific metrics. If deploying multiple clients onto the same node, the clients should not have conflicting ports | `9369` | +| `clients.proxyUrl` | Overrides the default proxyUrl setting of `http://pushprox-{{ .Values.component }}-proxy.{{ . Release.Namespace }}.svc.cluster.local:{{ .Values.proxy.port }}"` with the `proxyUrl` specified | `""` | +| `clients.useLocalhost` | Sets a flag on each client deployment to redirect scrapes directed to `HOST_IP` to `127.0.0.1` | `false` | +| `clients.https.enabled` | Enables scraping metrics via HTTPS using the provided TLS certs that exist on each host | `false` | +| `clients.https.forceHTTPSScheme` | Forces scraping metrics via HTTPS using the provided TLS certs that exist on each host | `false` | +| `clients.https.useServiceAccountCredentials` | If set to true, the client will create a service account with permissions to scrape `/metrics` endpoint of Kubernetes components. The client will use the service account token provided to make authorized scrape requests to the Kubernetes API | `false` | +| `clients.https.authenticationMethod.bearerTokenFile.enabled` | If set to true, the client will use service account credentials mounted at the configured path `clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath`. This requires permissions to scrape `/metrics` endpoint of Kubernetes components. This method is deprecated by the prometheus operator and may be removed in a future release | `false` | +| `clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath` | This is a volume mount on the pod with permissions to scrape `/metrics` endpoint of Kubernetes components | `"/var/run/secrets/kubernetes.io/serviceaccount/token"` | +| `clients.https.authenticationMethod.bearerTokenSecret.enabled` | If set to true, the client will use service account credentials to scrape `/metrics` endpoint of Kubernetes components. This method is deprecated by the prometheus operator and may be removed in a future release | `false` | +| `clients.https.authenticationMethod.authorization.enabled` | If set to true, the client will use service account credentials to scrape `/metrics` endpoint of Kubernetes components | `false` | +| `clients.https.authenticationMethod.authorization.type` | If set, the client will use this type of authorization in its client requests for metrics | `"bearer"` | +| `clients.https.authenticationMethod.authorization.credentials.key` | If set, the client will use this key in the secret created by `clients.https.useServiceAccountCredentials` for authorization in its client requests for metrics | `"token"` | +| `clients.https.authenticationMethod.authorization.credentials.optional` | If set to false, the client will fail if the key in the secret created by `clients.https.useServiceAccountCredentials` does not exist | `false` | +| `clients.https.insecureSkipVerify` | If set to true, the client will disable SSL security checks | `false` | +| `clients.https.certDir` | A `hostPath` where TLS certs can be found. This path is mounted as a volume on an `initContainer` which copies only the necessary files over to an EmptyDir volume used by each client. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.certFile` | The path to the TLS cert file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.keyFile` | The path to the TLS key file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.caCertFile` | The path to the TLS cacert file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.seLinuxOptions` | seLinuxOptions to be passed into the container that copies certs. Should define a container with permissions to read the files in the certDir provided on the host. Required and only used if `clients.https.enabled` is set and `clients.https.certDir` is provided. | `""` | +| `clients.metrics.enabled` | Whether the client should publish PushProx client-specific metrics. | `false` | +| `clients.rbac.additionalRules` | Additional permissions to provide to the ServiceAccount bound to the client. This can be used to provide additional permissions for the client to scrape metrics from the k8s API. Only enabled if clients.https.enabled and clients.https.useServiceAccountCredentials are true | `[]` | +| `clients.deployment.enabled` | Deploys the client as a Deployment (generally used if the underlying hostNetwork Pod that is being scraped is managed by a Deployment) | `false` | +| `clients.deployment.replicas` | The number of pods the Deployment has, it should match the number of pod the hostNetwork Deployment has. Required and only used if `client.deployment.enable` is set | `0` | +| `clients.deployment.affinity` | The affinity rules that allocate the pod to the node in which the hostNetwork Deployment's pods run. Required and only used if `client.deployment.enable` is set | `{}` | +| `clients.resources` | Set resource limits and requests for the client container | `{}` | +| `clients.nodeSelector` | Select which nodes to deploy the clients on | `{}` | +| `clients.tolerations` | Specify tolerations for clients | `[]` | +| `proxy.enabled` | Deploys the proxy that each client will register with | `true` | +| `proxy.port` | The port exposed by the proxy that each client will register with to allow metrics to be scraped from the host | `8080` | +| `proxy.resources` | Set resource limits and requests for the proxy container | `{}` | +| `proxy.nodeSelector` | Select which nodes the proxy can be deployed on | `{}` | +| `proxy.tolerations` | Specify tolerations (if necessary) to allow the proxy to be deployed on the selected node | `[]` | +| `kubeVersionOverrides` | A list of Semver constraint strings (defined by https://github.com/Masterminds/semver) and values.yaml overrides. For each key in kubeVersionOverrides, this chart will check to see if the current Kubernetes cluster's version matches any of the semver constraints provided as keys on the map. On seeing a match, the default value for each values.yaml field overridden will be updated with the new value. If multiple matches are encountered (due to overlapping semver ranges), the matches will be applied in order. | `[]` + +*Tip: The filepaths set in `clients.https.File` can include wildcard characters*. + +See [rancher-monitoring](https://github.com/rancher/charts/tree/gh-pages/packages/rancher-monitoring) for examples of how this chart can be used. diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2Proxy/templates/_helpers.tpl b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2Proxy/templates/_helpers.tpl new file mode 100644 index 00000000000..1ba50939441 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2Proxy/templates/_helpers.tpl @@ -0,0 +1,170 @@ +# Rancher + +{{- define "system_default_registry" -}} +{{- if .Values.global.cattle.systemDefaultRegistry -}} +{{- printf "%s/" .Values.global.cattle.systemDefaultRegistry -}} +{{- end -}} +{{- end -}} + +# Windows Support + +{{/* +Windows cluster will add default taint for linux nodes, +add below linux tolerations to workloads could be scheduled to those linux nodes +*/}} + +{{- define "linux-node-tolerations" -}} +- key: "cattle.io/os" + value: "linux" + effect: "NoSchedule" + operator: "Equal" +{{- end -}} + +{{- define "linux-node-selector" -}} +{{- if semverCompare "<1.14-0" .Capabilities.KubeVersion.GitVersion -}} +beta.kubernetes.io/os: linux +{{- else -}} +kubernetes.io/os: linux +{{- end -}} +{{- end -}} + +# General + +{{- define "applyKubeVersionOverrides" -}} +{{- $overrides := dict -}} +{{- range $override := .Values.kubeVersionOverrides -}} +{{- if semverCompare $override.constraint $.Capabilities.KubeVersion.Version -}} +{{- $_ := mergeOverwrite $overrides $override.values -}} +{{- end -}} +{{- end -}} +{{- $_ := mergeOverwrite .Values $overrides -}} +{{- end -}} + +{{- define "pushprox.namespace" -}} + {{- if .Values.namespaceOverride -}} + {{- .Values.namespaceOverride -}} + {{- else -}} + {{- .Release.Namespace -}} + {{- end -}} +{{- end -}} + +{{- define "pushProxy.commonLabels" -}} +release: {{ .Release.Name }} +component: {{ .Values.component | quote }} +provider: kubernetes +{{- end -}} + +{{- define "pushProxy.proxyUrl" -}} +{{- $_ := (required "Template requires either .Values.proxy.port or .Values.client.proxyUrl to set proxyUrl for client" (or .Values.clients.proxyUrl .Values.proxy.port)) -}} +{{- if .Values.clients.proxyUrl -}} +{{ printf "%s" .Values.clients.proxyUrl }} +{{- else -}} +{{ printf "http://%s.%s.svc:%d" (include "pushProxy.proxy.name" .) (include "pushprox.namespace" .) (int .Values.proxy.port) }} +{{- end -}}{{- end -}} + +# Client + +{{- define "pushProxy.client.name" -}} +{{- printf "pushprox-%s-client" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.client.serviceAccountTokenName" -}} +{{- printf "pushprox-%s-client-service-account-token" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.client.labels" -}} +k8s-app: {{ template "pushProxy.client.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +# Proxy + +{{- define "pushProxy.proxy.name" -}} +{{- printf "pushprox-%s-proxy" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.proxy.labels" -}} +k8s-app: {{ template "pushProxy.proxy.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +# ServiceMonitor + +{{- define "pushprox.serviceMonitor.name" -}} +{{- printf "%s-%s" .Release.Name (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.serviceMonitor.labels" -}} +app: {{ template "pushprox.serviceMonitor.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +{{- define "pushProxy.serviceMonitor.endpoints" -}} +{{- $proxyURL := (include "pushProxy.proxyUrl" .) -}} +{{- $useHTTPS := .Values.clients.https.enabled -}} +{{- $setHTTPSScheme := .Values.clients.https.forceHTTPSScheme -}} +{{- $insecureSkipVerify := .Values.clients.https.insecureSkipVerify -}} +{{- $useServiceAccountCredentials := .Values.clients.https.useServiceAccountCredentials -}} +{{- $serviceAccountTokenName := (include "pushProxy.client.serviceAccountTokenName" . ) -}} +{{- $metricRelabelings := list }} +{{- $endpoints := .Values.serviceMonitor.endpoints }} +{{- if .Values.proxy.enabled }} +{{- $_ := set . "proxyUrl" $proxyURL }} +{{- end }} +{{- range $endpoints }} +{{- if $.Values.proxy.enabled }} +{{- $_ := set . "proxyUrl" $proxyURL }} +{{- end }} +{{- $clusterIdRelabel := dict }} +{{- $metricRelabelings := list }} +{{- if $.Values.global.cattle.clusterId }} +{{- $_ := set $clusterIdRelabel "action" "replace" }} +{{- $_ := set $clusterIdRelabel "sourceLabels" (list "__address__") }} +{{- $_ := set $clusterIdRelabel "targetLabel" "cluster_id" }} +{{- $_ := set $clusterIdRelabel "replacement" $.Values.global.cattle.clusterId }} +{{- $metricRelabelings = append $metricRelabelings $clusterIdRelabel }} +{{- end }} +{{- $clusterNameRelabel := dict }} +{{- if $.Values.global.cattle.clusterName }} +{{- $_ := set $clusterNameRelabel "action" "replace" }} +{{- $_ := set $clusterNameRelabel "sourceLabels" (list "__address__") }} +{{- $_ := set $clusterNameRelabel "targetLabel" "cluster_name" }} +{{- $_ := set $clusterNameRelabel "replacement" $.Values.global.cattle.clusterName }} +{{- $metricRelabelings = append $metricRelabelings $clusterNameRelabel }} +{{- end }} +{{- if not (empty $metricRelabelings) }} +{{- $_ := set . "metricRelabelings" ($metricRelabelings)}} +{{- end }} +{{- if $setHTTPSScheme -}} +{{- $_ := set . "scheme" "https" }} +{{- end -}} +{{- if $useHTTPS -}} +{{- if (hasKey . "params") }} +{{- $_ := set (get . "params") "_scheme" (list "https") }} +{{- else }} +{{- $_ := set . "params" (dict "_scheme" (list "https")) }} +{{- end }} +{{- end }} +{{- if (hasKey . "tlsConfig") }} +{{- $_ := set (get . "tlsConfig") "insecureSkipVerify" $insecureSkipVerify }} +{{- else }} +{{- $_ := set . "tlsConfig" (dict "insecureSkipVerify" $insecureSkipVerify) }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.bearerTokenFile.enabled }} +{{- $_ := set . "bearerTokenFile" $.Values.clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.bearerTokenSecret.enabled }} +{{- $_ := set . "bearerTokenSecret" $serviceAccountTokenName }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.authorization.enabled }} +{{- if (hasKey . "authorization") }} +{{- $_ := set (get . "authorization") "type" $.Values.clients.https.authenticationMethod.authorization.type }} +{{- $_ := set (get . "authorization") "credentials" (dict "name" $serviceAccountTokenName "key" $.Values.clients.https.authenticationMethod.authorization.credentials.key "optional" $.Values.clients.https.authenticationMethod.authorization.credentials.optional) }} +{{- else }} +{{- $_ := set . "authorization" (dict "type" $.Values.clients.https.authenticationMethod.authorization.type) }} +{{- $_ := set . "authorization" (dict "credentials" (dict "name" $serviceAccountTokenName "key" $.Values.clients.https.authenticationMethod.authorization.credentials.key "optional" $.Values.clients.https.authenticationMethod.authorization.credentials.optional)) }} +{{- end }} +{{- end }} +{{- end }} +{{- toYaml $endpoints }} +{{- end -}} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2Proxy/templates/pushprox-clients-rbac.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2Proxy/templates/pushprox-clients-rbac.yaml new file mode 100644 index 00000000000..a8e27c37356 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2Proxy/templates/pushprox-clients-rbac.yaml @@ -0,0 +1,97 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.clients }}{{- if .Values.clients.enabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ template "pushProxy.client.name" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +rules: +{{- if .Values.global.cattle.psp.enabled }} +- apiGroups: ['policy'] + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - {{ template "pushProxy.client.name" . }} +{{- end }} +{{- if and .Values.clients.https.enabled .Values.clients.https.useServiceAccountCredentials }} +- nonResourceURLs: ["/metrics"] + verbs: ["get"] +{{- if .Values.clients.rbac.additionalRules }} +{{ toYaml .Values.clients.rbac.additionalRules }} +{{- end }} +{{- end }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ template "pushProxy.client.name" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "pushProxy.client.name" . }} +subjects: + - kind: ServiceAccount + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +--- +{{- if .Values.clients.https.useServiceAccountCredentials }} +apiVersion: v1 +kind: Secret +type: kubernetes.io/service-account-token +metadata: + name: {{ template "pushProxy.client.serviceAccountTokenName" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} + annotations: + kubernetes.io/service-account.name: {{ template "pushProxy.client.name" . }} +{{- end }} +--- +{{- if .Values.global.cattle.psp.enabled }} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +spec: + privileged: false + hostNetwork: true + hostIPC: false + hostPID: false + runAsUser: + rule: 'RunAsAny' + seLinux: + rule: 'RunAsAny' +{{- if and .Values.clients.https.enabled .Values.clients.https.certDir .Values.global.seLinux.enabled .Values.clients.https.seLinuxOptions }} + seLinuxOptions: {{ .Values.clients.https.seLinuxOptions | toYaml | nindent 6 }} +{{- end }} + supplementalGroups: + rule: 'MustRunAs' + ranges: + - min: 0 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + - min: 0 + max: 65535 + readOnlyRootFilesystem: false + volumes: + - 'secret' +{{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + - 'emptyDir' + - 'hostPath' + allowedHostPaths: + - pathPrefix: {{ required "Need access to volume on host with the SSL cert files to use HTTPs" .Values.clients.https.certDir }} + readOnly: true +{{- end }} +{{- end }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2Proxy/templates/pushprox-clients.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2Proxy/templates/pushprox-clients.yaml new file mode 100644 index 00000000000..e8fcfb38836 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2Proxy/templates/pushprox-clients.yaml @@ -0,0 +1,157 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.clients }}{{- if .Values.clients.enabled }} +apiVersion: apps/v1 +{{- if .Values.clients.deployment.enabled }} +kind: Deployment +{{- else }} +kind: DaemonSet +{{- end }} +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} + pushprox-exporter: "client" +spec: + {{- if .Values.clients.deployment.enabled }} + replicas: {{ .Values.clients.deployment.replicas }} + {{- end }} + selector: + matchLabels: {{ include "pushProxy.client.labels" . | nindent 6 }} + template: + metadata: + labels: {{ include "pushProxy.client.labels" . | nindent 8 }} + spec: + {{- if .Values.clients.affinity }} + affinity: {{ toYaml .Values.clients.affinity | nindent 8 }} + {{- end }} + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} +{{- if .Values.clients.nodeSelector }} +{{ toYaml .Values.clients.nodeSelector | indent 8 }} +{{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} +{{- if .Values.clients.tolerations }} +{{ toYaml .Values.clients.tolerations | indent 8 }} +{{- end }} + hostNetwork: true + dnsPolicy: ClusterFirstWithHostNet + serviceAccountName: {{ template "pushProxy.client.name" . }} + {{- if .Values.global.imagePullSecretName }} + imagePullSecrets: + - name: {{ .Values.global.imagePullSecretName }} + {{- end }} + containers: + - name: pushprox-client + image: {{ template "system_default_registry" . }}{{ .Values.clients.image.repository }}:{{ .Values.clients.image.tag }} + command: + {{- range .Values.clients.command }} + - {{ . | quote }} + {{- end }} + args: + - --fqdn=$(HOST_IP) + - --proxy-url=$(PROXY_URL) + {{- if .Values.clients.metrics.enabled }} + - --metrics-addr=$(PORT) + {{- end }} + - --allow-port={{ required "Need .Values.metricsPort to configure client to be allowed to scrape metrics at port" .Values.metricsPort}} + {{- if .Values.clients.useLocalhost }} + - --use-localhost + {{- end }} + {{- if .Values.clients.https.enabled }} + {{- if .Values.clients.https.insecureSkipVerify }} + - --insecure-skip-verify + {{- end }} + {{- if .Values.clients.https.useServiceAccountCredentials }} + - --token-path=/var/run/secrets/kubernetes.io/serviceaccount/token + {{- end }} + {{- if .Values.clients.https.certDir }} + - --tls.cert=/etc/ssl/push-proxy/push-proxy.pem + - --tls.key=/etc/ssl/push-proxy/push-proxy-key.pem + - --tls.cacert=/etc/ssl/push-proxy/push-proxy-ca-cert.pem + {{- end }} + {{- end }} + env: + - name: HOST_IP + valueFrom: + fieldRef: + fieldPath: status.hostIP + {{- if .Values.clients.metrics.enabled }} + - name: PORT + value: :{{ .Values.clients.port }} + {{- end }} + - name: PROXY_URL + value: {{ template "pushProxy.proxyUrl" . }} + securityContext: + runAsNonRoot: true + runAsUser: 1000 + {{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + volumeMounts: + - name: metrics-cert-dir + mountPath: /etc/ssl/push-proxy + {{- end }} + {{- if .Values.clients.resources }} + resources: {{ toYaml .Values.clients.resources | nindent 10 }} + {{- end }} + {{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + initContainers: + - name: copy-certs + image: {{ template "system_default_registry" . }}{{ .Values.clients.copyCertsImage.repository }}:{{ .Values.clients.copyCertsImage.tag }} + command: + - sh + - -c + - | + echo "Searching for files to copy within the source volume" + echo "cert: ${CERT_FILE_NAME}" + echo "key: ${KEY_FILE_NAME}" + echo "cacert: ${CACERT_FILE_NAME}" + + CERT_FILE_SOURCE=$(find /etc/source/ -type f -name "${CERT_FILE_NAME}" | sort -r | head -n 1) + KEY_FILE_SOURCE=$(find /etc/source/ -type f -name "${KEY_FILE_NAME}" | sort -r | head -n 1) + CACERT_FILE_SOURCE=$(find /etc/source/ -type f -name "${CACERT_FILE_NAME}" | sort -r | head -n 1) + + test -z ${CERT_FILE_SOURCE} && echo "Failed to find cert file" && exit 1 + test -z ${KEY_FILE_SOURCE} && echo "Failed to find key file" && exit 1 + test -z ${CACERT_FILE_SOURCE} && echo "Failed to find cacert file" && exit 1 + + echo "Copying cert file from $CERT_FILE_SOURCE to $CERT_FILE_TARGET" + cp $CERT_FILE_SOURCE $CERT_FILE_TARGET || exit 1 + chmod 444 $CERT_FILE_TARGET || exit 1 + + echo "Copying key file from $KEY_FILE_SOURCE to $KEY_FILE_TARGET" + cp $KEY_FILE_SOURCE $KEY_FILE_TARGET || exit 1 + chmod 444 $KEY_FILE_TARGET || exit 1 + + echo "Copying cacert file from $CACERT_FILE_SOURCE to $CACERT_FILE_TARGET" + cp $CACERT_FILE_SOURCE $CACERT_FILE_TARGET || exit 1 + chmod 444 $CACERT_FILE_TARGET || exit 1 + env: + - name: CERT_FILE_NAME + value: {{ required "Need a TLS cert file for scraping metrics endpoint over HTTPs" .Values.clients.https.certFile }} + - name: KEY_FILE_NAME + value: {{ required "Need a TLS key file for scraping metrics endpoint over HTTPs" .Values.clients.https.keyFile }} + - name: CACERT_FILE_NAME + value: {{ required "Need a TLS CA cert file for scraping metrics endpoint over HTTPs" .Values.clients.https.caCertFile }} + - name: CERT_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy.pem + - name: KEY_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy-key.pem + - name: CACERT_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy-ca-cert.pem + securityContext: + runAsNonRoot: false +{{- if and .Values.global.seLinux.enabled .Values.clients.https.seLinuxOptions }} + seLinuxOptions: {{ .Values.clients.https.seLinuxOptions | toYaml | nindent 12 }} +{{- end }} + volumeMounts: + - name: metrics-cert-dir-source + mountPath: /etc/source + readOnly: true + - name: metrics-cert-dir + mountPath: /etc/ssl/push-proxy + volumes: + - name: metrics-cert-dir-source + hostPath: + path: {{ required "Need access to volume on host with the SSL cert files to use HTTPs" .Values.clients.https.certDir }} + - name: metrics-cert-dir + emptyDir: {} + {{- end }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2Proxy/templates/pushprox-proxy-rbac.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2Proxy/templates/pushprox-proxy-rbac.yaml new file mode 100644 index 00000000000..eefe609058c --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2Proxy/templates/pushprox-proxy-rbac.yaml @@ -0,0 +1,68 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if and .Values.proxy }}{{ if .Values.proxy.enabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ template "pushProxy.proxy.name" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +rules: +{{- if .Values.global.cattle.psp.enabled }} +- apiGroups: ['policy'] + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - {{ template "pushProxy.proxy.name" . }} +{{- end }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ template "pushProxy.proxy.name" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "pushProxy.proxy.name" . }} +subjects: + - kind: ServiceAccount + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +--- +{{- if .Values.global.cattle.psp.enabled }} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +spec: + privileged: false + hostNetwork: false + hostIPC: false + hostPID: false + runAsUser: + rule: 'MustRunAsNonRoot' + seLinux: + rule: 'RunAsAny' + supplementalGroups: + rule: 'MustRunAs' + ranges: + - min: 1 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + - min: 1 + max: 65535 + readOnlyRootFilesystem: false + volumes: + - 'secret' +{{- end }}{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2Proxy/templates/pushprox-proxy.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2Proxy/templates/pushprox-proxy.yaml new file mode 100644 index 00000000000..723bbd6c00b --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2Proxy/templates/pushprox-proxy.yaml @@ -0,0 +1,57 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if and .Values.proxy }}{{ if .Values.proxy.enabled }} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} + pushprox-exporter: "proxy" +spec: + selector: + matchLabels: {{ include "pushProxy.proxy.labels" . | nindent 6 }} + template: + metadata: + labels: {{ include "pushProxy.proxy.labels" . | nindent 8 }} + spec: + securityContext: + runAsNonRoot: true + runAsUser: 1000 + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} +{{- if .Values.proxy.nodeSelector }} +{{ toYaml .Values.proxy.nodeSelector | indent 8 }} +{{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} +{{- if .Values.proxy.tolerations }} +{{ toYaml .Values.proxy.tolerations | indent 8 }} +{{- end }} + serviceAccountName: {{ template "pushProxy.proxy.name" . }} + {{- if .Values.global.imagePullSecretName }} + imagePullSecrets: + - name: {{ .Values.global.imagePullSecretName }} + {{- end }} + containers: + - name: pushprox-proxy + image: {{ template "system_default_registry" . }}{{ .Values.proxy.image.repository }}:{{ .Values.proxy.image.tag }} + command: + {{- range .Values.proxy.command }} + - {{ . | quote }} + {{- end }} + {{- if .Values.proxy.resources }} + resources: {{ toYaml .Values.proxy.resources | nindent 10 }} + {{- end }} +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +spec: + ports: + - name: pp-proxy + port: {{ required "Need .Values.proxy.port to configure proxy" .Values.proxy.port }} + protocol: TCP + targetPort: {{ .Values.proxy.port }} + selector: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2Proxy/templates/pushprox-servicemonitor.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2Proxy/templates/pushprox-servicemonitor.yaml new file mode 100644 index 00000000000..67eb2216b5a --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2Proxy/templates/pushprox-servicemonitor.yaml @@ -0,0 +1,45 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.serviceMonitor }}{{- if .Values.serviceMonitor.enabled }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ template "pushprox.serviceMonitor.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.serviceMonitor.labels" . | nindent 4 }} +spec: + endpoints: {{include "pushProxy.serviceMonitor.endpoints" . | nindent 4 }} + jobLabel: component + podTargetLabels: + - component + - pushprox-exporter + namespaceSelector: + matchNames: + - {{ template "pushprox.namespace" . }} + selector: + matchLabels: {{ include "pushProxy.client.labels" . | nindent 6 }} +--- +{{- $selector := "" }} +{{- if not (kindIs "invalid" .Values.service) }} +{{- if not (kindIs "invalid" .Values.service.selector) }} +{{ if .Values.service.selector }} +{{- if .Values.clients.enabled }} +{{- required (printf "Cannot override .Values.service.selector=%s when .Values.clients.enabled=true" (toJson .Values.service.selector)) "" }} +{{- end }} +{{- $selector = (toYaml .Values.service.selector) }} +{{- end }} +{{- end }} +{{- end }} +apiVersion: v1 +kind: Service +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +spec: + ports: + - name: metrics + port: {{ required "Need .Values.metricsPort to configure client to listen to metrics at port" .Values.metricsPort}} + protocol: TCP + targetPort: {{ .Values.metricsPort }} + selector: {{ default (include "pushProxy.client.labels" .) $selector | nindent 4 }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2Proxy/templates/validate-install-crd.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2Proxy/templates/validate-install-crd.yaml new file mode 100644 index 00000000000..16abc2fa83a --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2Proxy/templates/validate-install-crd.yaml @@ -0,0 +1,14 @@ +#{{- if gt (len (lookup "rbac.authorization.k8s.io/v1" "ClusterRole" "" "")) 0 -}} +# {{- $found := dict -}} +# {{- set $found "monitoring.coreos.com/v1/ServiceMonitor" false -}} +# {{- range .Capabilities.APIVersions -}} +# {{- if hasKey $found (toString .) -}} +# {{- set $found (toString .) true -}} +# {{- end -}} +# {{- end -}} +# {{- range $_, $exists := $found -}} +# {{- if (eq $exists false) -}} +# {{- required "Required CRDs are missing. Please install Prometheus Operator CRDs before installing this chart." "" -}} +# {{- end -}} +# {{- end -}} +#{{- end -}} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2Proxy/templates/validate-psp-install.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2Proxy/templates/validate-psp-install.yaml new file mode 100644 index 00000000000..a30c59d3b7c --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2Proxy/templates/validate-psp-install.yaml @@ -0,0 +1,7 @@ +#{{- if gt (len (lookup "rbac.authorization.k8s.io/v1" "ClusterRole" "" "")) 0 -}} +#{{- if .Values.global.cattle.psp.enabled }} +#{{- if not (.Capabilities.APIVersions.Has "policy/v1beta1/PodSecurityPolicy") }} +#{{- fail "The target cluster does not have the PodSecurityPolicy API resource. Please disable PSPs in this chart before proceeding." -}} +#{{- end }} +#{{- end }} +#{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2Proxy/values.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2Proxy/values.yaml new file mode 100644 index 00000000000..7f38498a1c5 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2Proxy/values.yaml @@ -0,0 +1,166 @@ +# Default values for rancher-pushprox. +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. + +# Default image containing both the proxy and the client was generated from the following Dockerfile +# https://github.com/prometheus-community/PushProx/blob/eeadbe766641699129920ccfaaaa30a85c67fe81/Dockerfile#L1-L15 + +# Configuration + +global: + cattle: + psp: + enabled: false + systemDefaultRegistry: "" + seLinux: + enabled: false + +# A list of Semver constraint strings (defined by https://github.com/Masterminds/semver) and values.yaml overrides. +# +# For each key in kubeVersionOverrides, this chart will check to see if the current Kubernetes cluster's version matches +# any of the semver constraints provided as keys on the map. +# +# On seeing a match, the default value for each values.yaml field overridden will be updated with the new value. +# +# If multiple matches are encountered (due to overlapping semver ranges), the matches will be applied in order. +# +# Notes: +# - On running a helm template, Helm generally assumes the kubeVersion is v1.20.0 +# - On running a helm install --dry-run, the correct kubeVersion should be chosen. +kubeVersionOverrides: [] +# - constraint: "< 1.21" +# values: +# metricsPort: 10252 +# clients: +# https: +# enabled: false +# insecureSkipVerify: false +# useServiceAccountCredentials: false + +namespaceOverride: "" + +# The component that is being monitored (i.e. etcd) +component: "component" + +# The port containing the metrics that need to be scraped +metricsPort: 2739 + +# Configure ServiceMonitor that monitors metrics from the metricsPort endpoint +serviceMonitor: + enabled: true + # A list of endpoints that will be added to the ServiceMonitor based on the Endpoint spec + # Source: https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#endpoint + # By default, proxyUrl and params._scheme will be overridden based on other values + endpoints: + - port: metrics + +# Configure Service that grabs scrape targets +service: + # The selector that is used to populate the Service's Endpoints object. + # The chart will error out on rendering templating if .Values.clients.enabled is set alongside this field, + # since it is expected that this service should point to the PushProx Clients Daemonset / Deployment + selector: {} + +clients: + enabled: true + # The port which the PushProx client will post PushProx metrics to + port: 9369 + # If unset, this will default to the URL for the proxy service: http://pushprox-{{component}}-proxy.{{namepsace}}.svc.cluster.local:{{proxy.port}} + # Should be modified if the clients are being deployed outside the cluster where the proxy rests, otherwise leave it null + proxyUrl: "" + # If set to true, the client will forward any requests from the host IP to 127.0.0.1 + # It will only allow proxy requests to the metricsPort specified + useLocalhost: false + # Configuration for accessing metrics via HTTPS + https: + # Does the client require https to access the metrics? + enabled: false + # Does the client require requests be sent to http or https? + forceHTTPSScheme: false + # If set to true, the client will create a service account with adequate permissions and set a flag + # on the client to use the service account token provided by it to make authorized scrape requests + useServiceAccountCredentials: false + # Configuration for authentication to metrics via https endpoint + authenticationMethod: + # Reads token from defined file in container + # This function is deprecated in the prometheus operator api and may be removed in a future version + bearerTokenFile: + enabled: false + bearerTokenFilePath: "/var/run/secrets/kubernetes.io/serviceaccount/token" + # Reads token from defined secret in namespace + # This function is deprecated in the prometheus operator api and may be removed in a future version + bearerTokenSecret: + enabled: false + # Reads token from defined secret in namespace + authorization: + enabled: false + type: "bearer" + credentials: + key: "token" + optional: false + # If set to true, the client will disable SSL security checks + insecureSkipVerify: false + # Directory on host where necessary TLS cert and key to scrape metrics can be found + certDir: "" + # Filenames for files located in .Values.clients.https.certDir that correspond to TLS settings + certFile: "" + keyFile: "" + caCertFile: "" + # seLinuxOptions to be passed into the container that copies certs. Should define a container with permissions to read the files in the certDir provided on the host. + # Required and only used if `clients.https.enabled` is set and `clients.https.certDir` is provided. + seLinuxOptions: {} + + metrics: + # Whether the client should publish PushProx client-specific metrics to .Values.clients.port + enabled: false + + rbac: + # Additional permissions to provide to the ServiceAccount bound to the client + # This can be used to provide additional permissions for the client to scrape metrics from the k8s API + # Only enabled if clients.https.enabled and clients.https.useServiceAccountCredentials are true + additionalRules: [] + + # Resource limits + resources: {} + + # Options to select all nodes to deploy client DaemonSet on + nodeSelector: {} + tolerations: [] + affinity: {} + + image: + repository: rancher/pushprox-client + tag: v0.1.4-rancher2-client + command: ["pushprox-client"] + + copyCertsImage: + repository: rancher/mirrored-library-busybox + tag: 1.31.1 + + # The default intention of rancher-pushprox clients is to scrape hostNetwork metrics across all nodes. + # This can be used to scrape internal Kubernetes components or DaemonSets of hostNetwork Pods in + # situations where a cloud provider firewall prevents Pod-To-Host communication but not Pod-To-Pod. + # However, if the underlying hostNetwork Pod that is being scraped is managed by a Deployment, + # this advanced option enables users to deploy the client as a Deployment instead of a DaemonSet. + # If a user deploys this feature and the underlying Deployment's number of replicas changes, the user will + # be responsible for upgrading this chart accordingly to the right number of replicas. + deployment: + enabled: false + replicas: 0 + +proxy: + enabled: true + # The port through which PushProx clients will communicate to the proxy + port: 8080 + + # Resource limits + resources: {} + + # Options to select a node to run a single proxy deployment on + nodeSelector: {} + tolerations: [] + + image: + repository: rancher/pushprox-proxy + tag: v0.1.4-rancher2-proxy + command: ["pushprox-proxy"] diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2Scheduler/.helmignore b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2Scheduler/.helmignore new file mode 100644 index 00000000000..0e8a0eb36f4 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2Scheduler/.helmignore @@ -0,0 +1,23 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*.orig +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2Scheduler/Chart.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2Scheduler/Chart.yaml new file mode 100644 index 00000000000..bffedd10b11 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2Scheduler/Chart.yaml @@ -0,0 +1,15 @@ +annotations: + catalog.cattle.io/hidden: "true" + catalog.cattle.io/kube-version: '>= 1.26.0-0 < 1.31.0-0' + catalog.cattle.io/os: linux + catalog.rancher.io/certified: rancher + catalog.rancher.io/namespace: cattle-monitoring-system + catalog.rancher.io/release-name: rancher-pushprox +apiVersion: v1 +appVersion: v0.1.4-rancher2 +description: Sets up a deployment of the PushProx proxy and a DaemonSet of PushProx + clients. +kubeVersion: '>=1.26.0-0' +name: rke2Scheduler +type: application +version: 0.2.1 diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2Scheduler/README.md b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2Scheduler/README.md new file mode 100644 index 00000000000..345002f48a2 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2Scheduler/README.md @@ -0,0 +1,90 @@ +# rancher-pushprox + +A Rancher chart based on Rancher [PushProx](https://github.com/rancher/PushProx) that sets up a Deployment of a PushProx proxy and a DaemonSet of PushProx clients on a Kubernetes cluster. + +Installs [rancher-pushprox](https://github.com/rancher/charts/tree/gh-pages/packages/rancher-pushprox) to create PushProx clients that can access their host's network and register with a PushProx proxy. A [Prometheus Operator](https://github.com/coreos/prometheus-operator) ServiceMonitor CR is also included that is configured to scrape the metrics from each of the clients through the proxy. + +Using an instance of this chart is suitable for the following scenarios: +- You need to scrape metrics from a port that should not be accessible outside of the host (e.g. scraping `etcd` metrics in a hardened cluster) +- You need to scrape metrics on a host that are not exposed outside of 127.0.0.1 (e.g. scraping `kube-proxy` metrics) +- You need to scrape metrics through HTTPS using certs hosted directly on `hostPath` +- You need to scrape metrics from Kubernetes components that require authorization via a service account (e.g. permissions to make request to `/metrics`) +- You need to scrape metrics without access to cacerts (i.e. enable `insecureSkipVerify`) + +The clients and proxy are created based on a Rancher fork of the [prometheus-community/PushProx](https://github.com/prometheus-community/PushProx) project. + +## Upgrading to Kubernetes v1.25+ + +Starting in Kubernetes v1.25, [Pod Security Policies](https://kubernetes.io/docs/concepts/security/pod-security-policy/) have been removed from the Kubernetes API. + +As a result, **before upgrading to Kubernetes v1.25** (or on a fresh install in a Kubernetes v1.25+ cluster), users are expected to perform an in-place upgrade of this chart with `global.cattle.psp.enabled` set to `false` if it has been previously set to `true`. +​ +> **Note:** +> In this chart release, any previous field that was associated with any PSP resources have been removed in favor of a single global field: `global.cattle.psp.enabled`. + +> **Note:** +> If you upgrade your cluster to Kubernetes v1.25+ before removing PSPs via a `helm upgrade` (even if you manually clean up resources), **it will leave the Helm release in a broken state within the cluster such that further Helm operations will not work (`helm uninstall`, `helm upgrade`, etc.).** +> +> If your charts get stuck in this state, please consult the Rancher docs on how to clean up your Helm release secrets. + +Upon setting `global.cattle.psp.enabled` to false, the chart will remove any PSP resources deployed on its behalf from the cluster. This is the default setting for this chart. + +As a replacement for PSPs, [Pod Security Admission](https://kubernetes.io/docs/concepts/security/pod-security-admission/) should be used. Please consult the Rancher docs for more details on how to configure your chart release namespaces to work with the new Pod Security Admission and apply Pod Security Standards. + +## Configuration + +The following tables list the configurable parameters of the rancher-pushprox chart and their default values. + +### General + +#### Required +| Parameter | Description | Example | +| ----- | ----------- | ------ | +| `component` | The component that is being monitored | `kube-etcd` +| `metricsPort` | The port on the host that contains the metrics you want to scrape (e.g. `http://:/metrics`) | `2379` | +| `namespaceOverride` | The namespace to install the chart | `""` + +#### Optional +| Parameter | Description | Default | +| ----- | ----------- | ------ | +| `serviceMonitor.enabled` | Deploys a [Prometheus Operator](https://github.com/coreos/prometheus-operator/blob/master/Documentation/api.md#servicemonitor) ServiceMonitor CR that is configured to scrape metrics on the hosts that the clients are deployed on via the proxy. Also deploys a Service that points to all pods with the expected client name that exposes the `metricsPort` selected | `true` | +| `serviceMonitor.endpoints` | A list of endpoints that will be added to the ServiceMonitor based on the [Endpoint spec](https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#endpoint) | `[{port: metrics}]` | +| `service.selector` | The selector that is used to populate the Service's Endpoints object. The chart will error out on rendering templating if `.Values.clients.enabled` is set alongside this field, since it is expected that this service should point to the PushProx Clients Daemonset / Deployment | `{}` | +| `clients.enabled` | Deploys a DaemonSet of clients that are each capable of scraping endpoints on the hostNetwork it is deployed on | `true` | +| `clients.port` | The port where the client will publish PushProx client-specific metrics. If deploying multiple clients onto the same node, the clients should not have conflicting ports | `9369` | +| `clients.proxyUrl` | Overrides the default proxyUrl setting of `http://pushprox-{{ .Values.component }}-proxy.{{ . Release.Namespace }}.svc.cluster.local:{{ .Values.proxy.port }}"` with the `proxyUrl` specified | `""` | +| `clients.useLocalhost` | Sets a flag on each client deployment to redirect scrapes directed to `HOST_IP` to `127.0.0.1` | `false` | +| `clients.https.enabled` | Enables scraping metrics via HTTPS using the provided TLS certs that exist on each host | `false` | +| `clients.https.forceHTTPSScheme` | Forces scraping metrics via HTTPS using the provided TLS certs that exist on each host | `false` | +| `clients.https.useServiceAccountCredentials` | If set to true, the client will create a service account with permissions to scrape `/metrics` endpoint of Kubernetes components. The client will use the service account token provided to make authorized scrape requests to the Kubernetes API | `false` | +| `clients.https.authenticationMethod.bearerTokenFile.enabled` | If set to true, the client will use service account credentials mounted at the configured path `clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath`. This requires permissions to scrape `/metrics` endpoint of Kubernetes components. This method is deprecated by the prometheus operator and may be removed in a future release | `false` | +| `clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath` | This is a volume mount on the pod with permissions to scrape `/metrics` endpoint of Kubernetes components | `"/var/run/secrets/kubernetes.io/serviceaccount/token"` | +| `clients.https.authenticationMethod.bearerTokenSecret.enabled` | If set to true, the client will use service account credentials to scrape `/metrics` endpoint of Kubernetes components. This method is deprecated by the prometheus operator and may be removed in a future release | `false` | +| `clients.https.authenticationMethod.authorization.enabled` | If set to true, the client will use service account credentials to scrape `/metrics` endpoint of Kubernetes components | `false` | +| `clients.https.authenticationMethod.authorization.type` | If set, the client will use this type of authorization in its client requests for metrics | `"bearer"` | +| `clients.https.authenticationMethod.authorization.credentials.key` | If set, the client will use this key in the secret created by `clients.https.useServiceAccountCredentials` for authorization in its client requests for metrics | `"token"` | +| `clients.https.authenticationMethod.authorization.credentials.optional` | If set to false, the client will fail if the key in the secret created by `clients.https.useServiceAccountCredentials` does not exist | `false` | +| `clients.https.insecureSkipVerify` | If set to true, the client will disable SSL security checks | `false` | +| `clients.https.certDir` | A `hostPath` where TLS certs can be found. This path is mounted as a volume on an `initContainer` which copies only the necessary files over to an EmptyDir volume used by each client. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.certFile` | The path to the TLS cert file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.keyFile` | The path to the TLS key file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.caCertFile` | The path to the TLS cacert file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.seLinuxOptions` | seLinuxOptions to be passed into the container that copies certs. Should define a container with permissions to read the files in the certDir provided on the host. Required and only used if `clients.https.enabled` is set and `clients.https.certDir` is provided. | `""` | +| `clients.metrics.enabled` | Whether the client should publish PushProx client-specific metrics. | `false` | +| `clients.rbac.additionalRules` | Additional permissions to provide to the ServiceAccount bound to the client. This can be used to provide additional permissions for the client to scrape metrics from the k8s API. Only enabled if clients.https.enabled and clients.https.useServiceAccountCredentials are true | `[]` | +| `clients.deployment.enabled` | Deploys the client as a Deployment (generally used if the underlying hostNetwork Pod that is being scraped is managed by a Deployment) | `false` | +| `clients.deployment.replicas` | The number of pods the Deployment has, it should match the number of pod the hostNetwork Deployment has. Required and only used if `client.deployment.enable` is set | `0` | +| `clients.deployment.affinity` | The affinity rules that allocate the pod to the node in which the hostNetwork Deployment's pods run. Required and only used if `client.deployment.enable` is set | `{}` | +| `clients.resources` | Set resource limits and requests for the client container | `{}` | +| `clients.nodeSelector` | Select which nodes to deploy the clients on | `{}` | +| `clients.tolerations` | Specify tolerations for clients | `[]` | +| `proxy.enabled` | Deploys the proxy that each client will register with | `true` | +| `proxy.port` | The port exposed by the proxy that each client will register with to allow metrics to be scraped from the host | `8080` | +| `proxy.resources` | Set resource limits and requests for the proxy container | `{}` | +| `proxy.nodeSelector` | Select which nodes the proxy can be deployed on | `{}` | +| `proxy.tolerations` | Specify tolerations (if necessary) to allow the proxy to be deployed on the selected node | `[]` | +| `kubeVersionOverrides` | A list of Semver constraint strings (defined by https://github.com/Masterminds/semver) and values.yaml overrides. For each key in kubeVersionOverrides, this chart will check to see if the current Kubernetes cluster's version matches any of the semver constraints provided as keys on the map. On seeing a match, the default value for each values.yaml field overridden will be updated with the new value. If multiple matches are encountered (due to overlapping semver ranges), the matches will be applied in order. | `[]` + +*Tip: The filepaths set in `clients.https.File` can include wildcard characters*. + +See [rancher-monitoring](https://github.com/rancher/charts/tree/gh-pages/packages/rancher-monitoring) for examples of how this chart can be used. diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2Scheduler/templates/_helpers.tpl b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2Scheduler/templates/_helpers.tpl new file mode 100644 index 00000000000..1ba50939441 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2Scheduler/templates/_helpers.tpl @@ -0,0 +1,170 @@ +# Rancher + +{{- define "system_default_registry" -}} +{{- if .Values.global.cattle.systemDefaultRegistry -}} +{{- printf "%s/" .Values.global.cattle.systemDefaultRegistry -}} +{{- end -}} +{{- end -}} + +# Windows Support + +{{/* +Windows cluster will add default taint for linux nodes, +add below linux tolerations to workloads could be scheduled to those linux nodes +*/}} + +{{- define "linux-node-tolerations" -}} +- key: "cattle.io/os" + value: "linux" + effect: "NoSchedule" + operator: "Equal" +{{- end -}} + +{{- define "linux-node-selector" -}} +{{- if semverCompare "<1.14-0" .Capabilities.KubeVersion.GitVersion -}} +beta.kubernetes.io/os: linux +{{- else -}} +kubernetes.io/os: linux +{{- end -}} +{{- end -}} + +# General + +{{- define "applyKubeVersionOverrides" -}} +{{- $overrides := dict -}} +{{- range $override := .Values.kubeVersionOverrides -}} +{{- if semverCompare $override.constraint $.Capabilities.KubeVersion.Version -}} +{{- $_ := mergeOverwrite $overrides $override.values -}} +{{- end -}} +{{- end -}} +{{- $_ := mergeOverwrite .Values $overrides -}} +{{- end -}} + +{{- define "pushprox.namespace" -}} + {{- if .Values.namespaceOverride -}} + {{- .Values.namespaceOverride -}} + {{- else -}} + {{- .Release.Namespace -}} + {{- end -}} +{{- end -}} + +{{- define "pushProxy.commonLabels" -}} +release: {{ .Release.Name }} +component: {{ .Values.component | quote }} +provider: kubernetes +{{- end -}} + +{{- define "pushProxy.proxyUrl" -}} +{{- $_ := (required "Template requires either .Values.proxy.port or .Values.client.proxyUrl to set proxyUrl for client" (or .Values.clients.proxyUrl .Values.proxy.port)) -}} +{{- if .Values.clients.proxyUrl -}} +{{ printf "%s" .Values.clients.proxyUrl }} +{{- else -}} +{{ printf "http://%s.%s.svc:%d" (include "pushProxy.proxy.name" .) (include "pushprox.namespace" .) (int .Values.proxy.port) }} +{{- end -}}{{- end -}} + +# Client + +{{- define "pushProxy.client.name" -}} +{{- printf "pushprox-%s-client" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.client.serviceAccountTokenName" -}} +{{- printf "pushprox-%s-client-service-account-token" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.client.labels" -}} +k8s-app: {{ template "pushProxy.client.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +# Proxy + +{{- define "pushProxy.proxy.name" -}} +{{- printf "pushprox-%s-proxy" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.proxy.labels" -}} +k8s-app: {{ template "pushProxy.proxy.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +# ServiceMonitor + +{{- define "pushprox.serviceMonitor.name" -}} +{{- printf "%s-%s" .Release.Name (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.serviceMonitor.labels" -}} +app: {{ template "pushprox.serviceMonitor.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +{{- define "pushProxy.serviceMonitor.endpoints" -}} +{{- $proxyURL := (include "pushProxy.proxyUrl" .) -}} +{{- $useHTTPS := .Values.clients.https.enabled -}} +{{- $setHTTPSScheme := .Values.clients.https.forceHTTPSScheme -}} +{{- $insecureSkipVerify := .Values.clients.https.insecureSkipVerify -}} +{{- $useServiceAccountCredentials := .Values.clients.https.useServiceAccountCredentials -}} +{{- $serviceAccountTokenName := (include "pushProxy.client.serviceAccountTokenName" . ) -}} +{{- $metricRelabelings := list }} +{{- $endpoints := .Values.serviceMonitor.endpoints }} +{{- if .Values.proxy.enabled }} +{{- $_ := set . "proxyUrl" $proxyURL }} +{{- end }} +{{- range $endpoints }} +{{- if $.Values.proxy.enabled }} +{{- $_ := set . "proxyUrl" $proxyURL }} +{{- end }} +{{- $clusterIdRelabel := dict }} +{{- $metricRelabelings := list }} +{{- if $.Values.global.cattle.clusterId }} +{{- $_ := set $clusterIdRelabel "action" "replace" }} +{{- $_ := set $clusterIdRelabel "sourceLabels" (list "__address__") }} +{{- $_ := set $clusterIdRelabel "targetLabel" "cluster_id" }} +{{- $_ := set $clusterIdRelabel "replacement" $.Values.global.cattle.clusterId }} +{{- $metricRelabelings = append $metricRelabelings $clusterIdRelabel }} +{{- end }} +{{- $clusterNameRelabel := dict }} +{{- if $.Values.global.cattle.clusterName }} +{{- $_ := set $clusterNameRelabel "action" "replace" }} +{{- $_ := set $clusterNameRelabel "sourceLabels" (list "__address__") }} +{{- $_ := set $clusterNameRelabel "targetLabel" "cluster_name" }} +{{- $_ := set $clusterNameRelabel "replacement" $.Values.global.cattle.clusterName }} +{{- $metricRelabelings = append $metricRelabelings $clusterNameRelabel }} +{{- end }} +{{- if not (empty $metricRelabelings) }} +{{- $_ := set . "metricRelabelings" ($metricRelabelings)}} +{{- end }} +{{- if $setHTTPSScheme -}} +{{- $_ := set . "scheme" "https" }} +{{- end -}} +{{- if $useHTTPS -}} +{{- if (hasKey . "params") }} +{{- $_ := set (get . "params") "_scheme" (list "https") }} +{{- else }} +{{- $_ := set . "params" (dict "_scheme" (list "https")) }} +{{- end }} +{{- end }} +{{- if (hasKey . "tlsConfig") }} +{{- $_ := set (get . "tlsConfig") "insecureSkipVerify" $insecureSkipVerify }} +{{- else }} +{{- $_ := set . "tlsConfig" (dict "insecureSkipVerify" $insecureSkipVerify) }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.bearerTokenFile.enabled }} +{{- $_ := set . "bearerTokenFile" $.Values.clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.bearerTokenSecret.enabled }} +{{- $_ := set . "bearerTokenSecret" $serviceAccountTokenName }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.authorization.enabled }} +{{- if (hasKey . "authorization") }} +{{- $_ := set (get . "authorization") "type" $.Values.clients.https.authenticationMethod.authorization.type }} +{{- $_ := set (get . "authorization") "credentials" (dict "name" $serviceAccountTokenName "key" $.Values.clients.https.authenticationMethod.authorization.credentials.key "optional" $.Values.clients.https.authenticationMethod.authorization.credentials.optional) }} +{{- else }} +{{- $_ := set . "authorization" (dict "type" $.Values.clients.https.authenticationMethod.authorization.type) }} +{{- $_ := set . "authorization" (dict "credentials" (dict "name" $serviceAccountTokenName "key" $.Values.clients.https.authenticationMethod.authorization.credentials.key "optional" $.Values.clients.https.authenticationMethod.authorization.credentials.optional)) }} +{{- end }} +{{- end }} +{{- end }} +{{- toYaml $endpoints }} +{{- end -}} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2Scheduler/templates/pushprox-clients-rbac.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2Scheduler/templates/pushprox-clients-rbac.yaml new file mode 100644 index 00000000000..a8e27c37356 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2Scheduler/templates/pushprox-clients-rbac.yaml @@ -0,0 +1,97 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.clients }}{{- if .Values.clients.enabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ template "pushProxy.client.name" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +rules: +{{- if .Values.global.cattle.psp.enabled }} +- apiGroups: ['policy'] + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - {{ template "pushProxy.client.name" . }} +{{- end }} +{{- if and .Values.clients.https.enabled .Values.clients.https.useServiceAccountCredentials }} +- nonResourceURLs: ["/metrics"] + verbs: ["get"] +{{- if .Values.clients.rbac.additionalRules }} +{{ toYaml .Values.clients.rbac.additionalRules }} +{{- end }} +{{- end }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ template "pushProxy.client.name" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "pushProxy.client.name" . }} +subjects: + - kind: ServiceAccount + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +--- +{{- if .Values.clients.https.useServiceAccountCredentials }} +apiVersion: v1 +kind: Secret +type: kubernetes.io/service-account-token +metadata: + name: {{ template "pushProxy.client.serviceAccountTokenName" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} + annotations: + kubernetes.io/service-account.name: {{ template "pushProxy.client.name" . }} +{{- end }} +--- +{{- if .Values.global.cattle.psp.enabled }} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +spec: + privileged: false + hostNetwork: true + hostIPC: false + hostPID: false + runAsUser: + rule: 'RunAsAny' + seLinux: + rule: 'RunAsAny' +{{- if and .Values.clients.https.enabled .Values.clients.https.certDir .Values.global.seLinux.enabled .Values.clients.https.seLinuxOptions }} + seLinuxOptions: {{ .Values.clients.https.seLinuxOptions | toYaml | nindent 6 }} +{{- end }} + supplementalGroups: + rule: 'MustRunAs' + ranges: + - min: 0 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + - min: 0 + max: 65535 + readOnlyRootFilesystem: false + volumes: + - 'secret' +{{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + - 'emptyDir' + - 'hostPath' + allowedHostPaths: + - pathPrefix: {{ required "Need access to volume on host with the SSL cert files to use HTTPs" .Values.clients.https.certDir }} + readOnly: true +{{- end }} +{{- end }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2Scheduler/templates/pushprox-clients.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2Scheduler/templates/pushprox-clients.yaml new file mode 100644 index 00000000000..e8fcfb38836 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2Scheduler/templates/pushprox-clients.yaml @@ -0,0 +1,157 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.clients }}{{- if .Values.clients.enabled }} +apiVersion: apps/v1 +{{- if .Values.clients.deployment.enabled }} +kind: Deployment +{{- else }} +kind: DaemonSet +{{- end }} +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} + pushprox-exporter: "client" +spec: + {{- if .Values.clients.deployment.enabled }} + replicas: {{ .Values.clients.deployment.replicas }} + {{- end }} + selector: + matchLabels: {{ include "pushProxy.client.labels" . | nindent 6 }} + template: + metadata: + labels: {{ include "pushProxy.client.labels" . | nindent 8 }} + spec: + {{- if .Values.clients.affinity }} + affinity: {{ toYaml .Values.clients.affinity | nindent 8 }} + {{- end }} + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} +{{- if .Values.clients.nodeSelector }} +{{ toYaml .Values.clients.nodeSelector | indent 8 }} +{{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} +{{- if .Values.clients.tolerations }} +{{ toYaml .Values.clients.tolerations | indent 8 }} +{{- end }} + hostNetwork: true + dnsPolicy: ClusterFirstWithHostNet + serviceAccountName: {{ template "pushProxy.client.name" . }} + {{- if .Values.global.imagePullSecretName }} + imagePullSecrets: + - name: {{ .Values.global.imagePullSecretName }} + {{- end }} + containers: + - name: pushprox-client + image: {{ template "system_default_registry" . }}{{ .Values.clients.image.repository }}:{{ .Values.clients.image.tag }} + command: + {{- range .Values.clients.command }} + - {{ . | quote }} + {{- end }} + args: + - --fqdn=$(HOST_IP) + - --proxy-url=$(PROXY_URL) + {{- if .Values.clients.metrics.enabled }} + - --metrics-addr=$(PORT) + {{- end }} + - --allow-port={{ required "Need .Values.metricsPort to configure client to be allowed to scrape metrics at port" .Values.metricsPort}} + {{- if .Values.clients.useLocalhost }} + - --use-localhost + {{- end }} + {{- if .Values.clients.https.enabled }} + {{- if .Values.clients.https.insecureSkipVerify }} + - --insecure-skip-verify + {{- end }} + {{- if .Values.clients.https.useServiceAccountCredentials }} + - --token-path=/var/run/secrets/kubernetes.io/serviceaccount/token + {{- end }} + {{- if .Values.clients.https.certDir }} + - --tls.cert=/etc/ssl/push-proxy/push-proxy.pem + - --tls.key=/etc/ssl/push-proxy/push-proxy-key.pem + - --tls.cacert=/etc/ssl/push-proxy/push-proxy-ca-cert.pem + {{- end }} + {{- end }} + env: + - name: HOST_IP + valueFrom: + fieldRef: + fieldPath: status.hostIP + {{- if .Values.clients.metrics.enabled }} + - name: PORT + value: :{{ .Values.clients.port }} + {{- end }} + - name: PROXY_URL + value: {{ template "pushProxy.proxyUrl" . }} + securityContext: + runAsNonRoot: true + runAsUser: 1000 + {{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + volumeMounts: + - name: metrics-cert-dir + mountPath: /etc/ssl/push-proxy + {{- end }} + {{- if .Values.clients.resources }} + resources: {{ toYaml .Values.clients.resources | nindent 10 }} + {{- end }} + {{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + initContainers: + - name: copy-certs + image: {{ template "system_default_registry" . }}{{ .Values.clients.copyCertsImage.repository }}:{{ .Values.clients.copyCertsImage.tag }} + command: + - sh + - -c + - | + echo "Searching for files to copy within the source volume" + echo "cert: ${CERT_FILE_NAME}" + echo "key: ${KEY_FILE_NAME}" + echo "cacert: ${CACERT_FILE_NAME}" + + CERT_FILE_SOURCE=$(find /etc/source/ -type f -name "${CERT_FILE_NAME}" | sort -r | head -n 1) + KEY_FILE_SOURCE=$(find /etc/source/ -type f -name "${KEY_FILE_NAME}" | sort -r | head -n 1) + CACERT_FILE_SOURCE=$(find /etc/source/ -type f -name "${CACERT_FILE_NAME}" | sort -r | head -n 1) + + test -z ${CERT_FILE_SOURCE} && echo "Failed to find cert file" && exit 1 + test -z ${KEY_FILE_SOURCE} && echo "Failed to find key file" && exit 1 + test -z ${CACERT_FILE_SOURCE} && echo "Failed to find cacert file" && exit 1 + + echo "Copying cert file from $CERT_FILE_SOURCE to $CERT_FILE_TARGET" + cp $CERT_FILE_SOURCE $CERT_FILE_TARGET || exit 1 + chmod 444 $CERT_FILE_TARGET || exit 1 + + echo "Copying key file from $KEY_FILE_SOURCE to $KEY_FILE_TARGET" + cp $KEY_FILE_SOURCE $KEY_FILE_TARGET || exit 1 + chmod 444 $KEY_FILE_TARGET || exit 1 + + echo "Copying cacert file from $CACERT_FILE_SOURCE to $CACERT_FILE_TARGET" + cp $CACERT_FILE_SOURCE $CACERT_FILE_TARGET || exit 1 + chmod 444 $CACERT_FILE_TARGET || exit 1 + env: + - name: CERT_FILE_NAME + value: {{ required "Need a TLS cert file for scraping metrics endpoint over HTTPs" .Values.clients.https.certFile }} + - name: KEY_FILE_NAME + value: {{ required "Need a TLS key file for scraping metrics endpoint over HTTPs" .Values.clients.https.keyFile }} + - name: CACERT_FILE_NAME + value: {{ required "Need a TLS CA cert file for scraping metrics endpoint over HTTPs" .Values.clients.https.caCertFile }} + - name: CERT_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy.pem + - name: KEY_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy-key.pem + - name: CACERT_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy-ca-cert.pem + securityContext: + runAsNonRoot: false +{{- if and .Values.global.seLinux.enabled .Values.clients.https.seLinuxOptions }} + seLinuxOptions: {{ .Values.clients.https.seLinuxOptions | toYaml | nindent 12 }} +{{- end }} + volumeMounts: + - name: metrics-cert-dir-source + mountPath: /etc/source + readOnly: true + - name: metrics-cert-dir + mountPath: /etc/ssl/push-proxy + volumes: + - name: metrics-cert-dir-source + hostPath: + path: {{ required "Need access to volume on host with the SSL cert files to use HTTPs" .Values.clients.https.certDir }} + - name: metrics-cert-dir + emptyDir: {} + {{- end }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2Scheduler/templates/pushprox-proxy-rbac.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2Scheduler/templates/pushprox-proxy-rbac.yaml new file mode 100644 index 00000000000..eefe609058c --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2Scheduler/templates/pushprox-proxy-rbac.yaml @@ -0,0 +1,68 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if and .Values.proxy }}{{ if .Values.proxy.enabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ template "pushProxy.proxy.name" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +rules: +{{- if .Values.global.cattle.psp.enabled }} +- apiGroups: ['policy'] + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - {{ template "pushProxy.proxy.name" . }} +{{- end }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ template "pushProxy.proxy.name" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "pushProxy.proxy.name" . }} +subjects: + - kind: ServiceAccount + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +--- +{{- if .Values.global.cattle.psp.enabled }} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +spec: + privileged: false + hostNetwork: false + hostIPC: false + hostPID: false + runAsUser: + rule: 'MustRunAsNonRoot' + seLinux: + rule: 'RunAsAny' + supplementalGroups: + rule: 'MustRunAs' + ranges: + - min: 1 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + - min: 1 + max: 65535 + readOnlyRootFilesystem: false + volumes: + - 'secret' +{{- end }}{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2Scheduler/templates/pushprox-proxy.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2Scheduler/templates/pushprox-proxy.yaml new file mode 100644 index 00000000000..723bbd6c00b --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2Scheduler/templates/pushprox-proxy.yaml @@ -0,0 +1,57 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if and .Values.proxy }}{{ if .Values.proxy.enabled }} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} + pushprox-exporter: "proxy" +spec: + selector: + matchLabels: {{ include "pushProxy.proxy.labels" . | nindent 6 }} + template: + metadata: + labels: {{ include "pushProxy.proxy.labels" . | nindent 8 }} + spec: + securityContext: + runAsNonRoot: true + runAsUser: 1000 + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} +{{- if .Values.proxy.nodeSelector }} +{{ toYaml .Values.proxy.nodeSelector | indent 8 }} +{{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} +{{- if .Values.proxy.tolerations }} +{{ toYaml .Values.proxy.tolerations | indent 8 }} +{{- end }} + serviceAccountName: {{ template "pushProxy.proxy.name" . }} + {{- if .Values.global.imagePullSecretName }} + imagePullSecrets: + - name: {{ .Values.global.imagePullSecretName }} + {{- end }} + containers: + - name: pushprox-proxy + image: {{ template "system_default_registry" . }}{{ .Values.proxy.image.repository }}:{{ .Values.proxy.image.tag }} + command: + {{- range .Values.proxy.command }} + - {{ . | quote }} + {{- end }} + {{- if .Values.proxy.resources }} + resources: {{ toYaml .Values.proxy.resources | nindent 10 }} + {{- end }} +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +spec: + ports: + - name: pp-proxy + port: {{ required "Need .Values.proxy.port to configure proxy" .Values.proxy.port }} + protocol: TCP + targetPort: {{ .Values.proxy.port }} + selector: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2Scheduler/templates/pushprox-servicemonitor.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2Scheduler/templates/pushprox-servicemonitor.yaml new file mode 100644 index 00000000000..67eb2216b5a --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2Scheduler/templates/pushprox-servicemonitor.yaml @@ -0,0 +1,45 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.serviceMonitor }}{{- if .Values.serviceMonitor.enabled }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ template "pushprox.serviceMonitor.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.serviceMonitor.labels" . | nindent 4 }} +spec: + endpoints: {{include "pushProxy.serviceMonitor.endpoints" . | nindent 4 }} + jobLabel: component + podTargetLabels: + - component + - pushprox-exporter + namespaceSelector: + matchNames: + - {{ template "pushprox.namespace" . }} + selector: + matchLabels: {{ include "pushProxy.client.labels" . | nindent 6 }} +--- +{{- $selector := "" }} +{{- if not (kindIs "invalid" .Values.service) }} +{{- if not (kindIs "invalid" .Values.service.selector) }} +{{ if .Values.service.selector }} +{{- if .Values.clients.enabled }} +{{- required (printf "Cannot override .Values.service.selector=%s when .Values.clients.enabled=true" (toJson .Values.service.selector)) "" }} +{{- end }} +{{- $selector = (toYaml .Values.service.selector) }} +{{- end }} +{{- end }} +{{- end }} +apiVersion: v1 +kind: Service +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +spec: + ports: + - name: metrics + port: {{ required "Need .Values.metricsPort to configure client to listen to metrics at port" .Values.metricsPort}} + protocol: TCP + targetPort: {{ .Values.metricsPort }} + selector: {{ default (include "pushProxy.client.labels" .) $selector | nindent 4 }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2Scheduler/templates/validate-install-crd.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2Scheduler/templates/validate-install-crd.yaml new file mode 100644 index 00000000000..16abc2fa83a --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2Scheduler/templates/validate-install-crd.yaml @@ -0,0 +1,14 @@ +#{{- if gt (len (lookup "rbac.authorization.k8s.io/v1" "ClusterRole" "" "")) 0 -}} +# {{- $found := dict -}} +# {{- set $found "monitoring.coreos.com/v1/ServiceMonitor" false -}} +# {{- range .Capabilities.APIVersions -}} +# {{- if hasKey $found (toString .) -}} +# {{- set $found (toString .) true -}} +# {{- end -}} +# {{- end -}} +# {{- range $_, $exists := $found -}} +# {{- if (eq $exists false) -}} +# {{- required "Required CRDs are missing. Please install Prometheus Operator CRDs before installing this chart." "" -}} +# {{- end -}} +# {{- end -}} +#{{- end -}} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2Scheduler/templates/validate-psp-install.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2Scheduler/templates/validate-psp-install.yaml new file mode 100644 index 00000000000..a30c59d3b7c --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2Scheduler/templates/validate-psp-install.yaml @@ -0,0 +1,7 @@ +#{{- if gt (len (lookup "rbac.authorization.k8s.io/v1" "ClusterRole" "" "")) 0 -}} +#{{- if .Values.global.cattle.psp.enabled }} +#{{- if not (.Capabilities.APIVersions.Has "policy/v1beta1/PodSecurityPolicy") }} +#{{- fail "The target cluster does not have the PodSecurityPolicy API resource. Please disable PSPs in this chart before proceeding." -}} +#{{- end }} +#{{- end }} +#{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2Scheduler/values.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2Scheduler/values.yaml new file mode 100644 index 00000000000..7f38498a1c5 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rke2Scheduler/values.yaml @@ -0,0 +1,166 @@ +# Default values for rancher-pushprox. +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. + +# Default image containing both the proxy and the client was generated from the following Dockerfile +# https://github.com/prometheus-community/PushProx/blob/eeadbe766641699129920ccfaaaa30a85c67fe81/Dockerfile#L1-L15 + +# Configuration + +global: + cattle: + psp: + enabled: false + systemDefaultRegistry: "" + seLinux: + enabled: false + +# A list of Semver constraint strings (defined by https://github.com/Masterminds/semver) and values.yaml overrides. +# +# For each key in kubeVersionOverrides, this chart will check to see if the current Kubernetes cluster's version matches +# any of the semver constraints provided as keys on the map. +# +# On seeing a match, the default value for each values.yaml field overridden will be updated with the new value. +# +# If multiple matches are encountered (due to overlapping semver ranges), the matches will be applied in order. +# +# Notes: +# - On running a helm template, Helm generally assumes the kubeVersion is v1.20.0 +# - On running a helm install --dry-run, the correct kubeVersion should be chosen. +kubeVersionOverrides: [] +# - constraint: "< 1.21" +# values: +# metricsPort: 10252 +# clients: +# https: +# enabled: false +# insecureSkipVerify: false +# useServiceAccountCredentials: false + +namespaceOverride: "" + +# The component that is being monitored (i.e. etcd) +component: "component" + +# The port containing the metrics that need to be scraped +metricsPort: 2739 + +# Configure ServiceMonitor that monitors metrics from the metricsPort endpoint +serviceMonitor: + enabled: true + # A list of endpoints that will be added to the ServiceMonitor based on the Endpoint spec + # Source: https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#endpoint + # By default, proxyUrl and params._scheme will be overridden based on other values + endpoints: + - port: metrics + +# Configure Service that grabs scrape targets +service: + # The selector that is used to populate the Service's Endpoints object. + # The chart will error out on rendering templating if .Values.clients.enabled is set alongside this field, + # since it is expected that this service should point to the PushProx Clients Daemonset / Deployment + selector: {} + +clients: + enabled: true + # The port which the PushProx client will post PushProx metrics to + port: 9369 + # If unset, this will default to the URL for the proxy service: http://pushprox-{{component}}-proxy.{{namepsace}}.svc.cluster.local:{{proxy.port}} + # Should be modified if the clients are being deployed outside the cluster where the proxy rests, otherwise leave it null + proxyUrl: "" + # If set to true, the client will forward any requests from the host IP to 127.0.0.1 + # It will only allow proxy requests to the metricsPort specified + useLocalhost: false + # Configuration for accessing metrics via HTTPS + https: + # Does the client require https to access the metrics? + enabled: false + # Does the client require requests be sent to http or https? + forceHTTPSScheme: false + # If set to true, the client will create a service account with adequate permissions and set a flag + # on the client to use the service account token provided by it to make authorized scrape requests + useServiceAccountCredentials: false + # Configuration for authentication to metrics via https endpoint + authenticationMethod: + # Reads token from defined file in container + # This function is deprecated in the prometheus operator api and may be removed in a future version + bearerTokenFile: + enabled: false + bearerTokenFilePath: "/var/run/secrets/kubernetes.io/serviceaccount/token" + # Reads token from defined secret in namespace + # This function is deprecated in the prometheus operator api and may be removed in a future version + bearerTokenSecret: + enabled: false + # Reads token from defined secret in namespace + authorization: + enabled: false + type: "bearer" + credentials: + key: "token" + optional: false + # If set to true, the client will disable SSL security checks + insecureSkipVerify: false + # Directory on host where necessary TLS cert and key to scrape metrics can be found + certDir: "" + # Filenames for files located in .Values.clients.https.certDir that correspond to TLS settings + certFile: "" + keyFile: "" + caCertFile: "" + # seLinuxOptions to be passed into the container that copies certs. Should define a container with permissions to read the files in the certDir provided on the host. + # Required and only used if `clients.https.enabled` is set and `clients.https.certDir` is provided. + seLinuxOptions: {} + + metrics: + # Whether the client should publish PushProx client-specific metrics to .Values.clients.port + enabled: false + + rbac: + # Additional permissions to provide to the ServiceAccount bound to the client + # This can be used to provide additional permissions for the client to scrape metrics from the k8s API + # Only enabled if clients.https.enabled and clients.https.useServiceAccountCredentials are true + additionalRules: [] + + # Resource limits + resources: {} + + # Options to select all nodes to deploy client DaemonSet on + nodeSelector: {} + tolerations: [] + affinity: {} + + image: + repository: rancher/pushprox-client + tag: v0.1.4-rancher2-client + command: ["pushprox-client"] + + copyCertsImage: + repository: rancher/mirrored-library-busybox + tag: 1.31.1 + + # The default intention of rancher-pushprox clients is to scrape hostNetwork metrics across all nodes. + # This can be used to scrape internal Kubernetes components or DaemonSets of hostNetwork Pods in + # situations where a cloud provider firewall prevents Pod-To-Host communication but not Pod-To-Pod. + # However, if the underlying hostNetwork Pod that is being scraped is managed by a Deployment, + # this advanced option enables users to deploy the client as a Deployment instead of a DaemonSet. + # If a user deploys this feature and the underlying Deployment's number of replicas changes, the user will + # be responsible for upgrading this chart accordingly to the right number of replicas. + deployment: + enabled: false + replicas: 0 + +proxy: + enabled: true + # The port through which PushProx clients will communicate to the proxy + port: 8080 + + # Resource limits + resources: {} + + # Options to select a node to run a single proxy deployment on + nodeSelector: {} + tolerations: [] + + image: + repository: rancher/pushprox-proxy + tag: v0.1.4-rancher2-proxy + command: ["pushprox-proxy"] diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeControllerManager/.helmignore b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeControllerManager/.helmignore new file mode 100644 index 00000000000..0e8a0eb36f4 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeControllerManager/.helmignore @@ -0,0 +1,23 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*.orig +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeControllerManager/Chart.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeControllerManager/Chart.yaml new file mode 100644 index 00000000000..76cc55be61e --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeControllerManager/Chart.yaml @@ -0,0 +1,15 @@ +annotations: + catalog.cattle.io/hidden: "true" + catalog.cattle.io/kube-version: '>= 1.26.0-0 < 1.31.0-0' + catalog.cattle.io/os: linux + catalog.rancher.io/certified: rancher + catalog.rancher.io/namespace: cattle-monitoring-system + catalog.rancher.io/release-name: rancher-pushprox +apiVersion: v1 +appVersion: v0.1.4-rancher2 +description: Sets up a deployment of the PushProx proxy and a DaemonSet of PushProx + clients. +kubeVersion: '>=1.26.0-0' +name: rkeControllerManager +type: application +version: 0.2.1 diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeControllerManager/README.md b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeControllerManager/README.md new file mode 100644 index 00000000000..345002f48a2 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeControllerManager/README.md @@ -0,0 +1,90 @@ +# rancher-pushprox + +A Rancher chart based on Rancher [PushProx](https://github.com/rancher/PushProx) that sets up a Deployment of a PushProx proxy and a DaemonSet of PushProx clients on a Kubernetes cluster. + +Installs [rancher-pushprox](https://github.com/rancher/charts/tree/gh-pages/packages/rancher-pushprox) to create PushProx clients that can access their host's network and register with a PushProx proxy. A [Prometheus Operator](https://github.com/coreos/prometheus-operator) ServiceMonitor CR is also included that is configured to scrape the metrics from each of the clients through the proxy. + +Using an instance of this chart is suitable for the following scenarios: +- You need to scrape metrics from a port that should not be accessible outside of the host (e.g. scraping `etcd` metrics in a hardened cluster) +- You need to scrape metrics on a host that are not exposed outside of 127.0.0.1 (e.g. scraping `kube-proxy` metrics) +- You need to scrape metrics through HTTPS using certs hosted directly on `hostPath` +- You need to scrape metrics from Kubernetes components that require authorization via a service account (e.g. permissions to make request to `/metrics`) +- You need to scrape metrics without access to cacerts (i.e. enable `insecureSkipVerify`) + +The clients and proxy are created based on a Rancher fork of the [prometheus-community/PushProx](https://github.com/prometheus-community/PushProx) project. + +## Upgrading to Kubernetes v1.25+ + +Starting in Kubernetes v1.25, [Pod Security Policies](https://kubernetes.io/docs/concepts/security/pod-security-policy/) have been removed from the Kubernetes API. + +As a result, **before upgrading to Kubernetes v1.25** (or on a fresh install in a Kubernetes v1.25+ cluster), users are expected to perform an in-place upgrade of this chart with `global.cattle.psp.enabled` set to `false` if it has been previously set to `true`. +​ +> **Note:** +> In this chart release, any previous field that was associated with any PSP resources have been removed in favor of a single global field: `global.cattle.psp.enabled`. + +> **Note:** +> If you upgrade your cluster to Kubernetes v1.25+ before removing PSPs via a `helm upgrade` (even if you manually clean up resources), **it will leave the Helm release in a broken state within the cluster such that further Helm operations will not work (`helm uninstall`, `helm upgrade`, etc.).** +> +> If your charts get stuck in this state, please consult the Rancher docs on how to clean up your Helm release secrets. + +Upon setting `global.cattle.psp.enabled` to false, the chart will remove any PSP resources deployed on its behalf from the cluster. This is the default setting for this chart. + +As a replacement for PSPs, [Pod Security Admission](https://kubernetes.io/docs/concepts/security/pod-security-admission/) should be used. Please consult the Rancher docs for more details on how to configure your chart release namespaces to work with the new Pod Security Admission and apply Pod Security Standards. + +## Configuration + +The following tables list the configurable parameters of the rancher-pushprox chart and their default values. + +### General + +#### Required +| Parameter | Description | Example | +| ----- | ----------- | ------ | +| `component` | The component that is being monitored | `kube-etcd` +| `metricsPort` | The port on the host that contains the metrics you want to scrape (e.g. `http://:/metrics`) | `2379` | +| `namespaceOverride` | The namespace to install the chart | `""` + +#### Optional +| Parameter | Description | Default | +| ----- | ----------- | ------ | +| `serviceMonitor.enabled` | Deploys a [Prometheus Operator](https://github.com/coreos/prometheus-operator/blob/master/Documentation/api.md#servicemonitor) ServiceMonitor CR that is configured to scrape metrics on the hosts that the clients are deployed on via the proxy. Also deploys a Service that points to all pods with the expected client name that exposes the `metricsPort` selected | `true` | +| `serviceMonitor.endpoints` | A list of endpoints that will be added to the ServiceMonitor based on the [Endpoint spec](https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#endpoint) | `[{port: metrics}]` | +| `service.selector` | The selector that is used to populate the Service's Endpoints object. The chart will error out on rendering templating if `.Values.clients.enabled` is set alongside this field, since it is expected that this service should point to the PushProx Clients Daemonset / Deployment | `{}` | +| `clients.enabled` | Deploys a DaemonSet of clients that are each capable of scraping endpoints on the hostNetwork it is deployed on | `true` | +| `clients.port` | The port where the client will publish PushProx client-specific metrics. If deploying multiple clients onto the same node, the clients should not have conflicting ports | `9369` | +| `clients.proxyUrl` | Overrides the default proxyUrl setting of `http://pushprox-{{ .Values.component }}-proxy.{{ . Release.Namespace }}.svc.cluster.local:{{ .Values.proxy.port }}"` with the `proxyUrl` specified | `""` | +| `clients.useLocalhost` | Sets a flag on each client deployment to redirect scrapes directed to `HOST_IP` to `127.0.0.1` | `false` | +| `clients.https.enabled` | Enables scraping metrics via HTTPS using the provided TLS certs that exist on each host | `false` | +| `clients.https.forceHTTPSScheme` | Forces scraping metrics via HTTPS using the provided TLS certs that exist on each host | `false` | +| `clients.https.useServiceAccountCredentials` | If set to true, the client will create a service account with permissions to scrape `/metrics` endpoint of Kubernetes components. The client will use the service account token provided to make authorized scrape requests to the Kubernetes API | `false` | +| `clients.https.authenticationMethod.bearerTokenFile.enabled` | If set to true, the client will use service account credentials mounted at the configured path `clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath`. This requires permissions to scrape `/metrics` endpoint of Kubernetes components. This method is deprecated by the prometheus operator and may be removed in a future release | `false` | +| `clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath` | This is a volume mount on the pod with permissions to scrape `/metrics` endpoint of Kubernetes components | `"/var/run/secrets/kubernetes.io/serviceaccount/token"` | +| `clients.https.authenticationMethod.bearerTokenSecret.enabled` | If set to true, the client will use service account credentials to scrape `/metrics` endpoint of Kubernetes components. This method is deprecated by the prometheus operator and may be removed in a future release | `false` | +| `clients.https.authenticationMethod.authorization.enabled` | If set to true, the client will use service account credentials to scrape `/metrics` endpoint of Kubernetes components | `false` | +| `clients.https.authenticationMethod.authorization.type` | If set, the client will use this type of authorization in its client requests for metrics | `"bearer"` | +| `clients.https.authenticationMethod.authorization.credentials.key` | If set, the client will use this key in the secret created by `clients.https.useServiceAccountCredentials` for authorization in its client requests for metrics | `"token"` | +| `clients.https.authenticationMethod.authorization.credentials.optional` | If set to false, the client will fail if the key in the secret created by `clients.https.useServiceAccountCredentials` does not exist | `false` | +| `clients.https.insecureSkipVerify` | If set to true, the client will disable SSL security checks | `false` | +| `clients.https.certDir` | A `hostPath` where TLS certs can be found. This path is mounted as a volume on an `initContainer` which copies only the necessary files over to an EmptyDir volume used by each client. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.certFile` | The path to the TLS cert file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.keyFile` | The path to the TLS key file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.caCertFile` | The path to the TLS cacert file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.seLinuxOptions` | seLinuxOptions to be passed into the container that copies certs. Should define a container with permissions to read the files in the certDir provided on the host. Required and only used if `clients.https.enabled` is set and `clients.https.certDir` is provided. | `""` | +| `clients.metrics.enabled` | Whether the client should publish PushProx client-specific metrics. | `false` | +| `clients.rbac.additionalRules` | Additional permissions to provide to the ServiceAccount bound to the client. This can be used to provide additional permissions for the client to scrape metrics from the k8s API. Only enabled if clients.https.enabled and clients.https.useServiceAccountCredentials are true | `[]` | +| `clients.deployment.enabled` | Deploys the client as a Deployment (generally used if the underlying hostNetwork Pod that is being scraped is managed by a Deployment) | `false` | +| `clients.deployment.replicas` | The number of pods the Deployment has, it should match the number of pod the hostNetwork Deployment has. Required and only used if `client.deployment.enable` is set | `0` | +| `clients.deployment.affinity` | The affinity rules that allocate the pod to the node in which the hostNetwork Deployment's pods run. Required and only used if `client.deployment.enable` is set | `{}` | +| `clients.resources` | Set resource limits and requests for the client container | `{}` | +| `clients.nodeSelector` | Select which nodes to deploy the clients on | `{}` | +| `clients.tolerations` | Specify tolerations for clients | `[]` | +| `proxy.enabled` | Deploys the proxy that each client will register with | `true` | +| `proxy.port` | The port exposed by the proxy that each client will register with to allow metrics to be scraped from the host | `8080` | +| `proxy.resources` | Set resource limits and requests for the proxy container | `{}` | +| `proxy.nodeSelector` | Select which nodes the proxy can be deployed on | `{}` | +| `proxy.tolerations` | Specify tolerations (if necessary) to allow the proxy to be deployed on the selected node | `[]` | +| `kubeVersionOverrides` | A list of Semver constraint strings (defined by https://github.com/Masterminds/semver) and values.yaml overrides. For each key in kubeVersionOverrides, this chart will check to see if the current Kubernetes cluster's version matches any of the semver constraints provided as keys on the map. On seeing a match, the default value for each values.yaml field overridden will be updated with the new value. If multiple matches are encountered (due to overlapping semver ranges), the matches will be applied in order. | `[]` + +*Tip: The filepaths set in `clients.https.File` can include wildcard characters*. + +See [rancher-monitoring](https://github.com/rancher/charts/tree/gh-pages/packages/rancher-monitoring) for examples of how this chart can be used. diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeControllerManager/templates/_helpers.tpl b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeControllerManager/templates/_helpers.tpl new file mode 100644 index 00000000000..1ba50939441 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeControllerManager/templates/_helpers.tpl @@ -0,0 +1,170 @@ +# Rancher + +{{- define "system_default_registry" -}} +{{- if .Values.global.cattle.systemDefaultRegistry -}} +{{- printf "%s/" .Values.global.cattle.systemDefaultRegistry -}} +{{- end -}} +{{- end -}} + +# Windows Support + +{{/* +Windows cluster will add default taint for linux nodes, +add below linux tolerations to workloads could be scheduled to those linux nodes +*/}} + +{{- define "linux-node-tolerations" -}} +- key: "cattle.io/os" + value: "linux" + effect: "NoSchedule" + operator: "Equal" +{{- end -}} + +{{- define "linux-node-selector" -}} +{{- if semverCompare "<1.14-0" .Capabilities.KubeVersion.GitVersion -}} +beta.kubernetes.io/os: linux +{{- else -}} +kubernetes.io/os: linux +{{- end -}} +{{- end -}} + +# General + +{{- define "applyKubeVersionOverrides" -}} +{{- $overrides := dict -}} +{{- range $override := .Values.kubeVersionOverrides -}} +{{- if semverCompare $override.constraint $.Capabilities.KubeVersion.Version -}} +{{- $_ := mergeOverwrite $overrides $override.values -}} +{{- end -}} +{{- end -}} +{{- $_ := mergeOverwrite .Values $overrides -}} +{{- end -}} + +{{- define "pushprox.namespace" -}} + {{- if .Values.namespaceOverride -}} + {{- .Values.namespaceOverride -}} + {{- else -}} + {{- .Release.Namespace -}} + {{- end -}} +{{- end -}} + +{{- define "pushProxy.commonLabels" -}} +release: {{ .Release.Name }} +component: {{ .Values.component | quote }} +provider: kubernetes +{{- end -}} + +{{- define "pushProxy.proxyUrl" -}} +{{- $_ := (required "Template requires either .Values.proxy.port or .Values.client.proxyUrl to set proxyUrl for client" (or .Values.clients.proxyUrl .Values.proxy.port)) -}} +{{- if .Values.clients.proxyUrl -}} +{{ printf "%s" .Values.clients.proxyUrl }} +{{- else -}} +{{ printf "http://%s.%s.svc:%d" (include "pushProxy.proxy.name" .) (include "pushprox.namespace" .) (int .Values.proxy.port) }} +{{- end -}}{{- end -}} + +# Client + +{{- define "pushProxy.client.name" -}} +{{- printf "pushprox-%s-client" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.client.serviceAccountTokenName" -}} +{{- printf "pushprox-%s-client-service-account-token" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.client.labels" -}} +k8s-app: {{ template "pushProxy.client.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +# Proxy + +{{- define "pushProxy.proxy.name" -}} +{{- printf "pushprox-%s-proxy" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.proxy.labels" -}} +k8s-app: {{ template "pushProxy.proxy.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +# ServiceMonitor + +{{- define "pushprox.serviceMonitor.name" -}} +{{- printf "%s-%s" .Release.Name (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.serviceMonitor.labels" -}} +app: {{ template "pushprox.serviceMonitor.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +{{- define "pushProxy.serviceMonitor.endpoints" -}} +{{- $proxyURL := (include "pushProxy.proxyUrl" .) -}} +{{- $useHTTPS := .Values.clients.https.enabled -}} +{{- $setHTTPSScheme := .Values.clients.https.forceHTTPSScheme -}} +{{- $insecureSkipVerify := .Values.clients.https.insecureSkipVerify -}} +{{- $useServiceAccountCredentials := .Values.clients.https.useServiceAccountCredentials -}} +{{- $serviceAccountTokenName := (include "pushProxy.client.serviceAccountTokenName" . ) -}} +{{- $metricRelabelings := list }} +{{- $endpoints := .Values.serviceMonitor.endpoints }} +{{- if .Values.proxy.enabled }} +{{- $_ := set . "proxyUrl" $proxyURL }} +{{- end }} +{{- range $endpoints }} +{{- if $.Values.proxy.enabled }} +{{- $_ := set . "proxyUrl" $proxyURL }} +{{- end }} +{{- $clusterIdRelabel := dict }} +{{- $metricRelabelings := list }} +{{- if $.Values.global.cattle.clusterId }} +{{- $_ := set $clusterIdRelabel "action" "replace" }} +{{- $_ := set $clusterIdRelabel "sourceLabels" (list "__address__") }} +{{- $_ := set $clusterIdRelabel "targetLabel" "cluster_id" }} +{{- $_ := set $clusterIdRelabel "replacement" $.Values.global.cattle.clusterId }} +{{- $metricRelabelings = append $metricRelabelings $clusterIdRelabel }} +{{- end }} +{{- $clusterNameRelabel := dict }} +{{- if $.Values.global.cattle.clusterName }} +{{- $_ := set $clusterNameRelabel "action" "replace" }} +{{- $_ := set $clusterNameRelabel "sourceLabels" (list "__address__") }} +{{- $_ := set $clusterNameRelabel "targetLabel" "cluster_name" }} +{{- $_ := set $clusterNameRelabel "replacement" $.Values.global.cattle.clusterName }} +{{- $metricRelabelings = append $metricRelabelings $clusterNameRelabel }} +{{- end }} +{{- if not (empty $metricRelabelings) }} +{{- $_ := set . "metricRelabelings" ($metricRelabelings)}} +{{- end }} +{{- if $setHTTPSScheme -}} +{{- $_ := set . "scheme" "https" }} +{{- end -}} +{{- if $useHTTPS -}} +{{- if (hasKey . "params") }} +{{- $_ := set (get . "params") "_scheme" (list "https") }} +{{- else }} +{{- $_ := set . "params" (dict "_scheme" (list "https")) }} +{{- end }} +{{- end }} +{{- if (hasKey . "tlsConfig") }} +{{- $_ := set (get . "tlsConfig") "insecureSkipVerify" $insecureSkipVerify }} +{{- else }} +{{- $_ := set . "tlsConfig" (dict "insecureSkipVerify" $insecureSkipVerify) }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.bearerTokenFile.enabled }} +{{- $_ := set . "bearerTokenFile" $.Values.clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.bearerTokenSecret.enabled }} +{{- $_ := set . "bearerTokenSecret" $serviceAccountTokenName }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.authorization.enabled }} +{{- if (hasKey . "authorization") }} +{{- $_ := set (get . "authorization") "type" $.Values.clients.https.authenticationMethod.authorization.type }} +{{- $_ := set (get . "authorization") "credentials" (dict "name" $serviceAccountTokenName "key" $.Values.clients.https.authenticationMethod.authorization.credentials.key "optional" $.Values.clients.https.authenticationMethod.authorization.credentials.optional) }} +{{- else }} +{{- $_ := set . "authorization" (dict "type" $.Values.clients.https.authenticationMethod.authorization.type) }} +{{- $_ := set . "authorization" (dict "credentials" (dict "name" $serviceAccountTokenName "key" $.Values.clients.https.authenticationMethod.authorization.credentials.key "optional" $.Values.clients.https.authenticationMethod.authorization.credentials.optional)) }} +{{- end }} +{{- end }} +{{- end }} +{{- toYaml $endpoints }} +{{- end -}} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeControllerManager/templates/pushprox-clients-rbac.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeControllerManager/templates/pushprox-clients-rbac.yaml new file mode 100644 index 00000000000..a8e27c37356 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeControllerManager/templates/pushprox-clients-rbac.yaml @@ -0,0 +1,97 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.clients }}{{- if .Values.clients.enabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ template "pushProxy.client.name" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +rules: +{{- if .Values.global.cattle.psp.enabled }} +- apiGroups: ['policy'] + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - {{ template "pushProxy.client.name" . }} +{{- end }} +{{- if and .Values.clients.https.enabled .Values.clients.https.useServiceAccountCredentials }} +- nonResourceURLs: ["/metrics"] + verbs: ["get"] +{{- if .Values.clients.rbac.additionalRules }} +{{ toYaml .Values.clients.rbac.additionalRules }} +{{- end }} +{{- end }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ template "pushProxy.client.name" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "pushProxy.client.name" . }} +subjects: + - kind: ServiceAccount + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +--- +{{- if .Values.clients.https.useServiceAccountCredentials }} +apiVersion: v1 +kind: Secret +type: kubernetes.io/service-account-token +metadata: + name: {{ template "pushProxy.client.serviceAccountTokenName" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} + annotations: + kubernetes.io/service-account.name: {{ template "pushProxy.client.name" . }} +{{- end }} +--- +{{- if .Values.global.cattle.psp.enabled }} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +spec: + privileged: false + hostNetwork: true + hostIPC: false + hostPID: false + runAsUser: + rule: 'RunAsAny' + seLinux: + rule: 'RunAsAny' +{{- if and .Values.clients.https.enabled .Values.clients.https.certDir .Values.global.seLinux.enabled .Values.clients.https.seLinuxOptions }} + seLinuxOptions: {{ .Values.clients.https.seLinuxOptions | toYaml | nindent 6 }} +{{- end }} + supplementalGroups: + rule: 'MustRunAs' + ranges: + - min: 0 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + - min: 0 + max: 65535 + readOnlyRootFilesystem: false + volumes: + - 'secret' +{{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + - 'emptyDir' + - 'hostPath' + allowedHostPaths: + - pathPrefix: {{ required "Need access to volume on host with the SSL cert files to use HTTPs" .Values.clients.https.certDir }} + readOnly: true +{{- end }} +{{- end }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeControllerManager/templates/pushprox-clients.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeControllerManager/templates/pushprox-clients.yaml new file mode 100644 index 00000000000..e8fcfb38836 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeControllerManager/templates/pushprox-clients.yaml @@ -0,0 +1,157 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.clients }}{{- if .Values.clients.enabled }} +apiVersion: apps/v1 +{{- if .Values.clients.deployment.enabled }} +kind: Deployment +{{- else }} +kind: DaemonSet +{{- end }} +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} + pushprox-exporter: "client" +spec: + {{- if .Values.clients.deployment.enabled }} + replicas: {{ .Values.clients.deployment.replicas }} + {{- end }} + selector: + matchLabels: {{ include "pushProxy.client.labels" . | nindent 6 }} + template: + metadata: + labels: {{ include "pushProxy.client.labels" . | nindent 8 }} + spec: + {{- if .Values.clients.affinity }} + affinity: {{ toYaml .Values.clients.affinity | nindent 8 }} + {{- end }} + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} +{{- if .Values.clients.nodeSelector }} +{{ toYaml .Values.clients.nodeSelector | indent 8 }} +{{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} +{{- if .Values.clients.tolerations }} +{{ toYaml .Values.clients.tolerations | indent 8 }} +{{- end }} + hostNetwork: true + dnsPolicy: ClusterFirstWithHostNet + serviceAccountName: {{ template "pushProxy.client.name" . }} + {{- if .Values.global.imagePullSecretName }} + imagePullSecrets: + - name: {{ .Values.global.imagePullSecretName }} + {{- end }} + containers: + - name: pushprox-client + image: {{ template "system_default_registry" . }}{{ .Values.clients.image.repository }}:{{ .Values.clients.image.tag }} + command: + {{- range .Values.clients.command }} + - {{ . | quote }} + {{- end }} + args: + - --fqdn=$(HOST_IP) + - --proxy-url=$(PROXY_URL) + {{- if .Values.clients.metrics.enabled }} + - --metrics-addr=$(PORT) + {{- end }} + - --allow-port={{ required "Need .Values.metricsPort to configure client to be allowed to scrape metrics at port" .Values.metricsPort}} + {{- if .Values.clients.useLocalhost }} + - --use-localhost + {{- end }} + {{- if .Values.clients.https.enabled }} + {{- if .Values.clients.https.insecureSkipVerify }} + - --insecure-skip-verify + {{- end }} + {{- if .Values.clients.https.useServiceAccountCredentials }} + - --token-path=/var/run/secrets/kubernetes.io/serviceaccount/token + {{- end }} + {{- if .Values.clients.https.certDir }} + - --tls.cert=/etc/ssl/push-proxy/push-proxy.pem + - --tls.key=/etc/ssl/push-proxy/push-proxy-key.pem + - --tls.cacert=/etc/ssl/push-proxy/push-proxy-ca-cert.pem + {{- end }} + {{- end }} + env: + - name: HOST_IP + valueFrom: + fieldRef: + fieldPath: status.hostIP + {{- if .Values.clients.metrics.enabled }} + - name: PORT + value: :{{ .Values.clients.port }} + {{- end }} + - name: PROXY_URL + value: {{ template "pushProxy.proxyUrl" . }} + securityContext: + runAsNonRoot: true + runAsUser: 1000 + {{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + volumeMounts: + - name: metrics-cert-dir + mountPath: /etc/ssl/push-proxy + {{- end }} + {{- if .Values.clients.resources }} + resources: {{ toYaml .Values.clients.resources | nindent 10 }} + {{- end }} + {{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + initContainers: + - name: copy-certs + image: {{ template "system_default_registry" . }}{{ .Values.clients.copyCertsImage.repository }}:{{ .Values.clients.copyCertsImage.tag }} + command: + - sh + - -c + - | + echo "Searching for files to copy within the source volume" + echo "cert: ${CERT_FILE_NAME}" + echo "key: ${KEY_FILE_NAME}" + echo "cacert: ${CACERT_FILE_NAME}" + + CERT_FILE_SOURCE=$(find /etc/source/ -type f -name "${CERT_FILE_NAME}" | sort -r | head -n 1) + KEY_FILE_SOURCE=$(find /etc/source/ -type f -name "${KEY_FILE_NAME}" | sort -r | head -n 1) + CACERT_FILE_SOURCE=$(find /etc/source/ -type f -name "${CACERT_FILE_NAME}" | sort -r | head -n 1) + + test -z ${CERT_FILE_SOURCE} && echo "Failed to find cert file" && exit 1 + test -z ${KEY_FILE_SOURCE} && echo "Failed to find key file" && exit 1 + test -z ${CACERT_FILE_SOURCE} && echo "Failed to find cacert file" && exit 1 + + echo "Copying cert file from $CERT_FILE_SOURCE to $CERT_FILE_TARGET" + cp $CERT_FILE_SOURCE $CERT_FILE_TARGET || exit 1 + chmod 444 $CERT_FILE_TARGET || exit 1 + + echo "Copying key file from $KEY_FILE_SOURCE to $KEY_FILE_TARGET" + cp $KEY_FILE_SOURCE $KEY_FILE_TARGET || exit 1 + chmod 444 $KEY_FILE_TARGET || exit 1 + + echo "Copying cacert file from $CACERT_FILE_SOURCE to $CACERT_FILE_TARGET" + cp $CACERT_FILE_SOURCE $CACERT_FILE_TARGET || exit 1 + chmod 444 $CACERT_FILE_TARGET || exit 1 + env: + - name: CERT_FILE_NAME + value: {{ required "Need a TLS cert file for scraping metrics endpoint over HTTPs" .Values.clients.https.certFile }} + - name: KEY_FILE_NAME + value: {{ required "Need a TLS key file for scraping metrics endpoint over HTTPs" .Values.clients.https.keyFile }} + - name: CACERT_FILE_NAME + value: {{ required "Need a TLS CA cert file for scraping metrics endpoint over HTTPs" .Values.clients.https.caCertFile }} + - name: CERT_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy.pem + - name: KEY_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy-key.pem + - name: CACERT_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy-ca-cert.pem + securityContext: + runAsNonRoot: false +{{- if and .Values.global.seLinux.enabled .Values.clients.https.seLinuxOptions }} + seLinuxOptions: {{ .Values.clients.https.seLinuxOptions | toYaml | nindent 12 }} +{{- end }} + volumeMounts: + - name: metrics-cert-dir-source + mountPath: /etc/source + readOnly: true + - name: metrics-cert-dir + mountPath: /etc/ssl/push-proxy + volumes: + - name: metrics-cert-dir-source + hostPath: + path: {{ required "Need access to volume on host with the SSL cert files to use HTTPs" .Values.clients.https.certDir }} + - name: metrics-cert-dir + emptyDir: {} + {{- end }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeControllerManager/templates/pushprox-proxy-rbac.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeControllerManager/templates/pushprox-proxy-rbac.yaml new file mode 100644 index 00000000000..eefe609058c --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeControllerManager/templates/pushprox-proxy-rbac.yaml @@ -0,0 +1,68 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if and .Values.proxy }}{{ if .Values.proxy.enabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ template "pushProxy.proxy.name" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +rules: +{{- if .Values.global.cattle.psp.enabled }} +- apiGroups: ['policy'] + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - {{ template "pushProxy.proxy.name" . }} +{{- end }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ template "pushProxy.proxy.name" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "pushProxy.proxy.name" . }} +subjects: + - kind: ServiceAccount + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +--- +{{- if .Values.global.cattle.psp.enabled }} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +spec: + privileged: false + hostNetwork: false + hostIPC: false + hostPID: false + runAsUser: + rule: 'MustRunAsNonRoot' + seLinux: + rule: 'RunAsAny' + supplementalGroups: + rule: 'MustRunAs' + ranges: + - min: 1 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + - min: 1 + max: 65535 + readOnlyRootFilesystem: false + volumes: + - 'secret' +{{- end }}{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeControllerManager/templates/pushprox-proxy.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeControllerManager/templates/pushprox-proxy.yaml new file mode 100644 index 00000000000..723bbd6c00b --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeControllerManager/templates/pushprox-proxy.yaml @@ -0,0 +1,57 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if and .Values.proxy }}{{ if .Values.proxy.enabled }} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} + pushprox-exporter: "proxy" +spec: + selector: + matchLabels: {{ include "pushProxy.proxy.labels" . | nindent 6 }} + template: + metadata: + labels: {{ include "pushProxy.proxy.labels" . | nindent 8 }} + spec: + securityContext: + runAsNonRoot: true + runAsUser: 1000 + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} +{{- if .Values.proxy.nodeSelector }} +{{ toYaml .Values.proxy.nodeSelector | indent 8 }} +{{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} +{{- if .Values.proxy.tolerations }} +{{ toYaml .Values.proxy.tolerations | indent 8 }} +{{- end }} + serviceAccountName: {{ template "pushProxy.proxy.name" . }} + {{- if .Values.global.imagePullSecretName }} + imagePullSecrets: + - name: {{ .Values.global.imagePullSecretName }} + {{- end }} + containers: + - name: pushprox-proxy + image: {{ template "system_default_registry" . }}{{ .Values.proxy.image.repository }}:{{ .Values.proxy.image.tag }} + command: + {{- range .Values.proxy.command }} + - {{ . | quote }} + {{- end }} + {{- if .Values.proxy.resources }} + resources: {{ toYaml .Values.proxy.resources | nindent 10 }} + {{- end }} +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +spec: + ports: + - name: pp-proxy + port: {{ required "Need .Values.proxy.port to configure proxy" .Values.proxy.port }} + protocol: TCP + targetPort: {{ .Values.proxy.port }} + selector: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeControllerManager/templates/pushprox-servicemonitor.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeControllerManager/templates/pushprox-servicemonitor.yaml new file mode 100644 index 00000000000..67eb2216b5a --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeControllerManager/templates/pushprox-servicemonitor.yaml @@ -0,0 +1,45 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.serviceMonitor }}{{- if .Values.serviceMonitor.enabled }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ template "pushprox.serviceMonitor.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.serviceMonitor.labels" . | nindent 4 }} +spec: + endpoints: {{include "pushProxy.serviceMonitor.endpoints" . | nindent 4 }} + jobLabel: component + podTargetLabels: + - component + - pushprox-exporter + namespaceSelector: + matchNames: + - {{ template "pushprox.namespace" . }} + selector: + matchLabels: {{ include "pushProxy.client.labels" . | nindent 6 }} +--- +{{- $selector := "" }} +{{- if not (kindIs "invalid" .Values.service) }} +{{- if not (kindIs "invalid" .Values.service.selector) }} +{{ if .Values.service.selector }} +{{- if .Values.clients.enabled }} +{{- required (printf "Cannot override .Values.service.selector=%s when .Values.clients.enabled=true" (toJson .Values.service.selector)) "" }} +{{- end }} +{{- $selector = (toYaml .Values.service.selector) }} +{{- end }} +{{- end }} +{{- end }} +apiVersion: v1 +kind: Service +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +spec: + ports: + - name: metrics + port: {{ required "Need .Values.metricsPort to configure client to listen to metrics at port" .Values.metricsPort}} + protocol: TCP + targetPort: {{ .Values.metricsPort }} + selector: {{ default (include "pushProxy.client.labels" .) $selector | nindent 4 }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeControllerManager/templates/validate-install-crd.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeControllerManager/templates/validate-install-crd.yaml new file mode 100644 index 00000000000..16abc2fa83a --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeControllerManager/templates/validate-install-crd.yaml @@ -0,0 +1,14 @@ +#{{- if gt (len (lookup "rbac.authorization.k8s.io/v1" "ClusterRole" "" "")) 0 -}} +# {{- $found := dict -}} +# {{- set $found "monitoring.coreos.com/v1/ServiceMonitor" false -}} +# {{- range .Capabilities.APIVersions -}} +# {{- if hasKey $found (toString .) -}} +# {{- set $found (toString .) true -}} +# {{- end -}} +# {{- end -}} +# {{- range $_, $exists := $found -}} +# {{- if (eq $exists false) -}} +# {{- required "Required CRDs are missing. Please install Prometheus Operator CRDs before installing this chart." "" -}} +# {{- end -}} +# {{- end -}} +#{{- end -}} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeControllerManager/templates/validate-psp-install.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeControllerManager/templates/validate-psp-install.yaml new file mode 100644 index 00000000000..a30c59d3b7c --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeControllerManager/templates/validate-psp-install.yaml @@ -0,0 +1,7 @@ +#{{- if gt (len (lookup "rbac.authorization.k8s.io/v1" "ClusterRole" "" "")) 0 -}} +#{{- if .Values.global.cattle.psp.enabled }} +#{{- if not (.Capabilities.APIVersions.Has "policy/v1beta1/PodSecurityPolicy") }} +#{{- fail "The target cluster does not have the PodSecurityPolicy API resource. Please disable PSPs in this chart before proceeding." -}} +#{{- end }} +#{{- end }} +#{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeControllerManager/values.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeControllerManager/values.yaml new file mode 100644 index 00000000000..7f38498a1c5 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeControllerManager/values.yaml @@ -0,0 +1,166 @@ +# Default values for rancher-pushprox. +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. + +# Default image containing both the proxy and the client was generated from the following Dockerfile +# https://github.com/prometheus-community/PushProx/blob/eeadbe766641699129920ccfaaaa30a85c67fe81/Dockerfile#L1-L15 + +# Configuration + +global: + cattle: + psp: + enabled: false + systemDefaultRegistry: "" + seLinux: + enabled: false + +# A list of Semver constraint strings (defined by https://github.com/Masterminds/semver) and values.yaml overrides. +# +# For each key in kubeVersionOverrides, this chart will check to see if the current Kubernetes cluster's version matches +# any of the semver constraints provided as keys on the map. +# +# On seeing a match, the default value for each values.yaml field overridden will be updated with the new value. +# +# If multiple matches are encountered (due to overlapping semver ranges), the matches will be applied in order. +# +# Notes: +# - On running a helm template, Helm generally assumes the kubeVersion is v1.20.0 +# - On running a helm install --dry-run, the correct kubeVersion should be chosen. +kubeVersionOverrides: [] +# - constraint: "< 1.21" +# values: +# metricsPort: 10252 +# clients: +# https: +# enabled: false +# insecureSkipVerify: false +# useServiceAccountCredentials: false + +namespaceOverride: "" + +# The component that is being monitored (i.e. etcd) +component: "component" + +# The port containing the metrics that need to be scraped +metricsPort: 2739 + +# Configure ServiceMonitor that monitors metrics from the metricsPort endpoint +serviceMonitor: + enabled: true + # A list of endpoints that will be added to the ServiceMonitor based on the Endpoint spec + # Source: https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#endpoint + # By default, proxyUrl and params._scheme will be overridden based on other values + endpoints: + - port: metrics + +# Configure Service that grabs scrape targets +service: + # The selector that is used to populate the Service's Endpoints object. + # The chart will error out on rendering templating if .Values.clients.enabled is set alongside this field, + # since it is expected that this service should point to the PushProx Clients Daemonset / Deployment + selector: {} + +clients: + enabled: true + # The port which the PushProx client will post PushProx metrics to + port: 9369 + # If unset, this will default to the URL for the proxy service: http://pushprox-{{component}}-proxy.{{namepsace}}.svc.cluster.local:{{proxy.port}} + # Should be modified if the clients are being deployed outside the cluster where the proxy rests, otherwise leave it null + proxyUrl: "" + # If set to true, the client will forward any requests from the host IP to 127.0.0.1 + # It will only allow proxy requests to the metricsPort specified + useLocalhost: false + # Configuration for accessing metrics via HTTPS + https: + # Does the client require https to access the metrics? + enabled: false + # Does the client require requests be sent to http or https? + forceHTTPSScheme: false + # If set to true, the client will create a service account with adequate permissions and set a flag + # on the client to use the service account token provided by it to make authorized scrape requests + useServiceAccountCredentials: false + # Configuration for authentication to metrics via https endpoint + authenticationMethod: + # Reads token from defined file in container + # This function is deprecated in the prometheus operator api and may be removed in a future version + bearerTokenFile: + enabled: false + bearerTokenFilePath: "/var/run/secrets/kubernetes.io/serviceaccount/token" + # Reads token from defined secret in namespace + # This function is deprecated in the prometheus operator api and may be removed in a future version + bearerTokenSecret: + enabled: false + # Reads token from defined secret in namespace + authorization: + enabled: false + type: "bearer" + credentials: + key: "token" + optional: false + # If set to true, the client will disable SSL security checks + insecureSkipVerify: false + # Directory on host where necessary TLS cert and key to scrape metrics can be found + certDir: "" + # Filenames for files located in .Values.clients.https.certDir that correspond to TLS settings + certFile: "" + keyFile: "" + caCertFile: "" + # seLinuxOptions to be passed into the container that copies certs. Should define a container with permissions to read the files in the certDir provided on the host. + # Required and only used if `clients.https.enabled` is set and `clients.https.certDir` is provided. + seLinuxOptions: {} + + metrics: + # Whether the client should publish PushProx client-specific metrics to .Values.clients.port + enabled: false + + rbac: + # Additional permissions to provide to the ServiceAccount bound to the client + # This can be used to provide additional permissions for the client to scrape metrics from the k8s API + # Only enabled if clients.https.enabled and clients.https.useServiceAccountCredentials are true + additionalRules: [] + + # Resource limits + resources: {} + + # Options to select all nodes to deploy client DaemonSet on + nodeSelector: {} + tolerations: [] + affinity: {} + + image: + repository: rancher/pushprox-client + tag: v0.1.4-rancher2-client + command: ["pushprox-client"] + + copyCertsImage: + repository: rancher/mirrored-library-busybox + tag: 1.31.1 + + # The default intention of rancher-pushprox clients is to scrape hostNetwork metrics across all nodes. + # This can be used to scrape internal Kubernetes components or DaemonSets of hostNetwork Pods in + # situations where a cloud provider firewall prevents Pod-To-Host communication but not Pod-To-Pod. + # However, if the underlying hostNetwork Pod that is being scraped is managed by a Deployment, + # this advanced option enables users to deploy the client as a Deployment instead of a DaemonSet. + # If a user deploys this feature and the underlying Deployment's number of replicas changes, the user will + # be responsible for upgrading this chart accordingly to the right number of replicas. + deployment: + enabled: false + replicas: 0 + +proxy: + enabled: true + # The port through which PushProx clients will communicate to the proxy + port: 8080 + + # Resource limits + resources: {} + + # Options to select a node to run a single proxy deployment on + nodeSelector: {} + tolerations: [] + + image: + repository: rancher/pushprox-proxy + tag: v0.1.4-rancher2-proxy + command: ["pushprox-proxy"] diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeEtcd/.helmignore b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeEtcd/.helmignore new file mode 100644 index 00000000000..0e8a0eb36f4 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeEtcd/.helmignore @@ -0,0 +1,23 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*.orig +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeEtcd/Chart.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeEtcd/Chart.yaml new file mode 100644 index 00000000000..7f08ee0ba72 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeEtcd/Chart.yaml @@ -0,0 +1,15 @@ +annotations: + catalog.cattle.io/hidden: "true" + catalog.cattle.io/kube-version: '>= 1.26.0-0 < 1.31.0-0' + catalog.cattle.io/os: linux + catalog.rancher.io/certified: rancher + catalog.rancher.io/namespace: cattle-monitoring-system + catalog.rancher.io/release-name: rancher-pushprox +apiVersion: v1 +appVersion: v0.1.4-rancher2 +description: Sets up a deployment of the PushProx proxy and a DaemonSet of PushProx + clients. +kubeVersion: '>=1.26.0-0' +name: rkeEtcd +type: application +version: 0.2.1 diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeEtcd/README.md b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeEtcd/README.md new file mode 100644 index 00000000000..345002f48a2 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeEtcd/README.md @@ -0,0 +1,90 @@ +# rancher-pushprox + +A Rancher chart based on Rancher [PushProx](https://github.com/rancher/PushProx) that sets up a Deployment of a PushProx proxy and a DaemonSet of PushProx clients on a Kubernetes cluster. + +Installs [rancher-pushprox](https://github.com/rancher/charts/tree/gh-pages/packages/rancher-pushprox) to create PushProx clients that can access their host's network and register with a PushProx proxy. A [Prometheus Operator](https://github.com/coreos/prometheus-operator) ServiceMonitor CR is also included that is configured to scrape the metrics from each of the clients through the proxy. + +Using an instance of this chart is suitable for the following scenarios: +- You need to scrape metrics from a port that should not be accessible outside of the host (e.g. scraping `etcd` metrics in a hardened cluster) +- You need to scrape metrics on a host that are not exposed outside of 127.0.0.1 (e.g. scraping `kube-proxy` metrics) +- You need to scrape metrics through HTTPS using certs hosted directly on `hostPath` +- You need to scrape metrics from Kubernetes components that require authorization via a service account (e.g. permissions to make request to `/metrics`) +- You need to scrape metrics without access to cacerts (i.e. enable `insecureSkipVerify`) + +The clients and proxy are created based on a Rancher fork of the [prometheus-community/PushProx](https://github.com/prometheus-community/PushProx) project. + +## Upgrading to Kubernetes v1.25+ + +Starting in Kubernetes v1.25, [Pod Security Policies](https://kubernetes.io/docs/concepts/security/pod-security-policy/) have been removed from the Kubernetes API. + +As a result, **before upgrading to Kubernetes v1.25** (or on a fresh install in a Kubernetes v1.25+ cluster), users are expected to perform an in-place upgrade of this chart with `global.cattle.psp.enabled` set to `false` if it has been previously set to `true`. +​ +> **Note:** +> In this chart release, any previous field that was associated with any PSP resources have been removed in favor of a single global field: `global.cattle.psp.enabled`. + +> **Note:** +> If you upgrade your cluster to Kubernetes v1.25+ before removing PSPs via a `helm upgrade` (even if you manually clean up resources), **it will leave the Helm release in a broken state within the cluster such that further Helm operations will not work (`helm uninstall`, `helm upgrade`, etc.).** +> +> If your charts get stuck in this state, please consult the Rancher docs on how to clean up your Helm release secrets. + +Upon setting `global.cattle.psp.enabled` to false, the chart will remove any PSP resources deployed on its behalf from the cluster. This is the default setting for this chart. + +As a replacement for PSPs, [Pod Security Admission](https://kubernetes.io/docs/concepts/security/pod-security-admission/) should be used. Please consult the Rancher docs for more details on how to configure your chart release namespaces to work with the new Pod Security Admission and apply Pod Security Standards. + +## Configuration + +The following tables list the configurable parameters of the rancher-pushprox chart and their default values. + +### General + +#### Required +| Parameter | Description | Example | +| ----- | ----------- | ------ | +| `component` | The component that is being monitored | `kube-etcd` +| `metricsPort` | The port on the host that contains the metrics you want to scrape (e.g. `http://:/metrics`) | `2379` | +| `namespaceOverride` | The namespace to install the chart | `""` + +#### Optional +| Parameter | Description | Default | +| ----- | ----------- | ------ | +| `serviceMonitor.enabled` | Deploys a [Prometheus Operator](https://github.com/coreos/prometheus-operator/blob/master/Documentation/api.md#servicemonitor) ServiceMonitor CR that is configured to scrape metrics on the hosts that the clients are deployed on via the proxy. Also deploys a Service that points to all pods with the expected client name that exposes the `metricsPort` selected | `true` | +| `serviceMonitor.endpoints` | A list of endpoints that will be added to the ServiceMonitor based on the [Endpoint spec](https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#endpoint) | `[{port: metrics}]` | +| `service.selector` | The selector that is used to populate the Service's Endpoints object. The chart will error out on rendering templating if `.Values.clients.enabled` is set alongside this field, since it is expected that this service should point to the PushProx Clients Daemonset / Deployment | `{}` | +| `clients.enabled` | Deploys a DaemonSet of clients that are each capable of scraping endpoints on the hostNetwork it is deployed on | `true` | +| `clients.port` | The port where the client will publish PushProx client-specific metrics. If deploying multiple clients onto the same node, the clients should not have conflicting ports | `9369` | +| `clients.proxyUrl` | Overrides the default proxyUrl setting of `http://pushprox-{{ .Values.component }}-proxy.{{ . Release.Namespace }}.svc.cluster.local:{{ .Values.proxy.port }}"` with the `proxyUrl` specified | `""` | +| `clients.useLocalhost` | Sets a flag on each client deployment to redirect scrapes directed to `HOST_IP` to `127.0.0.1` | `false` | +| `clients.https.enabled` | Enables scraping metrics via HTTPS using the provided TLS certs that exist on each host | `false` | +| `clients.https.forceHTTPSScheme` | Forces scraping metrics via HTTPS using the provided TLS certs that exist on each host | `false` | +| `clients.https.useServiceAccountCredentials` | If set to true, the client will create a service account with permissions to scrape `/metrics` endpoint of Kubernetes components. The client will use the service account token provided to make authorized scrape requests to the Kubernetes API | `false` | +| `clients.https.authenticationMethod.bearerTokenFile.enabled` | If set to true, the client will use service account credentials mounted at the configured path `clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath`. This requires permissions to scrape `/metrics` endpoint of Kubernetes components. This method is deprecated by the prometheus operator and may be removed in a future release | `false` | +| `clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath` | This is a volume mount on the pod with permissions to scrape `/metrics` endpoint of Kubernetes components | `"/var/run/secrets/kubernetes.io/serviceaccount/token"` | +| `clients.https.authenticationMethod.bearerTokenSecret.enabled` | If set to true, the client will use service account credentials to scrape `/metrics` endpoint of Kubernetes components. This method is deprecated by the prometheus operator and may be removed in a future release | `false` | +| `clients.https.authenticationMethod.authorization.enabled` | If set to true, the client will use service account credentials to scrape `/metrics` endpoint of Kubernetes components | `false` | +| `clients.https.authenticationMethod.authorization.type` | If set, the client will use this type of authorization in its client requests for metrics | `"bearer"` | +| `clients.https.authenticationMethod.authorization.credentials.key` | If set, the client will use this key in the secret created by `clients.https.useServiceAccountCredentials` for authorization in its client requests for metrics | `"token"` | +| `clients.https.authenticationMethod.authorization.credentials.optional` | If set to false, the client will fail if the key in the secret created by `clients.https.useServiceAccountCredentials` does not exist | `false` | +| `clients.https.insecureSkipVerify` | If set to true, the client will disable SSL security checks | `false` | +| `clients.https.certDir` | A `hostPath` where TLS certs can be found. This path is mounted as a volume on an `initContainer` which copies only the necessary files over to an EmptyDir volume used by each client. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.certFile` | The path to the TLS cert file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.keyFile` | The path to the TLS key file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.caCertFile` | The path to the TLS cacert file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.seLinuxOptions` | seLinuxOptions to be passed into the container that copies certs. Should define a container with permissions to read the files in the certDir provided on the host. Required and only used if `clients.https.enabled` is set and `clients.https.certDir` is provided. | `""` | +| `clients.metrics.enabled` | Whether the client should publish PushProx client-specific metrics. | `false` | +| `clients.rbac.additionalRules` | Additional permissions to provide to the ServiceAccount bound to the client. This can be used to provide additional permissions for the client to scrape metrics from the k8s API. Only enabled if clients.https.enabled and clients.https.useServiceAccountCredentials are true | `[]` | +| `clients.deployment.enabled` | Deploys the client as a Deployment (generally used if the underlying hostNetwork Pod that is being scraped is managed by a Deployment) | `false` | +| `clients.deployment.replicas` | The number of pods the Deployment has, it should match the number of pod the hostNetwork Deployment has. Required and only used if `client.deployment.enable` is set | `0` | +| `clients.deployment.affinity` | The affinity rules that allocate the pod to the node in which the hostNetwork Deployment's pods run. Required and only used if `client.deployment.enable` is set | `{}` | +| `clients.resources` | Set resource limits and requests for the client container | `{}` | +| `clients.nodeSelector` | Select which nodes to deploy the clients on | `{}` | +| `clients.tolerations` | Specify tolerations for clients | `[]` | +| `proxy.enabled` | Deploys the proxy that each client will register with | `true` | +| `proxy.port` | The port exposed by the proxy that each client will register with to allow metrics to be scraped from the host | `8080` | +| `proxy.resources` | Set resource limits and requests for the proxy container | `{}` | +| `proxy.nodeSelector` | Select which nodes the proxy can be deployed on | `{}` | +| `proxy.tolerations` | Specify tolerations (if necessary) to allow the proxy to be deployed on the selected node | `[]` | +| `kubeVersionOverrides` | A list of Semver constraint strings (defined by https://github.com/Masterminds/semver) and values.yaml overrides. For each key in kubeVersionOverrides, this chart will check to see if the current Kubernetes cluster's version matches any of the semver constraints provided as keys on the map. On seeing a match, the default value for each values.yaml field overridden will be updated with the new value. If multiple matches are encountered (due to overlapping semver ranges), the matches will be applied in order. | `[]` + +*Tip: The filepaths set in `clients.https.File` can include wildcard characters*. + +See [rancher-monitoring](https://github.com/rancher/charts/tree/gh-pages/packages/rancher-monitoring) for examples of how this chart can be used. diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeEtcd/templates/_helpers.tpl b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeEtcd/templates/_helpers.tpl new file mode 100644 index 00000000000..1ba50939441 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeEtcd/templates/_helpers.tpl @@ -0,0 +1,170 @@ +# Rancher + +{{- define "system_default_registry" -}} +{{- if .Values.global.cattle.systemDefaultRegistry -}} +{{- printf "%s/" .Values.global.cattle.systemDefaultRegistry -}} +{{- end -}} +{{- end -}} + +# Windows Support + +{{/* +Windows cluster will add default taint for linux nodes, +add below linux tolerations to workloads could be scheduled to those linux nodes +*/}} + +{{- define "linux-node-tolerations" -}} +- key: "cattle.io/os" + value: "linux" + effect: "NoSchedule" + operator: "Equal" +{{- end -}} + +{{- define "linux-node-selector" -}} +{{- if semverCompare "<1.14-0" .Capabilities.KubeVersion.GitVersion -}} +beta.kubernetes.io/os: linux +{{- else -}} +kubernetes.io/os: linux +{{- end -}} +{{- end -}} + +# General + +{{- define "applyKubeVersionOverrides" -}} +{{- $overrides := dict -}} +{{- range $override := .Values.kubeVersionOverrides -}} +{{- if semverCompare $override.constraint $.Capabilities.KubeVersion.Version -}} +{{- $_ := mergeOverwrite $overrides $override.values -}} +{{- end -}} +{{- end -}} +{{- $_ := mergeOverwrite .Values $overrides -}} +{{- end -}} + +{{- define "pushprox.namespace" -}} + {{- if .Values.namespaceOverride -}} + {{- .Values.namespaceOverride -}} + {{- else -}} + {{- .Release.Namespace -}} + {{- end -}} +{{- end -}} + +{{- define "pushProxy.commonLabels" -}} +release: {{ .Release.Name }} +component: {{ .Values.component | quote }} +provider: kubernetes +{{- end -}} + +{{- define "pushProxy.proxyUrl" -}} +{{- $_ := (required "Template requires either .Values.proxy.port or .Values.client.proxyUrl to set proxyUrl for client" (or .Values.clients.proxyUrl .Values.proxy.port)) -}} +{{- if .Values.clients.proxyUrl -}} +{{ printf "%s" .Values.clients.proxyUrl }} +{{- else -}} +{{ printf "http://%s.%s.svc:%d" (include "pushProxy.proxy.name" .) (include "pushprox.namespace" .) (int .Values.proxy.port) }} +{{- end -}}{{- end -}} + +# Client + +{{- define "pushProxy.client.name" -}} +{{- printf "pushprox-%s-client" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.client.serviceAccountTokenName" -}} +{{- printf "pushprox-%s-client-service-account-token" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.client.labels" -}} +k8s-app: {{ template "pushProxy.client.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +# Proxy + +{{- define "pushProxy.proxy.name" -}} +{{- printf "pushprox-%s-proxy" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.proxy.labels" -}} +k8s-app: {{ template "pushProxy.proxy.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +# ServiceMonitor + +{{- define "pushprox.serviceMonitor.name" -}} +{{- printf "%s-%s" .Release.Name (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.serviceMonitor.labels" -}} +app: {{ template "pushprox.serviceMonitor.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +{{- define "pushProxy.serviceMonitor.endpoints" -}} +{{- $proxyURL := (include "pushProxy.proxyUrl" .) -}} +{{- $useHTTPS := .Values.clients.https.enabled -}} +{{- $setHTTPSScheme := .Values.clients.https.forceHTTPSScheme -}} +{{- $insecureSkipVerify := .Values.clients.https.insecureSkipVerify -}} +{{- $useServiceAccountCredentials := .Values.clients.https.useServiceAccountCredentials -}} +{{- $serviceAccountTokenName := (include "pushProxy.client.serviceAccountTokenName" . ) -}} +{{- $metricRelabelings := list }} +{{- $endpoints := .Values.serviceMonitor.endpoints }} +{{- if .Values.proxy.enabled }} +{{- $_ := set . "proxyUrl" $proxyURL }} +{{- end }} +{{- range $endpoints }} +{{- if $.Values.proxy.enabled }} +{{- $_ := set . "proxyUrl" $proxyURL }} +{{- end }} +{{- $clusterIdRelabel := dict }} +{{- $metricRelabelings := list }} +{{- if $.Values.global.cattle.clusterId }} +{{- $_ := set $clusterIdRelabel "action" "replace" }} +{{- $_ := set $clusterIdRelabel "sourceLabels" (list "__address__") }} +{{- $_ := set $clusterIdRelabel "targetLabel" "cluster_id" }} +{{- $_ := set $clusterIdRelabel "replacement" $.Values.global.cattle.clusterId }} +{{- $metricRelabelings = append $metricRelabelings $clusterIdRelabel }} +{{- end }} +{{- $clusterNameRelabel := dict }} +{{- if $.Values.global.cattle.clusterName }} +{{- $_ := set $clusterNameRelabel "action" "replace" }} +{{- $_ := set $clusterNameRelabel "sourceLabels" (list "__address__") }} +{{- $_ := set $clusterNameRelabel "targetLabel" "cluster_name" }} +{{- $_ := set $clusterNameRelabel "replacement" $.Values.global.cattle.clusterName }} +{{- $metricRelabelings = append $metricRelabelings $clusterNameRelabel }} +{{- end }} +{{- if not (empty $metricRelabelings) }} +{{- $_ := set . "metricRelabelings" ($metricRelabelings)}} +{{- end }} +{{- if $setHTTPSScheme -}} +{{- $_ := set . "scheme" "https" }} +{{- end -}} +{{- if $useHTTPS -}} +{{- if (hasKey . "params") }} +{{- $_ := set (get . "params") "_scheme" (list "https") }} +{{- else }} +{{- $_ := set . "params" (dict "_scheme" (list "https")) }} +{{- end }} +{{- end }} +{{- if (hasKey . "tlsConfig") }} +{{- $_ := set (get . "tlsConfig") "insecureSkipVerify" $insecureSkipVerify }} +{{- else }} +{{- $_ := set . "tlsConfig" (dict "insecureSkipVerify" $insecureSkipVerify) }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.bearerTokenFile.enabled }} +{{- $_ := set . "bearerTokenFile" $.Values.clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.bearerTokenSecret.enabled }} +{{- $_ := set . "bearerTokenSecret" $serviceAccountTokenName }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.authorization.enabled }} +{{- if (hasKey . "authorization") }} +{{- $_ := set (get . "authorization") "type" $.Values.clients.https.authenticationMethod.authorization.type }} +{{- $_ := set (get . "authorization") "credentials" (dict "name" $serviceAccountTokenName "key" $.Values.clients.https.authenticationMethod.authorization.credentials.key "optional" $.Values.clients.https.authenticationMethod.authorization.credentials.optional) }} +{{- else }} +{{- $_ := set . "authorization" (dict "type" $.Values.clients.https.authenticationMethod.authorization.type) }} +{{- $_ := set . "authorization" (dict "credentials" (dict "name" $serviceAccountTokenName "key" $.Values.clients.https.authenticationMethod.authorization.credentials.key "optional" $.Values.clients.https.authenticationMethod.authorization.credentials.optional)) }} +{{- end }} +{{- end }} +{{- end }} +{{- toYaml $endpoints }} +{{- end -}} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeEtcd/templates/pushprox-clients-rbac.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeEtcd/templates/pushprox-clients-rbac.yaml new file mode 100644 index 00000000000..a8e27c37356 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeEtcd/templates/pushprox-clients-rbac.yaml @@ -0,0 +1,97 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.clients }}{{- if .Values.clients.enabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ template "pushProxy.client.name" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +rules: +{{- if .Values.global.cattle.psp.enabled }} +- apiGroups: ['policy'] + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - {{ template "pushProxy.client.name" . }} +{{- end }} +{{- if and .Values.clients.https.enabled .Values.clients.https.useServiceAccountCredentials }} +- nonResourceURLs: ["/metrics"] + verbs: ["get"] +{{- if .Values.clients.rbac.additionalRules }} +{{ toYaml .Values.clients.rbac.additionalRules }} +{{- end }} +{{- end }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ template "pushProxy.client.name" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "pushProxy.client.name" . }} +subjects: + - kind: ServiceAccount + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +--- +{{- if .Values.clients.https.useServiceAccountCredentials }} +apiVersion: v1 +kind: Secret +type: kubernetes.io/service-account-token +metadata: + name: {{ template "pushProxy.client.serviceAccountTokenName" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} + annotations: + kubernetes.io/service-account.name: {{ template "pushProxy.client.name" . }} +{{- end }} +--- +{{- if .Values.global.cattle.psp.enabled }} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +spec: + privileged: false + hostNetwork: true + hostIPC: false + hostPID: false + runAsUser: + rule: 'RunAsAny' + seLinux: + rule: 'RunAsAny' +{{- if and .Values.clients.https.enabled .Values.clients.https.certDir .Values.global.seLinux.enabled .Values.clients.https.seLinuxOptions }} + seLinuxOptions: {{ .Values.clients.https.seLinuxOptions | toYaml | nindent 6 }} +{{- end }} + supplementalGroups: + rule: 'MustRunAs' + ranges: + - min: 0 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + - min: 0 + max: 65535 + readOnlyRootFilesystem: false + volumes: + - 'secret' +{{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + - 'emptyDir' + - 'hostPath' + allowedHostPaths: + - pathPrefix: {{ required "Need access to volume on host with the SSL cert files to use HTTPs" .Values.clients.https.certDir }} + readOnly: true +{{- end }} +{{- end }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeEtcd/templates/pushprox-clients.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeEtcd/templates/pushprox-clients.yaml new file mode 100644 index 00000000000..e8fcfb38836 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeEtcd/templates/pushprox-clients.yaml @@ -0,0 +1,157 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.clients }}{{- if .Values.clients.enabled }} +apiVersion: apps/v1 +{{- if .Values.clients.deployment.enabled }} +kind: Deployment +{{- else }} +kind: DaemonSet +{{- end }} +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} + pushprox-exporter: "client" +spec: + {{- if .Values.clients.deployment.enabled }} + replicas: {{ .Values.clients.deployment.replicas }} + {{- end }} + selector: + matchLabels: {{ include "pushProxy.client.labels" . | nindent 6 }} + template: + metadata: + labels: {{ include "pushProxy.client.labels" . | nindent 8 }} + spec: + {{- if .Values.clients.affinity }} + affinity: {{ toYaml .Values.clients.affinity | nindent 8 }} + {{- end }} + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} +{{- if .Values.clients.nodeSelector }} +{{ toYaml .Values.clients.nodeSelector | indent 8 }} +{{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} +{{- if .Values.clients.tolerations }} +{{ toYaml .Values.clients.tolerations | indent 8 }} +{{- end }} + hostNetwork: true + dnsPolicy: ClusterFirstWithHostNet + serviceAccountName: {{ template "pushProxy.client.name" . }} + {{- if .Values.global.imagePullSecretName }} + imagePullSecrets: + - name: {{ .Values.global.imagePullSecretName }} + {{- end }} + containers: + - name: pushprox-client + image: {{ template "system_default_registry" . }}{{ .Values.clients.image.repository }}:{{ .Values.clients.image.tag }} + command: + {{- range .Values.clients.command }} + - {{ . | quote }} + {{- end }} + args: + - --fqdn=$(HOST_IP) + - --proxy-url=$(PROXY_URL) + {{- if .Values.clients.metrics.enabled }} + - --metrics-addr=$(PORT) + {{- end }} + - --allow-port={{ required "Need .Values.metricsPort to configure client to be allowed to scrape metrics at port" .Values.metricsPort}} + {{- if .Values.clients.useLocalhost }} + - --use-localhost + {{- end }} + {{- if .Values.clients.https.enabled }} + {{- if .Values.clients.https.insecureSkipVerify }} + - --insecure-skip-verify + {{- end }} + {{- if .Values.clients.https.useServiceAccountCredentials }} + - --token-path=/var/run/secrets/kubernetes.io/serviceaccount/token + {{- end }} + {{- if .Values.clients.https.certDir }} + - --tls.cert=/etc/ssl/push-proxy/push-proxy.pem + - --tls.key=/etc/ssl/push-proxy/push-proxy-key.pem + - --tls.cacert=/etc/ssl/push-proxy/push-proxy-ca-cert.pem + {{- end }} + {{- end }} + env: + - name: HOST_IP + valueFrom: + fieldRef: + fieldPath: status.hostIP + {{- if .Values.clients.metrics.enabled }} + - name: PORT + value: :{{ .Values.clients.port }} + {{- end }} + - name: PROXY_URL + value: {{ template "pushProxy.proxyUrl" . }} + securityContext: + runAsNonRoot: true + runAsUser: 1000 + {{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + volumeMounts: + - name: metrics-cert-dir + mountPath: /etc/ssl/push-proxy + {{- end }} + {{- if .Values.clients.resources }} + resources: {{ toYaml .Values.clients.resources | nindent 10 }} + {{- end }} + {{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + initContainers: + - name: copy-certs + image: {{ template "system_default_registry" . }}{{ .Values.clients.copyCertsImage.repository }}:{{ .Values.clients.copyCertsImage.tag }} + command: + - sh + - -c + - | + echo "Searching for files to copy within the source volume" + echo "cert: ${CERT_FILE_NAME}" + echo "key: ${KEY_FILE_NAME}" + echo "cacert: ${CACERT_FILE_NAME}" + + CERT_FILE_SOURCE=$(find /etc/source/ -type f -name "${CERT_FILE_NAME}" | sort -r | head -n 1) + KEY_FILE_SOURCE=$(find /etc/source/ -type f -name "${KEY_FILE_NAME}" | sort -r | head -n 1) + CACERT_FILE_SOURCE=$(find /etc/source/ -type f -name "${CACERT_FILE_NAME}" | sort -r | head -n 1) + + test -z ${CERT_FILE_SOURCE} && echo "Failed to find cert file" && exit 1 + test -z ${KEY_FILE_SOURCE} && echo "Failed to find key file" && exit 1 + test -z ${CACERT_FILE_SOURCE} && echo "Failed to find cacert file" && exit 1 + + echo "Copying cert file from $CERT_FILE_SOURCE to $CERT_FILE_TARGET" + cp $CERT_FILE_SOURCE $CERT_FILE_TARGET || exit 1 + chmod 444 $CERT_FILE_TARGET || exit 1 + + echo "Copying key file from $KEY_FILE_SOURCE to $KEY_FILE_TARGET" + cp $KEY_FILE_SOURCE $KEY_FILE_TARGET || exit 1 + chmod 444 $KEY_FILE_TARGET || exit 1 + + echo "Copying cacert file from $CACERT_FILE_SOURCE to $CACERT_FILE_TARGET" + cp $CACERT_FILE_SOURCE $CACERT_FILE_TARGET || exit 1 + chmod 444 $CACERT_FILE_TARGET || exit 1 + env: + - name: CERT_FILE_NAME + value: {{ required "Need a TLS cert file for scraping metrics endpoint over HTTPs" .Values.clients.https.certFile }} + - name: KEY_FILE_NAME + value: {{ required "Need a TLS key file for scraping metrics endpoint over HTTPs" .Values.clients.https.keyFile }} + - name: CACERT_FILE_NAME + value: {{ required "Need a TLS CA cert file for scraping metrics endpoint over HTTPs" .Values.clients.https.caCertFile }} + - name: CERT_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy.pem + - name: KEY_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy-key.pem + - name: CACERT_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy-ca-cert.pem + securityContext: + runAsNonRoot: false +{{- if and .Values.global.seLinux.enabled .Values.clients.https.seLinuxOptions }} + seLinuxOptions: {{ .Values.clients.https.seLinuxOptions | toYaml | nindent 12 }} +{{- end }} + volumeMounts: + - name: metrics-cert-dir-source + mountPath: /etc/source + readOnly: true + - name: metrics-cert-dir + mountPath: /etc/ssl/push-proxy + volumes: + - name: metrics-cert-dir-source + hostPath: + path: {{ required "Need access to volume on host with the SSL cert files to use HTTPs" .Values.clients.https.certDir }} + - name: metrics-cert-dir + emptyDir: {} + {{- end }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeEtcd/templates/pushprox-proxy-rbac.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeEtcd/templates/pushprox-proxy-rbac.yaml new file mode 100644 index 00000000000..eefe609058c --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeEtcd/templates/pushprox-proxy-rbac.yaml @@ -0,0 +1,68 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if and .Values.proxy }}{{ if .Values.proxy.enabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ template "pushProxy.proxy.name" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +rules: +{{- if .Values.global.cattle.psp.enabled }} +- apiGroups: ['policy'] + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - {{ template "pushProxy.proxy.name" . }} +{{- end }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ template "pushProxy.proxy.name" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "pushProxy.proxy.name" . }} +subjects: + - kind: ServiceAccount + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +--- +{{- if .Values.global.cattle.psp.enabled }} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +spec: + privileged: false + hostNetwork: false + hostIPC: false + hostPID: false + runAsUser: + rule: 'MustRunAsNonRoot' + seLinux: + rule: 'RunAsAny' + supplementalGroups: + rule: 'MustRunAs' + ranges: + - min: 1 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + - min: 1 + max: 65535 + readOnlyRootFilesystem: false + volumes: + - 'secret' +{{- end }}{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeEtcd/templates/pushprox-proxy.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeEtcd/templates/pushprox-proxy.yaml new file mode 100644 index 00000000000..723bbd6c00b --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeEtcd/templates/pushprox-proxy.yaml @@ -0,0 +1,57 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if and .Values.proxy }}{{ if .Values.proxy.enabled }} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} + pushprox-exporter: "proxy" +spec: + selector: + matchLabels: {{ include "pushProxy.proxy.labels" . | nindent 6 }} + template: + metadata: + labels: {{ include "pushProxy.proxy.labels" . | nindent 8 }} + spec: + securityContext: + runAsNonRoot: true + runAsUser: 1000 + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} +{{- if .Values.proxy.nodeSelector }} +{{ toYaml .Values.proxy.nodeSelector | indent 8 }} +{{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} +{{- if .Values.proxy.tolerations }} +{{ toYaml .Values.proxy.tolerations | indent 8 }} +{{- end }} + serviceAccountName: {{ template "pushProxy.proxy.name" . }} + {{- if .Values.global.imagePullSecretName }} + imagePullSecrets: + - name: {{ .Values.global.imagePullSecretName }} + {{- end }} + containers: + - name: pushprox-proxy + image: {{ template "system_default_registry" . }}{{ .Values.proxy.image.repository }}:{{ .Values.proxy.image.tag }} + command: + {{- range .Values.proxy.command }} + - {{ . | quote }} + {{- end }} + {{- if .Values.proxy.resources }} + resources: {{ toYaml .Values.proxy.resources | nindent 10 }} + {{- end }} +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +spec: + ports: + - name: pp-proxy + port: {{ required "Need .Values.proxy.port to configure proxy" .Values.proxy.port }} + protocol: TCP + targetPort: {{ .Values.proxy.port }} + selector: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeEtcd/templates/pushprox-servicemonitor.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeEtcd/templates/pushprox-servicemonitor.yaml new file mode 100644 index 00000000000..67eb2216b5a --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeEtcd/templates/pushprox-servicemonitor.yaml @@ -0,0 +1,45 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.serviceMonitor }}{{- if .Values.serviceMonitor.enabled }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ template "pushprox.serviceMonitor.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.serviceMonitor.labels" . | nindent 4 }} +spec: + endpoints: {{include "pushProxy.serviceMonitor.endpoints" . | nindent 4 }} + jobLabel: component + podTargetLabels: + - component + - pushprox-exporter + namespaceSelector: + matchNames: + - {{ template "pushprox.namespace" . }} + selector: + matchLabels: {{ include "pushProxy.client.labels" . | nindent 6 }} +--- +{{- $selector := "" }} +{{- if not (kindIs "invalid" .Values.service) }} +{{- if not (kindIs "invalid" .Values.service.selector) }} +{{ if .Values.service.selector }} +{{- if .Values.clients.enabled }} +{{- required (printf "Cannot override .Values.service.selector=%s when .Values.clients.enabled=true" (toJson .Values.service.selector)) "" }} +{{- end }} +{{- $selector = (toYaml .Values.service.selector) }} +{{- end }} +{{- end }} +{{- end }} +apiVersion: v1 +kind: Service +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +spec: + ports: + - name: metrics + port: {{ required "Need .Values.metricsPort to configure client to listen to metrics at port" .Values.metricsPort}} + protocol: TCP + targetPort: {{ .Values.metricsPort }} + selector: {{ default (include "pushProxy.client.labels" .) $selector | nindent 4 }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeEtcd/templates/validate-install-crd.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeEtcd/templates/validate-install-crd.yaml new file mode 100644 index 00000000000..16abc2fa83a --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeEtcd/templates/validate-install-crd.yaml @@ -0,0 +1,14 @@ +#{{- if gt (len (lookup "rbac.authorization.k8s.io/v1" "ClusterRole" "" "")) 0 -}} +# {{- $found := dict -}} +# {{- set $found "monitoring.coreos.com/v1/ServiceMonitor" false -}} +# {{- range .Capabilities.APIVersions -}} +# {{- if hasKey $found (toString .) -}} +# {{- set $found (toString .) true -}} +# {{- end -}} +# {{- end -}} +# {{- range $_, $exists := $found -}} +# {{- if (eq $exists false) -}} +# {{- required "Required CRDs are missing. Please install Prometheus Operator CRDs before installing this chart." "" -}} +# {{- end -}} +# {{- end -}} +#{{- end -}} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeEtcd/templates/validate-psp-install.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeEtcd/templates/validate-psp-install.yaml new file mode 100644 index 00000000000..a30c59d3b7c --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeEtcd/templates/validate-psp-install.yaml @@ -0,0 +1,7 @@ +#{{- if gt (len (lookup "rbac.authorization.k8s.io/v1" "ClusterRole" "" "")) 0 -}} +#{{- if .Values.global.cattle.psp.enabled }} +#{{- if not (.Capabilities.APIVersions.Has "policy/v1beta1/PodSecurityPolicy") }} +#{{- fail "The target cluster does not have the PodSecurityPolicy API resource. Please disable PSPs in this chart before proceeding." -}} +#{{- end }} +#{{- end }} +#{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeEtcd/values.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeEtcd/values.yaml new file mode 100644 index 00000000000..7f38498a1c5 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeEtcd/values.yaml @@ -0,0 +1,166 @@ +# Default values for rancher-pushprox. +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. + +# Default image containing both the proxy and the client was generated from the following Dockerfile +# https://github.com/prometheus-community/PushProx/blob/eeadbe766641699129920ccfaaaa30a85c67fe81/Dockerfile#L1-L15 + +# Configuration + +global: + cattle: + psp: + enabled: false + systemDefaultRegistry: "" + seLinux: + enabled: false + +# A list of Semver constraint strings (defined by https://github.com/Masterminds/semver) and values.yaml overrides. +# +# For each key in kubeVersionOverrides, this chart will check to see if the current Kubernetes cluster's version matches +# any of the semver constraints provided as keys on the map. +# +# On seeing a match, the default value for each values.yaml field overridden will be updated with the new value. +# +# If multiple matches are encountered (due to overlapping semver ranges), the matches will be applied in order. +# +# Notes: +# - On running a helm template, Helm generally assumes the kubeVersion is v1.20.0 +# - On running a helm install --dry-run, the correct kubeVersion should be chosen. +kubeVersionOverrides: [] +# - constraint: "< 1.21" +# values: +# metricsPort: 10252 +# clients: +# https: +# enabled: false +# insecureSkipVerify: false +# useServiceAccountCredentials: false + +namespaceOverride: "" + +# The component that is being monitored (i.e. etcd) +component: "component" + +# The port containing the metrics that need to be scraped +metricsPort: 2739 + +# Configure ServiceMonitor that monitors metrics from the metricsPort endpoint +serviceMonitor: + enabled: true + # A list of endpoints that will be added to the ServiceMonitor based on the Endpoint spec + # Source: https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#endpoint + # By default, proxyUrl and params._scheme will be overridden based on other values + endpoints: + - port: metrics + +# Configure Service that grabs scrape targets +service: + # The selector that is used to populate the Service's Endpoints object. + # The chart will error out on rendering templating if .Values.clients.enabled is set alongside this field, + # since it is expected that this service should point to the PushProx Clients Daemonset / Deployment + selector: {} + +clients: + enabled: true + # The port which the PushProx client will post PushProx metrics to + port: 9369 + # If unset, this will default to the URL for the proxy service: http://pushprox-{{component}}-proxy.{{namepsace}}.svc.cluster.local:{{proxy.port}} + # Should be modified if the clients are being deployed outside the cluster where the proxy rests, otherwise leave it null + proxyUrl: "" + # If set to true, the client will forward any requests from the host IP to 127.0.0.1 + # It will only allow proxy requests to the metricsPort specified + useLocalhost: false + # Configuration for accessing metrics via HTTPS + https: + # Does the client require https to access the metrics? + enabled: false + # Does the client require requests be sent to http or https? + forceHTTPSScheme: false + # If set to true, the client will create a service account with adequate permissions and set a flag + # on the client to use the service account token provided by it to make authorized scrape requests + useServiceAccountCredentials: false + # Configuration for authentication to metrics via https endpoint + authenticationMethod: + # Reads token from defined file in container + # This function is deprecated in the prometheus operator api and may be removed in a future version + bearerTokenFile: + enabled: false + bearerTokenFilePath: "/var/run/secrets/kubernetes.io/serviceaccount/token" + # Reads token from defined secret in namespace + # This function is deprecated in the prometheus operator api and may be removed in a future version + bearerTokenSecret: + enabled: false + # Reads token from defined secret in namespace + authorization: + enabled: false + type: "bearer" + credentials: + key: "token" + optional: false + # If set to true, the client will disable SSL security checks + insecureSkipVerify: false + # Directory on host where necessary TLS cert and key to scrape metrics can be found + certDir: "" + # Filenames for files located in .Values.clients.https.certDir that correspond to TLS settings + certFile: "" + keyFile: "" + caCertFile: "" + # seLinuxOptions to be passed into the container that copies certs. Should define a container with permissions to read the files in the certDir provided on the host. + # Required and only used if `clients.https.enabled` is set and `clients.https.certDir` is provided. + seLinuxOptions: {} + + metrics: + # Whether the client should publish PushProx client-specific metrics to .Values.clients.port + enabled: false + + rbac: + # Additional permissions to provide to the ServiceAccount bound to the client + # This can be used to provide additional permissions for the client to scrape metrics from the k8s API + # Only enabled if clients.https.enabled and clients.https.useServiceAccountCredentials are true + additionalRules: [] + + # Resource limits + resources: {} + + # Options to select all nodes to deploy client DaemonSet on + nodeSelector: {} + tolerations: [] + affinity: {} + + image: + repository: rancher/pushprox-client + tag: v0.1.4-rancher2-client + command: ["pushprox-client"] + + copyCertsImage: + repository: rancher/mirrored-library-busybox + tag: 1.31.1 + + # The default intention of rancher-pushprox clients is to scrape hostNetwork metrics across all nodes. + # This can be used to scrape internal Kubernetes components or DaemonSets of hostNetwork Pods in + # situations where a cloud provider firewall prevents Pod-To-Host communication but not Pod-To-Pod. + # However, if the underlying hostNetwork Pod that is being scraped is managed by a Deployment, + # this advanced option enables users to deploy the client as a Deployment instead of a DaemonSet. + # If a user deploys this feature and the underlying Deployment's number of replicas changes, the user will + # be responsible for upgrading this chart accordingly to the right number of replicas. + deployment: + enabled: false + replicas: 0 + +proxy: + enabled: true + # The port through which PushProx clients will communicate to the proxy + port: 8080 + + # Resource limits + resources: {} + + # Options to select a node to run a single proxy deployment on + nodeSelector: {} + tolerations: [] + + image: + repository: rancher/pushprox-proxy + tag: v0.1.4-rancher2-proxy + command: ["pushprox-proxy"] diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeIngressNginx/.helmignore b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeIngressNginx/.helmignore new file mode 100644 index 00000000000..0e8a0eb36f4 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeIngressNginx/.helmignore @@ -0,0 +1,23 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*.orig +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeIngressNginx/Chart.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeIngressNginx/Chart.yaml new file mode 100644 index 00000000000..4fd95623942 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeIngressNginx/Chart.yaml @@ -0,0 +1,15 @@ +annotations: + catalog.cattle.io/hidden: "true" + catalog.cattle.io/kube-version: '>= 1.26.0-0 < 1.31.0-0' + catalog.cattle.io/os: linux + catalog.rancher.io/certified: rancher + catalog.rancher.io/namespace: cattle-monitoring-system + catalog.rancher.io/release-name: rancher-pushprox +apiVersion: v1 +appVersion: v0.1.4-rancher2 +description: Sets up a deployment of the PushProx proxy and a DaemonSet of PushProx + clients. +kubeVersion: '>=1.26.0-0' +name: rkeIngressNginx +type: application +version: 0.2.1 diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeIngressNginx/README.md b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeIngressNginx/README.md new file mode 100644 index 00000000000..345002f48a2 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeIngressNginx/README.md @@ -0,0 +1,90 @@ +# rancher-pushprox + +A Rancher chart based on Rancher [PushProx](https://github.com/rancher/PushProx) that sets up a Deployment of a PushProx proxy and a DaemonSet of PushProx clients on a Kubernetes cluster. + +Installs [rancher-pushprox](https://github.com/rancher/charts/tree/gh-pages/packages/rancher-pushprox) to create PushProx clients that can access their host's network and register with a PushProx proxy. A [Prometheus Operator](https://github.com/coreos/prometheus-operator) ServiceMonitor CR is also included that is configured to scrape the metrics from each of the clients through the proxy. + +Using an instance of this chart is suitable for the following scenarios: +- You need to scrape metrics from a port that should not be accessible outside of the host (e.g. scraping `etcd` metrics in a hardened cluster) +- You need to scrape metrics on a host that are not exposed outside of 127.0.0.1 (e.g. scraping `kube-proxy` metrics) +- You need to scrape metrics through HTTPS using certs hosted directly on `hostPath` +- You need to scrape metrics from Kubernetes components that require authorization via a service account (e.g. permissions to make request to `/metrics`) +- You need to scrape metrics without access to cacerts (i.e. enable `insecureSkipVerify`) + +The clients and proxy are created based on a Rancher fork of the [prometheus-community/PushProx](https://github.com/prometheus-community/PushProx) project. + +## Upgrading to Kubernetes v1.25+ + +Starting in Kubernetes v1.25, [Pod Security Policies](https://kubernetes.io/docs/concepts/security/pod-security-policy/) have been removed from the Kubernetes API. + +As a result, **before upgrading to Kubernetes v1.25** (or on a fresh install in a Kubernetes v1.25+ cluster), users are expected to perform an in-place upgrade of this chart with `global.cattle.psp.enabled` set to `false` if it has been previously set to `true`. +​ +> **Note:** +> In this chart release, any previous field that was associated with any PSP resources have been removed in favor of a single global field: `global.cattle.psp.enabled`. + +> **Note:** +> If you upgrade your cluster to Kubernetes v1.25+ before removing PSPs via a `helm upgrade` (even if you manually clean up resources), **it will leave the Helm release in a broken state within the cluster such that further Helm operations will not work (`helm uninstall`, `helm upgrade`, etc.).** +> +> If your charts get stuck in this state, please consult the Rancher docs on how to clean up your Helm release secrets. + +Upon setting `global.cattle.psp.enabled` to false, the chart will remove any PSP resources deployed on its behalf from the cluster. This is the default setting for this chart. + +As a replacement for PSPs, [Pod Security Admission](https://kubernetes.io/docs/concepts/security/pod-security-admission/) should be used. Please consult the Rancher docs for more details on how to configure your chart release namespaces to work with the new Pod Security Admission and apply Pod Security Standards. + +## Configuration + +The following tables list the configurable parameters of the rancher-pushprox chart and their default values. + +### General + +#### Required +| Parameter | Description | Example | +| ----- | ----------- | ------ | +| `component` | The component that is being monitored | `kube-etcd` +| `metricsPort` | The port on the host that contains the metrics you want to scrape (e.g. `http://:/metrics`) | `2379` | +| `namespaceOverride` | The namespace to install the chart | `""` + +#### Optional +| Parameter | Description | Default | +| ----- | ----------- | ------ | +| `serviceMonitor.enabled` | Deploys a [Prometheus Operator](https://github.com/coreos/prometheus-operator/blob/master/Documentation/api.md#servicemonitor) ServiceMonitor CR that is configured to scrape metrics on the hosts that the clients are deployed on via the proxy. Also deploys a Service that points to all pods with the expected client name that exposes the `metricsPort` selected | `true` | +| `serviceMonitor.endpoints` | A list of endpoints that will be added to the ServiceMonitor based on the [Endpoint spec](https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#endpoint) | `[{port: metrics}]` | +| `service.selector` | The selector that is used to populate the Service's Endpoints object. The chart will error out on rendering templating if `.Values.clients.enabled` is set alongside this field, since it is expected that this service should point to the PushProx Clients Daemonset / Deployment | `{}` | +| `clients.enabled` | Deploys a DaemonSet of clients that are each capable of scraping endpoints on the hostNetwork it is deployed on | `true` | +| `clients.port` | The port where the client will publish PushProx client-specific metrics. If deploying multiple clients onto the same node, the clients should not have conflicting ports | `9369` | +| `clients.proxyUrl` | Overrides the default proxyUrl setting of `http://pushprox-{{ .Values.component }}-proxy.{{ . Release.Namespace }}.svc.cluster.local:{{ .Values.proxy.port }}"` with the `proxyUrl` specified | `""` | +| `clients.useLocalhost` | Sets a flag on each client deployment to redirect scrapes directed to `HOST_IP` to `127.0.0.1` | `false` | +| `clients.https.enabled` | Enables scraping metrics via HTTPS using the provided TLS certs that exist on each host | `false` | +| `clients.https.forceHTTPSScheme` | Forces scraping metrics via HTTPS using the provided TLS certs that exist on each host | `false` | +| `clients.https.useServiceAccountCredentials` | If set to true, the client will create a service account with permissions to scrape `/metrics` endpoint of Kubernetes components. The client will use the service account token provided to make authorized scrape requests to the Kubernetes API | `false` | +| `clients.https.authenticationMethod.bearerTokenFile.enabled` | If set to true, the client will use service account credentials mounted at the configured path `clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath`. This requires permissions to scrape `/metrics` endpoint of Kubernetes components. This method is deprecated by the prometheus operator and may be removed in a future release | `false` | +| `clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath` | This is a volume mount on the pod with permissions to scrape `/metrics` endpoint of Kubernetes components | `"/var/run/secrets/kubernetes.io/serviceaccount/token"` | +| `clients.https.authenticationMethod.bearerTokenSecret.enabled` | If set to true, the client will use service account credentials to scrape `/metrics` endpoint of Kubernetes components. This method is deprecated by the prometheus operator and may be removed in a future release | `false` | +| `clients.https.authenticationMethod.authorization.enabled` | If set to true, the client will use service account credentials to scrape `/metrics` endpoint of Kubernetes components | `false` | +| `clients.https.authenticationMethod.authorization.type` | If set, the client will use this type of authorization in its client requests for metrics | `"bearer"` | +| `clients.https.authenticationMethod.authorization.credentials.key` | If set, the client will use this key in the secret created by `clients.https.useServiceAccountCredentials` for authorization in its client requests for metrics | `"token"` | +| `clients.https.authenticationMethod.authorization.credentials.optional` | If set to false, the client will fail if the key in the secret created by `clients.https.useServiceAccountCredentials` does not exist | `false` | +| `clients.https.insecureSkipVerify` | If set to true, the client will disable SSL security checks | `false` | +| `clients.https.certDir` | A `hostPath` where TLS certs can be found. This path is mounted as a volume on an `initContainer` which copies only the necessary files over to an EmptyDir volume used by each client. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.certFile` | The path to the TLS cert file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.keyFile` | The path to the TLS key file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.caCertFile` | The path to the TLS cacert file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.seLinuxOptions` | seLinuxOptions to be passed into the container that copies certs. Should define a container with permissions to read the files in the certDir provided on the host. Required and only used if `clients.https.enabled` is set and `clients.https.certDir` is provided. | `""` | +| `clients.metrics.enabled` | Whether the client should publish PushProx client-specific metrics. | `false` | +| `clients.rbac.additionalRules` | Additional permissions to provide to the ServiceAccount bound to the client. This can be used to provide additional permissions for the client to scrape metrics from the k8s API. Only enabled if clients.https.enabled and clients.https.useServiceAccountCredentials are true | `[]` | +| `clients.deployment.enabled` | Deploys the client as a Deployment (generally used if the underlying hostNetwork Pod that is being scraped is managed by a Deployment) | `false` | +| `clients.deployment.replicas` | The number of pods the Deployment has, it should match the number of pod the hostNetwork Deployment has. Required and only used if `client.deployment.enable` is set | `0` | +| `clients.deployment.affinity` | The affinity rules that allocate the pod to the node in which the hostNetwork Deployment's pods run. Required and only used if `client.deployment.enable` is set | `{}` | +| `clients.resources` | Set resource limits and requests for the client container | `{}` | +| `clients.nodeSelector` | Select which nodes to deploy the clients on | `{}` | +| `clients.tolerations` | Specify tolerations for clients | `[]` | +| `proxy.enabled` | Deploys the proxy that each client will register with | `true` | +| `proxy.port` | The port exposed by the proxy that each client will register with to allow metrics to be scraped from the host | `8080` | +| `proxy.resources` | Set resource limits and requests for the proxy container | `{}` | +| `proxy.nodeSelector` | Select which nodes the proxy can be deployed on | `{}` | +| `proxy.tolerations` | Specify tolerations (if necessary) to allow the proxy to be deployed on the selected node | `[]` | +| `kubeVersionOverrides` | A list of Semver constraint strings (defined by https://github.com/Masterminds/semver) and values.yaml overrides. For each key in kubeVersionOverrides, this chart will check to see if the current Kubernetes cluster's version matches any of the semver constraints provided as keys on the map. On seeing a match, the default value for each values.yaml field overridden will be updated with the new value. If multiple matches are encountered (due to overlapping semver ranges), the matches will be applied in order. | `[]` + +*Tip: The filepaths set in `clients.https.File` can include wildcard characters*. + +See [rancher-monitoring](https://github.com/rancher/charts/tree/gh-pages/packages/rancher-monitoring) for examples of how this chart can be used. diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeIngressNginx/templates/_helpers.tpl b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeIngressNginx/templates/_helpers.tpl new file mode 100644 index 00000000000..1ba50939441 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeIngressNginx/templates/_helpers.tpl @@ -0,0 +1,170 @@ +# Rancher + +{{- define "system_default_registry" -}} +{{- if .Values.global.cattle.systemDefaultRegistry -}} +{{- printf "%s/" .Values.global.cattle.systemDefaultRegistry -}} +{{- end -}} +{{- end -}} + +# Windows Support + +{{/* +Windows cluster will add default taint for linux nodes, +add below linux tolerations to workloads could be scheduled to those linux nodes +*/}} + +{{- define "linux-node-tolerations" -}} +- key: "cattle.io/os" + value: "linux" + effect: "NoSchedule" + operator: "Equal" +{{- end -}} + +{{- define "linux-node-selector" -}} +{{- if semverCompare "<1.14-0" .Capabilities.KubeVersion.GitVersion -}} +beta.kubernetes.io/os: linux +{{- else -}} +kubernetes.io/os: linux +{{- end -}} +{{- end -}} + +# General + +{{- define "applyKubeVersionOverrides" -}} +{{- $overrides := dict -}} +{{- range $override := .Values.kubeVersionOverrides -}} +{{- if semverCompare $override.constraint $.Capabilities.KubeVersion.Version -}} +{{- $_ := mergeOverwrite $overrides $override.values -}} +{{- end -}} +{{- end -}} +{{- $_ := mergeOverwrite .Values $overrides -}} +{{- end -}} + +{{- define "pushprox.namespace" -}} + {{- if .Values.namespaceOverride -}} + {{- .Values.namespaceOverride -}} + {{- else -}} + {{- .Release.Namespace -}} + {{- end -}} +{{- end -}} + +{{- define "pushProxy.commonLabels" -}} +release: {{ .Release.Name }} +component: {{ .Values.component | quote }} +provider: kubernetes +{{- end -}} + +{{- define "pushProxy.proxyUrl" -}} +{{- $_ := (required "Template requires either .Values.proxy.port or .Values.client.proxyUrl to set proxyUrl for client" (or .Values.clients.proxyUrl .Values.proxy.port)) -}} +{{- if .Values.clients.proxyUrl -}} +{{ printf "%s" .Values.clients.proxyUrl }} +{{- else -}} +{{ printf "http://%s.%s.svc:%d" (include "pushProxy.proxy.name" .) (include "pushprox.namespace" .) (int .Values.proxy.port) }} +{{- end -}}{{- end -}} + +# Client + +{{- define "pushProxy.client.name" -}} +{{- printf "pushprox-%s-client" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.client.serviceAccountTokenName" -}} +{{- printf "pushprox-%s-client-service-account-token" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.client.labels" -}} +k8s-app: {{ template "pushProxy.client.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +# Proxy + +{{- define "pushProxy.proxy.name" -}} +{{- printf "pushprox-%s-proxy" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.proxy.labels" -}} +k8s-app: {{ template "pushProxy.proxy.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +# ServiceMonitor + +{{- define "pushprox.serviceMonitor.name" -}} +{{- printf "%s-%s" .Release.Name (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.serviceMonitor.labels" -}} +app: {{ template "pushprox.serviceMonitor.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +{{- define "pushProxy.serviceMonitor.endpoints" -}} +{{- $proxyURL := (include "pushProxy.proxyUrl" .) -}} +{{- $useHTTPS := .Values.clients.https.enabled -}} +{{- $setHTTPSScheme := .Values.clients.https.forceHTTPSScheme -}} +{{- $insecureSkipVerify := .Values.clients.https.insecureSkipVerify -}} +{{- $useServiceAccountCredentials := .Values.clients.https.useServiceAccountCredentials -}} +{{- $serviceAccountTokenName := (include "pushProxy.client.serviceAccountTokenName" . ) -}} +{{- $metricRelabelings := list }} +{{- $endpoints := .Values.serviceMonitor.endpoints }} +{{- if .Values.proxy.enabled }} +{{- $_ := set . "proxyUrl" $proxyURL }} +{{- end }} +{{- range $endpoints }} +{{- if $.Values.proxy.enabled }} +{{- $_ := set . "proxyUrl" $proxyURL }} +{{- end }} +{{- $clusterIdRelabel := dict }} +{{- $metricRelabelings := list }} +{{- if $.Values.global.cattle.clusterId }} +{{- $_ := set $clusterIdRelabel "action" "replace" }} +{{- $_ := set $clusterIdRelabel "sourceLabels" (list "__address__") }} +{{- $_ := set $clusterIdRelabel "targetLabel" "cluster_id" }} +{{- $_ := set $clusterIdRelabel "replacement" $.Values.global.cattle.clusterId }} +{{- $metricRelabelings = append $metricRelabelings $clusterIdRelabel }} +{{- end }} +{{- $clusterNameRelabel := dict }} +{{- if $.Values.global.cattle.clusterName }} +{{- $_ := set $clusterNameRelabel "action" "replace" }} +{{- $_ := set $clusterNameRelabel "sourceLabels" (list "__address__") }} +{{- $_ := set $clusterNameRelabel "targetLabel" "cluster_name" }} +{{- $_ := set $clusterNameRelabel "replacement" $.Values.global.cattle.clusterName }} +{{- $metricRelabelings = append $metricRelabelings $clusterNameRelabel }} +{{- end }} +{{- if not (empty $metricRelabelings) }} +{{- $_ := set . "metricRelabelings" ($metricRelabelings)}} +{{- end }} +{{- if $setHTTPSScheme -}} +{{- $_ := set . "scheme" "https" }} +{{- end -}} +{{- if $useHTTPS -}} +{{- if (hasKey . "params") }} +{{- $_ := set (get . "params") "_scheme" (list "https") }} +{{- else }} +{{- $_ := set . "params" (dict "_scheme" (list "https")) }} +{{- end }} +{{- end }} +{{- if (hasKey . "tlsConfig") }} +{{- $_ := set (get . "tlsConfig") "insecureSkipVerify" $insecureSkipVerify }} +{{- else }} +{{- $_ := set . "tlsConfig" (dict "insecureSkipVerify" $insecureSkipVerify) }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.bearerTokenFile.enabled }} +{{- $_ := set . "bearerTokenFile" $.Values.clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.bearerTokenSecret.enabled }} +{{- $_ := set . "bearerTokenSecret" $serviceAccountTokenName }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.authorization.enabled }} +{{- if (hasKey . "authorization") }} +{{- $_ := set (get . "authorization") "type" $.Values.clients.https.authenticationMethod.authorization.type }} +{{- $_ := set (get . "authorization") "credentials" (dict "name" $serviceAccountTokenName "key" $.Values.clients.https.authenticationMethod.authorization.credentials.key "optional" $.Values.clients.https.authenticationMethod.authorization.credentials.optional) }} +{{- else }} +{{- $_ := set . "authorization" (dict "type" $.Values.clients.https.authenticationMethod.authorization.type) }} +{{- $_ := set . "authorization" (dict "credentials" (dict "name" $serviceAccountTokenName "key" $.Values.clients.https.authenticationMethod.authorization.credentials.key "optional" $.Values.clients.https.authenticationMethod.authorization.credentials.optional)) }} +{{- end }} +{{- end }} +{{- end }} +{{- toYaml $endpoints }} +{{- end -}} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeIngressNginx/templates/pushprox-clients-rbac.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeIngressNginx/templates/pushprox-clients-rbac.yaml new file mode 100644 index 00000000000..a8e27c37356 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeIngressNginx/templates/pushprox-clients-rbac.yaml @@ -0,0 +1,97 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.clients }}{{- if .Values.clients.enabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ template "pushProxy.client.name" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +rules: +{{- if .Values.global.cattle.psp.enabled }} +- apiGroups: ['policy'] + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - {{ template "pushProxy.client.name" . }} +{{- end }} +{{- if and .Values.clients.https.enabled .Values.clients.https.useServiceAccountCredentials }} +- nonResourceURLs: ["/metrics"] + verbs: ["get"] +{{- if .Values.clients.rbac.additionalRules }} +{{ toYaml .Values.clients.rbac.additionalRules }} +{{- end }} +{{- end }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ template "pushProxy.client.name" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "pushProxy.client.name" . }} +subjects: + - kind: ServiceAccount + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +--- +{{- if .Values.clients.https.useServiceAccountCredentials }} +apiVersion: v1 +kind: Secret +type: kubernetes.io/service-account-token +metadata: + name: {{ template "pushProxy.client.serviceAccountTokenName" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} + annotations: + kubernetes.io/service-account.name: {{ template "pushProxy.client.name" . }} +{{- end }} +--- +{{- if .Values.global.cattle.psp.enabled }} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +spec: + privileged: false + hostNetwork: true + hostIPC: false + hostPID: false + runAsUser: + rule: 'RunAsAny' + seLinux: + rule: 'RunAsAny' +{{- if and .Values.clients.https.enabled .Values.clients.https.certDir .Values.global.seLinux.enabled .Values.clients.https.seLinuxOptions }} + seLinuxOptions: {{ .Values.clients.https.seLinuxOptions | toYaml | nindent 6 }} +{{- end }} + supplementalGroups: + rule: 'MustRunAs' + ranges: + - min: 0 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + - min: 0 + max: 65535 + readOnlyRootFilesystem: false + volumes: + - 'secret' +{{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + - 'emptyDir' + - 'hostPath' + allowedHostPaths: + - pathPrefix: {{ required "Need access to volume on host with the SSL cert files to use HTTPs" .Values.clients.https.certDir }} + readOnly: true +{{- end }} +{{- end }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeIngressNginx/templates/pushprox-clients.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeIngressNginx/templates/pushprox-clients.yaml new file mode 100644 index 00000000000..e8fcfb38836 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeIngressNginx/templates/pushprox-clients.yaml @@ -0,0 +1,157 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.clients }}{{- if .Values.clients.enabled }} +apiVersion: apps/v1 +{{- if .Values.clients.deployment.enabled }} +kind: Deployment +{{- else }} +kind: DaemonSet +{{- end }} +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} + pushprox-exporter: "client" +spec: + {{- if .Values.clients.deployment.enabled }} + replicas: {{ .Values.clients.deployment.replicas }} + {{- end }} + selector: + matchLabels: {{ include "pushProxy.client.labels" . | nindent 6 }} + template: + metadata: + labels: {{ include "pushProxy.client.labels" . | nindent 8 }} + spec: + {{- if .Values.clients.affinity }} + affinity: {{ toYaml .Values.clients.affinity | nindent 8 }} + {{- end }} + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} +{{- if .Values.clients.nodeSelector }} +{{ toYaml .Values.clients.nodeSelector | indent 8 }} +{{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} +{{- if .Values.clients.tolerations }} +{{ toYaml .Values.clients.tolerations | indent 8 }} +{{- end }} + hostNetwork: true + dnsPolicy: ClusterFirstWithHostNet + serviceAccountName: {{ template "pushProxy.client.name" . }} + {{- if .Values.global.imagePullSecretName }} + imagePullSecrets: + - name: {{ .Values.global.imagePullSecretName }} + {{- end }} + containers: + - name: pushprox-client + image: {{ template "system_default_registry" . }}{{ .Values.clients.image.repository }}:{{ .Values.clients.image.tag }} + command: + {{- range .Values.clients.command }} + - {{ . | quote }} + {{- end }} + args: + - --fqdn=$(HOST_IP) + - --proxy-url=$(PROXY_URL) + {{- if .Values.clients.metrics.enabled }} + - --metrics-addr=$(PORT) + {{- end }} + - --allow-port={{ required "Need .Values.metricsPort to configure client to be allowed to scrape metrics at port" .Values.metricsPort}} + {{- if .Values.clients.useLocalhost }} + - --use-localhost + {{- end }} + {{- if .Values.clients.https.enabled }} + {{- if .Values.clients.https.insecureSkipVerify }} + - --insecure-skip-verify + {{- end }} + {{- if .Values.clients.https.useServiceAccountCredentials }} + - --token-path=/var/run/secrets/kubernetes.io/serviceaccount/token + {{- end }} + {{- if .Values.clients.https.certDir }} + - --tls.cert=/etc/ssl/push-proxy/push-proxy.pem + - --tls.key=/etc/ssl/push-proxy/push-proxy-key.pem + - --tls.cacert=/etc/ssl/push-proxy/push-proxy-ca-cert.pem + {{- end }} + {{- end }} + env: + - name: HOST_IP + valueFrom: + fieldRef: + fieldPath: status.hostIP + {{- if .Values.clients.metrics.enabled }} + - name: PORT + value: :{{ .Values.clients.port }} + {{- end }} + - name: PROXY_URL + value: {{ template "pushProxy.proxyUrl" . }} + securityContext: + runAsNonRoot: true + runAsUser: 1000 + {{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + volumeMounts: + - name: metrics-cert-dir + mountPath: /etc/ssl/push-proxy + {{- end }} + {{- if .Values.clients.resources }} + resources: {{ toYaml .Values.clients.resources | nindent 10 }} + {{- end }} + {{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + initContainers: + - name: copy-certs + image: {{ template "system_default_registry" . }}{{ .Values.clients.copyCertsImage.repository }}:{{ .Values.clients.copyCertsImage.tag }} + command: + - sh + - -c + - | + echo "Searching for files to copy within the source volume" + echo "cert: ${CERT_FILE_NAME}" + echo "key: ${KEY_FILE_NAME}" + echo "cacert: ${CACERT_FILE_NAME}" + + CERT_FILE_SOURCE=$(find /etc/source/ -type f -name "${CERT_FILE_NAME}" | sort -r | head -n 1) + KEY_FILE_SOURCE=$(find /etc/source/ -type f -name "${KEY_FILE_NAME}" | sort -r | head -n 1) + CACERT_FILE_SOURCE=$(find /etc/source/ -type f -name "${CACERT_FILE_NAME}" | sort -r | head -n 1) + + test -z ${CERT_FILE_SOURCE} && echo "Failed to find cert file" && exit 1 + test -z ${KEY_FILE_SOURCE} && echo "Failed to find key file" && exit 1 + test -z ${CACERT_FILE_SOURCE} && echo "Failed to find cacert file" && exit 1 + + echo "Copying cert file from $CERT_FILE_SOURCE to $CERT_FILE_TARGET" + cp $CERT_FILE_SOURCE $CERT_FILE_TARGET || exit 1 + chmod 444 $CERT_FILE_TARGET || exit 1 + + echo "Copying key file from $KEY_FILE_SOURCE to $KEY_FILE_TARGET" + cp $KEY_FILE_SOURCE $KEY_FILE_TARGET || exit 1 + chmod 444 $KEY_FILE_TARGET || exit 1 + + echo "Copying cacert file from $CACERT_FILE_SOURCE to $CACERT_FILE_TARGET" + cp $CACERT_FILE_SOURCE $CACERT_FILE_TARGET || exit 1 + chmod 444 $CACERT_FILE_TARGET || exit 1 + env: + - name: CERT_FILE_NAME + value: {{ required "Need a TLS cert file for scraping metrics endpoint over HTTPs" .Values.clients.https.certFile }} + - name: KEY_FILE_NAME + value: {{ required "Need a TLS key file for scraping metrics endpoint over HTTPs" .Values.clients.https.keyFile }} + - name: CACERT_FILE_NAME + value: {{ required "Need a TLS CA cert file for scraping metrics endpoint over HTTPs" .Values.clients.https.caCertFile }} + - name: CERT_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy.pem + - name: KEY_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy-key.pem + - name: CACERT_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy-ca-cert.pem + securityContext: + runAsNonRoot: false +{{- if and .Values.global.seLinux.enabled .Values.clients.https.seLinuxOptions }} + seLinuxOptions: {{ .Values.clients.https.seLinuxOptions | toYaml | nindent 12 }} +{{- end }} + volumeMounts: + - name: metrics-cert-dir-source + mountPath: /etc/source + readOnly: true + - name: metrics-cert-dir + mountPath: /etc/ssl/push-proxy + volumes: + - name: metrics-cert-dir-source + hostPath: + path: {{ required "Need access to volume on host with the SSL cert files to use HTTPs" .Values.clients.https.certDir }} + - name: metrics-cert-dir + emptyDir: {} + {{- end }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeIngressNginx/templates/pushprox-proxy-rbac.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeIngressNginx/templates/pushprox-proxy-rbac.yaml new file mode 100644 index 00000000000..eefe609058c --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeIngressNginx/templates/pushprox-proxy-rbac.yaml @@ -0,0 +1,68 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if and .Values.proxy }}{{ if .Values.proxy.enabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ template "pushProxy.proxy.name" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +rules: +{{- if .Values.global.cattle.psp.enabled }} +- apiGroups: ['policy'] + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - {{ template "pushProxy.proxy.name" . }} +{{- end }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ template "pushProxy.proxy.name" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "pushProxy.proxy.name" . }} +subjects: + - kind: ServiceAccount + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +--- +{{- if .Values.global.cattle.psp.enabled }} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +spec: + privileged: false + hostNetwork: false + hostIPC: false + hostPID: false + runAsUser: + rule: 'MustRunAsNonRoot' + seLinux: + rule: 'RunAsAny' + supplementalGroups: + rule: 'MustRunAs' + ranges: + - min: 1 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + - min: 1 + max: 65535 + readOnlyRootFilesystem: false + volumes: + - 'secret' +{{- end }}{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeIngressNginx/templates/pushprox-proxy.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeIngressNginx/templates/pushprox-proxy.yaml new file mode 100644 index 00000000000..723bbd6c00b --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeIngressNginx/templates/pushprox-proxy.yaml @@ -0,0 +1,57 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if and .Values.proxy }}{{ if .Values.proxy.enabled }} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} + pushprox-exporter: "proxy" +spec: + selector: + matchLabels: {{ include "pushProxy.proxy.labels" . | nindent 6 }} + template: + metadata: + labels: {{ include "pushProxy.proxy.labels" . | nindent 8 }} + spec: + securityContext: + runAsNonRoot: true + runAsUser: 1000 + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} +{{- if .Values.proxy.nodeSelector }} +{{ toYaml .Values.proxy.nodeSelector | indent 8 }} +{{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} +{{- if .Values.proxy.tolerations }} +{{ toYaml .Values.proxy.tolerations | indent 8 }} +{{- end }} + serviceAccountName: {{ template "pushProxy.proxy.name" . }} + {{- if .Values.global.imagePullSecretName }} + imagePullSecrets: + - name: {{ .Values.global.imagePullSecretName }} + {{- end }} + containers: + - name: pushprox-proxy + image: {{ template "system_default_registry" . }}{{ .Values.proxy.image.repository }}:{{ .Values.proxy.image.tag }} + command: + {{- range .Values.proxy.command }} + - {{ . | quote }} + {{- end }} + {{- if .Values.proxy.resources }} + resources: {{ toYaml .Values.proxy.resources | nindent 10 }} + {{- end }} +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +spec: + ports: + - name: pp-proxy + port: {{ required "Need .Values.proxy.port to configure proxy" .Values.proxy.port }} + protocol: TCP + targetPort: {{ .Values.proxy.port }} + selector: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeIngressNginx/templates/pushprox-servicemonitor.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeIngressNginx/templates/pushprox-servicemonitor.yaml new file mode 100644 index 00000000000..67eb2216b5a --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeIngressNginx/templates/pushprox-servicemonitor.yaml @@ -0,0 +1,45 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.serviceMonitor }}{{- if .Values.serviceMonitor.enabled }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ template "pushprox.serviceMonitor.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.serviceMonitor.labels" . | nindent 4 }} +spec: + endpoints: {{include "pushProxy.serviceMonitor.endpoints" . | nindent 4 }} + jobLabel: component + podTargetLabels: + - component + - pushprox-exporter + namespaceSelector: + matchNames: + - {{ template "pushprox.namespace" . }} + selector: + matchLabels: {{ include "pushProxy.client.labels" . | nindent 6 }} +--- +{{- $selector := "" }} +{{- if not (kindIs "invalid" .Values.service) }} +{{- if not (kindIs "invalid" .Values.service.selector) }} +{{ if .Values.service.selector }} +{{- if .Values.clients.enabled }} +{{- required (printf "Cannot override .Values.service.selector=%s when .Values.clients.enabled=true" (toJson .Values.service.selector)) "" }} +{{- end }} +{{- $selector = (toYaml .Values.service.selector) }} +{{- end }} +{{- end }} +{{- end }} +apiVersion: v1 +kind: Service +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +spec: + ports: + - name: metrics + port: {{ required "Need .Values.metricsPort to configure client to listen to metrics at port" .Values.metricsPort}} + protocol: TCP + targetPort: {{ .Values.metricsPort }} + selector: {{ default (include "pushProxy.client.labels" .) $selector | nindent 4 }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeIngressNginx/templates/validate-install-crd.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeIngressNginx/templates/validate-install-crd.yaml new file mode 100644 index 00000000000..16abc2fa83a --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeIngressNginx/templates/validate-install-crd.yaml @@ -0,0 +1,14 @@ +#{{- if gt (len (lookup "rbac.authorization.k8s.io/v1" "ClusterRole" "" "")) 0 -}} +# {{- $found := dict -}} +# {{- set $found "monitoring.coreos.com/v1/ServiceMonitor" false -}} +# {{- range .Capabilities.APIVersions -}} +# {{- if hasKey $found (toString .) -}} +# {{- set $found (toString .) true -}} +# {{- end -}} +# {{- end -}} +# {{- range $_, $exists := $found -}} +# {{- if (eq $exists false) -}} +# {{- required "Required CRDs are missing. Please install Prometheus Operator CRDs before installing this chart." "" -}} +# {{- end -}} +# {{- end -}} +#{{- end -}} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeIngressNginx/templates/validate-psp-install.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeIngressNginx/templates/validate-psp-install.yaml new file mode 100644 index 00000000000..a30c59d3b7c --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeIngressNginx/templates/validate-psp-install.yaml @@ -0,0 +1,7 @@ +#{{- if gt (len (lookup "rbac.authorization.k8s.io/v1" "ClusterRole" "" "")) 0 -}} +#{{- if .Values.global.cattle.psp.enabled }} +#{{- if not (.Capabilities.APIVersions.Has "policy/v1beta1/PodSecurityPolicy") }} +#{{- fail "The target cluster does not have the PodSecurityPolicy API resource. Please disable PSPs in this chart before proceeding." -}} +#{{- end }} +#{{- end }} +#{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeIngressNginx/values.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeIngressNginx/values.yaml new file mode 100644 index 00000000000..7f38498a1c5 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeIngressNginx/values.yaml @@ -0,0 +1,166 @@ +# Default values for rancher-pushprox. +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. + +# Default image containing both the proxy and the client was generated from the following Dockerfile +# https://github.com/prometheus-community/PushProx/blob/eeadbe766641699129920ccfaaaa30a85c67fe81/Dockerfile#L1-L15 + +# Configuration + +global: + cattle: + psp: + enabled: false + systemDefaultRegistry: "" + seLinux: + enabled: false + +# A list of Semver constraint strings (defined by https://github.com/Masterminds/semver) and values.yaml overrides. +# +# For each key in kubeVersionOverrides, this chart will check to see if the current Kubernetes cluster's version matches +# any of the semver constraints provided as keys on the map. +# +# On seeing a match, the default value for each values.yaml field overridden will be updated with the new value. +# +# If multiple matches are encountered (due to overlapping semver ranges), the matches will be applied in order. +# +# Notes: +# - On running a helm template, Helm generally assumes the kubeVersion is v1.20.0 +# - On running a helm install --dry-run, the correct kubeVersion should be chosen. +kubeVersionOverrides: [] +# - constraint: "< 1.21" +# values: +# metricsPort: 10252 +# clients: +# https: +# enabled: false +# insecureSkipVerify: false +# useServiceAccountCredentials: false + +namespaceOverride: "" + +# The component that is being monitored (i.e. etcd) +component: "component" + +# The port containing the metrics that need to be scraped +metricsPort: 2739 + +# Configure ServiceMonitor that monitors metrics from the metricsPort endpoint +serviceMonitor: + enabled: true + # A list of endpoints that will be added to the ServiceMonitor based on the Endpoint spec + # Source: https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#endpoint + # By default, proxyUrl and params._scheme will be overridden based on other values + endpoints: + - port: metrics + +# Configure Service that grabs scrape targets +service: + # The selector that is used to populate the Service's Endpoints object. + # The chart will error out on rendering templating if .Values.clients.enabled is set alongside this field, + # since it is expected that this service should point to the PushProx Clients Daemonset / Deployment + selector: {} + +clients: + enabled: true + # The port which the PushProx client will post PushProx metrics to + port: 9369 + # If unset, this will default to the URL for the proxy service: http://pushprox-{{component}}-proxy.{{namepsace}}.svc.cluster.local:{{proxy.port}} + # Should be modified if the clients are being deployed outside the cluster where the proxy rests, otherwise leave it null + proxyUrl: "" + # If set to true, the client will forward any requests from the host IP to 127.0.0.1 + # It will only allow proxy requests to the metricsPort specified + useLocalhost: false + # Configuration for accessing metrics via HTTPS + https: + # Does the client require https to access the metrics? + enabled: false + # Does the client require requests be sent to http or https? + forceHTTPSScheme: false + # If set to true, the client will create a service account with adequate permissions and set a flag + # on the client to use the service account token provided by it to make authorized scrape requests + useServiceAccountCredentials: false + # Configuration for authentication to metrics via https endpoint + authenticationMethod: + # Reads token from defined file in container + # This function is deprecated in the prometheus operator api and may be removed in a future version + bearerTokenFile: + enabled: false + bearerTokenFilePath: "/var/run/secrets/kubernetes.io/serviceaccount/token" + # Reads token from defined secret in namespace + # This function is deprecated in the prometheus operator api and may be removed in a future version + bearerTokenSecret: + enabled: false + # Reads token from defined secret in namespace + authorization: + enabled: false + type: "bearer" + credentials: + key: "token" + optional: false + # If set to true, the client will disable SSL security checks + insecureSkipVerify: false + # Directory on host where necessary TLS cert and key to scrape metrics can be found + certDir: "" + # Filenames for files located in .Values.clients.https.certDir that correspond to TLS settings + certFile: "" + keyFile: "" + caCertFile: "" + # seLinuxOptions to be passed into the container that copies certs. Should define a container with permissions to read the files in the certDir provided on the host. + # Required and only used if `clients.https.enabled` is set and `clients.https.certDir` is provided. + seLinuxOptions: {} + + metrics: + # Whether the client should publish PushProx client-specific metrics to .Values.clients.port + enabled: false + + rbac: + # Additional permissions to provide to the ServiceAccount bound to the client + # This can be used to provide additional permissions for the client to scrape metrics from the k8s API + # Only enabled if clients.https.enabled and clients.https.useServiceAccountCredentials are true + additionalRules: [] + + # Resource limits + resources: {} + + # Options to select all nodes to deploy client DaemonSet on + nodeSelector: {} + tolerations: [] + affinity: {} + + image: + repository: rancher/pushprox-client + tag: v0.1.4-rancher2-client + command: ["pushprox-client"] + + copyCertsImage: + repository: rancher/mirrored-library-busybox + tag: 1.31.1 + + # The default intention of rancher-pushprox clients is to scrape hostNetwork metrics across all nodes. + # This can be used to scrape internal Kubernetes components or DaemonSets of hostNetwork Pods in + # situations where a cloud provider firewall prevents Pod-To-Host communication but not Pod-To-Pod. + # However, if the underlying hostNetwork Pod that is being scraped is managed by a Deployment, + # this advanced option enables users to deploy the client as a Deployment instead of a DaemonSet. + # If a user deploys this feature and the underlying Deployment's number of replicas changes, the user will + # be responsible for upgrading this chart accordingly to the right number of replicas. + deployment: + enabled: false + replicas: 0 + +proxy: + enabled: true + # The port through which PushProx clients will communicate to the proxy + port: 8080 + + # Resource limits + resources: {} + + # Options to select a node to run a single proxy deployment on + nodeSelector: {} + tolerations: [] + + image: + repository: rancher/pushprox-proxy + tag: v0.1.4-rancher2-proxy + command: ["pushprox-proxy"] diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeProxy/.helmignore b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeProxy/.helmignore new file mode 100644 index 00000000000..0e8a0eb36f4 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeProxy/.helmignore @@ -0,0 +1,23 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*.orig +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeProxy/Chart.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeProxy/Chart.yaml new file mode 100644 index 00000000000..5ff503cf7cd --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeProxy/Chart.yaml @@ -0,0 +1,15 @@ +annotations: + catalog.cattle.io/hidden: "true" + catalog.cattle.io/kube-version: '>= 1.26.0-0 < 1.31.0-0' + catalog.cattle.io/os: linux + catalog.rancher.io/certified: rancher + catalog.rancher.io/namespace: cattle-monitoring-system + catalog.rancher.io/release-name: rancher-pushprox +apiVersion: v1 +appVersion: v0.1.4-rancher2 +description: Sets up a deployment of the PushProx proxy and a DaemonSet of PushProx + clients. +kubeVersion: '>=1.26.0-0' +name: rkeProxy +type: application +version: 0.2.1 diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeProxy/README.md b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeProxy/README.md new file mode 100644 index 00000000000..345002f48a2 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeProxy/README.md @@ -0,0 +1,90 @@ +# rancher-pushprox + +A Rancher chart based on Rancher [PushProx](https://github.com/rancher/PushProx) that sets up a Deployment of a PushProx proxy and a DaemonSet of PushProx clients on a Kubernetes cluster. + +Installs [rancher-pushprox](https://github.com/rancher/charts/tree/gh-pages/packages/rancher-pushprox) to create PushProx clients that can access their host's network and register with a PushProx proxy. A [Prometheus Operator](https://github.com/coreos/prometheus-operator) ServiceMonitor CR is also included that is configured to scrape the metrics from each of the clients through the proxy. + +Using an instance of this chart is suitable for the following scenarios: +- You need to scrape metrics from a port that should not be accessible outside of the host (e.g. scraping `etcd` metrics in a hardened cluster) +- You need to scrape metrics on a host that are not exposed outside of 127.0.0.1 (e.g. scraping `kube-proxy` metrics) +- You need to scrape metrics through HTTPS using certs hosted directly on `hostPath` +- You need to scrape metrics from Kubernetes components that require authorization via a service account (e.g. permissions to make request to `/metrics`) +- You need to scrape metrics without access to cacerts (i.e. enable `insecureSkipVerify`) + +The clients and proxy are created based on a Rancher fork of the [prometheus-community/PushProx](https://github.com/prometheus-community/PushProx) project. + +## Upgrading to Kubernetes v1.25+ + +Starting in Kubernetes v1.25, [Pod Security Policies](https://kubernetes.io/docs/concepts/security/pod-security-policy/) have been removed from the Kubernetes API. + +As a result, **before upgrading to Kubernetes v1.25** (or on a fresh install in a Kubernetes v1.25+ cluster), users are expected to perform an in-place upgrade of this chart with `global.cattle.psp.enabled` set to `false` if it has been previously set to `true`. +​ +> **Note:** +> In this chart release, any previous field that was associated with any PSP resources have been removed in favor of a single global field: `global.cattle.psp.enabled`. + +> **Note:** +> If you upgrade your cluster to Kubernetes v1.25+ before removing PSPs via a `helm upgrade` (even if you manually clean up resources), **it will leave the Helm release in a broken state within the cluster such that further Helm operations will not work (`helm uninstall`, `helm upgrade`, etc.).** +> +> If your charts get stuck in this state, please consult the Rancher docs on how to clean up your Helm release secrets. + +Upon setting `global.cattle.psp.enabled` to false, the chart will remove any PSP resources deployed on its behalf from the cluster. This is the default setting for this chart. + +As a replacement for PSPs, [Pod Security Admission](https://kubernetes.io/docs/concepts/security/pod-security-admission/) should be used. Please consult the Rancher docs for more details on how to configure your chart release namespaces to work with the new Pod Security Admission and apply Pod Security Standards. + +## Configuration + +The following tables list the configurable parameters of the rancher-pushprox chart and their default values. + +### General + +#### Required +| Parameter | Description | Example | +| ----- | ----------- | ------ | +| `component` | The component that is being monitored | `kube-etcd` +| `metricsPort` | The port on the host that contains the metrics you want to scrape (e.g. `http://:/metrics`) | `2379` | +| `namespaceOverride` | The namespace to install the chart | `""` + +#### Optional +| Parameter | Description | Default | +| ----- | ----------- | ------ | +| `serviceMonitor.enabled` | Deploys a [Prometheus Operator](https://github.com/coreos/prometheus-operator/blob/master/Documentation/api.md#servicemonitor) ServiceMonitor CR that is configured to scrape metrics on the hosts that the clients are deployed on via the proxy. Also deploys a Service that points to all pods with the expected client name that exposes the `metricsPort` selected | `true` | +| `serviceMonitor.endpoints` | A list of endpoints that will be added to the ServiceMonitor based on the [Endpoint spec](https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#endpoint) | `[{port: metrics}]` | +| `service.selector` | The selector that is used to populate the Service's Endpoints object. The chart will error out on rendering templating if `.Values.clients.enabled` is set alongside this field, since it is expected that this service should point to the PushProx Clients Daemonset / Deployment | `{}` | +| `clients.enabled` | Deploys a DaemonSet of clients that are each capable of scraping endpoints on the hostNetwork it is deployed on | `true` | +| `clients.port` | The port where the client will publish PushProx client-specific metrics. If deploying multiple clients onto the same node, the clients should not have conflicting ports | `9369` | +| `clients.proxyUrl` | Overrides the default proxyUrl setting of `http://pushprox-{{ .Values.component }}-proxy.{{ . Release.Namespace }}.svc.cluster.local:{{ .Values.proxy.port }}"` with the `proxyUrl` specified | `""` | +| `clients.useLocalhost` | Sets a flag on each client deployment to redirect scrapes directed to `HOST_IP` to `127.0.0.1` | `false` | +| `clients.https.enabled` | Enables scraping metrics via HTTPS using the provided TLS certs that exist on each host | `false` | +| `clients.https.forceHTTPSScheme` | Forces scraping metrics via HTTPS using the provided TLS certs that exist on each host | `false` | +| `clients.https.useServiceAccountCredentials` | If set to true, the client will create a service account with permissions to scrape `/metrics` endpoint of Kubernetes components. The client will use the service account token provided to make authorized scrape requests to the Kubernetes API | `false` | +| `clients.https.authenticationMethod.bearerTokenFile.enabled` | If set to true, the client will use service account credentials mounted at the configured path `clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath`. This requires permissions to scrape `/metrics` endpoint of Kubernetes components. This method is deprecated by the prometheus operator and may be removed in a future release | `false` | +| `clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath` | This is a volume mount on the pod with permissions to scrape `/metrics` endpoint of Kubernetes components | `"/var/run/secrets/kubernetes.io/serviceaccount/token"` | +| `clients.https.authenticationMethod.bearerTokenSecret.enabled` | If set to true, the client will use service account credentials to scrape `/metrics` endpoint of Kubernetes components. This method is deprecated by the prometheus operator and may be removed in a future release | `false` | +| `clients.https.authenticationMethod.authorization.enabled` | If set to true, the client will use service account credentials to scrape `/metrics` endpoint of Kubernetes components | `false` | +| `clients.https.authenticationMethod.authorization.type` | If set, the client will use this type of authorization in its client requests for metrics | `"bearer"` | +| `clients.https.authenticationMethod.authorization.credentials.key` | If set, the client will use this key in the secret created by `clients.https.useServiceAccountCredentials` for authorization in its client requests for metrics | `"token"` | +| `clients.https.authenticationMethod.authorization.credentials.optional` | If set to false, the client will fail if the key in the secret created by `clients.https.useServiceAccountCredentials` does not exist | `false` | +| `clients.https.insecureSkipVerify` | If set to true, the client will disable SSL security checks | `false` | +| `clients.https.certDir` | A `hostPath` where TLS certs can be found. This path is mounted as a volume on an `initContainer` which copies only the necessary files over to an EmptyDir volume used by each client. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.certFile` | The path to the TLS cert file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.keyFile` | The path to the TLS key file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.caCertFile` | The path to the TLS cacert file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.seLinuxOptions` | seLinuxOptions to be passed into the container that copies certs. Should define a container with permissions to read the files in the certDir provided on the host. Required and only used if `clients.https.enabled` is set and `clients.https.certDir` is provided. | `""` | +| `clients.metrics.enabled` | Whether the client should publish PushProx client-specific metrics. | `false` | +| `clients.rbac.additionalRules` | Additional permissions to provide to the ServiceAccount bound to the client. This can be used to provide additional permissions for the client to scrape metrics from the k8s API. Only enabled if clients.https.enabled and clients.https.useServiceAccountCredentials are true | `[]` | +| `clients.deployment.enabled` | Deploys the client as a Deployment (generally used if the underlying hostNetwork Pod that is being scraped is managed by a Deployment) | `false` | +| `clients.deployment.replicas` | The number of pods the Deployment has, it should match the number of pod the hostNetwork Deployment has. Required and only used if `client.deployment.enable` is set | `0` | +| `clients.deployment.affinity` | The affinity rules that allocate the pod to the node in which the hostNetwork Deployment's pods run. Required and only used if `client.deployment.enable` is set | `{}` | +| `clients.resources` | Set resource limits and requests for the client container | `{}` | +| `clients.nodeSelector` | Select which nodes to deploy the clients on | `{}` | +| `clients.tolerations` | Specify tolerations for clients | `[]` | +| `proxy.enabled` | Deploys the proxy that each client will register with | `true` | +| `proxy.port` | The port exposed by the proxy that each client will register with to allow metrics to be scraped from the host | `8080` | +| `proxy.resources` | Set resource limits and requests for the proxy container | `{}` | +| `proxy.nodeSelector` | Select which nodes the proxy can be deployed on | `{}` | +| `proxy.tolerations` | Specify tolerations (if necessary) to allow the proxy to be deployed on the selected node | `[]` | +| `kubeVersionOverrides` | A list of Semver constraint strings (defined by https://github.com/Masterminds/semver) and values.yaml overrides. For each key in kubeVersionOverrides, this chart will check to see if the current Kubernetes cluster's version matches any of the semver constraints provided as keys on the map. On seeing a match, the default value for each values.yaml field overridden will be updated with the new value. If multiple matches are encountered (due to overlapping semver ranges), the matches will be applied in order. | `[]` + +*Tip: The filepaths set in `clients.https.File` can include wildcard characters*. + +See [rancher-monitoring](https://github.com/rancher/charts/tree/gh-pages/packages/rancher-monitoring) for examples of how this chart can be used. diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeProxy/templates/_helpers.tpl b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeProxy/templates/_helpers.tpl new file mode 100644 index 00000000000..1ba50939441 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeProxy/templates/_helpers.tpl @@ -0,0 +1,170 @@ +# Rancher + +{{- define "system_default_registry" -}} +{{- if .Values.global.cattle.systemDefaultRegistry -}} +{{- printf "%s/" .Values.global.cattle.systemDefaultRegistry -}} +{{- end -}} +{{- end -}} + +# Windows Support + +{{/* +Windows cluster will add default taint for linux nodes, +add below linux tolerations to workloads could be scheduled to those linux nodes +*/}} + +{{- define "linux-node-tolerations" -}} +- key: "cattle.io/os" + value: "linux" + effect: "NoSchedule" + operator: "Equal" +{{- end -}} + +{{- define "linux-node-selector" -}} +{{- if semverCompare "<1.14-0" .Capabilities.KubeVersion.GitVersion -}} +beta.kubernetes.io/os: linux +{{- else -}} +kubernetes.io/os: linux +{{- end -}} +{{- end -}} + +# General + +{{- define "applyKubeVersionOverrides" -}} +{{- $overrides := dict -}} +{{- range $override := .Values.kubeVersionOverrides -}} +{{- if semverCompare $override.constraint $.Capabilities.KubeVersion.Version -}} +{{- $_ := mergeOverwrite $overrides $override.values -}} +{{- end -}} +{{- end -}} +{{- $_ := mergeOverwrite .Values $overrides -}} +{{- end -}} + +{{- define "pushprox.namespace" -}} + {{- if .Values.namespaceOverride -}} + {{- .Values.namespaceOverride -}} + {{- else -}} + {{- .Release.Namespace -}} + {{- end -}} +{{- end -}} + +{{- define "pushProxy.commonLabels" -}} +release: {{ .Release.Name }} +component: {{ .Values.component | quote }} +provider: kubernetes +{{- end -}} + +{{- define "pushProxy.proxyUrl" -}} +{{- $_ := (required "Template requires either .Values.proxy.port or .Values.client.proxyUrl to set proxyUrl for client" (or .Values.clients.proxyUrl .Values.proxy.port)) -}} +{{- if .Values.clients.proxyUrl -}} +{{ printf "%s" .Values.clients.proxyUrl }} +{{- else -}} +{{ printf "http://%s.%s.svc:%d" (include "pushProxy.proxy.name" .) (include "pushprox.namespace" .) (int .Values.proxy.port) }} +{{- end -}}{{- end -}} + +# Client + +{{- define "pushProxy.client.name" -}} +{{- printf "pushprox-%s-client" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.client.serviceAccountTokenName" -}} +{{- printf "pushprox-%s-client-service-account-token" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.client.labels" -}} +k8s-app: {{ template "pushProxy.client.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +# Proxy + +{{- define "pushProxy.proxy.name" -}} +{{- printf "pushprox-%s-proxy" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.proxy.labels" -}} +k8s-app: {{ template "pushProxy.proxy.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +# ServiceMonitor + +{{- define "pushprox.serviceMonitor.name" -}} +{{- printf "%s-%s" .Release.Name (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.serviceMonitor.labels" -}} +app: {{ template "pushprox.serviceMonitor.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +{{- define "pushProxy.serviceMonitor.endpoints" -}} +{{- $proxyURL := (include "pushProxy.proxyUrl" .) -}} +{{- $useHTTPS := .Values.clients.https.enabled -}} +{{- $setHTTPSScheme := .Values.clients.https.forceHTTPSScheme -}} +{{- $insecureSkipVerify := .Values.clients.https.insecureSkipVerify -}} +{{- $useServiceAccountCredentials := .Values.clients.https.useServiceAccountCredentials -}} +{{- $serviceAccountTokenName := (include "pushProxy.client.serviceAccountTokenName" . ) -}} +{{- $metricRelabelings := list }} +{{- $endpoints := .Values.serviceMonitor.endpoints }} +{{- if .Values.proxy.enabled }} +{{- $_ := set . "proxyUrl" $proxyURL }} +{{- end }} +{{- range $endpoints }} +{{- if $.Values.proxy.enabled }} +{{- $_ := set . "proxyUrl" $proxyURL }} +{{- end }} +{{- $clusterIdRelabel := dict }} +{{- $metricRelabelings := list }} +{{- if $.Values.global.cattle.clusterId }} +{{- $_ := set $clusterIdRelabel "action" "replace" }} +{{- $_ := set $clusterIdRelabel "sourceLabels" (list "__address__") }} +{{- $_ := set $clusterIdRelabel "targetLabel" "cluster_id" }} +{{- $_ := set $clusterIdRelabel "replacement" $.Values.global.cattle.clusterId }} +{{- $metricRelabelings = append $metricRelabelings $clusterIdRelabel }} +{{- end }} +{{- $clusterNameRelabel := dict }} +{{- if $.Values.global.cattle.clusterName }} +{{- $_ := set $clusterNameRelabel "action" "replace" }} +{{- $_ := set $clusterNameRelabel "sourceLabels" (list "__address__") }} +{{- $_ := set $clusterNameRelabel "targetLabel" "cluster_name" }} +{{- $_ := set $clusterNameRelabel "replacement" $.Values.global.cattle.clusterName }} +{{- $metricRelabelings = append $metricRelabelings $clusterNameRelabel }} +{{- end }} +{{- if not (empty $metricRelabelings) }} +{{- $_ := set . "metricRelabelings" ($metricRelabelings)}} +{{- end }} +{{- if $setHTTPSScheme -}} +{{- $_ := set . "scheme" "https" }} +{{- end -}} +{{- if $useHTTPS -}} +{{- if (hasKey . "params") }} +{{- $_ := set (get . "params") "_scheme" (list "https") }} +{{- else }} +{{- $_ := set . "params" (dict "_scheme" (list "https")) }} +{{- end }} +{{- end }} +{{- if (hasKey . "tlsConfig") }} +{{- $_ := set (get . "tlsConfig") "insecureSkipVerify" $insecureSkipVerify }} +{{- else }} +{{- $_ := set . "tlsConfig" (dict "insecureSkipVerify" $insecureSkipVerify) }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.bearerTokenFile.enabled }} +{{- $_ := set . "bearerTokenFile" $.Values.clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.bearerTokenSecret.enabled }} +{{- $_ := set . "bearerTokenSecret" $serviceAccountTokenName }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.authorization.enabled }} +{{- if (hasKey . "authorization") }} +{{- $_ := set (get . "authorization") "type" $.Values.clients.https.authenticationMethod.authorization.type }} +{{- $_ := set (get . "authorization") "credentials" (dict "name" $serviceAccountTokenName "key" $.Values.clients.https.authenticationMethod.authorization.credentials.key "optional" $.Values.clients.https.authenticationMethod.authorization.credentials.optional) }} +{{- else }} +{{- $_ := set . "authorization" (dict "type" $.Values.clients.https.authenticationMethod.authorization.type) }} +{{- $_ := set . "authorization" (dict "credentials" (dict "name" $serviceAccountTokenName "key" $.Values.clients.https.authenticationMethod.authorization.credentials.key "optional" $.Values.clients.https.authenticationMethod.authorization.credentials.optional)) }} +{{- end }} +{{- end }} +{{- end }} +{{- toYaml $endpoints }} +{{- end -}} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeProxy/templates/pushprox-clients-rbac.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeProxy/templates/pushprox-clients-rbac.yaml new file mode 100644 index 00000000000..a8e27c37356 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeProxy/templates/pushprox-clients-rbac.yaml @@ -0,0 +1,97 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.clients }}{{- if .Values.clients.enabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ template "pushProxy.client.name" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +rules: +{{- if .Values.global.cattle.psp.enabled }} +- apiGroups: ['policy'] + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - {{ template "pushProxy.client.name" . }} +{{- end }} +{{- if and .Values.clients.https.enabled .Values.clients.https.useServiceAccountCredentials }} +- nonResourceURLs: ["/metrics"] + verbs: ["get"] +{{- if .Values.clients.rbac.additionalRules }} +{{ toYaml .Values.clients.rbac.additionalRules }} +{{- end }} +{{- end }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ template "pushProxy.client.name" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "pushProxy.client.name" . }} +subjects: + - kind: ServiceAccount + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +--- +{{- if .Values.clients.https.useServiceAccountCredentials }} +apiVersion: v1 +kind: Secret +type: kubernetes.io/service-account-token +metadata: + name: {{ template "pushProxy.client.serviceAccountTokenName" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} + annotations: + kubernetes.io/service-account.name: {{ template "pushProxy.client.name" . }} +{{- end }} +--- +{{- if .Values.global.cattle.psp.enabled }} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +spec: + privileged: false + hostNetwork: true + hostIPC: false + hostPID: false + runAsUser: + rule: 'RunAsAny' + seLinux: + rule: 'RunAsAny' +{{- if and .Values.clients.https.enabled .Values.clients.https.certDir .Values.global.seLinux.enabled .Values.clients.https.seLinuxOptions }} + seLinuxOptions: {{ .Values.clients.https.seLinuxOptions | toYaml | nindent 6 }} +{{- end }} + supplementalGroups: + rule: 'MustRunAs' + ranges: + - min: 0 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + - min: 0 + max: 65535 + readOnlyRootFilesystem: false + volumes: + - 'secret' +{{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + - 'emptyDir' + - 'hostPath' + allowedHostPaths: + - pathPrefix: {{ required "Need access to volume on host with the SSL cert files to use HTTPs" .Values.clients.https.certDir }} + readOnly: true +{{- end }} +{{- end }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeProxy/templates/pushprox-clients.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeProxy/templates/pushprox-clients.yaml new file mode 100644 index 00000000000..e8fcfb38836 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeProxy/templates/pushprox-clients.yaml @@ -0,0 +1,157 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.clients }}{{- if .Values.clients.enabled }} +apiVersion: apps/v1 +{{- if .Values.clients.deployment.enabled }} +kind: Deployment +{{- else }} +kind: DaemonSet +{{- end }} +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} + pushprox-exporter: "client" +spec: + {{- if .Values.clients.deployment.enabled }} + replicas: {{ .Values.clients.deployment.replicas }} + {{- end }} + selector: + matchLabels: {{ include "pushProxy.client.labels" . | nindent 6 }} + template: + metadata: + labels: {{ include "pushProxy.client.labels" . | nindent 8 }} + spec: + {{- if .Values.clients.affinity }} + affinity: {{ toYaml .Values.clients.affinity | nindent 8 }} + {{- end }} + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} +{{- if .Values.clients.nodeSelector }} +{{ toYaml .Values.clients.nodeSelector | indent 8 }} +{{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} +{{- if .Values.clients.tolerations }} +{{ toYaml .Values.clients.tolerations | indent 8 }} +{{- end }} + hostNetwork: true + dnsPolicy: ClusterFirstWithHostNet + serviceAccountName: {{ template "pushProxy.client.name" . }} + {{- if .Values.global.imagePullSecretName }} + imagePullSecrets: + - name: {{ .Values.global.imagePullSecretName }} + {{- end }} + containers: + - name: pushprox-client + image: {{ template "system_default_registry" . }}{{ .Values.clients.image.repository }}:{{ .Values.clients.image.tag }} + command: + {{- range .Values.clients.command }} + - {{ . | quote }} + {{- end }} + args: + - --fqdn=$(HOST_IP) + - --proxy-url=$(PROXY_URL) + {{- if .Values.clients.metrics.enabled }} + - --metrics-addr=$(PORT) + {{- end }} + - --allow-port={{ required "Need .Values.metricsPort to configure client to be allowed to scrape metrics at port" .Values.metricsPort}} + {{- if .Values.clients.useLocalhost }} + - --use-localhost + {{- end }} + {{- if .Values.clients.https.enabled }} + {{- if .Values.clients.https.insecureSkipVerify }} + - --insecure-skip-verify + {{- end }} + {{- if .Values.clients.https.useServiceAccountCredentials }} + - --token-path=/var/run/secrets/kubernetes.io/serviceaccount/token + {{- end }} + {{- if .Values.clients.https.certDir }} + - --tls.cert=/etc/ssl/push-proxy/push-proxy.pem + - --tls.key=/etc/ssl/push-proxy/push-proxy-key.pem + - --tls.cacert=/etc/ssl/push-proxy/push-proxy-ca-cert.pem + {{- end }} + {{- end }} + env: + - name: HOST_IP + valueFrom: + fieldRef: + fieldPath: status.hostIP + {{- if .Values.clients.metrics.enabled }} + - name: PORT + value: :{{ .Values.clients.port }} + {{- end }} + - name: PROXY_URL + value: {{ template "pushProxy.proxyUrl" . }} + securityContext: + runAsNonRoot: true + runAsUser: 1000 + {{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + volumeMounts: + - name: metrics-cert-dir + mountPath: /etc/ssl/push-proxy + {{- end }} + {{- if .Values.clients.resources }} + resources: {{ toYaml .Values.clients.resources | nindent 10 }} + {{- end }} + {{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + initContainers: + - name: copy-certs + image: {{ template "system_default_registry" . }}{{ .Values.clients.copyCertsImage.repository }}:{{ .Values.clients.copyCertsImage.tag }} + command: + - sh + - -c + - | + echo "Searching for files to copy within the source volume" + echo "cert: ${CERT_FILE_NAME}" + echo "key: ${KEY_FILE_NAME}" + echo "cacert: ${CACERT_FILE_NAME}" + + CERT_FILE_SOURCE=$(find /etc/source/ -type f -name "${CERT_FILE_NAME}" | sort -r | head -n 1) + KEY_FILE_SOURCE=$(find /etc/source/ -type f -name "${KEY_FILE_NAME}" | sort -r | head -n 1) + CACERT_FILE_SOURCE=$(find /etc/source/ -type f -name "${CACERT_FILE_NAME}" | sort -r | head -n 1) + + test -z ${CERT_FILE_SOURCE} && echo "Failed to find cert file" && exit 1 + test -z ${KEY_FILE_SOURCE} && echo "Failed to find key file" && exit 1 + test -z ${CACERT_FILE_SOURCE} && echo "Failed to find cacert file" && exit 1 + + echo "Copying cert file from $CERT_FILE_SOURCE to $CERT_FILE_TARGET" + cp $CERT_FILE_SOURCE $CERT_FILE_TARGET || exit 1 + chmod 444 $CERT_FILE_TARGET || exit 1 + + echo "Copying key file from $KEY_FILE_SOURCE to $KEY_FILE_TARGET" + cp $KEY_FILE_SOURCE $KEY_FILE_TARGET || exit 1 + chmod 444 $KEY_FILE_TARGET || exit 1 + + echo "Copying cacert file from $CACERT_FILE_SOURCE to $CACERT_FILE_TARGET" + cp $CACERT_FILE_SOURCE $CACERT_FILE_TARGET || exit 1 + chmod 444 $CACERT_FILE_TARGET || exit 1 + env: + - name: CERT_FILE_NAME + value: {{ required "Need a TLS cert file for scraping metrics endpoint over HTTPs" .Values.clients.https.certFile }} + - name: KEY_FILE_NAME + value: {{ required "Need a TLS key file for scraping metrics endpoint over HTTPs" .Values.clients.https.keyFile }} + - name: CACERT_FILE_NAME + value: {{ required "Need a TLS CA cert file for scraping metrics endpoint over HTTPs" .Values.clients.https.caCertFile }} + - name: CERT_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy.pem + - name: KEY_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy-key.pem + - name: CACERT_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy-ca-cert.pem + securityContext: + runAsNonRoot: false +{{- if and .Values.global.seLinux.enabled .Values.clients.https.seLinuxOptions }} + seLinuxOptions: {{ .Values.clients.https.seLinuxOptions | toYaml | nindent 12 }} +{{- end }} + volumeMounts: + - name: metrics-cert-dir-source + mountPath: /etc/source + readOnly: true + - name: metrics-cert-dir + mountPath: /etc/ssl/push-proxy + volumes: + - name: metrics-cert-dir-source + hostPath: + path: {{ required "Need access to volume on host with the SSL cert files to use HTTPs" .Values.clients.https.certDir }} + - name: metrics-cert-dir + emptyDir: {} + {{- end }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeProxy/templates/pushprox-proxy-rbac.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeProxy/templates/pushprox-proxy-rbac.yaml new file mode 100644 index 00000000000..eefe609058c --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeProxy/templates/pushprox-proxy-rbac.yaml @@ -0,0 +1,68 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if and .Values.proxy }}{{ if .Values.proxy.enabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ template "pushProxy.proxy.name" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +rules: +{{- if .Values.global.cattle.psp.enabled }} +- apiGroups: ['policy'] + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - {{ template "pushProxy.proxy.name" . }} +{{- end }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ template "pushProxy.proxy.name" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "pushProxy.proxy.name" . }} +subjects: + - kind: ServiceAccount + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +--- +{{- if .Values.global.cattle.psp.enabled }} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +spec: + privileged: false + hostNetwork: false + hostIPC: false + hostPID: false + runAsUser: + rule: 'MustRunAsNonRoot' + seLinux: + rule: 'RunAsAny' + supplementalGroups: + rule: 'MustRunAs' + ranges: + - min: 1 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + - min: 1 + max: 65535 + readOnlyRootFilesystem: false + volumes: + - 'secret' +{{- end }}{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeProxy/templates/pushprox-proxy.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeProxy/templates/pushprox-proxy.yaml new file mode 100644 index 00000000000..723bbd6c00b --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeProxy/templates/pushprox-proxy.yaml @@ -0,0 +1,57 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if and .Values.proxy }}{{ if .Values.proxy.enabled }} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} + pushprox-exporter: "proxy" +spec: + selector: + matchLabels: {{ include "pushProxy.proxy.labels" . | nindent 6 }} + template: + metadata: + labels: {{ include "pushProxy.proxy.labels" . | nindent 8 }} + spec: + securityContext: + runAsNonRoot: true + runAsUser: 1000 + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} +{{- if .Values.proxy.nodeSelector }} +{{ toYaml .Values.proxy.nodeSelector | indent 8 }} +{{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} +{{- if .Values.proxy.tolerations }} +{{ toYaml .Values.proxy.tolerations | indent 8 }} +{{- end }} + serviceAccountName: {{ template "pushProxy.proxy.name" . }} + {{- if .Values.global.imagePullSecretName }} + imagePullSecrets: + - name: {{ .Values.global.imagePullSecretName }} + {{- end }} + containers: + - name: pushprox-proxy + image: {{ template "system_default_registry" . }}{{ .Values.proxy.image.repository }}:{{ .Values.proxy.image.tag }} + command: + {{- range .Values.proxy.command }} + - {{ . | quote }} + {{- end }} + {{- if .Values.proxy.resources }} + resources: {{ toYaml .Values.proxy.resources | nindent 10 }} + {{- end }} +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +spec: + ports: + - name: pp-proxy + port: {{ required "Need .Values.proxy.port to configure proxy" .Values.proxy.port }} + protocol: TCP + targetPort: {{ .Values.proxy.port }} + selector: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeProxy/templates/pushprox-servicemonitor.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeProxy/templates/pushprox-servicemonitor.yaml new file mode 100644 index 00000000000..67eb2216b5a --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeProxy/templates/pushprox-servicemonitor.yaml @@ -0,0 +1,45 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.serviceMonitor }}{{- if .Values.serviceMonitor.enabled }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ template "pushprox.serviceMonitor.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.serviceMonitor.labels" . | nindent 4 }} +spec: + endpoints: {{include "pushProxy.serviceMonitor.endpoints" . | nindent 4 }} + jobLabel: component + podTargetLabels: + - component + - pushprox-exporter + namespaceSelector: + matchNames: + - {{ template "pushprox.namespace" . }} + selector: + matchLabels: {{ include "pushProxy.client.labels" . | nindent 6 }} +--- +{{- $selector := "" }} +{{- if not (kindIs "invalid" .Values.service) }} +{{- if not (kindIs "invalid" .Values.service.selector) }} +{{ if .Values.service.selector }} +{{- if .Values.clients.enabled }} +{{- required (printf "Cannot override .Values.service.selector=%s when .Values.clients.enabled=true" (toJson .Values.service.selector)) "" }} +{{- end }} +{{- $selector = (toYaml .Values.service.selector) }} +{{- end }} +{{- end }} +{{- end }} +apiVersion: v1 +kind: Service +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +spec: + ports: + - name: metrics + port: {{ required "Need .Values.metricsPort to configure client to listen to metrics at port" .Values.metricsPort}} + protocol: TCP + targetPort: {{ .Values.metricsPort }} + selector: {{ default (include "pushProxy.client.labels" .) $selector | nindent 4 }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeProxy/templates/validate-install-crd.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeProxy/templates/validate-install-crd.yaml new file mode 100644 index 00000000000..16abc2fa83a --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeProxy/templates/validate-install-crd.yaml @@ -0,0 +1,14 @@ +#{{- if gt (len (lookup "rbac.authorization.k8s.io/v1" "ClusterRole" "" "")) 0 -}} +# {{- $found := dict -}} +# {{- set $found "monitoring.coreos.com/v1/ServiceMonitor" false -}} +# {{- range .Capabilities.APIVersions -}} +# {{- if hasKey $found (toString .) -}} +# {{- set $found (toString .) true -}} +# {{- end -}} +# {{- end -}} +# {{- range $_, $exists := $found -}} +# {{- if (eq $exists false) -}} +# {{- required "Required CRDs are missing. Please install Prometheus Operator CRDs before installing this chart." "" -}} +# {{- end -}} +# {{- end -}} +#{{- end -}} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeProxy/templates/validate-psp-install.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeProxy/templates/validate-psp-install.yaml new file mode 100644 index 00000000000..a30c59d3b7c --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeProxy/templates/validate-psp-install.yaml @@ -0,0 +1,7 @@ +#{{- if gt (len (lookup "rbac.authorization.k8s.io/v1" "ClusterRole" "" "")) 0 -}} +#{{- if .Values.global.cattle.psp.enabled }} +#{{- if not (.Capabilities.APIVersions.Has "policy/v1beta1/PodSecurityPolicy") }} +#{{- fail "The target cluster does not have the PodSecurityPolicy API resource. Please disable PSPs in this chart before proceeding." -}} +#{{- end }} +#{{- end }} +#{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeProxy/values.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeProxy/values.yaml new file mode 100644 index 00000000000..7f38498a1c5 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeProxy/values.yaml @@ -0,0 +1,166 @@ +# Default values for rancher-pushprox. +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. + +# Default image containing both the proxy and the client was generated from the following Dockerfile +# https://github.com/prometheus-community/PushProx/blob/eeadbe766641699129920ccfaaaa30a85c67fe81/Dockerfile#L1-L15 + +# Configuration + +global: + cattle: + psp: + enabled: false + systemDefaultRegistry: "" + seLinux: + enabled: false + +# A list of Semver constraint strings (defined by https://github.com/Masterminds/semver) and values.yaml overrides. +# +# For each key in kubeVersionOverrides, this chart will check to see if the current Kubernetes cluster's version matches +# any of the semver constraints provided as keys on the map. +# +# On seeing a match, the default value for each values.yaml field overridden will be updated with the new value. +# +# If multiple matches are encountered (due to overlapping semver ranges), the matches will be applied in order. +# +# Notes: +# - On running a helm template, Helm generally assumes the kubeVersion is v1.20.0 +# - On running a helm install --dry-run, the correct kubeVersion should be chosen. +kubeVersionOverrides: [] +# - constraint: "< 1.21" +# values: +# metricsPort: 10252 +# clients: +# https: +# enabled: false +# insecureSkipVerify: false +# useServiceAccountCredentials: false + +namespaceOverride: "" + +# The component that is being monitored (i.e. etcd) +component: "component" + +# The port containing the metrics that need to be scraped +metricsPort: 2739 + +# Configure ServiceMonitor that monitors metrics from the metricsPort endpoint +serviceMonitor: + enabled: true + # A list of endpoints that will be added to the ServiceMonitor based on the Endpoint spec + # Source: https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#endpoint + # By default, proxyUrl and params._scheme will be overridden based on other values + endpoints: + - port: metrics + +# Configure Service that grabs scrape targets +service: + # The selector that is used to populate the Service's Endpoints object. + # The chart will error out on rendering templating if .Values.clients.enabled is set alongside this field, + # since it is expected that this service should point to the PushProx Clients Daemonset / Deployment + selector: {} + +clients: + enabled: true + # The port which the PushProx client will post PushProx metrics to + port: 9369 + # If unset, this will default to the URL for the proxy service: http://pushprox-{{component}}-proxy.{{namepsace}}.svc.cluster.local:{{proxy.port}} + # Should be modified if the clients are being deployed outside the cluster where the proxy rests, otherwise leave it null + proxyUrl: "" + # If set to true, the client will forward any requests from the host IP to 127.0.0.1 + # It will only allow proxy requests to the metricsPort specified + useLocalhost: false + # Configuration for accessing metrics via HTTPS + https: + # Does the client require https to access the metrics? + enabled: false + # Does the client require requests be sent to http or https? + forceHTTPSScheme: false + # If set to true, the client will create a service account with adequate permissions and set a flag + # on the client to use the service account token provided by it to make authorized scrape requests + useServiceAccountCredentials: false + # Configuration for authentication to metrics via https endpoint + authenticationMethod: + # Reads token from defined file in container + # This function is deprecated in the prometheus operator api and may be removed in a future version + bearerTokenFile: + enabled: false + bearerTokenFilePath: "/var/run/secrets/kubernetes.io/serviceaccount/token" + # Reads token from defined secret in namespace + # This function is deprecated in the prometheus operator api and may be removed in a future version + bearerTokenSecret: + enabled: false + # Reads token from defined secret in namespace + authorization: + enabled: false + type: "bearer" + credentials: + key: "token" + optional: false + # If set to true, the client will disable SSL security checks + insecureSkipVerify: false + # Directory on host where necessary TLS cert and key to scrape metrics can be found + certDir: "" + # Filenames for files located in .Values.clients.https.certDir that correspond to TLS settings + certFile: "" + keyFile: "" + caCertFile: "" + # seLinuxOptions to be passed into the container that copies certs. Should define a container with permissions to read the files in the certDir provided on the host. + # Required and only used if `clients.https.enabled` is set and `clients.https.certDir` is provided. + seLinuxOptions: {} + + metrics: + # Whether the client should publish PushProx client-specific metrics to .Values.clients.port + enabled: false + + rbac: + # Additional permissions to provide to the ServiceAccount bound to the client + # This can be used to provide additional permissions for the client to scrape metrics from the k8s API + # Only enabled if clients.https.enabled and clients.https.useServiceAccountCredentials are true + additionalRules: [] + + # Resource limits + resources: {} + + # Options to select all nodes to deploy client DaemonSet on + nodeSelector: {} + tolerations: [] + affinity: {} + + image: + repository: rancher/pushprox-client + tag: v0.1.4-rancher2-client + command: ["pushprox-client"] + + copyCertsImage: + repository: rancher/mirrored-library-busybox + tag: 1.31.1 + + # The default intention of rancher-pushprox clients is to scrape hostNetwork metrics across all nodes. + # This can be used to scrape internal Kubernetes components or DaemonSets of hostNetwork Pods in + # situations where a cloud provider firewall prevents Pod-To-Host communication but not Pod-To-Pod. + # However, if the underlying hostNetwork Pod that is being scraped is managed by a Deployment, + # this advanced option enables users to deploy the client as a Deployment instead of a DaemonSet. + # If a user deploys this feature and the underlying Deployment's number of replicas changes, the user will + # be responsible for upgrading this chart accordingly to the right number of replicas. + deployment: + enabled: false + replicas: 0 + +proxy: + enabled: true + # The port through which PushProx clients will communicate to the proxy + port: 8080 + + # Resource limits + resources: {} + + # Options to select a node to run a single proxy deployment on + nodeSelector: {} + tolerations: [] + + image: + repository: rancher/pushprox-proxy + tag: v0.1.4-rancher2-proxy + command: ["pushprox-proxy"] diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeScheduler/.helmignore b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeScheduler/.helmignore new file mode 100644 index 00000000000..0e8a0eb36f4 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeScheduler/.helmignore @@ -0,0 +1,23 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*.orig +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeScheduler/Chart.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeScheduler/Chart.yaml new file mode 100644 index 00000000000..7ff647b0c29 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeScheduler/Chart.yaml @@ -0,0 +1,15 @@ +annotations: + catalog.cattle.io/hidden: "true" + catalog.cattle.io/kube-version: '>= 1.26.0-0 < 1.31.0-0' + catalog.cattle.io/os: linux + catalog.rancher.io/certified: rancher + catalog.rancher.io/namespace: cattle-monitoring-system + catalog.rancher.io/release-name: rancher-pushprox +apiVersion: v1 +appVersion: v0.1.4-rancher2 +description: Sets up a deployment of the PushProx proxy and a DaemonSet of PushProx + clients. +kubeVersion: '>=1.26.0-0' +name: rkeScheduler +type: application +version: 0.2.1 diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeScheduler/README.md b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeScheduler/README.md new file mode 100644 index 00000000000..345002f48a2 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeScheduler/README.md @@ -0,0 +1,90 @@ +# rancher-pushprox + +A Rancher chart based on Rancher [PushProx](https://github.com/rancher/PushProx) that sets up a Deployment of a PushProx proxy and a DaemonSet of PushProx clients on a Kubernetes cluster. + +Installs [rancher-pushprox](https://github.com/rancher/charts/tree/gh-pages/packages/rancher-pushprox) to create PushProx clients that can access their host's network and register with a PushProx proxy. A [Prometheus Operator](https://github.com/coreos/prometheus-operator) ServiceMonitor CR is also included that is configured to scrape the metrics from each of the clients through the proxy. + +Using an instance of this chart is suitable for the following scenarios: +- You need to scrape metrics from a port that should not be accessible outside of the host (e.g. scraping `etcd` metrics in a hardened cluster) +- You need to scrape metrics on a host that are not exposed outside of 127.0.0.1 (e.g. scraping `kube-proxy` metrics) +- You need to scrape metrics through HTTPS using certs hosted directly on `hostPath` +- You need to scrape metrics from Kubernetes components that require authorization via a service account (e.g. permissions to make request to `/metrics`) +- You need to scrape metrics without access to cacerts (i.e. enable `insecureSkipVerify`) + +The clients and proxy are created based on a Rancher fork of the [prometheus-community/PushProx](https://github.com/prometheus-community/PushProx) project. + +## Upgrading to Kubernetes v1.25+ + +Starting in Kubernetes v1.25, [Pod Security Policies](https://kubernetes.io/docs/concepts/security/pod-security-policy/) have been removed from the Kubernetes API. + +As a result, **before upgrading to Kubernetes v1.25** (or on a fresh install in a Kubernetes v1.25+ cluster), users are expected to perform an in-place upgrade of this chart with `global.cattle.psp.enabled` set to `false` if it has been previously set to `true`. +​ +> **Note:** +> In this chart release, any previous field that was associated with any PSP resources have been removed in favor of a single global field: `global.cattle.psp.enabled`. + +> **Note:** +> If you upgrade your cluster to Kubernetes v1.25+ before removing PSPs via a `helm upgrade` (even if you manually clean up resources), **it will leave the Helm release in a broken state within the cluster such that further Helm operations will not work (`helm uninstall`, `helm upgrade`, etc.).** +> +> If your charts get stuck in this state, please consult the Rancher docs on how to clean up your Helm release secrets. + +Upon setting `global.cattle.psp.enabled` to false, the chart will remove any PSP resources deployed on its behalf from the cluster. This is the default setting for this chart. + +As a replacement for PSPs, [Pod Security Admission](https://kubernetes.io/docs/concepts/security/pod-security-admission/) should be used. Please consult the Rancher docs for more details on how to configure your chart release namespaces to work with the new Pod Security Admission and apply Pod Security Standards. + +## Configuration + +The following tables list the configurable parameters of the rancher-pushprox chart and their default values. + +### General + +#### Required +| Parameter | Description | Example | +| ----- | ----------- | ------ | +| `component` | The component that is being monitored | `kube-etcd` +| `metricsPort` | The port on the host that contains the metrics you want to scrape (e.g. `http://:/metrics`) | `2379` | +| `namespaceOverride` | The namespace to install the chart | `""` + +#### Optional +| Parameter | Description | Default | +| ----- | ----------- | ------ | +| `serviceMonitor.enabled` | Deploys a [Prometheus Operator](https://github.com/coreos/prometheus-operator/blob/master/Documentation/api.md#servicemonitor) ServiceMonitor CR that is configured to scrape metrics on the hosts that the clients are deployed on via the proxy. Also deploys a Service that points to all pods with the expected client name that exposes the `metricsPort` selected | `true` | +| `serviceMonitor.endpoints` | A list of endpoints that will be added to the ServiceMonitor based on the [Endpoint spec](https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#endpoint) | `[{port: metrics}]` | +| `service.selector` | The selector that is used to populate the Service's Endpoints object. The chart will error out on rendering templating if `.Values.clients.enabled` is set alongside this field, since it is expected that this service should point to the PushProx Clients Daemonset / Deployment | `{}` | +| `clients.enabled` | Deploys a DaemonSet of clients that are each capable of scraping endpoints on the hostNetwork it is deployed on | `true` | +| `clients.port` | The port where the client will publish PushProx client-specific metrics. If deploying multiple clients onto the same node, the clients should not have conflicting ports | `9369` | +| `clients.proxyUrl` | Overrides the default proxyUrl setting of `http://pushprox-{{ .Values.component }}-proxy.{{ . Release.Namespace }}.svc.cluster.local:{{ .Values.proxy.port }}"` with the `proxyUrl` specified | `""` | +| `clients.useLocalhost` | Sets a flag on each client deployment to redirect scrapes directed to `HOST_IP` to `127.0.0.1` | `false` | +| `clients.https.enabled` | Enables scraping metrics via HTTPS using the provided TLS certs that exist on each host | `false` | +| `clients.https.forceHTTPSScheme` | Forces scraping metrics via HTTPS using the provided TLS certs that exist on each host | `false` | +| `clients.https.useServiceAccountCredentials` | If set to true, the client will create a service account with permissions to scrape `/metrics` endpoint of Kubernetes components. The client will use the service account token provided to make authorized scrape requests to the Kubernetes API | `false` | +| `clients.https.authenticationMethod.bearerTokenFile.enabled` | If set to true, the client will use service account credentials mounted at the configured path `clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath`. This requires permissions to scrape `/metrics` endpoint of Kubernetes components. This method is deprecated by the prometheus operator and may be removed in a future release | `false` | +| `clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath` | This is a volume mount on the pod with permissions to scrape `/metrics` endpoint of Kubernetes components | `"/var/run/secrets/kubernetes.io/serviceaccount/token"` | +| `clients.https.authenticationMethod.bearerTokenSecret.enabled` | If set to true, the client will use service account credentials to scrape `/metrics` endpoint of Kubernetes components. This method is deprecated by the prometheus operator and may be removed in a future release | `false` | +| `clients.https.authenticationMethod.authorization.enabled` | If set to true, the client will use service account credentials to scrape `/metrics` endpoint of Kubernetes components | `false` | +| `clients.https.authenticationMethod.authorization.type` | If set, the client will use this type of authorization in its client requests for metrics | `"bearer"` | +| `clients.https.authenticationMethod.authorization.credentials.key` | If set, the client will use this key in the secret created by `clients.https.useServiceAccountCredentials` for authorization in its client requests for metrics | `"token"` | +| `clients.https.authenticationMethod.authorization.credentials.optional` | If set to false, the client will fail if the key in the secret created by `clients.https.useServiceAccountCredentials` does not exist | `false` | +| `clients.https.insecureSkipVerify` | If set to true, the client will disable SSL security checks | `false` | +| `clients.https.certDir` | A `hostPath` where TLS certs can be found. This path is mounted as a volume on an `initContainer` which copies only the necessary files over to an EmptyDir volume used by each client. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.certFile` | The path to the TLS cert file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.keyFile` | The path to the TLS key file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.caCertFile` | The path to the TLS cacert file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.seLinuxOptions` | seLinuxOptions to be passed into the container that copies certs. Should define a container with permissions to read the files in the certDir provided on the host. Required and only used if `clients.https.enabled` is set and `clients.https.certDir` is provided. | `""` | +| `clients.metrics.enabled` | Whether the client should publish PushProx client-specific metrics. | `false` | +| `clients.rbac.additionalRules` | Additional permissions to provide to the ServiceAccount bound to the client. This can be used to provide additional permissions for the client to scrape metrics from the k8s API. Only enabled if clients.https.enabled and clients.https.useServiceAccountCredentials are true | `[]` | +| `clients.deployment.enabled` | Deploys the client as a Deployment (generally used if the underlying hostNetwork Pod that is being scraped is managed by a Deployment) | `false` | +| `clients.deployment.replicas` | The number of pods the Deployment has, it should match the number of pod the hostNetwork Deployment has. Required and only used if `client.deployment.enable` is set | `0` | +| `clients.deployment.affinity` | The affinity rules that allocate the pod to the node in which the hostNetwork Deployment's pods run. Required and only used if `client.deployment.enable` is set | `{}` | +| `clients.resources` | Set resource limits and requests for the client container | `{}` | +| `clients.nodeSelector` | Select which nodes to deploy the clients on | `{}` | +| `clients.tolerations` | Specify tolerations for clients | `[]` | +| `proxy.enabled` | Deploys the proxy that each client will register with | `true` | +| `proxy.port` | The port exposed by the proxy that each client will register with to allow metrics to be scraped from the host | `8080` | +| `proxy.resources` | Set resource limits and requests for the proxy container | `{}` | +| `proxy.nodeSelector` | Select which nodes the proxy can be deployed on | `{}` | +| `proxy.tolerations` | Specify tolerations (if necessary) to allow the proxy to be deployed on the selected node | `[]` | +| `kubeVersionOverrides` | A list of Semver constraint strings (defined by https://github.com/Masterminds/semver) and values.yaml overrides. For each key in kubeVersionOverrides, this chart will check to see if the current Kubernetes cluster's version matches any of the semver constraints provided as keys on the map. On seeing a match, the default value for each values.yaml field overridden will be updated with the new value. If multiple matches are encountered (due to overlapping semver ranges), the matches will be applied in order. | `[]` + +*Tip: The filepaths set in `clients.https.File` can include wildcard characters*. + +See [rancher-monitoring](https://github.com/rancher/charts/tree/gh-pages/packages/rancher-monitoring) for examples of how this chart can be used. diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeScheduler/templates/_helpers.tpl b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeScheduler/templates/_helpers.tpl new file mode 100644 index 00000000000..1ba50939441 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeScheduler/templates/_helpers.tpl @@ -0,0 +1,170 @@ +# Rancher + +{{- define "system_default_registry" -}} +{{- if .Values.global.cattle.systemDefaultRegistry -}} +{{- printf "%s/" .Values.global.cattle.systemDefaultRegistry -}} +{{- end -}} +{{- end -}} + +# Windows Support + +{{/* +Windows cluster will add default taint for linux nodes, +add below linux tolerations to workloads could be scheduled to those linux nodes +*/}} + +{{- define "linux-node-tolerations" -}} +- key: "cattle.io/os" + value: "linux" + effect: "NoSchedule" + operator: "Equal" +{{- end -}} + +{{- define "linux-node-selector" -}} +{{- if semverCompare "<1.14-0" .Capabilities.KubeVersion.GitVersion -}} +beta.kubernetes.io/os: linux +{{- else -}} +kubernetes.io/os: linux +{{- end -}} +{{- end -}} + +# General + +{{- define "applyKubeVersionOverrides" -}} +{{- $overrides := dict -}} +{{- range $override := .Values.kubeVersionOverrides -}} +{{- if semverCompare $override.constraint $.Capabilities.KubeVersion.Version -}} +{{- $_ := mergeOverwrite $overrides $override.values -}} +{{- end -}} +{{- end -}} +{{- $_ := mergeOverwrite .Values $overrides -}} +{{- end -}} + +{{- define "pushprox.namespace" -}} + {{- if .Values.namespaceOverride -}} + {{- .Values.namespaceOverride -}} + {{- else -}} + {{- .Release.Namespace -}} + {{- end -}} +{{- end -}} + +{{- define "pushProxy.commonLabels" -}} +release: {{ .Release.Name }} +component: {{ .Values.component | quote }} +provider: kubernetes +{{- end -}} + +{{- define "pushProxy.proxyUrl" -}} +{{- $_ := (required "Template requires either .Values.proxy.port or .Values.client.proxyUrl to set proxyUrl for client" (or .Values.clients.proxyUrl .Values.proxy.port)) -}} +{{- if .Values.clients.proxyUrl -}} +{{ printf "%s" .Values.clients.proxyUrl }} +{{- else -}} +{{ printf "http://%s.%s.svc:%d" (include "pushProxy.proxy.name" .) (include "pushprox.namespace" .) (int .Values.proxy.port) }} +{{- end -}}{{- end -}} + +# Client + +{{- define "pushProxy.client.name" -}} +{{- printf "pushprox-%s-client" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.client.serviceAccountTokenName" -}} +{{- printf "pushprox-%s-client-service-account-token" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.client.labels" -}} +k8s-app: {{ template "pushProxy.client.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +# Proxy + +{{- define "pushProxy.proxy.name" -}} +{{- printf "pushprox-%s-proxy" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.proxy.labels" -}} +k8s-app: {{ template "pushProxy.proxy.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +# ServiceMonitor + +{{- define "pushprox.serviceMonitor.name" -}} +{{- printf "%s-%s" .Release.Name (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.serviceMonitor.labels" -}} +app: {{ template "pushprox.serviceMonitor.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +{{- define "pushProxy.serviceMonitor.endpoints" -}} +{{- $proxyURL := (include "pushProxy.proxyUrl" .) -}} +{{- $useHTTPS := .Values.clients.https.enabled -}} +{{- $setHTTPSScheme := .Values.clients.https.forceHTTPSScheme -}} +{{- $insecureSkipVerify := .Values.clients.https.insecureSkipVerify -}} +{{- $useServiceAccountCredentials := .Values.clients.https.useServiceAccountCredentials -}} +{{- $serviceAccountTokenName := (include "pushProxy.client.serviceAccountTokenName" . ) -}} +{{- $metricRelabelings := list }} +{{- $endpoints := .Values.serviceMonitor.endpoints }} +{{- if .Values.proxy.enabled }} +{{- $_ := set . "proxyUrl" $proxyURL }} +{{- end }} +{{- range $endpoints }} +{{- if $.Values.proxy.enabled }} +{{- $_ := set . "proxyUrl" $proxyURL }} +{{- end }} +{{- $clusterIdRelabel := dict }} +{{- $metricRelabelings := list }} +{{- if $.Values.global.cattle.clusterId }} +{{- $_ := set $clusterIdRelabel "action" "replace" }} +{{- $_ := set $clusterIdRelabel "sourceLabels" (list "__address__") }} +{{- $_ := set $clusterIdRelabel "targetLabel" "cluster_id" }} +{{- $_ := set $clusterIdRelabel "replacement" $.Values.global.cattle.clusterId }} +{{- $metricRelabelings = append $metricRelabelings $clusterIdRelabel }} +{{- end }} +{{- $clusterNameRelabel := dict }} +{{- if $.Values.global.cattle.clusterName }} +{{- $_ := set $clusterNameRelabel "action" "replace" }} +{{- $_ := set $clusterNameRelabel "sourceLabels" (list "__address__") }} +{{- $_ := set $clusterNameRelabel "targetLabel" "cluster_name" }} +{{- $_ := set $clusterNameRelabel "replacement" $.Values.global.cattle.clusterName }} +{{- $metricRelabelings = append $metricRelabelings $clusterNameRelabel }} +{{- end }} +{{- if not (empty $metricRelabelings) }} +{{- $_ := set . "metricRelabelings" ($metricRelabelings)}} +{{- end }} +{{- if $setHTTPSScheme -}} +{{- $_ := set . "scheme" "https" }} +{{- end -}} +{{- if $useHTTPS -}} +{{- if (hasKey . "params") }} +{{- $_ := set (get . "params") "_scheme" (list "https") }} +{{- else }} +{{- $_ := set . "params" (dict "_scheme" (list "https")) }} +{{- end }} +{{- end }} +{{- if (hasKey . "tlsConfig") }} +{{- $_ := set (get . "tlsConfig") "insecureSkipVerify" $insecureSkipVerify }} +{{- else }} +{{- $_ := set . "tlsConfig" (dict "insecureSkipVerify" $insecureSkipVerify) }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.bearerTokenFile.enabled }} +{{- $_ := set . "bearerTokenFile" $.Values.clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.bearerTokenSecret.enabled }} +{{- $_ := set . "bearerTokenSecret" $serviceAccountTokenName }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.authorization.enabled }} +{{- if (hasKey . "authorization") }} +{{- $_ := set (get . "authorization") "type" $.Values.clients.https.authenticationMethod.authorization.type }} +{{- $_ := set (get . "authorization") "credentials" (dict "name" $serviceAccountTokenName "key" $.Values.clients.https.authenticationMethod.authorization.credentials.key "optional" $.Values.clients.https.authenticationMethod.authorization.credentials.optional) }} +{{- else }} +{{- $_ := set . "authorization" (dict "type" $.Values.clients.https.authenticationMethod.authorization.type) }} +{{- $_ := set . "authorization" (dict "credentials" (dict "name" $serviceAccountTokenName "key" $.Values.clients.https.authenticationMethod.authorization.credentials.key "optional" $.Values.clients.https.authenticationMethod.authorization.credentials.optional)) }} +{{- end }} +{{- end }} +{{- end }} +{{- toYaml $endpoints }} +{{- end -}} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeScheduler/templates/pushprox-clients-rbac.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeScheduler/templates/pushprox-clients-rbac.yaml new file mode 100644 index 00000000000..a8e27c37356 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeScheduler/templates/pushprox-clients-rbac.yaml @@ -0,0 +1,97 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.clients }}{{- if .Values.clients.enabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ template "pushProxy.client.name" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +rules: +{{- if .Values.global.cattle.psp.enabled }} +- apiGroups: ['policy'] + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - {{ template "pushProxy.client.name" . }} +{{- end }} +{{- if and .Values.clients.https.enabled .Values.clients.https.useServiceAccountCredentials }} +- nonResourceURLs: ["/metrics"] + verbs: ["get"] +{{- if .Values.clients.rbac.additionalRules }} +{{ toYaml .Values.clients.rbac.additionalRules }} +{{- end }} +{{- end }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ template "pushProxy.client.name" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "pushProxy.client.name" . }} +subjects: + - kind: ServiceAccount + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +--- +{{- if .Values.clients.https.useServiceAccountCredentials }} +apiVersion: v1 +kind: Secret +type: kubernetes.io/service-account-token +metadata: + name: {{ template "pushProxy.client.serviceAccountTokenName" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} + annotations: + kubernetes.io/service-account.name: {{ template "pushProxy.client.name" . }} +{{- end }} +--- +{{- if .Values.global.cattle.psp.enabled }} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +spec: + privileged: false + hostNetwork: true + hostIPC: false + hostPID: false + runAsUser: + rule: 'RunAsAny' + seLinux: + rule: 'RunAsAny' +{{- if and .Values.clients.https.enabled .Values.clients.https.certDir .Values.global.seLinux.enabled .Values.clients.https.seLinuxOptions }} + seLinuxOptions: {{ .Values.clients.https.seLinuxOptions | toYaml | nindent 6 }} +{{- end }} + supplementalGroups: + rule: 'MustRunAs' + ranges: + - min: 0 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + - min: 0 + max: 65535 + readOnlyRootFilesystem: false + volumes: + - 'secret' +{{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + - 'emptyDir' + - 'hostPath' + allowedHostPaths: + - pathPrefix: {{ required "Need access to volume on host with the SSL cert files to use HTTPs" .Values.clients.https.certDir }} + readOnly: true +{{- end }} +{{- end }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeScheduler/templates/pushprox-clients.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeScheduler/templates/pushprox-clients.yaml new file mode 100644 index 00000000000..e8fcfb38836 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeScheduler/templates/pushprox-clients.yaml @@ -0,0 +1,157 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.clients }}{{- if .Values.clients.enabled }} +apiVersion: apps/v1 +{{- if .Values.clients.deployment.enabled }} +kind: Deployment +{{- else }} +kind: DaemonSet +{{- end }} +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} + pushprox-exporter: "client" +spec: + {{- if .Values.clients.deployment.enabled }} + replicas: {{ .Values.clients.deployment.replicas }} + {{- end }} + selector: + matchLabels: {{ include "pushProxy.client.labels" . | nindent 6 }} + template: + metadata: + labels: {{ include "pushProxy.client.labels" . | nindent 8 }} + spec: + {{- if .Values.clients.affinity }} + affinity: {{ toYaml .Values.clients.affinity | nindent 8 }} + {{- end }} + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} +{{- if .Values.clients.nodeSelector }} +{{ toYaml .Values.clients.nodeSelector | indent 8 }} +{{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} +{{- if .Values.clients.tolerations }} +{{ toYaml .Values.clients.tolerations | indent 8 }} +{{- end }} + hostNetwork: true + dnsPolicy: ClusterFirstWithHostNet + serviceAccountName: {{ template "pushProxy.client.name" . }} + {{- if .Values.global.imagePullSecretName }} + imagePullSecrets: + - name: {{ .Values.global.imagePullSecretName }} + {{- end }} + containers: + - name: pushprox-client + image: {{ template "system_default_registry" . }}{{ .Values.clients.image.repository }}:{{ .Values.clients.image.tag }} + command: + {{- range .Values.clients.command }} + - {{ . | quote }} + {{- end }} + args: + - --fqdn=$(HOST_IP) + - --proxy-url=$(PROXY_URL) + {{- if .Values.clients.metrics.enabled }} + - --metrics-addr=$(PORT) + {{- end }} + - --allow-port={{ required "Need .Values.metricsPort to configure client to be allowed to scrape metrics at port" .Values.metricsPort}} + {{- if .Values.clients.useLocalhost }} + - --use-localhost + {{- end }} + {{- if .Values.clients.https.enabled }} + {{- if .Values.clients.https.insecureSkipVerify }} + - --insecure-skip-verify + {{- end }} + {{- if .Values.clients.https.useServiceAccountCredentials }} + - --token-path=/var/run/secrets/kubernetes.io/serviceaccount/token + {{- end }} + {{- if .Values.clients.https.certDir }} + - --tls.cert=/etc/ssl/push-proxy/push-proxy.pem + - --tls.key=/etc/ssl/push-proxy/push-proxy-key.pem + - --tls.cacert=/etc/ssl/push-proxy/push-proxy-ca-cert.pem + {{- end }} + {{- end }} + env: + - name: HOST_IP + valueFrom: + fieldRef: + fieldPath: status.hostIP + {{- if .Values.clients.metrics.enabled }} + - name: PORT + value: :{{ .Values.clients.port }} + {{- end }} + - name: PROXY_URL + value: {{ template "pushProxy.proxyUrl" . }} + securityContext: + runAsNonRoot: true + runAsUser: 1000 + {{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + volumeMounts: + - name: metrics-cert-dir + mountPath: /etc/ssl/push-proxy + {{- end }} + {{- if .Values.clients.resources }} + resources: {{ toYaml .Values.clients.resources | nindent 10 }} + {{- end }} + {{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + initContainers: + - name: copy-certs + image: {{ template "system_default_registry" . }}{{ .Values.clients.copyCertsImage.repository }}:{{ .Values.clients.copyCertsImage.tag }} + command: + - sh + - -c + - | + echo "Searching for files to copy within the source volume" + echo "cert: ${CERT_FILE_NAME}" + echo "key: ${KEY_FILE_NAME}" + echo "cacert: ${CACERT_FILE_NAME}" + + CERT_FILE_SOURCE=$(find /etc/source/ -type f -name "${CERT_FILE_NAME}" | sort -r | head -n 1) + KEY_FILE_SOURCE=$(find /etc/source/ -type f -name "${KEY_FILE_NAME}" | sort -r | head -n 1) + CACERT_FILE_SOURCE=$(find /etc/source/ -type f -name "${CACERT_FILE_NAME}" | sort -r | head -n 1) + + test -z ${CERT_FILE_SOURCE} && echo "Failed to find cert file" && exit 1 + test -z ${KEY_FILE_SOURCE} && echo "Failed to find key file" && exit 1 + test -z ${CACERT_FILE_SOURCE} && echo "Failed to find cacert file" && exit 1 + + echo "Copying cert file from $CERT_FILE_SOURCE to $CERT_FILE_TARGET" + cp $CERT_FILE_SOURCE $CERT_FILE_TARGET || exit 1 + chmod 444 $CERT_FILE_TARGET || exit 1 + + echo "Copying key file from $KEY_FILE_SOURCE to $KEY_FILE_TARGET" + cp $KEY_FILE_SOURCE $KEY_FILE_TARGET || exit 1 + chmod 444 $KEY_FILE_TARGET || exit 1 + + echo "Copying cacert file from $CACERT_FILE_SOURCE to $CACERT_FILE_TARGET" + cp $CACERT_FILE_SOURCE $CACERT_FILE_TARGET || exit 1 + chmod 444 $CACERT_FILE_TARGET || exit 1 + env: + - name: CERT_FILE_NAME + value: {{ required "Need a TLS cert file for scraping metrics endpoint over HTTPs" .Values.clients.https.certFile }} + - name: KEY_FILE_NAME + value: {{ required "Need a TLS key file for scraping metrics endpoint over HTTPs" .Values.clients.https.keyFile }} + - name: CACERT_FILE_NAME + value: {{ required "Need a TLS CA cert file for scraping metrics endpoint over HTTPs" .Values.clients.https.caCertFile }} + - name: CERT_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy.pem + - name: KEY_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy-key.pem + - name: CACERT_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy-ca-cert.pem + securityContext: + runAsNonRoot: false +{{- if and .Values.global.seLinux.enabled .Values.clients.https.seLinuxOptions }} + seLinuxOptions: {{ .Values.clients.https.seLinuxOptions | toYaml | nindent 12 }} +{{- end }} + volumeMounts: + - name: metrics-cert-dir-source + mountPath: /etc/source + readOnly: true + - name: metrics-cert-dir + mountPath: /etc/ssl/push-proxy + volumes: + - name: metrics-cert-dir-source + hostPath: + path: {{ required "Need access to volume on host with the SSL cert files to use HTTPs" .Values.clients.https.certDir }} + - name: metrics-cert-dir + emptyDir: {} + {{- end }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeScheduler/templates/pushprox-proxy-rbac.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeScheduler/templates/pushprox-proxy-rbac.yaml new file mode 100644 index 00000000000..eefe609058c --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeScheduler/templates/pushprox-proxy-rbac.yaml @@ -0,0 +1,68 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if and .Values.proxy }}{{ if .Values.proxy.enabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ template "pushProxy.proxy.name" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +rules: +{{- if .Values.global.cattle.psp.enabled }} +- apiGroups: ['policy'] + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - {{ template "pushProxy.proxy.name" . }} +{{- end }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ template "pushProxy.proxy.name" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "pushProxy.proxy.name" . }} +subjects: + - kind: ServiceAccount + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +--- +{{- if .Values.global.cattle.psp.enabled }} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +spec: + privileged: false + hostNetwork: false + hostIPC: false + hostPID: false + runAsUser: + rule: 'MustRunAsNonRoot' + seLinux: + rule: 'RunAsAny' + supplementalGroups: + rule: 'MustRunAs' + ranges: + - min: 1 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + - min: 1 + max: 65535 + readOnlyRootFilesystem: false + volumes: + - 'secret' +{{- end }}{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeScheduler/templates/pushprox-proxy.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeScheduler/templates/pushprox-proxy.yaml new file mode 100644 index 00000000000..723bbd6c00b --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeScheduler/templates/pushprox-proxy.yaml @@ -0,0 +1,57 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if and .Values.proxy }}{{ if .Values.proxy.enabled }} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} + pushprox-exporter: "proxy" +spec: + selector: + matchLabels: {{ include "pushProxy.proxy.labels" . | nindent 6 }} + template: + metadata: + labels: {{ include "pushProxy.proxy.labels" . | nindent 8 }} + spec: + securityContext: + runAsNonRoot: true + runAsUser: 1000 + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} +{{- if .Values.proxy.nodeSelector }} +{{ toYaml .Values.proxy.nodeSelector | indent 8 }} +{{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} +{{- if .Values.proxy.tolerations }} +{{ toYaml .Values.proxy.tolerations | indent 8 }} +{{- end }} + serviceAccountName: {{ template "pushProxy.proxy.name" . }} + {{- if .Values.global.imagePullSecretName }} + imagePullSecrets: + - name: {{ .Values.global.imagePullSecretName }} + {{- end }} + containers: + - name: pushprox-proxy + image: {{ template "system_default_registry" . }}{{ .Values.proxy.image.repository }}:{{ .Values.proxy.image.tag }} + command: + {{- range .Values.proxy.command }} + - {{ . | quote }} + {{- end }} + {{- if .Values.proxy.resources }} + resources: {{ toYaml .Values.proxy.resources | nindent 10 }} + {{- end }} +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +spec: + ports: + - name: pp-proxy + port: {{ required "Need .Values.proxy.port to configure proxy" .Values.proxy.port }} + protocol: TCP + targetPort: {{ .Values.proxy.port }} + selector: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeScheduler/templates/pushprox-servicemonitor.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeScheduler/templates/pushprox-servicemonitor.yaml new file mode 100644 index 00000000000..67eb2216b5a --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeScheduler/templates/pushprox-servicemonitor.yaml @@ -0,0 +1,45 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.serviceMonitor }}{{- if .Values.serviceMonitor.enabled }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ template "pushprox.serviceMonitor.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.serviceMonitor.labels" . | nindent 4 }} +spec: + endpoints: {{include "pushProxy.serviceMonitor.endpoints" . | nindent 4 }} + jobLabel: component + podTargetLabels: + - component + - pushprox-exporter + namespaceSelector: + matchNames: + - {{ template "pushprox.namespace" . }} + selector: + matchLabels: {{ include "pushProxy.client.labels" . | nindent 6 }} +--- +{{- $selector := "" }} +{{- if not (kindIs "invalid" .Values.service) }} +{{- if not (kindIs "invalid" .Values.service.selector) }} +{{ if .Values.service.selector }} +{{- if .Values.clients.enabled }} +{{- required (printf "Cannot override .Values.service.selector=%s when .Values.clients.enabled=true" (toJson .Values.service.selector)) "" }} +{{- end }} +{{- $selector = (toYaml .Values.service.selector) }} +{{- end }} +{{- end }} +{{- end }} +apiVersion: v1 +kind: Service +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +spec: + ports: + - name: metrics + port: {{ required "Need .Values.metricsPort to configure client to listen to metrics at port" .Values.metricsPort}} + protocol: TCP + targetPort: {{ .Values.metricsPort }} + selector: {{ default (include "pushProxy.client.labels" .) $selector | nindent 4 }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeScheduler/templates/validate-install-crd.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeScheduler/templates/validate-install-crd.yaml new file mode 100644 index 00000000000..16abc2fa83a --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeScheduler/templates/validate-install-crd.yaml @@ -0,0 +1,14 @@ +#{{- if gt (len (lookup "rbac.authorization.k8s.io/v1" "ClusterRole" "" "")) 0 -}} +# {{- $found := dict -}} +# {{- set $found "monitoring.coreos.com/v1/ServiceMonitor" false -}} +# {{- range .Capabilities.APIVersions -}} +# {{- if hasKey $found (toString .) -}} +# {{- set $found (toString .) true -}} +# {{- end -}} +# {{- end -}} +# {{- range $_, $exists := $found -}} +# {{- if (eq $exists false) -}} +# {{- required "Required CRDs are missing. Please install Prometheus Operator CRDs before installing this chart." "" -}} +# {{- end -}} +# {{- end -}} +#{{- end -}} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeScheduler/templates/validate-psp-install.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeScheduler/templates/validate-psp-install.yaml new file mode 100644 index 00000000000..a30c59d3b7c --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeScheduler/templates/validate-psp-install.yaml @@ -0,0 +1,7 @@ +#{{- if gt (len (lookup "rbac.authorization.k8s.io/v1" "ClusterRole" "" "")) 0 -}} +#{{- if .Values.global.cattle.psp.enabled }} +#{{- if not (.Capabilities.APIVersions.Has "policy/v1beta1/PodSecurityPolicy") }} +#{{- fail "The target cluster does not have the PodSecurityPolicy API resource. Please disable PSPs in this chart before proceeding." -}} +#{{- end }} +#{{- end }} +#{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeScheduler/values.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeScheduler/values.yaml new file mode 100644 index 00000000000..7f38498a1c5 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/rkeScheduler/values.yaml @@ -0,0 +1,166 @@ +# Default values for rancher-pushprox. +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. + +# Default image containing both the proxy and the client was generated from the following Dockerfile +# https://github.com/prometheus-community/PushProx/blob/eeadbe766641699129920ccfaaaa30a85c67fe81/Dockerfile#L1-L15 + +# Configuration + +global: + cattle: + psp: + enabled: false + systemDefaultRegistry: "" + seLinux: + enabled: false + +# A list of Semver constraint strings (defined by https://github.com/Masterminds/semver) and values.yaml overrides. +# +# For each key in kubeVersionOverrides, this chart will check to see if the current Kubernetes cluster's version matches +# any of the semver constraints provided as keys on the map. +# +# On seeing a match, the default value for each values.yaml field overridden will be updated with the new value. +# +# If multiple matches are encountered (due to overlapping semver ranges), the matches will be applied in order. +# +# Notes: +# - On running a helm template, Helm generally assumes the kubeVersion is v1.20.0 +# - On running a helm install --dry-run, the correct kubeVersion should be chosen. +kubeVersionOverrides: [] +# - constraint: "< 1.21" +# values: +# metricsPort: 10252 +# clients: +# https: +# enabled: false +# insecureSkipVerify: false +# useServiceAccountCredentials: false + +namespaceOverride: "" + +# The component that is being monitored (i.e. etcd) +component: "component" + +# The port containing the metrics that need to be scraped +metricsPort: 2739 + +# Configure ServiceMonitor that monitors metrics from the metricsPort endpoint +serviceMonitor: + enabled: true + # A list of endpoints that will be added to the ServiceMonitor based on the Endpoint spec + # Source: https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#endpoint + # By default, proxyUrl and params._scheme will be overridden based on other values + endpoints: + - port: metrics + +# Configure Service that grabs scrape targets +service: + # The selector that is used to populate the Service's Endpoints object. + # The chart will error out on rendering templating if .Values.clients.enabled is set alongside this field, + # since it is expected that this service should point to the PushProx Clients Daemonset / Deployment + selector: {} + +clients: + enabled: true + # The port which the PushProx client will post PushProx metrics to + port: 9369 + # If unset, this will default to the URL for the proxy service: http://pushprox-{{component}}-proxy.{{namepsace}}.svc.cluster.local:{{proxy.port}} + # Should be modified if the clients are being deployed outside the cluster where the proxy rests, otherwise leave it null + proxyUrl: "" + # If set to true, the client will forward any requests from the host IP to 127.0.0.1 + # It will only allow proxy requests to the metricsPort specified + useLocalhost: false + # Configuration for accessing metrics via HTTPS + https: + # Does the client require https to access the metrics? + enabled: false + # Does the client require requests be sent to http or https? + forceHTTPSScheme: false + # If set to true, the client will create a service account with adequate permissions and set a flag + # on the client to use the service account token provided by it to make authorized scrape requests + useServiceAccountCredentials: false + # Configuration for authentication to metrics via https endpoint + authenticationMethod: + # Reads token from defined file in container + # This function is deprecated in the prometheus operator api and may be removed in a future version + bearerTokenFile: + enabled: false + bearerTokenFilePath: "/var/run/secrets/kubernetes.io/serviceaccount/token" + # Reads token from defined secret in namespace + # This function is deprecated in the prometheus operator api and may be removed in a future version + bearerTokenSecret: + enabled: false + # Reads token from defined secret in namespace + authorization: + enabled: false + type: "bearer" + credentials: + key: "token" + optional: false + # If set to true, the client will disable SSL security checks + insecureSkipVerify: false + # Directory on host where necessary TLS cert and key to scrape metrics can be found + certDir: "" + # Filenames for files located in .Values.clients.https.certDir that correspond to TLS settings + certFile: "" + keyFile: "" + caCertFile: "" + # seLinuxOptions to be passed into the container that copies certs. Should define a container with permissions to read the files in the certDir provided on the host. + # Required and only used if `clients.https.enabled` is set and `clients.https.certDir` is provided. + seLinuxOptions: {} + + metrics: + # Whether the client should publish PushProx client-specific metrics to .Values.clients.port + enabled: false + + rbac: + # Additional permissions to provide to the ServiceAccount bound to the client + # This can be used to provide additional permissions for the client to scrape metrics from the k8s API + # Only enabled if clients.https.enabled and clients.https.useServiceAccountCredentials are true + additionalRules: [] + + # Resource limits + resources: {} + + # Options to select all nodes to deploy client DaemonSet on + nodeSelector: {} + tolerations: [] + affinity: {} + + image: + repository: rancher/pushprox-client + tag: v0.1.4-rancher2-client + command: ["pushprox-client"] + + copyCertsImage: + repository: rancher/mirrored-library-busybox + tag: 1.31.1 + + # The default intention of rancher-pushprox clients is to scrape hostNetwork metrics across all nodes. + # This can be used to scrape internal Kubernetes components or DaemonSets of hostNetwork Pods in + # situations where a cloud provider firewall prevents Pod-To-Host communication but not Pod-To-Pod. + # However, if the underlying hostNetwork Pod that is being scraped is managed by a Deployment, + # this advanced option enables users to deploy the client as a Deployment instead of a DaemonSet. + # If a user deploys this feature and the underlying Deployment's number of replicas changes, the user will + # be responsible for upgrading this chart accordingly to the right number of replicas. + deployment: + enabled: false + replicas: 0 + +proxy: + enabled: true + # The port through which PushProx clients will communicate to the proxy + port: 8080 + + # Resource limits + resources: {} + + # Options to select a node to run a single proxy deployment on + nodeSelector: {} + tolerations: [] + + image: + repository: rancher/pushprox-proxy + tag: v0.1.4-rancher2-proxy + command: ["pushprox-proxy"] diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/windowsExporter/.helmignore b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/windowsExporter/.helmignore new file mode 100644 index 00000000000..f0c13194444 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/windowsExporter/.helmignore @@ -0,0 +1,21 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*~ +# Various IDEs +.project +.idea/ +*.tmproj diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/windowsExporter/Chart.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/windowsExporter/Chart.yaml new file mode 100644 index 00000000000..784bb0ec7e0 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/windowsExporter/Chart.yaml @@ -0,0 +1,17 @@ +apiVersion: v2 +appVersion: 0.25.1 +description: A Helm chart for prometheus windows-exporter +home: https://github.com/prometheus-community/windows_exporter/ +keywords: +- windows-exporter +- windows +- prometheus +- exporter +maintainers: +- email: github@jkroepke.de + name: jkroepke +name: windowsExporter +sources: +- https://github.com/prometheus-community/windows_exporter/ +type: application +version: 0.3.1 diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/windowsExporter/README.md b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/windowsExporter/README.md new file mode 100644 index 00000000000..1da1c64e12b --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/windowsExporter/README.md @@ -0,0 +1,42 @@ +# Prometheus `Windows Exporter` + +Prometheus exporter for hardware and OS metrics exposed by Windows kernels, written in Go with pluggable metric collectors. + +This chart bootstraps a prometheus [`Windows Exporter`](http://github.com/prometheus-community/windows_exporter) daemonset on a [Kubernetes](http://kubernetes.io) cluster using the [Helm](https://helm.sh) package manager. + +## Get Repository Info + +```console +helm repo add prometheus-community https://prometheus-community.github.io/helm-charts +helm repo update +``` + +_See [`helm repo`](https://helm.sh/docs/helm/helm_repo/) for command documentation._ + +## Install Chart + +```console +helm install [RELEASE_NAME] prometheus-community/prometheus-windows-exporter +``` + +_See [configuration](#configuring) below._ + +_See [helm install](https://helm.sh/docs/helm/helm_install/) for command documentation._ + +## Uninstall Chart + +```console +helm uninstall [RELEASE_NAME] +``` + +This removes all the Kubernetes components associated with the chart and deletes the release. + +_See [helm uninstall](https://helm.sh/docs/helm/helm_uninstall/) for command documentation._ + +## Configuring + +See [Customizing the Chart Before Installing](https://helm.sh/docs/intro/using_helm/#customizing-the-chart-before-installing). To see all configurable options with detailed comments, visit the chart's [values.yaml](./values.yaml), or run these configuration commands: + +```console +helm show values prometheus-community/prometheus-windows-exporter +``` diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/windowsExporter/scripts/configure-firewall.ps1 b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/windowsExporter/scripts/configure-firewall.ps1 new file mode 100644 index 00000000000..9cbed7112de --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/windowsExporter/scripts/configure-firewall.ps1 @@ -0,0 +1,31 @@ +$ErrorActionPreference = 'Continue' + +function CheckFirewallRuleError { + # We hit an error. This can happen for a number of reasons, including if the rule already exists + if ($error[0]) { + if (($error[0].Exception.NativeErrorCode) -and ($error[0].Exception.NativeErrorCode.ToString() -eq "AlreadyExists")) { + # Previous versions of monitoring may have already created this Firewall Rule + # Because of this, if the rule alreadys exists there is no need to delete and recreate it. + Write-Host "Detected Existing Firewall Rule, Nothing To Do" + } else { + Write-Host "Error Encountered Setting Up Required Firewall Rule" + $error[0].Exception + exit 1 + } + } +} + +Write-Host "Attempting To Configure Firewall Rules For Ports 9796, 10250" + +# This is the exact same firewall rule that has historically been created by rancher-wins +# https://github.com/rancher/wins/blob/91f670c47f19c6d9fe97d8f66a695d3081ad994f/pkg/apis/process_service_mgmt.go#L149 +New-NetFirewallRule -DisplayName rancher-wins-windows-exporter-TCP-9796 -Name rancher-wins-windows-exporter-TCP-9796 -Action Allow -Protocol TCP -LocalPort 9796 -Enabled True -PolicyStore ActiveStore +CheckFirewallRuleError +Write-Host "Windows Node Exporter Firewall Rule Successfully Created" + +# This rule is required in order to have the Rancher UI display node metrics in the 'Nodes' tab of the cluster explorer +New-NetFirewallRule -DisplayName rancher-wins-windows-exporter-TCP-10250 -Name rancher-wins-windows-exporter-TCP-10250 -Action Allow -Protocol TCP -LocalPort 10250 -Enabled True -PolicyStore ActiveStore +CheckFirewallRuleError +Write-Host "Windows Prometheus Metrics Firewall Rule Successfully Created" + +Write-Host "All Firewall Rules Successfully Configured" diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/windowsExporter/templates/_helpers.tpl b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/windowsExporter/templates/_helpers.tpl new file mode 100644 index 00000000000..c9a5d6db8cd --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/windowsExporter/templates/_helpers.tpl @@ -0,0 +1,216 @@ +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +The components in this chart create additional resources that expand the longest created name strings. +The longest name that gets created adds and extra 37 characters, so truncation should be 63-35=26. +*/}} +{{- define "prometheus-windows-exporter.fullname" -}} +{{ printf "%s-windows-exporter" .Release.Name }} +{{- end -}} + +{{- define "system_default_registry" -}} +{{- if .Values.global.cattle.systemDefaultRegistry -}} +{{- printf "%s/" .Values.global.cattle.systemDefaultRegistry -}} +{{- end -}} +{{- end -}} + +{{- define "windowsExporter.renamedMetricsRelabeling" -}} +{{- range $original, $new := (include "windowsExporter.renamedMetrics" . | fromJson) -}} +- sourceLabels: [__name__] + regex: {{ $original }} + replacement: '{{ $new }}' + targetLabel: __name__ +{{ end -}} +{{- end -}} + +{{- define "windowsExporter.labels" -}} +k8s-app: {{ template "prometheus-windows-exporter.fullname" . }} +release: {{ .Release.Name }} +component: "windows-exporter" +provider: kubernetes +{{- end -}} + +{{- define "windowsExporter.renamedMetrics" -}} +{{- $renamed := dict -}} +{{/* v0.15.0 */}} +{{- $_ := set $renamed "windows_mssql_transactions_active_total" "windows_mssql_transactions_active" -}} +{{/* v0.16.0 */}} +{{- $_ := set $renamed "windows_adfs_ad_login_connection_failures" "windows_adfs_ad_login_connection_failures_total" -}} +{{- $_ := set $renamed "windows_adfs_certificate_authentications" "windows_adfs_certificate_authentications_total" -}} +{{- $_ := set $renamed "windows_adfs_device_authentications" "windows_adfs_device_authentications_total" -}} +{{- $_ := set $renamed "windows_adfs_extranet_account_lockouts" "windows_adfs_extranet_account_lockouts_total" -}} +{{- $_ := set $renamed "windows_adfs_federated_authentications" "windows_adfs_federated_authentications_total" -}} +{{- $_ := set $renamed "windows_adfs_passport_authentications" "windows_adfs_passport_authentications_total" -}} +{{- $_ := set $renamed "windows_adfs_password_change_failed" "windows_adfs_password_change_failed_total" -}} +{{- $_ := set $renamed "windows_adfs_password_change_succeeded" "windows_adfs_password_change_succeeded_total" -}} +{{- $_ := set $renamed "windows_adfs_token_requests" "windows_adfs_token_requests_total" -}} +{{- $_ := set $renamed "windows_adfs_windows_integrated_authentications" "windows_adfs_windows_integrated_authentications_total" -}} +{{- $_ := set $renamed "windows_net_packets_outbound_errors" "windows_net_packets_outbound_errors_total" -}} +{{- $_ := set $renamed "windows_net_packets_received_discarded" "windows_net_packets_received_discarded_total" -}} +{{- $_ := set $renamed "windows_net_packets_received_errors" "windows_net_packets_received_errors_total" -}} +{{- $_ := set $renamed "windows_net_packets_received_total" "windows_net_packets_received_total_total" -}} +{{- $_ := set $renamed "windows_net_packets_received_unknown" "windows_net_packets_received_unknown_total" -}} +{{- $_ := set $renamed "windows_dns_memory_used_bytes_total" "windows_dns_memory_used_bytes" -}} +{{- $renamed | toJson -}} +{{- end -}} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "prometheus-windows-exporter.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Common labels +*/}} +{{- define "prometheus-windows-exporter.labels" -}} +helm.sh/chart: {{ include "prometheus-windows-exporter.chart" . }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +app.kubernetes.io/component: metrics +app.kubernetes.io/part-of: {{ include "prometheus-windows-exporter.name" . }} +{{ include "prometheus-windows-exporter.selectorLabels" . }} +{{- with .Chart.AppVersion }} +app.kubernetes.io/version: {{ . | quote }} +{{- end }} +{{- with .Values.podLabels }} +{{ toYaml . }} +{{- end }} +{{- if .Values.releaseLabel }} +release: {{ .Release.Name }} +{{- end }} +{{- end }} + +{{/* +Selector labels +*/}} +{{- define "prometheus-windows-exporter.selectorLabels" -}} +app.kubernetes.io/name: {{ include "prometheus-windows-exporter.fullname" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end }} + + +{{/* +Create the name of the service account to use +*/}} +{{- define "prometheus-windows-exporter.serviceAccountName" -}} +{{- if .Values.serviceAccount.create }} +{{- default (include "prometheus-windows-exporter.fullname" .) .Values.serviceAccount.name }} +{{- else }} +{{- default "default" .Values.serviceAccount.name }} +{{- end }} +{{- end }} + +{{/* +The image to use +*/}} +{{- define "prometheus-windows-exporter.image" -}} +{{- if .Values.image.sha }} +{{- fail "image.sha forbidden. Use image.digest instead" }} +{{- else if .Values.image.digest }} +{{- if .Values.global.cattle.systemDefaultRegistry }} +{{- printf "%s/%s:%s@%s" .Values.global.cattle.systemDefaultRegistry .Values.image.repository (default .Chart.AppVersion .Values.image.tag) .Values.image.digest }} +{{- else }} +{{- printf "%s/%s:%s@%s" .Values.image.registry .Values.image.repository (default .Chart.AppVersion .Values.image.tag) .Values.image.digest }} +{{- end }} +{{- else }} +{{- if .Values.global.cattle.systemDefaultRegistry }} +{{- printf "%s/%s:%s" .Values.global.cattle.systemDefaultRegistry .Values.image.repository (default .Chart.AppVersion .Values.image.tag) }} +{{- else }} +{{- printf "%s/%s:%s" .Values.image.registry .Values.image.repository (default .Chart.AppVersion .Values.image.tag) }} +{{- end }} +{{- end }} +{{- end }} + +{{/* +Allow the release namespace to be overridden for multi-namespace deployments in combined charts +*/}} +{{- define "prometheus-windows-exporter.namespace" -}} +{{- if .Values.namespaceOverride }} +{{- .Values.namespaceOverride }} +{{- else }} +{{- .Release.Namespace }} +{{- end }} +{{- end }} + +{{/* +Create the namespace name of the service monitor +*/}} +{{- define "prometheus-windows-exporter.monitor-namespace" -}} +{{- if .Values.namespaceOverride }} +{{- .Values.namespaceOverride }} +{{- else }} +{{- if .Values.prometheus.monitor.namespace }} +{{- .Values.prometheus.monitor.namespace }} +{{- else }} +{{- .Release.Namespace }} +{{- end }} +{{- end }} +{{- end }} + +{{/* Sets default scrape limits for servicemonitor */}} +{{- define "servicemonitor.scrapeLimits" -}} +{{- with .sampleLimit }} +sampleLimit: {{ . }} +{{- end }} +{{- with .targetLimit }} +targetLimit: {{ . }} +{{- end }} +{{- with .labelLimit }} +labelLimit: {{ . }} +{{- end }} +{{- with .labelNameLengthLimit }} +labelNameLengthLimit: {{ . }} +{{- end }} +{{- with .labelValueLengthLimit }} +labelValueLengthLimit: {{ . }} +{{- end }} +{{- end }} + +{{/* +Formats imagePullSecrets. Input is (dict "Values" .Values "imagePullSecrets" .{specific imagePullSecrets}) +*/}} +{{- define "prometheus-windows-exporter.imagePullSecrets" -}} +{{- range (concat .Values.global.imagePullSecrets .imagePullSecrets) }} + {{- if eq (typeOf .) "map[string]interface {}" }} +- {{ toYaml . | trim }} + {{- else }} +- name: {{ . }} + {{- end }} +{{- end }} +{{- end -}} + +{{/* +Create the namespace name of the pod monitor +*/}} +{{- define "prometheus-windows-exporter.podmonitor-namespace" -}} +{{- if .Values.namespaceOverride }} +{{- .Values.namespaceOverride }} +{{- else }} +{{- if .Values.prometheus.podMonitor.namespace }} +{{- .Values.prometheus.podMonitor.namespace }} +{{- else }} +{{- .Release.Namespace }} +{{- end }} +{{- end }} +{{- end }} + +{{/* Sets default scrape limits for podmonitor */}} +{{- define "podmonitor.scrapeLimits" -}} +{{- with .sampleLimit }} +sampleLimit: {{ . }} +{{- end }} +{{- with .targetLimit }} +targetLimit: {{ . }} +{{- end }} +{{- with .labelLimit }} +labelLimit: {{ . }} +{{- end }} +{{- with .labelNameLengthLimit }} +labelNameLengthLimit: {{ . }} +{{- end }} +{{- with .labelValueLengthLimit }} +labelValueLengthLimit: {{ . }} +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/windowsExporter/templates/config.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/windowsExporter/templates/config.yaml new file mode 100644 index 00000000000..25f1fa69c28 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/windowsExporter/templates/config.yaml @@ -0,0 +1,14 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ include "prometheus-windows-exporter.fullname" . }} + namespace: {{ include "prometheus-windows-exporter.namespace" . }} + labels: + {{- include "windowsExporter.labels" $ | nindent 4 }} + {{- with .Values.service.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +data: + config.yml: | + {{- .Values.config | nindent 4 }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/windowsExporter/templates/daemonset.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/windowsExporter/templates/daemonset.yaml new file mode 100644 index 00000000000..be7feb3ed1f --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/windowsExporter/templates/daemonset.yaml @@ -0,0 +1,200 @@ +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: {{ include "prometheus-windows-exporter.fullname" . }} + namespace: {{ include "prometheus-windows-exporter.namespace" . }} + labels: + {{- include "windowsExporter.labels" . | nindent 4 }} + {{- with .Values.daemonsetAnnotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + selector: + matchLabels: + {{- include "windowsExporter.labels" . | nindent 6 }} + {{- with .Values.updateStrategy }} + updateStrategy: + {{- toYaml . | nindent 4 }} + {{- end }} + template: + metadata: + {{- with .Values.podAnnotations }} + annotations: + {{- toYaml . | nindent 8 }} + {{- end }} + labels: + {{- include "windowsExporter.labels" . | nindent 8 }} + spec: + automountServiceAccountToken: {{ .Values.serviceAccount.automountServiceAccountToken }} + {{- with .Values.securityContext }} + securityContext: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.priorityClassName }} + priorityClassName: {{ . }} + {{- end }} + initContainers: + - name: configure-firewall + image: {{ include "prometheus-windows-exporter.image" . }} + command: + - C:\WINDOWS\System32\WindowsPowerShell\v1.0\powershell.exe + args: ["-f", "scripts/configure-firewall.ps1"] + volumeMounts: + - mountPath: /scripts + name: exporter-scripts + {{- with .Values.extraInitContainers }} + {{- toYaml . | nindent 8 }} + {{- end }} + serviceAccountName: {{ include "prometheus-windows-exporter.fullname" . }} + containers: + - name: windows-exporter + image: {{ include "prometheus-windows-exporter.image" . }} + imagePullPolicy: {{ .Values.image.pullPolicy }} + args: + - --config.file=%CONTAINER_SANDBOX_MOUNT_POINT%/config.yml + - --collector.textfile.directories=%CONTAINER_SANDBOX_MOUNT_POINT% + - --web.listen-address=:{{ .Values.service.port }} + {{- with .Values.extraArgs }} + {{- toYaml . | nindent 12 }} + {{- end }} + {{- with .Values.securityContext }} + securityContext: + {{- toYaml . | nindent 12 }} + {{- end }} + env: + {{- range $key, $value := .Values.env }} + - name: {{ $key }} + value: {{ $value | quote }} + {{- end }} + ports: + - name: http + containerPort: {{ .Values.service.port }} + hostPort: {{ .Values.service.port }} + protocol: TCP + livenessProbe: + failureThreshold: {{ .Values.livenessProbe.failureThreshold }} + httpGet: + httpHeaders: + {{- range $_, $header := .Values.livenessProbe.httpGet.httpHeaders }} + - name: {{ $header.name }} + value: {{ $header.value }} + {{- end }} + path: / + port: {{ .Values.service.port }} + scheme: {{ upper .Values.livenessProbe.httpGet.scheme }} + initialDelaySeconds: {{ .Values.livenessProbe.initialDelaySeconds }} + periodSeconds: {{ .Values.livenessProbe.periodSeconds }} + successThreshold: {{ .Values.livenessProbe.successThreshold }} + timeoutSeconds: {{ .Values.livenessProbe.timeoutSeconds }} + readinessProbe: + failureThreshold: {{ .Values.readinessProbe.failureThreshold }} + httpGet: + httpHeaders: + {{- range $_, $header := .Values.readinessProbe.httpGet.httpHeaders }} + - name: {{ $header.name }} + value: {{ $header.value }} + {{- end }} + path: / + port: {{ .Values.service.port }} + scheme: {{ upper .Values.readinessProbe.httpGet.scheme }} + initialDelaySeconds: {{ .Values.readinessProbe.initialDelaySeconds }} + periodSeconds: {{ .Values.readinessProbe.periodSeconds }} + successThreshold: {{ .Values.readinessProbe.successThreshold }} + timeoutSeconds: {{ .Values.readinessProbe.timeoutSeconds }} + {{- with .Values.resources }} + resources: + {{- toYaml . | nindent 12 }} + {{- end }} + volumeMounts: + - name: config + mountPath: /config.yml + subPath: config.yml + {{- range $_, $mount := .Values.extraHostVolumeMounts }} + - name: {{ $mount.name }} + mountPath: {{ $mount.mountPath }} + readOnly: {{ $mount.readOnly }} + {{- end }} + {{- range $_, $mount := .Values.sidecarVolumeMount }} + - name: {{ $mount.name }} + mountPath: {{ $mount.mountPath }} + readOnly: true + {{- end }} + {{- range $_, $mount := .Values.configmaps }} + - name: {{ $mount.name }} + mountPath: {{ $mount.mountPath }} + {{- end }} + {{- range $_, $mount := .Values.secrets }} + - name: {{ .name }} + mountPath: {{ .mountPath }} + {{- end }} + {{- with .Values.sidecars }} + {{- toYaml . | nindent 8 }} + {{- if or .Values.sidecarVolumeMount .Values.sidecarHostVolumeMounts }} + volumeMounts: + {{- range $_, $mount := .Values.sidecarVolumeMount }} + - name: {{ $mount.name }} + mountPath: {{ $mount.mountPath }} + readOnly: {{ $mount.readOnly }} + {{- end }} + {{- range $_, $mount := .Values.sidecarHostVolumeMounts }} + - name: {{ $mount.name }} + mountPath: {{ $mount.mountPath }} + readOnly: {{ $mount.readOnly }} + {{- end }} + {{- end }} + {{- end }} + {{- if or .Values.imagePullSecrets .Values.global.imagePullSecrets }} + imagePullSecrets: + {{- include "prometheus-windows-exporter.imagePullSecrets" (dict "Values" .Values "imagePullSecrets" .Values.imagePullSecrets) | indent 8 }} + {{- end }} + hostNetwork: {{ .Values.hostNetwork }} + hostPID: {{ .Values.hostPID }} + {{- with .Values.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.dnsConfig }} + dnsConfig: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} + volumes: + - name: exporter-scripts + configMap: + name: {{ include "prometheus-windows-exporter.fullname" . }}-scripts + - name: config + configMap: + name: {{ include "prometheus-windows-exporter.fullname" . }} + {{- range $_, $mount := .Values.extraHostVolumeMounts }} + - name: {{ $mount.name }} + hostPath: + path: {{ $mount.hostPath }} + {{- end }} + {{- range $_, $mount := .Values.sidecarVolumeMount }} + - name: {{ $mount.name }} + emptyDir: + medium: Memory + {{- end }} + {{- range $_, $mount := .Values.sidecarHostVolumeMounts }} + - name: {{ $mount.name }} + hostPath: + path: {{ $mount.hostPath }} + {{- end }} + {{- range $_, $mount := .Values.configmaps }} + - name: {{ $mount.name }} + configMap: + name: {{ $mount.name }} + {{- end }} + {{- range $_, $mount := .Values.secrets }} + - name: {{ $mount.name }} + secret: + secretName: {{ $mount.name }} + {{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/windowsExporter/templates/podmonitor.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/windowsExporter/templates/podmonitor.yaml new file mode 100644 index 00000000000..bbb6c393402 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/windowsExporter/templates/podmonitor.yaml @@ -0,0 +1,91 @@ +{{- if .Values.prometheus.podMonitor.enabled }} +apiVersion: {{ .Values.prometheus.podMonitor.apiVersion | default "monitoring.coreos.com/v1" }} +kind: PodMonitor +metadata: + name: {{ include "prometheus-windows-exporter.fullname" . }} + namespace: {{ include "prometheus-windows-exporter.podmonitor-namespace" . }} + labels: + {{- include "windowsExporter.labels" . | nindent 4 }} + {{- with .Values.prometheus.podMonitor.additionalLabels }} + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + jobLabel: {{ default "app.kubernetes.io/name" .Values.prometheus.podMonitor.jobLabel }} + {{- include "podmonitor.scrapeLimits" .Values.prometheus.podMonitor | nindent 2 }} + selector: + matchLabels: + {{- with .Values.prometheus.podMonitor.selectorOverride }} + {{- toYaml . | nindent 6 }} + {{- else }} + {{- include "prometheus-windows-exporter.selectorLabels" . | nindent 6 }} + {{- end }} + namespaceSelector: + matchNames: + - {{ include "prometheus-windows-exporter.namespace" . }} + {{- with .Values.prometheus.podMonitor.attachMetadata }} + attachMetadata: + {{- toYaml . | nindent 4 }} + {{- end }} + {{- with .Values.prometheus.podMonitor.podTargetLabels }} + podTargetLabels: + {{- toYaml . | nindent 4 }} + {{- end }} + podMetricsEndpoints: + - port: {{ .Values.service.portName }} + {{- with .Values.prometheus.podMonitor.scheme }} + scheme: {{ . }} + {{- end }} + {{- with .Values.prometheus.podMonitor.path }} + path: {{ . }} + {{- end }} + {{- with .Values.prometheus.podMonitor.basicAuth }} + basicAuth: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.prometheus.podMonitor.bearerTokenSecret }} + bearerTokenSecret: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.prometheus.podMonitor.tlsConfig }} + tlsConfig: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.prometheus.podMonitor.authorization }} + authorization: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.prometheus.podMonitor.oauth2 }} + oauth2: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.prometheus.podMonitor.proxyUrl }} + proxyUrl: {{ . }} + {{- end }} + {{- with .Values.prometheus.podMonitor.interval }} + interval: {{ . }} + {{- end }} + {{- with .Values.prometheus.podMonitor.honorTimestamps }} + honorTimestamps: {{ . }} + {{- end }} + {{- with .Values.prometheus.podMonitor.honorLabels }} + honorLabels: {{ . }} + {{- end }} + {{- with .Values.prometheus.podMonitor.scrapeTimeout }} + scrapeTimeout: {{ . }} + {{- end }} + {{- with .Values.prometheus.podMonitor.relabelings }} + relabelings: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.prometheus.podMonitor.metricRelabelings }} + metricRelabelings: + {{- toYaml . | nindent 8 }} + {{- end }} + enableHttp2: {{ default false .Values.prometheus.podMonitor.enableHttp2 }} + filterRunning: {{ default true .Values.prometheus.podMonitor.filterRunning }} + followRedirects: {{ default false .Values.prometheus.podMonitor.followRedirects }} + {{- with .Values.prometheus.podMonitor.params }} + params: + {{- toYaml . | nindent 8 }} + {{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/windowsExporter/templates/scriptConfig.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/windowsExporter/templates/scriptConfig.yaml new file mode 100644 index 00000000000..f514c8161a5 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/windowsExporter/templates/scriptConfig.yaml @@ -0,0 +1,14 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ include "prometheus-windows-exporter.fullname" . }}-scripts + namespace: {{ include "prometheus-windows-exporter.namespace" . }} + labels: + {{- include "windowsExporter.labels" $ | nindent 4 }} + {{- with .Values.service.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +data: +{{ (.Files.Glob "scripts/*").AsConfig | indent 2 }} + diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/windowsExporter/templates/service.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/windowsExporter/templates/service.yaml new file mode 100644 index 00000000000..267b796f633 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/windowsExporter/templates/service.yaml @@ -0,0 +1,32 @@ +apiVersion: v1 +kind: Service +metadata: + name: {{ include "prometheus-windows-exporter.fullname" . }} + namespace: {{ include "prometheus-windows-exporter.namespace" . }} + labels: + {{- include "windowsExporter.labels" $ | nindent 4 }} + {{- if or .Values.prometheus.monitor.enabled .Values.prometheus.podMonitor.enabled }} + {{- with .Values.service.annotations }} + annotations: + {{- unset . "prometheus.io/scrape" | toYaml | nindent 4 }} + {{- end }} + {{- else }} + annotations: + prometheus.io/scrape: "true" + {{- with .Values.service.annotations }} + {{- toYaml . | nindent 4 }} + {{- end }} + {{- end }} +spec: + type: {{ .Values.service.type }} + ports: + - port: {{ .Values.service.port }} + {{- if ( and (eq .Values.service.type "NodePort" ) (not (empty .Values.service.nodePort)) ) }} + nodePort: {{ .Values.service.nodePort }} + {{- end }} + targetPort: {{ .Values.service.port }} + protocol: TCP + appProtocol: http + name: {{ .Values.service.portName }} + selector: + {{- include "windowsExporter.labels" . | nindent 4 }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/windowsExporter/templates/serviceaccount.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/windowsExporter/templates/serviceaccount.yaml new file mode 100644 index 00000000000..14c1c46807d --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/windowsExporter/templates/serviceaccount.yaml @@ -0,0 +1,17 @@ +{{- if and .Values.rbac.create .Values.serviceAccount.create -}} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ include "prometheus-windows-exporter.serviceAccountName" . }} + namespace: {{ include "prometheus-windows-exporter.namespace" . }} + labels: + {{- include "windowsExporter.labels" . | nindent 4 }} + {{- with .Values.serviceAccount.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +{{- if or .Values.serviceAccount.imagePullSecrets .Values.global.imagePullSecrets }} +imagePullSecrets: + {{- include "prometheus-windows-exporter.imagePullSecrets" (dict "Values" .Values "imagePullSecrets" .Values.serviceAccount.imagePullSecrets) | indent 2 }} +{{- end }} +{{- end -}} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/windowsExporter/templates/servicemonitor.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/windowsExporter/templates/servicemonitor.yaml new file mode 100644 index 00000000000..2effc07758b --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/windowsExporter/templates/servicemonitor.yaml @@ -0,0 +1,75 @@ +{{- if .Values.prometheus.monitor.enabled }} +apiVersion: {{ .Values.prometheus.monitor.apiVersion | default "monitoring.coreos.com/v1" }} +kind: ServiceMonitor +metadata: + name: {{ include "prometheus-windows-exporter.fullname" . }} + namespace: {{ include "prometheus-windows-exporter.monitor-namespace" . }} + labels: + {{- include "windowsExporter.labels" . | nindent 4 }} + {{- with .Values.prometheus.monitor.additionalLabels }} + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + jobLabel: {{ default "app.kubernetes.io/name" .Values.prometheus.monitor.jobLabel }} + {{- include "servicemonitor.scrapeLimits" .Values.prometheus.monitor | nindent 2 }} + {{- with .Values.prometheus.monitor.podTargetLabels }} + podTargetLabels: + {{- toYaml . | nindent 4 }} + {{- end }} + selector: + matchLabels: + {{- with .Values.prometheus.monitor.selectorOverride }} + {{- toYaml . | nindent 6 }} + {{- else }} + {{- include "windowsExporter.labels" . | nindent 6 }} + {{- end }} + {{- with .Values.prometheus.monitor.attachMetadata }} + attachMetadata: + {{- toYaml . | nindent 4 }} + {{- end }} + endpoints: + - port: {{ .Values.service.portName }} + scheme: {{ .Values.prometheus.monitor.scheme }} + {{- with .Values.prometheus.monitor.basicAuth }} + basicAuth: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.prometheus.monitor.bearerTokenFile }} + bearerTokenFile: {{ . }} + {{- end }} + {{- with .Values.prometheus.monitor.tlsConfig }} + tlsConfig: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.prometheus.monitor.proxyUrl }} + proxyUrl: {{ . }} + {{- end }} + {{- with .Values.prometheus.monitor.interval }} + interval: {{ . }} + {{- end }} + {{- with .Values.prometheus.monitor.scrapeTimeout }} + scrapeTimeout: {{ . }} + {{- end }} + metricRelabelings: +{{- include "windowsExporter.renamedMetricsRelabeling" . | nindent 6 -}} + - sourceLabels: [__name__] + regex: 'wmi_(.*)' + replacement: 'windows_$1' + targetLabel: __name__ + - sourceLabels: [volume, nic] + regex: (.*);(.*) + separator: '' + targetLabel: device + action: replace + replacement: $1$2 + - sourceLabels: [__name__] + regex: windows_cs_logical_processors + replacement: 'system' + targetLabel: mode + relabelings: + - separator: ':' + sourceLabels: + - __meta_kubernetes_pod_host_ip + - __meta_kubernetes_pod_container_port_number + targetLabel: instance +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/charts/windowsExporter/values.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/windowsExporter/values.yaml new file mode 100644 index 00000000000..04569505d62 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/charts/windowsExporter/values.yaml @@ -0,0 +1,366 @@ +# Default values for prometheus-windows-exporter. +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. + +image: + registry: docker.io + repository: rancher/mirrored-prometheus-windows-exporter + # Overrides the image tag whose default is {{ printf "v%s" .Chart.AppVersion }} + tag: "0.25.1" + pullPolicy: IfNotPresent + digest: "" + +config: |- + collectors: + enabled: '[defaults],tcp,memory,container' + +imagePullSecrets: [] +# - name: "image-pull-secret" +nameOverride: "" +fullnameOverride: "" + +global: + # To help compatibility with other charts which use global.imagePullSecrets. + # Allow either an array of {name: pullSecret} maps (k8s-style), or an array of strings (more common helm-style). + # global: + # imagePullSecrets: + # - name: pullSecret1 + # - name: pullSecret2 + # or + # global: + # imagePullSecrets: + # - pullSecret1 + # - pullSecret2 + imagePullSecrets: [] + cattle: + systemDefaultRegistry: "" + +service: + type: ClusterIP + port: 9796 + nodePort: + portName: windows-metrics + annotations: {} + +# Additional environment variables that will be passed to the daemonset +env: {} +## env: +## VARIABLE: value + +prometheus: + monitor: + enabled: true + additionalLabels: {} + namespace: "" + + jobLabel: "component" + + # List of pod labels to add to windows exporter metrics + # https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#servicemonitor + podTargetLabels: ["component"] + + scheme: http + basicAuth: {} + bearerTokenFile: + tlsConfig: {} + + ## proxyUrl: URL of a proxy that should be used for scraping. + ## + proxyUrl: "" + + ## Override serviceMonitor selector + ## + selectorOverride: {} + + ## Attach node metadata to discovered targets. Requires Prometheus v2.35.0 and above. + ## + attachMetadata: + node: false + + relabelings: [] + metricRelabelings: [] + interval: "" + scrapeTimeout: 10s + ## prometheus.monitor.apiVersion ApiVersion for the serviceMonitor Resource(defaults to "monitoring.coreos.com/v1") + apiVersion: "" + + ## SampleLimit defines per-scrape limit on number of scraped samples that will be accepted. + ## + sampleLimit: 0 + + ## TargetLimit defines a limit on the number of scraped targets that will be accepted. + ## + targetLimit: 0 + + ## Per-scrape limit on number of labels that will be accepted for a sample. Only valid in Prometheus versions 2.27.0 and newer. + ## + labelLimit: 0 + + ## Per-scrape limit on length of labels name that will be accepted for a sample. Only valid in Prometheus versions 2.27.0 and newer. + ## + labelNameLengthLimit: 0 + + ## Per-scrape limit on length of labels value that will be accepted for a sample. Only valid in Prometheus versions 2.27.0 and newer. + ## + labelValueLengthLimit: 0 + + # PodMonitor defines monitoring for a set of pods. + # ref. https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#monitoring.coreos.com/v1.PodMonitor + # Using a PodMonitor may be preferred in some environments where there is very large number + # of Windows Exporter endpoints (1000+) behind a single service. + # The PodMonitor is disabled by default. When switching from ServiceMonitor to PodMonitor, + # the time series resulting from the configuration through PodMonitor may have different labels. + # For instance, there will not be the service label any longer which might + # affect PromQL queries selecting that label. + podMonitor: + enabled: false + # Namespace in which to deploy the pod monitor. Defaults to the release namespace. + namespace: "" + # Additional labels, e.g. setting a label for pod monitor selector as set in prometheus + additionalLabels: {} + # release: kube-prometheus-stack + # PodTargetLabels transfers labels of the Kubernetes Pod onto the target. + podTargetLabels: [] + # apiVersion defaults to monitoring.coreos.com/v1. + apiVersion: "" + # Override pod selector to select pod objects. + selectorOverride: {} + # Attach node metadata to discovered targets. Requires Prometheus v2.35.0 and above. + attachMetadata: + node: false + # The label to use to retrieve the job name from. Defaults to label app.kubernetes.io/name. + jobLabel: "" + + # Scheme/protocol to use for scraping. + scheme: "http" + # Path to scrape metrics at. + path: "/metrics" + + # BasicAuth allow an endpoint to authenticate over basic authentication. + # More info: https://prometheus.io/docs/operating/configuration/#endpoint + basicAuth: {} + # Secret to mount to read bearer token for scraping targets. + # The secret needs to be in the same namespace as the pod monitor and accessible by the Prometheus Operator. + # https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.24/#secretkeyselector-v1-core + bearerTokenSecret: {} + # TLS configuration to use when scraping the endpoint. + tlsConfig: {} + # Authorization section for this endpoint. + # https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#monitoring.coreos.com/v1.SafeAuthorization + authorization: {} + # OAuth2 for the URL. Only valid in Prometheus versions 2.27.0 and newer. + # https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#monitoring.coreos.com/v1.OAuth2 + oauth2: {} + + # ProxyURL eg http://proxyserver:2195. Directs scrapes through proxy to this endpoint. + proxyUrl: "" + # Interval at which endpoints should be scraped. If not specified Prometheus’ global scrape interval is used. + interval: "" + # Timeout after which the scrape is ended. If not specified, the Prometheus global scrape interval is used. + scrapeTimeout: "" + # HonorTimestamps controls whether Prometheus respects the timestamps present in scraped data. + honorTimestamps: true + # HonorLabels chooses the metric’s labels on collisions with target labels. + honorLabels: true + # Whether to enable HTTP2. Default false. + enableHttp2: "" + # Drop pods that are not running. (Failed, Succeeded). + # Enabled by default. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle/#pod-phase + filterRunning: "" + # FollowRedirects configures whether scrape requests follow HTTP 3xx redirects. Default false. + followRedirects: "" + # Optional HTTP URL parameters + params: {} + + # RelabelConfigs to apply to samples before scraping. Prometheus Operator automatically adds + # relabelings for a few standard Kubernetes fields. The original scrape job’s name + # is available via the __tmp_prometheus_job_name label. + # More info: https://prometheus.io/docs/prometheus/latest/configuration/configuration/#relabel_config + relabelings: [] + # MetricRelabelConfigs to apply to samples before ingestion. + metricRelabelings: [] + + # SampleLimit defines per-scrape limit on number of scraped samples that will be accepted. + sampleLimit: 0 + # TargetLimit defines a limit on the number of scraped targets that will be accepted. + targetLimit: 0 + # Per-scrape limit on number of labels that will be accepted for a sample. + # Only valid in Prometheus versions 2.27.0 and newer. + labelLimit: 0 + # Per-scrape limit on length of labels name that will be accepted for a sample. + # Only valid in Prometheus versions 2.27.0 and newer. + labelNameLengthLimit: 0 + # Per-scrape limit on length of labels value that will be accepted for a sample. + # Only valid in Prometheus versions 2.27.0 and newer. + labelValueLengthLimit: 0 + +## Customize the updateStrategy if set +updateStrategy: + type: RollingUpdate + rollingUpdate: + maxUnavailable: 1 + +resources: {} + # We usually recommend not to specify default resources and to leave this as a conscious + # choice for the user. This also increases chances charts run on environments with little + # resources, such as Minikube. If you do want to specify resources, uncomment the following + # lines, adjust them as necessary, and remove the curly braces after 'resources:'. + # limits: + # cpu: 200m + # memory: 50Mi + # requests: + # cpu: 100m +# memory: 30Mi + +serviceAccount: + # Specifies whether a ServiceAccount should be created + create: true + # The name of the ServiceAccount to use. + # If not set and create is true, a name is generated using the fullname template + name: + annotations: {} + imagePullSecrets: [] + automountServiceAccountToken: false + +securityContext: + windowsOptions: + hostProcess: true + runAsUserName: "NT AUTHORITY\\system" + +rbac: + ## If true, create & use RBAC resources + ## + create: true + +# Expose the service to the host network +hostNetwork: true + +# Share the host process ID namespace +hostPID: true + +## Assign a group of affinity scheduling rules +## +affinity: {} +# nodeAffinity: +# requiredDuringSchedulingIgnoredDuringExecution: +# nodeSelectorTerms: +# - matchFields: +# - key: metadata.name +# operator: In +# values: +# - target-host-name + +# Annotations to be added to windows exporter pods +podAnnotations: + # Fix for very slow GKE cluster upgrades + cluster-autoscaler.kubernetes.io/safe-to-evict: "true" + +# Extra labels to be added to windows exporter pods +podLabels: {} + +# Annotations to be added to windows exporter daemonset +daemonsetAnnotations: {} + +## set to true to add the release label so scraping of the servicemonitor with kube-prometheus-stack works out of the box +releaseLabel: false + +# Custom DNS configuration to be added to prometheus-windows-exporter pods +dnsConfig: {} +# nameservers: +# - 1.2.3.4 +# searches: +# - ns1.svc.cluster-domain.example +# - my.dns.search.suffix +# options: +# - name: ndots +# value: "2" +# - name: edns0 + +## Assign a nodeSelector if operating a hybrid cluster +## +nodeSelector: + kubernetes.io/os: windows + # kubernetes.io/arch: amd64 + +tolerations: + - effect: NoSchedule + operator: Exists + +## Assign a PriorityClassName to pods if set +# priorityClassName: "" + +## Additional container arguments +## +extraArgs: [] +# - --collector.service.services-where +# - "Name LIKE 'sql%'" + +## Additional mounts from the host to windows-exporter container +## +extraHostVolumeMounts: [] +# - name: +# hostPath: +# mountPath: +# readOnly: true|false + +## Additional configmaps to be mounted. +## +configmaps: [] +# - name: +# mountPath: +secrets: [] +# - name: +# mountPath: +## Override the deployment namespace +## +namespaceOverride: "" + +## Additional containers for export metrics to text file +## +sidecars: [] +## - name: nvidia-dcgm-exporter +## image: nvidia/dcgm-exporter:1.4.3 + +## Volume for sidecar containers +## +sidecarVolumeMount: [] +## - name: collector-textfiles +## mountPath: /run/prometheus +## readOnly: false + +## Additional mounts from the host to sidecar containers +## +sidecarHostVolumeMounts: [] +# - name: +# hostPath: +# mountPath: +# readOnly: true|false +# mountPropagation: None|HostToContainer|Bidirectional + +## Additional InitContainers to initialize the pod +## +extraInitContainers: [] + +## Liveness probe +## +livenessProbe: + failureThreshold: 3 + httpGet: + httpHeaders: [] + scheme: http + initialDelaySeconds: 0 + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 1 + +## Readiness probe +## +readinessProbe: + failureThreshold: 3 + httpGet: + httpHeaders: [] + scheme: http + initialDelaySeconds: 0 + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 1 diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/files/ingress-nginx/nginx.json b/charts/rancher-monitoring/103.2.2+up57.0.3/files/ingress-nginx/nginx.json new file mode 100644 index 00000000000..565352235ae --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/files/ingress-nginx/nginx.json @@ -0,0 +1,1445 @@ +{ + "__inputs": [ + + ], + "__requires": [ + + ], + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + }, + { + "datasource": "$datasource", + "enable": true, + "expr": "sum(changes(nginx_ingress_controller_config_last_reload_successful_timestamp_seconds{instance!=\"unknown\",controller_class=~\"$controller_class\",namespace=~\"$namespace\"}[30s])) by (controller_class)", + "hide": false, + "iconColor": "rgba(255, 96, 96, 1)", + "limit": 100, + "name": "Config Reloads", + "showIn": 0, + "step": "30s", + "tagKeys": "controller_class", + "tags": [], + "titleFormat": "Config Reloaded", + "type": "tags" + } + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "iteration": 1534359654832, + "links": [], + "panels": [ + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "datasource": "$datasource", + "format": "ops", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 6, + "x": 0, + "y": 0 + }, + "id": 20, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": true, + "lineColor": "rgb(31, 120, 193)", + "show": true + }, + "tableColumn": "", + "targets": [ + { + "expr": "round(sum(irate(nginx_ingress_controller_requests{controller_pod=~\"$controller\",controller_class=~\"$controller_class\",namespace=~\"$namespace\"}[2m])), 0.001)", + "format": "time_series", + "intervalFactor": 1, + "refId": "A", + "step": 4 + } + ], + "thresholds": "", + "title": "Controller Request Volume", + "transparent": false, + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "avg" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "datasource": "$datasource", + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 6, + "x": 6, + "y": 0 + }, + "id": 82, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": true, + "lineColor": "rgb(31, 120, 193)", + "show": true + }, + "tableColumn": "", + "targets": [ + { + "expr": "sum(avg_over_time(nginx_ingress_controller_nginx_process_connections{controller_pod=~\"$controller\",controller_class=~\"$controller_class\",controller_namespace=~\"$namespace\",state=\"active\"}[2m]))", + "format": "time_series", + "instant": false, + "intervalFactor": 1, + "refId": "A", + "step": 4 + } + ], + "thresholds": "", + "title": "Controller Connections", + "transparent": false, + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "avg" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "datasource": "$datasource", + "format": "percentunit", + "gauge": { + "maxValue": 100, + "minValue": 80, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": false + }, + "gridPos": { + "h": 3, + "w": 6, + "x": 12, + "y": 0 + }, + "id": 21, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": true, + "lineColor": "rgb(31, 120, 193)", + "show": true + }, + "tableColumn": "", + "targets": [ + { + "expr": "sum(rate(nginx_ingress_controller_requests{controller_pod=~\"$controller\",controller_class=~\"$controller_class\",namespace=~\"$namespace\",status!~\"[4-5].*\"}[2m])) / sum(rate(nginx_ingress_controller_requests{controller_pod=~\"$controller\",controller_class=~\"$controller_class\",namespace=~\"$namespace\"}[2m]))", + "format": "time_series", + "intervalFactor": 1, + "refId": "A", + "step": 4 + } + ], + "thresholds": "95, 99, 99.5", + "title": "Controller Success Rate (non-4|5xx responses)", + "transparent": false, + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "avg" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "datasource": "$datasource", + "decimals": 0, + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 3, + "x": 18, + "y": 0 + }, + "id": 81, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": true, + "lineColor": "rgb(31, 120, 193)", + "show": true + }, + "tableColumn": "", + "targets": [ + { + "expr": "avg(irate(nginx_ingress_controller_success{controller_pod=~\"$controller\",controller_class=~\"$controller_class\",controller_namespace=~\"$namespace\"}[1m])) * 60", + "format": "time_series", + "instant": false, + "intervalFactor": 1, + "refId": "A", + "step": 4 + } + ], + "thresholds": "", + "title": "Config Reloads", + "transparent": false, + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "total" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "datasource": "$datasource", + "decimals": 0, + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 3, + "x": 21, + "y": 0 + }, + "id": 83, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": true, + "lineColor": "rgb(31, 120, 193)", + "show": true + }, + "tableColumn": "", + "targets": [ + { + "expr": "count(nginx_ingress_controller_config_last_reload_successful{controller_pod=~\"$controller\",controller_namespace=~\"$namespace\"} == 0)", + "format": "time_series", + "instant": true, + "intervalFactor": 1, + "refId": "A", + "step": 4 + } + ], + "thresholds": "", + "title": "Last Config Failed", + "transparent": false, + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "avg" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "decimals": 2, + "editable": true, + "error": false, + "fill": 1, + "grid": {}, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 3 + }, + "height": "200px", + "id": 86, + "isNew": true, + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "hideEmpty": false, + "hideZero": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": 300, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "repeatDirection": "h", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "round(sum(irate(nginx_ingress_controller_requests{controller_pod=~\"$controller\",controller_class=~\"$controller_class\",controller_namespace=~\"$namespace\",ingress=~\"$ingress\"}[2m])) by (ingress), 0.001)", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{ ingress }}", + "metric": "network", + "refId": "A", + "step": 10 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Ingress Request Volume", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 2, + "value_type": "cumulative" + }, + "transparent": false, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "reqps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "max - istio-proxy": "#890f02", + "max - master": "#bf1b00", + "max - prometheus": "#bf1b00" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "decimals": 2, + "editable": false, + "error": false, + "fill": 0, + "grid": {}, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 3 + }, + "id": 87, + "isNew": true, + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "hideEmpty": true, + "hideZero": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": 300, + "sort": "avg", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(nginx_ingress_controller_requests{controller_pod=~\"$controller\",controller_class=~\"$controller_class\",namespace=~\"$namespace\",ingress=~\"$ingress\",status!~\"[4-5].*\"}[2m])) by (ingress) / sum(rate(nginx_ingress_controller_requests{controller_pod=~\"$controller\",controller_class=~\"$controller_class\",namespace=~\"$namespace\",ingress=~\"$ingress\"}[2m])) by (ingress)", + "format": "time_series", + "instant": false, + "interval": "10s", + "intervalFactor": 1, + "legendFormat": "{{ ingress }}", + "metric": "container_memory_usage:sort_desc", + "refId": "A", + "step": 10 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Ingress Success Rate (non-4|5xx responses)", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 1, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "decimals": 2, + "editable": true, + "error": false, + "fill": 1, + "grid": {}, + "gridPos": { + "h": 6, + "w": 8, + "x": 0, + "y": 10 + }, + "height": "200px", + "id": 32, + "isNew": true, + "legend": { + "alignAsTable": false, + "avg": true, + "current": true, + "max": false, + "min": false, + "rightSide": false, + "show": false, + "sideWidth": 200, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum (irate (nginx_ingress_controller_request_size_sum{controller_pod=~\"$controller\",controller_class=~\"$controller_class\",controller_namespace=~\"$namespace\"}[2m]))", + "format": "time_series", + "instant": false, + "interval": "10s", + "intervalFactor": 1, + "legendFormat": "Received", + "metric": "network", + "refId": "A", + "step": 10 + }, + { + "expr": "- sum (irate (nginx_ingress_controller_response_size_sum{controller_pod=~\"$controller\",controller_class=~\"$controller_class\",controller_namespace=~\"$namespace\"}[2m]))", + "format": "time_series", + "hide": false, + "interval": "10s", + "intervalFactor": 1, + "legendFormat": "Sent", + "metric": "network", + "refId": "B", + "step": 10 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Network I/O pressure", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "transparent": false, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "max - istio-proxy": "#890f02", + "max - master": "#bf1b00", + "max - prometheus": "#bf1b00" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "decimals": 2, + "editable": false, + "error": false, + "fill": 0, + "grid": {}, + "gridPos": { + "h": 6, + "w": 8, + "x": 8, + "y": 10 + }, + "id": 77, + "isNew": true, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": false, + "min": false, + "rightSide": false, + "show": false, + "sideWidth": 200, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "avg(nginx_ingress_controller_nginx_process_resident_memory_bytes{controller_pod=~\"$controller\",controller_class=~\"$controller_class\",controller_namespace=~\"$namespace\"}) ", + "format": "time_series", + "instant": false, + "interval": "10s", + "intervalFactor": 1, + "legendFormat": "nginx", + "metric": "container_memory_usage:sort_desc", + "refId": "A", + "step": 10 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Average Memory Usage", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 2, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "max - istio-proxy": "#890f02", + "max - master": "#bf1b00" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "decimals": 3, + "editable": false, + "error": false, + "fill": 0, + "grid": {}, + "gridPos": { + "h": 6, + "w": 8, + "x": 16, + "y": 10 + }, + "height": "", + "id": 79, + "isNew": true, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": false, + "min": false, + "rightSide": false, + "show": false, + "sort": null, + "sortDesc": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "avg (rate (nginx_ingress_controller_nginx_process_cpu_seconds_total{controller_pod=~\"$controller\",controller_class=~\"$controller_class\",controller_namespace=~\"$namespace\"}[2m])) ", + "format": "time_series", + "interval": "10s", + "intervalFactor": 1, + "legendFormat": "nginx", + "metric": "container_cpu", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + { + "colorMode": "critical", + "fill": true, + "line": true, + "op": "gt" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Average CPU Usage", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 2, + "value_type": "cumulative" + }, + "transparent": false, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "none", + "label": "cores", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "columns": [], + "datasource": "$datasource", + "fontSize": "100%", + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 16 + }, + "hideTimeOverride": false, + "id": 75, + "links": [], + "pageSize": 7, + "repeat": null, + "repeatDirection": "h", + "scroll": true, + "showHeader": true, + "sort": { + "col": 1, + "desc": true + }, + "styles": [ + { + "alias": "Ingress", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "pattern": "ingress", + "preserveFormat": false, + "sanitize": false, + "thresholds": [], + "type": "string", + "unit": "short" + }, + { + "alias": "Requests", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "pattern": "Value #A", + "thresholds": [ + "" + ], + "type": "number", + "unit": "ops" + }, + { + "alias": "Errors", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "pattern": "Value #B", + "thresholds": [], + "type": "number", + "unit": "ops" + }, + { + "alias": "P50 Latency", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 0, + "link": false, + "pattern": "Value #C", + "thresholds": [], + "type": "number", + "unit": "dtdurations" + }, + { + "alias": "P90 Latency", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 0, + "pattern": "Value #D", + "thresholds": [], + "type": "number", + "unit": "dtdurations" + }, + { + "alias": "P99 Latency", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 0, + "pattern": "Value #E", + "thresholds": [], + "type": "number", + "unit": "dtdurations" + }, + { + "alias": "IN", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "pattern": "Value #F", + "thresholds": [ + "" + ], + "type": "number", + "unit": "Bps" + }, + { + "alias": "", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "pattern": "Time", + "thresholds": [], + "type": "hidden", + "unit": "short" + }, + { + "alias": "OUT", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "mappingType": 1, + "pattern": "Value #G", + "thresholds": [], + "type": "number", + "unit": "Bps" + } + ], + "targets": [ + { + "expr": "histogram_quantile(0.50, sum(rate(nginx_ingress_controller_request_duration_seconds_bucket{ingress!=\"\",controller_pod=~\"$controller\",controller_class=~\"$controller_class\",controller_namespace=~\"$namespace\",ingress=~\"$ingress\"}[2m])) by (le, ingress))", + "format": "table", + "hide": false, + "instant": true, + "intervalFactor": 1, + "legendFormat": "{{ ingress }}", + "refId": "C" + }, + { + "expr": "histogram_quantile(0.90, sum(rate(nginx_ingress_controller_request_duration_seconds_bucket{ingress!=\"\",controller_pod=~\"$controller\",controller_class=~\"$controller_class\",controller_namespace=~\"$namespace\",ingress=~\"$ingress\"}[2m])) by (le, ingress))", + "format": "table", + "hide": false, + "instant": true, + "intervalFactor": 1, + "legendFormat": "{{ ingress }}", + "refId": "D" + }, + { + "expr": "histogram_quantile(0.99, sum(rate(nginx_ingress_controller_request_duration_seconds_bucket{ingress!=\"\",controller_pod=~\"$controller\",controller_class=~\"$controller_class\",controller_namespace=~\"$namespace\",ingress=~\"$ingress\"}[2m])) by (le, ingress))", + "format": "table", + "hide": false, + "instant": true, + "intervalFactor": 1, + "legendFormat": "{{ destination_service }}", + "refId": "E" + }, + { + "expr": "sum(irate(nginx_ingress_controller_request_size_sum{ingress!=\"\",controller_pod=~\"$controller\",controller_class=~\"$controller_class\",controller_namespace=~\"$namespace\",ingress=~\"$ingress\"}[2m])) by (ingress)", + "format": "table", + "hide": false, + "instant": true, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{ ingress }}", + "refId": "F" + }, + { + "expr": "sum(irate(nginx_ingress_controller_response_size_sum{ingress!=\"\",controller_pod=~\"$controller\",controller_class=~\"$controller_class\",controller_namespace=~\"$namespace\",ingress=~\"$ingress\"}[2m])) by (ingress)", + "format": "table", + "instant": true, + "intervalFactor": 1, + "legendFormat": "{{ ingress }}", + "refId": "G" + } + ], + "timeFrom": null, + "title": "Ingress Percentile Response Times and Transfer Rates", + "transform": "table", + "transparent": false, + "type": "table" + }, + { + "columns": [ + { + "text": "Current", + "value": "current" + } + ], + "datasource": "$datasource", + "fontSize": "100%", + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 24 + }, + "height": "1024", + "id": 85, + "links": [], + "pageSize": 7, + "scroll": true, + "showHeader": true, + "sort": { + "col": 1, + "desc": false + }, + "styles": [ + { + "alias": "Time", + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "pattern": "Time", + "type": "date" + }, + { + "alias": "TTL", + "colorMode": "cell", + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 0, + "pattern": "Current", + "thresholds": [ + "0", + "691200" + ], + "type": "number", + "unit": "s" + }, + { + "alias": "", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "decimals": 2, + "pattern": "/.*/", + "thresholds": [], + "type": "number", + "unit": "short" + } + ], + "targets": [ + { + "expr": "avg(nginx_ingress_controller_ssl_expire_time_seconds{kubernetes_pod_name=~\"$controller\",namespace=~\"$namespace\",ingress=~\"$ingress\"}) by (host) - time()", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{ host }}", + "metric": "gke_letsencrypt_cert_expiration", + "refId": "A", + "step": 1 + } + ], + "title": "Ingress Certificate Expiry", + "transform": "timeseries_aggregations", + "type": "table" + } + ], + "refresh": "5s", + "schemaVersion": 16, + "style": "dark", + "tags": [ + "nginx" + ], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": "Data Source", + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": ".*", + "current": { + "text": "All", + "value": "$__all" + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": true, + "label": "Namespace", + "multi": false, + "name": "namespace", + "options": [], + "query": "label_values(nginx_ingress_controller_config_hash, controller_namespace)", + "refresh": 1, + "regex": "", + "sort": 0, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": ".*", + "current": { + "text": "All", + "value": "$__all" + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": true, + "label": "Controller Class", + "multi": false, + "name": "controller_class", + "options": [], + "query": "label_values(nginx_ingress_controller_config_hash{namespace=~\"$namespace\"}, controller_class) ", + "refresh": 1, + "regex": "", + "sort": 0, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": ".*", + "current": { + "text": "All", + "value": "$__all" + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": true, + "label": "Controller", + "multi": false, + "name": "controller", + "options": [], + "query": "label_values(nginx_ingress_controller_config_hash{namespace=~\"$namespace\",controller_class=~\"$controller_class\"}, controller_pod) ", + "refresh": 1, + "regex": "", + "sort": 0, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": ".*", + "current": { + "tags": [], + "text": "All", + "value": "$__all" + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": true, + "label": "Ingress", + "multi": false, + "name": "ingress", + "options": [], + "query": "label_values(nginx_ingress_controller_requests{namespace=~\"$namespace\",controller_class=~\"$controller_class\",controller_pod=~\"$controller\"}, ingress) ", + "refresh": 1, + "regex": "", + "sort": 2, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "2m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "browser", + "title": "NGINX / Ingress Controller", + "uid": "nginx", + "version": 1 +} \ No newline at end of file diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/files/ingress-nginx/request-handling-performance.json b/charts/rancher-monitoring/103.2.2+up57.0.3/files/ingress-nginx/request-handling-performance.json new file mode 100644 index 00000000000..156e33123d4 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/files/ingress-nginx/request-handling-performance.json @@ -0,0 +1,963 @@ +{ + "__inputs": [ + + ], + "__requires": [ + + ], + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "description": "", + "editable": true, + "gnetId": 9614, + "graphTooltip": 1, + "id": null, + "iteration": 1582146566338, + "links": [], + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "Total time taken for nginx and upstream servers to process a request and send a response", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 0 + }, + "hiddenSeries": false, + "id": 91, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(\n 0.5,\n sum by (le)(\n rate(\n nginx_ingress_controller_request_duration_seconds_bucket{\n ingress =~ \"$ingress\"\n }[1m]\n )\n )\n)", + "interval": "", + "legendFormat": ".5", + "refId": "D" + }, + { + "expr": "histogram_quantile(\n 0.95,\n sum by (le)(\n rate(\n nginx_ingress_controller_request_duration_seconds_bucket{\n ingress =~ \"$ingress\"\n }[1m]\n )\n )\n)", + "interval": "", + "legendFormat": ".95", + "refId": "B" + }, + { + "expr": "histogram_quantile(\n 0.99,\n sum by (le)(\n rate(\n nginx_ingress_controller_request_duration_seconds_bucket{\n ingress =~ \"$ingress\"\n }[1m]\n )\n )\n)", + "interval": "", + "legendFormat": ".99", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Total request handling time", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "The time spent on receiving the response from the upstream server", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 0 + }, + "hiddenSeries": false, + "id": 94, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(\n 0.5,\n sum by (le)(\n rate(\n nginx_ingress_controller_response_duration_seconds_bucket{\n ingress =~ \"$ingress\"\n }[1m]\n )\n )\n)", + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": ".5", + "refId": "D" + }, + { + "expr": "histogram_quantile(\n 0.95,\n sum by (le)(\n rate(\n nginx_ingress_controller_response_duration_seconds_bucket{\n ingress =~ \"$ingress\"\n }[1m]\n )\n )\n)", + "interval": "", + "legendFormat": ".95", + "refId": "B" + }, + { + "expr": "histogram_quantile(\n 0.99,\n sum by (le)(\n rate(\n nginx_ingress_controller_response_duration_seconds_bucket{\n ingress =~ \"$ingress\"\n }[1m]\n )\n )\n)", + "interval": "", + "legendFormat": ".99", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Upstream response time", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 8 + }, + "hiddenSeries": false, + "id": 93, + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": " sum by (path)(\n rate(\n nginx_ingress_controller_request_duration_seconds_count{\n ingress =~ \"$ingress\"\n }[1m]\n )\n )\n", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{ path }}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Request volume by Path", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "reqps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "For each path observed, its median upstream response time", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 8 + }, + "hiddenSeries": false, + "id": 98, + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(\n .5,\n sum by (le, path)(\n rate(\n nginx_ingress_controller_response_duration_seconds_bucket{\n ingress =~ \"$ingress\"\n }[1m]\n )\n )\n)", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{ path }}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Median upstream response time by Path", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "Percentage of 4xx and 5xx responses among all responses.", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 16 + }, + "hiddenSeries": false, + "id": 100, + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null as zero", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum by (path) (rate(nginx_ingress_controller_request_duration_seconds_count{\n ingress =~ \"$ingress\",\n status =~ \"[4-5].*\"\n}[1m])) / sum by (path) (rate(nginx_ingress_controller_request_duration_seconds_count{\n ingress =~ \"$ingress\",\n}[1m]))", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{ path }}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Response error rate by Path", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "For each path observed, the sum of upstream request time", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 16 + }, + "hiddenSeries": false, + "id": 102, + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum by (path) (rate(nginx_ingress_controller_response_duration_seconds_sum{ingress =~ \"$ingress\"}[1m]))", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{ path }}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Upstream time consumed by Path", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 24 + }, + "hiddenSeries": false, + "id": 101, + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": " sum (\n rate(\n nginx_ingress_controller_request_duration_seconds_count{\n ingress =~ \"$ingress\",\n status =~\"[4-5].*\",\n }[1m]\n )\n ) by(path, status)\n", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{ path }} {{ status }}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Response error volume by Path", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "reqps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 24 + }, + "hiddenSeries": false, + "id": 99, + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum (\n rate (\n nginx_ingress_controller_response_size_sum {\n ingress =~ \"$ingress\",\n }[1m]\n )\n) by (path) / sum (\n rate(\n nginx_ingress_controller_response_size_count {\n ingress =~ \"$ingress\",\n }[1m]\n )\n) by (path)\n", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{ path }}", + "refId": "D" + }, + { + "expr": " sum (rate(nginx_ingress_controller_response_size_bucket{\n ingress =~ \"$ingress\",\n }[1m])) by (le)\n", + "hide": true, + "legendFormat": "{{le}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Average response size by Path", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "decbytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 32 + }, + "hiddenSeries": false, + "id": 96, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum (\n rate(\n nginx_ingress_controller_ingress_upstream_latency_seconds_sum {\n ingress =~ \"$ingress\",\n }[1m]\n)) / sum (\n rate(\n nginx_ingress_controller_ingress_upstream_latency_seconds_count {\n ingress =~ \"$ingress\",\n }[1m]\n )\n)\n", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "average", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Upstream service latency", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + } + ], + "refresh": "30s", + "schemaVersion": 22, + "style": "dark", + "tags": [ + "nginx" + ], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": "Data Source", + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": ".*", + "current": {}, + "datasource": "$datasource", + "definition": "label_values(nginx_ingress_controller_requests, ingress) ", + "hide": 0, + "includeAll": true, + "label": "Service Ingress", + "multi": false, + "name": "ingress", + "options": [], + "query": "label_values(nginx_ingress_controller_requests, ingress) ", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 2, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-15m", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "2m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "browser", + "title": "NGINX / Request Handling Performance", + "uid": "4GFbkOsZk", + "version": 1 +} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/files/rancher/cluster/rancher-cluster-nodes.json b/charts/rancher-monitoring/103.2.2+up57.0.3/files/rancher/cluster/rancher-cluster-nodes.json new file mode 100644 index 00000000000..1d4943501ba --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/files/rancher/cluster/rancher-cluster-nodes.json @@ -0,0 +1,793 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "id": 28, + "links": [], + "panels": [ + { + "aliasColors": { + "{{instance}}": "#3797d5" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 0 + }, + "hiddenSeries": false, + "id": 2, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "1 - avg(irate({__name__=~\"node_cpu_seconds_total|windows_cpu_time_total\",mode=\"idle\"}[$__rate_interval])) by (instance)", + "interval": "", + "legendFormat": "{{instance}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "CPU Utilization", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "percentunit", + "label": null, + "logBase": 1, + "max": "1", + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "{{instance}}": "#3797d5" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Load[5m] ({{instance}})" + }, + "properties": [] + } + ] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 0 + }, + "hiddenSeries": false, + "id": 3, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(node_load5 OR avg_over_time(windows_system_processor_queue_length[5m])) by (instance)", + "interval": "", + "legendFormat": "Load[5m] ({{instance}})", + "refId": "A" + }, + { + "expr": "sum(node_load1 OR avg_over_time(windows_system_processor_queue_length[1m])) by (instance)", + "interval": "", + "legendFormat": "Load[1m] ({{instance}})", + "refId": "B" + }, + { + "expr": "sum(node_load15 OR avg_over_time(windows_system_processor_queue_length[15m])) by (instance)", + "interval": "", + "legendFormat": "Load[15m] ({{instance}})", + "refId": "C" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Load Average", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "{{instance}}": "#3797d5" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 0 + }, + "hiddenSeries": false, + "id": 4, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "1 - sum(node_memory_MemAvailable_bytes OR windows_os_physical_memory_free_bytes) by (instance) / sum(node_memory_MemTotal_bytes OR windows_cs_physical_memory_bytes) by (instance) ", + "interval": "", + "legendFormat": "{{instance}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Memory Utilization", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "percentunit", + "label": null, + "logBase": 1, + "max": "1", + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "{{instance}}": "#3797d5" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 7 + }, + "hiddenSeries": false, + "id": 5, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "1 - (sum(node_filesystem_free_bytes{device!~\"rootfs|HarddiskVolume.+\"} OR windows_logical_disk_free_bytes{volume!~\"(HarddiskVolume.+|[A-Z]:.+)\"}) by (instance) / sum(node_filesystem_size_bytes{device!~\"rootfs|HarddiskVolume.+\"} OR windows_logical_disk_size_bytes{volume!~\"(HarddiskVolume.+|[A-Z]:.+)\"}) by (instance))", + "interval": "", + "legendFormat": "{{instance}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Disk Utilization", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "percentunit", + "label": null, + "logBase": 1, + "max": "1", + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "{{instance}}": "#3797d5" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 7 + }, + "hiddenSeries": false, + "id": 6, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(node_disk_read_bytes_total[$__rate_interval]) OR rate(windows_logical_disk_read_bytes_total[$__rate_interval])) by (instance)", + "interval": "", + "legendFormat": "Read ({{instance}})", + "refId": "A" + }, + { + "expr": "sum(rate(node_disk_written_bytes_total[$__rate_interval]) OR rate(windows_logical_disk_write_bytes_total[$__rate_interval])) by (instance)", + "interval": "", + "legendFormat": "Write ({{instance}})", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Disk I/O", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "{{instance}}": "#3797d5" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 7 + }, + "hiddenSeries": false, + "id": 7, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(node_network_receive_errs_total{device!~\"lo|veth.*|docker.*|flannel.*|cali.*|cbr.*\"}[$__rate_interval])) by (instance) OR sum(rate(windows_net_packets_received_errors_total{nic!~'.*isatap.*|.*VPN.*|.*Pseudo.*|.*tunneling.*'}[$__rate_interval])) by (instance)", + "interval": "", + "legendFormat": "Receive Errors ({{instance}})", + "refId": "A" + }, + { + "expr": "sum(rate(node_network_receive_packets_total{device!~\"lo|veth.*|docker.*|flannel.*|cali.*|cbr.*\"}[$__rate_interval])) by (instance) OR sum(rate(windows_net_packets_received_total_total{nic!~'.*isatap.*|.*VPN.*|.*Pseudo.*|.*tunneling.*'}[$__rate_interval])) by (instance)", + "interval": "", + "legendFormat": "Receive Total ({{instance}})", + "refId": "B" + }, + { + "expr": "sum(rate(node_network_transmit_errs_total{device!~\"lo|veth.*|docker.*|flannel.*|cali.*|cbr.*\"}[$__rate_interval])) by (instance) OR sum(rate(windows_net_packets_outbound_errors_total{nic!~'.*isatap.*|.*VPN.*|.*Pseudo.*|.*tunneling.*'}[$__rate_interval])) by (instance)", + "interval": "", + "legendFormat": "Transmit Errors ({{instance}})", + "refId": "C" + }, + { + "expr": "sum(rate(node_network_receive_drop_total{device!~\"lo|veth.*|docker.*|flannel.*|cali.*|cbr.*\"}[$__rate_interval])) by (instance) OR sum(rate(windows_net_packets_received_discarded_total{nic!~'.*isatap.*|.*VPN.*|.*Pseudo.*|.*tunneling.*'}[$__rate_interval])) by (instance)", + "interval": "", + "legendFormat": "Receive Dropped ({{instance}})", + "refId": "D" + }, + { + "expr": "sum(rate(node_network_transmit_drop_total{device!~\"lo|veth.*|docker.*|flannel.*|cali.*|cbr.*\"}[$__rate_interval])) by (instance) OR sum(rate(windows_net_packets_outbound_discarded{nic!~'.*isatap.*|.*VPN.*|.*Pseudo.*|.*tunneling.*'}[$__rate_interval])) by (instance)", + "interval": "", + "legendFormat": "Transmit Dropped ({{instance}})", + "refId": "E" + }, + { + "expr": "sum(rate(node_network_transmit_packets_total{device!~\"lo|veth.*|docker.*|flannel.*|cali.*|cbr.*\"}[$__rate_interval])) by (instance) OR sum(rate(windows_net_packets_sent_total{nic!~'.*isatap.*|.*VPN.*|.*Pseudo.*|.*tunneling.*'}[$__rate_interval])) by (instance)", + "interval": "", + "legendFormat": "Transmit Total ({{instance}})", + "refId": "F" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Network Traffic", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "{{instance}}": "#3797d5" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 14 + }, + "hiddenSeries": false, + "id": 8, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(node_network_transmit_bytes_total{device!~\"lo|veth.*|docker.*|flannel.*|cali.*|cbr.*\"}[$__rate_interval]) OR rate(windows_net_packets_sent_total{nic!~'.*isatap.*|.*VPN.*|.*Pseudo.*|.*tunneling.*'}[$__rate_interval])) by (instance)", + "interval": "", + "legendFormat": "Transmit Total ({{instance}})", + "refId": "A" + }, + { + "expr": "sum(rate(node_network_receive_bytes_total{device!~\"lo|veth.*|docker.*|flannel.*|cali.*|cbr.*\"}[$__rate_interval]) OR rate(windows_net_packets_received_total_total{nic!~'.*isatap.*|.*VPN.*|.*Pseudo.*|.*tunneling.*'}[$__rate_interval])) by (instance)", + "interval": "", + "legendFormat": "Receive Total ({{instance}})", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Network I/O", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + } + ], + "schemaVersion": 26, + "style": "dark", + "tags": [], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": "Data Source", + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ] + }, + "timezone": "", + "title": "Rancher / Cluster (Nodes)", + "uid": "rancher-cluster-nodes-1", + "version": 3 +} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/files/rancher/cluster/rancher-cluster.json b/charts/rancher-monitoring/103.2.2+up57.0.3/files/rancher/cluster/rancher-cluster.json new file mode 100644 index 00000000000..24385a237af --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/files/rancher/cluster/rancher-cluster.json @@ -0,0 +1,776 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "id": 28, + "links": [], + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 0 + }, + "hiddenSeries": false, + "id": 2, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "1 - avg(irate({__name__=~\"node_cpu_seconds_total|windows_cpu_time_total\",mode=\"idle\"}[$__rate_interval]))", + "legendFormat": "Total", + "interval": "", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "CPU Utilization", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "percentunit", + "label": null, + "logBase": 1, + "max": "1", + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Load[5m]" + }, + "properties": [] + } + ] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 0 + }, + "hiddenSeries": false, + "id": 3, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(node_load5 OR avg_over_time(windows_system_processor_queue_length[5m]))", + "interval": "", + "legendFormat": "Load[5m]", + "refId": "A" + }, + { + "expr": "sum(node_load1 OR avg_over_time(windows_system_processor_queue_length[1m]))", + "interval": "", + "legendFormat": "Load[1m]", + "refId": "B" + }, + { + "expr": "sum(node_load15 OR avg_over_time(windows_system_processor_queue_length[15m]))", + "interval": "", + "legendFormat": "Load[15m]", + "refId": "C" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Load Average", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 0 + }, + "hiddenSeries": false, + "id": 4, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "1 - sum(node_memory_MemAvailable_bytes OR windows_os_physical_memory_free_bytes) / sum(node_memory_MemTotal_bytes OR windows_cs_physical_memory_bytes)", + "legendFormat": "Total", + "interval": "", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Memory Utilization", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "percentunit", + "label": null, + "logBase": 1, + "max": "1", + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 7 + }, + "hiddenSeries": false, + "id": 5, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "1 - (sum(node_filesystem_free_bytes{device!~\"rootfs|HarddiskVolume.+\"} OR windows_logical_disk_free_bytes{volume!~\"(HarddiskVolume.+|[A-Z]:.+)\"}) / sum(node_filesystem_size_bytes{device!~\"rootfs|HarddiskVolume.+\"} OR windows_logical_disk_size_bytes{volume!~\"(HarddiskVolume.+|[A-Z]:.+)\"}))", + "legendFormat": "Total", + "interval": "", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Disk Utilization", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "percentunit", + "label": null, + "logBase": 1, + "max": "1", + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 7 + }, + "hiddenSeries": false, + "id": 6, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(node_disk_read_bytes_total[$__rate_interval]) OR rate(windows_logical_disk_read_bytes_total[$__rate_interval]))", + "interval": "", + "legendFormat": "Read", + "refId": "A" + }, + { + "expr": "sum(rate(node_disk_written_bytes_total[$__rate_interval]) OR rate(windows_logical_disk_write_bytes_total[$__rate_interval]))", + "interval": "", + "legendFormat": "Write", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Disk I/O", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 7 + }, + "hiddenSeries": false, + "id": 7, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "(sum(rate(node_network_receive_errs_total{device!~\"lo|veth.*|docker.*|flannel.*|cali.*|cbr.*\"}[$__rate_interval])) OR on() vector(0)) + (sum(rate(windows_net_packets_received_errors_total{nic!~'.*isatap.*|.*VPN.*|.*Pseudo.*|.*tunneling.*'}[$__rate_interval])) OR on() vector(0))", + "interval": "", + "legendFormat": "Receive Errors", + "refId": "A" + }, + { + "expr": "(sum(rate(node_network_receive_packets_total{device!~\"lo|veth.*|docker.*|flannel.*|cali.*|cbr.*\"}[$__rate_interval])) OR on() vector(0)) + (sum(rate(windows_net_packets_received_total_total{nic!~'.*isatap.*|.*VPN.*|.*Pseudo.*|.*tunneling.*'}[$__rate_interval])) OR on() vector(0))", + "interval": "", + "legendFormat": "Receive Total", + "refId": "B" + }, + { + "expr": "(sum(rate(node_network_transmit_errs_total{device!~\"lo|veth.*|docker.*|flannel.*|cali.*|cbr.*\"}[$__rate_interval])) OR on() vector(0)) + (sum(rate(windows_net_packets_outbound_errors_total{nic!~'.*isatap.*|.*VPN.*|.*Pseudo.*|.*tunneling.*'}[$__rate_interval])) OR on() vector(0))", + "interval": "", + "legendFormat": "Transmit Errors", + "refId": "C" + }, + { + "expr": "(sum(rate(node_network_receive_drop_total{device!~\"lo|veth.*|docker.*|flannel.*|cali.*|cbr.*\"}[$__rate_interval])) OR on() vector(0)) + (sum(rate(windows_net_packets_received_discarded_total{nic!~'.*isatap.*|.*VPN.*|.*Pseudo.*|.*tunneling.*'}[$__rate_interval])) OR on() vector(0))", + "interval": "", + "legendFormat": "Receive Dropped", + "refId": "D" + }, + { + "expr": "(sum(rate(node_network_transmit_drop_total{device!~\"lo|veth.*|docker.*|flannel.*|cali.*|cbr.*\"}[$__rate_interval])) OR on() vector(0)) + (sum(rate(windows_net_packets_outbound_discarded{nic!~'.*isatap.*|.*VPN.*|.*Pseudo.*|.*tunneling.*'}[$__rate_interval])) OR on() vector(0))", + "interval": "", + "legendFormat": "Transmit Dropped", + "refId": "E" + }, + { + "expr": "(sum(rate(node_network_transmit_packets_total{device!~\"lo|veth.*|docker.*|flannel.*|cali.*|cbr.*\"}[$__rate_interval])) OR on() vector(0)) + (sum(rate(windows_net_packets_sent_total{nic!~'.*isatap.*|.*VPN.*|.*Pseudo.*|.*tunneling.*'}[$__rate_interval])) OR on() vector(0))", + "interval": "", + "legendFormat": "Transmit Total", + "refId": "F" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Network Traffic", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 14 + }, + "hiddenSeries": false, + "id": 8, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(node_network_transmit_bytes_total{device!~\"lo|veth.*|docker.*|flannel.*|cali.*|cbr.*\"}[$__rate_interval]) OR rate(windows_net_packets_sent_total{nic!~'.*isatap.*|.*VPN.*|.*Pseudo.*|.*tunneling.*'}[$__rate_interval]))", + "interval": "", + "legendFormat": "Transmit Total", + "refId": "A" + }, + { + "expr": "sum(rate(node_network_receive_bytes_total{device!~\"lo|veth.*|docker.*|flannel.*|cali.*|cbr.*\"}[$__rate_interval]) OR rate(windows_net_packets_received_total_total{nic!~'.*isatap.*|.*VPN.*|.*Pseudo.*|.*tunneling.*'}[$__rate_interval]))", + "interval": "", + "legendFormat": "Receive Total", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Network I/O", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + } + ], + "schemaVersion": 26, + "style": "dark", + "tags": [], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": "Data Source", + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ] + }, + "timezone": "", + "title": "Rancher / Cluster", + "uid": "rancher-cluster-1", + "version": 3 +} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/files/rancher/fleet/bundle.json b/charts/rancher-monitoring/103.2.2+up57.0.3/files/rancher/fleet/bundle.json new file mode 100644 index 00000000000..698f48aeed5 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/files/rancher/fleet/bundle.json @@ -0,0 +1,246 @@ +{ + "description": "Bundle", + "graphTooltip": 1, + "panels": [ + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "fieldConfig": { + "defaults": { + "decimals": 0, + "unit": "percentunit" + } + }, + "gridPos": { + "h": 5, + "w": 7, + "x": 0, + "y": 0 + }, + "id": 1, + "pluginVersion": "v11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_bundle_ready{exported_namespace=\"$namespace\",name=~\"$name\"}) / sum(fleet_bundle_desired_ready{exported_namespace=\"$namespace\",name=~\"$name\"})" + } + ], + "title": "Ready Bundles", + "type": "stat" + }, + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "fieldConfig": { + "defaults": { + "decimals": 0, + "unit": null + } + }, + "gridPos": { + "h": 5, + "w": 17, + "x": 7, + "y": 0 + }, + "id": 2, + "pluginVersion": "v11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_bundle_desired_ready{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Desired Ready" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_bundle_ready{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Ready" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_bundle_not_ready{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Not Ready" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_bundle_out_of_sync{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Out of Sync" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_bundle_err_applied{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Err Applied" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_bundle_modified{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Modified" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_bundle_pending{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Pending" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_bundle_wait_applied{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Wait Applied" + } + ], + "title": "Bundles", + "type": "stat" + }, + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "fieldConfig": { + "defaults": { + "decimals": 0, + "unit": null + } + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 8 + }, + "id": 3, + "pluginVersion": "v11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_bundle_desired_ready{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Desired Ready" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_bundle_ready{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Ready" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_bundle_not_ready{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Not Ready" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_bundle_out_of_sync{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Out of Sync" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_bundle_err_applied{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Err Applied" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_bundle_modified{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Modified" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_bundle_pending{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Pending" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_bundle_wait_applied{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Wait Applied" + } + ], + "title": "Bundles", + "type": "timeseries" + } + ], + "schemaVersion": 39, + "templating": { + "list": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "name": "namespace", + "query": "label_values(fleet_bundle_desired_ready, exported_namespace)", + "refresh": 2, + "type": "query" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "includeAll": true, + "name": "name", + "query": "label_values(fleet_bundle_desired_ready{exported_namespace=~\"$namespace\"}, name)", + "refresh": 2, + "type": "query" + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timezone": "utc", + "title": "Fleet / Bundle", + "uid": "fleet-bundle" +} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/files/rancher/fleet/bundledeployment.json b/charts/rancher-monitoring/103.2.2+up57.0.3/files/rancher/fleet/bundledeployment.json new file mode 100644 index 00000000000..c81f7a62128 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/files/rancher/fleet/bundledeployment.json @@ -0,0 +1,219 @@ +{ + "description": "BundleDeployment", + "graphTooltip": 1, + "panels": [ + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "fieldConfig": { + "defaults": { + "decimals": 0, + "unit": "percentunit" + } + }, + "gridPos": { + "h": 5, + "w": 7, + "x": 0, + "y": 0 + }, + "id": 1, + "pluginVersion": "v11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_bundledeployment_state{cluster_namespace=~\"$namespace\",state=\"Ready\"}) / sum(fleet_bundledeployment_state{cluster_namespace=~\"$namespace\"})" + } + ], + "title": "Ready BundleDeployments", + "type": "stat" + }, + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "fieldConfig": { + "defaults": { + "decimals": 0, + "unit": null + } + }, + "gridPos": { + "h": 5, + "w": 17, + "x": 7, + "y": 0 + }, + "id": 2, + "pluginVersion": "v11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_bundledeployment_state{cluster_namespace=~\"$namespace\",state=\"Ready\"})", + "legendFormat": "Ready" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_bundledeployment_state{cluster_namespace=~\"$namespace\",state=\"NotReady\"})", + "legendFormat": "Not Ready" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_bundledeployment_state{cluster_namespace=~\"$namespace\",state=\"WaitApplied\"})", + "legendFormat": "Wait Applied" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_bundledeployment_state{cluster_namespace=~\"$namespace\",state=\"ErrApplied\"})", + "legendFormat": "Err Applied" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_bundledeployment_state{cluster_namespace=~\"$namespace\",state=\"OutOfSync\"})", + "legendFormat": "OutOfSync" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_bundledeployment_state{cluster_namespace=~\"$namespace\",state=\"Pending\"})", + "legendFormat": "Pending" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_bundledeployment_state{cluster_namespace=~\"$namespace\",state=\"Modified\"})", + "legendFormat": "Modified" + } + ], + "title": "BundleDeployments", + "type": "stat" + }, + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "fieldConfig": { + "defaults": { + "decimals": 0, + "unit": null + } + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 8 + }, + "id": 3, + "pluginVersion": "v11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_bundledeployment_state{cluster_namespace=~\"$namespace\",state=\"Ready\"})", + "legendFormat": "Ready" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_bundledeployment_state{cluster_namespace=~\"$namespace\",state=\"NotReady\"})", + "legendFormat": "Not Ready" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_bundledeployment_state{cluster_namespace=~\"$namespace\",state=\"WaitApplied\"})", + "legendFormat": "Wait Applied" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_bundledeployment_state{cluster_namespace=~\"$namespace\",state=\"ErrApplied\"})", + "legendFormat": "Err Applied" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_bundledeployment_state{cluster_namespace=~\"$namespace\",state=\"OutOfSync\"})", + "legendFormat": "OutOfSync" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_bundledeployment_state{cluster_namespace=~\"$namespace\",state=\"Pending\"})", + "legendFormat": "Pending" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_bundledeployment_state{cluster_namespace=~\"$namespace\",state=\"Modified\"})", + "legendFormat": "Modified" + } + ], + "title": "BundleDeployments", + "type": "timeseries" + } + ], + "schemaVersion": 39, + "templating": { + "list": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "name": "namespace", + "query": "label_values(fleet_bundledeployment_state, cluster_namespace)", + "refresh": 2, + "type": "query" + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timezone": "utc", + "title": "Fleet / BundleDeployment", + "uid": "fleet-bundledeployment" +} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/files/rancher/fleet/cluster.json b/charts/rancher-monitoring/103.2.2+up57.0.3/files/rancher/fleet/cluster.json new file mode 100644 index 00000000000..73bdea48342 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/files/rancher/fleet/cluster.json @@ -0,0 +1,484 @@ +{ + "description": "Cluster", + "graphTooltip": 1, + "panels": [ + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "fieldConfig": { + "defaults": { + "decimals": 0, + "unit": "percentunit" + } + }, + "gridPos": { + "h": 5, + "w": 7, + "x": 0, + "y": 0 + }, + "id": 1, + "pluginVersion": "v11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_cluster_ready_git_repos{exported_namespace=\"$namespace\",name=~\"$name\"}) / sum(fleet_cluster_desired_ready_git_repos{exported_namespace=\"$namespace\",name=~\"$name\"})" + } + ], + "title": "Ready Git Repos", + "type": "stat" + }, + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "fieldConfig": { + "defaults": { + "decimals": 0, + "unit": null + } + }, + "gridPos": { + "h": 5, + "w": 17, + "x": 7, + "y": 0 + }, + "id": 2, + "pluginVersion": "v11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_cluster_desired_ready_git_repos{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Desired Ready" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_cluster_ready_git_repos{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Ready" + } + ], + "title": "Git Repos", + "type": "stat" + }, + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "fieldConfig": { + "defaults": { + "decimals": 0, + "unit": null + } + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 8 + }, + "id": 3, + "pluginVersion": "v11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_cluster_desired_ready_git_repos{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Desired Ready" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_cluster_ready_git_repos{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Ready" + } + ], + "title": "Git Repos", + "type": "timeseries" + }, + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "fieldConfig": { + "defaults": { + "decimals": 0, + "unit": "percentunit" + } + }, + "gridPos": { + "h": 5, + "w": 7, + "x": 0, + "y": 13 + }, + "id": 4, + "pluginVersion": "v11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_cluster_resources_count_ready{exported_namespace=\"$namespace\",name=~\"$name\"}) / sum(fleet_cluster_resources_count_desiredready{exported_namespace=\"$namespace\",name=~\"$name\"})" + } + ], + "title": "Ready Resources", + "type": "stat" + }, + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "fieldConfig": { + "defaults": { + "decimals": 0, + "unit": null + } + }, + "gridPos": { + "h": 5, + "w": 17, + "x": 7, + "y": 13 + }, + "id": 5, + "pluginVersion": "v11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_cluster_resources_count_desiredready{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Desired Ready" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_cluster_resources_count_ready{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Ready" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_cluster_resources_count_notready{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Not Ready" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_cluster_resources_count_missing{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Missing" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_cluster_resources_count_modified{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Modified" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_cluster_resources_count_unknown{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Unknown" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_cluster_resources_count_orphaned{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Orphaned" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_cluster_resources_count_waitapplied{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Wait Applied" + } + ], + "title": "Resources", + "type": "stat" + }, + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "fieldConfig": { + "defaults": { + "decimals": 0, + "unit": null + } + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 21 + }, + "id": 6, + "pluginVersion": "v11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_cluster_resources_count_desiredready{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Desired Ready" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_cluster_resources_count_ready{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Ready" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_cluster_resources_count_notready{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Not Ready" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_cluster_resources_count_missing{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Missing" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_cluster_resources_count_modified{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Modified" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_cluster_resources_count_unknown{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Unknown" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_cluster_resources_count_orphaned{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Orphaned" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_cluster_resources_count_waitapplied{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Wait Applied" + } + ], + "title": "Resources", + "type": "timeseries" + }, + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "fieldConfig": { + "defaults": { + "decimals": 0, + "unit": "percentunit" + } + }, + "gridPos": { + "h": 5, + "w": 7, + "x": 0, + "y": 26 + }, + "id": 7, + "pluginVersion": "v11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_cluster_state{exported_namespace=\"$namespace\",name=~\"$name\",state=\"Ready\"}) / sum(fleet_cluster_state{exported_namespace=\"$namespace\",name=~\"$name\"})" + } + ], + "title": "Ready Clusters", + "type": "stat" + }, + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "fieldConfig": { + "defaults": { + "decimals": 0, + "unit": null + } + }, + "gridPos": { + "h": 5, + "w": 17, + "x": 7, + "y": 26 + }, + "id": 8, + "pluginVersion": "v11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_cluster_state{exported_namespace=\"$namespace\",name=~\"$name\",state=\"Ready\"})", + "legendFormat": "Ready" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_cluster_state{exported_namespace=\"$namespace\",name=~\"$name\",state=\"NotReady\"})", + "legendFormat": "Not Ready" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_cluster_state{exported_namespace=\"$namespace\",name=~\"$name\",state=\"WaitCheckIn\"})", + "legendFormat": "Wait Check In" + } + ], + "title": "Clusters", + "type": "stat" + }, + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "fieldConfig": { + "defaults": { + "decimals": 0, + "unit": null + } + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 34 + }, + "id": 9, + "pluginVersion": "v11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_cluster_state{exported_namespace=\"$namespace\",name=~\"$name\",state=\"Ready\"})", + "legendFormat": "Ready" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_cluster_state{exported_namespace=\"$namespace\",name=~\"$name\",state=\"NotReady\"})", + "legendFormat": "Not Ready" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_cluster_state{exported_namespace=\"$namespace\",name=~\"$name\",state=\"WaitCheckIn\"})", + "legendFormat": "Wait Check In" + } + ], + "title": "Clusters", + "type": "timeseries" + } + ], + "schemaVersion": 39, + "templating": { + "list": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "name": "namespace", + "query": "label_values(fleet_cluster_desired_ready_git_repos, exported_namespace)", + "refresh": 2, + "type": "query" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "includeAll": true, + "name": "name", + "query": "label_values(fleet_cluster_desired_ready_git_repos{exported_namespace=~\"$namespace\"}, name)", + "refresh": 2, + "type": "query" + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timezone": "utc", + "title": "Fleet / Cluster", + "uid": "fleet-cluster" +} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/files/rancher/fleet/clustergroup.json b/charts/rancher-monitoring/103.2.2+up57.0.3/files/rancher/fleet/clustergroup.json new file mode 100644 index 00000000000..ce3df87b21d --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/files/rancher/fleet/clustergroup.json @@ -0,0 +1,468 @@ +{ + "description": "ClusterGroup", + "graphTooltip": 1, + "panels": [ + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "fieldConfig": { + "defaults": { + "decimals": 0, + "unit": "percentunit" + } + }, + "gridPos": { + "h": 5, + "w": 7, + "x": 0, + "y": 0 + }, + "id": 1, + "pluginVersion": "v11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_cluster_group_bundle_ready{exported_namespace=\"$namespace\",name=~\"$name\"}) / sum(fleet_cluster_group_bundle_desired_ready{exported_namespace=\"$namespace\",name=~\"$name\"})" + } + ], + "title": "Ready Bundles", + "type": "stat" + }, + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "fieldConfig": { + "defaults": { + "decimals": 0, + "unit": null + } + }, + "gridPos": { + "h": 5, + "w": 17, + "x": 7, + "y": 0 + }, + "id": 2, + "pluginVersion": "v11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_cluster_group_bundle_desired_ready{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Desired Ready" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_cluster_group_bundle_ready{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Ready" + } + ], + "title": "Bundles", + "type": "stat" + }, + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "fieldConfig": { + "defaults": { + "decimals": 0, + "unit": null + } + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 8 + }, + "id": 3, + "pluginVersion": "v11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_cluster_group_bundle_desired_ready{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Desired Ready" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_cluster_group_bundle_ready{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Ready" + } + ], + "title": "Bundles", + "type": "timeseries" + }, + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "fieldConfig": { + "defaults": { + "decimals": 0, + "unit": "percentunit" + } + }, + "gridPos": { + "h": 5, + "w": 7, + "x": 0, + "y": 13 + }, + "id": 4, + "pluginVersion": "v11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "(sum(fleet_cluster_group_cluster_count{exported_namespace=\"$namespace\",name=~\"$name\"}) - sum(fleet_cluster_group_non_ready_cluster_count{exported_namespace=\"$namespace\",name=~\"$name\"})) / sum(fleet_cluster_group_cluster_count{exported_namespace=\"$namespace\",name=~\"$name\"})" + } + ], + "title": "Ready Clusters", + "type": "stat" + }, + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "fieldConfig": { + "defaults": { + "decimals": 0, + "unit": null + } + }, + "gridPos": { + "h": 5, + "w": 17, + "x": 7, + "y": 13 + }, + "id": 5, + "pluginVersion": "v11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_cluster_group_cluster_count{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Total" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_cluster_group_non_ready_cluster_count{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Non Ready" + } + ], + "title": "Clusters", + "type": "stat" + }, + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "fieldConfig": { + "defaults": { + "decimals": 0, + "unit": null + } + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 21 + }, + "id": 6, + "pluginVersion": "v11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_cluster_group_cluster_count{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Total" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_cluster_group_non_ready_cluster_count{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Non Ready" + } + ], + "title": "Clusters", + "type": "timeseries" + }, + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "fieldConfig": { + "defaults": { + "decimals": 0, + "unit": "percentunit" + } + }, + "gridPos": { + "h": 5, + "w": 7, + "x": 0, + "y": 26 + }, + "id": 7, + "pluginVersion": "v11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_cluster_group_resource_count_ready{exported_namespace=\"$namespace\",name=~\"$name\"}) / sum(fleet_cluster_group_resource_count_desired_ready{exported_namespace=\"$namespace\",name=~\"$name\"})" + } + ], + "title": "Ready Resources", + "type": "stat" + }, + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "fieldConfig": { + "defaults": { + "decimals": 0, + "unit": null + } + }, + "gridPos": { + "h": 5, + "w": 17, + "x": 7, + "y": 26 + }, + "id": 8, + "pluginVersion": "v11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_cluster_group_resource_count_desired_ready{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Desired Ready" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_cluster_group_resource_count_ready{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Ready" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_cluster_group_resource_count_notready{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Not Ready" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_cluster_group_resource_count_missing{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Missing" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_cluster_group_resource_count_modified{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Modified" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_cluster_group_resource_count_orphaned{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Orphaned" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_cluster_group_resource_count_unknown{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Unknown" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_cluster_group_resource_count_waitapplied{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Wait Applied" + } + ], + "title": "Resources", + "type": "stat" + }, + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "fieldConfig": { + "defaults": { + "decimals": 0, + "unit": null + } + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 34 + }, + "id": 9, + "pluginVersion": "v11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_cluster_group_resource_count_desired_ready{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Desired Ready" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_cluster_group_resource_count_ready{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Ready" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_cluster_group_resource_count_notready{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Not Ready" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_cluster_group_resource_count_missing{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Missing" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_cluster_group_resource_count_modified{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Modified" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_cluster_group_resource_count_orphaned{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Orphaned" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_cluster_group_resource_count_unknown{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Unknown" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_cluster_group_resource_count_waitapplied{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Wait Applied" + } + ], + "title": "Resources", + "type": "timeseries" + } + ], + "schemaVersion": 39, + "templating": { + "list": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "name": "namespace", + "query": "label_values(fleet_cluster_group_bundle_desired_ready, exported_namespace)", + "refresh": 2, + "type": "query" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "includeAll": true, + "name": "name", + "query": "label_values(fleet_cluster_group_bundle_desired_ready{exported_namespace=~\"$namespace\"}, name)", + "refresh": 2, + "type": "query" + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timezone": "utc", + "title": "Fleet / ClusterGroup", + "uid": "fleet-cluster-group" +} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/files/rancher/fleet/controller-runtime.json b/charts/rancher-monitoring/103.2.2+up57.0.3/files/rancher/fleet/controller-runtime.json new file mode 100644 index 00000000000..23a81f2a8c5 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/files/rancher/fleet/controller-runtime.json @@ -0,0 +1,454 @@ +{ + "description": "Controller Runtime", + "graphTooltip": 1, + "panels": [ + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "fieldConfig": { + "defaults": { + "decimals": 0, + "unit": null + } + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 1, + "pluginVersion": "v11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "controller_runtime_active_workers{job=\"$job\", namespace=\"$namespace\"}", + "legendFormat": "{{controller}} {{instance}}" + } + ], + "title": "Number of Workers in Use", + "type": "stat" + }, + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "fieldConfig": { + "defaults": { + "decimals": null, + "unit": null + } + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 8 + }, + "id": 2, + "pluginVersion": "v11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(rate(controller_runtime_reconcile_errors_total{job=\"$job\", namespace=\"$namespace\"}[5m])) by (instance, pod)", + "legendFormat": "{{instance}} {{pod}}" + } + ], + "title": "Reconciliation Error Count per Controller", + "type": "timeseries" + }, + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "fieldConfig": { + "defaults": { + "decimals": null, + "unit": null + } + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 16 + }, + "id": 3, + "pluginVersion": "v11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(rate(controller_runtime_reconcile_total{job=\"$job\", namespace=\"$namespace\"}[5m])) by (instance, pod)", + "legendFormat": "{{instance}} {{pod}}" + } + ], + "title": "Total Reconciliation Count per Controller", + "type": "timeseries" + }, + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "fieldConfig": { + "defaults": { + "decimals": 0, + "unit": null + } + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 24 + }, + "id": 4, + "pluginVersion": "v11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "workqueue_depth{job=\"$job\", namespace=\"$namespace\"}", + "legendFormat": "{{instance}} {{pod}}" + } + ], + "title": "WorkQueue Depth", + "type": "stat" + }, + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "fieldConfig": { + "defaults": { + "decimals": null, + "unit": null + } + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 32 + }, + "id": 5, + "pluginVersion": "v11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "histogram_quantile(0.50, sum(rate(workqueue_queue_duration_seconds_bucket{job=\"$job\", namespace=\"$namespace\"}[5m])) by (instance, name, le))", + "legendFormat": "P50 {{name}}" + } + ], + "title": "Seconds for Items Stay in Queue (before being requested) P50", + "type": "timeseries" + }, + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "fieldConfig": { + "defaults": { + "decimals": null, + "unit": null + } + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 40 + }, + "id": 6, + "pluginVersion": "v11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "histogram_quantile(0.90, sum(rate(workqueue_queue_duration_seconds_bucket{job=\"$job\", namespace=\"$namespace\"}[5m])) by (instance, name, le))", + "legendFormat": "P90 {{name}}" + } + ], + "title": "Seconds for Items Stay in Queue (before being requested) P90", + "type": "timeseries" + }, + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "fieldConfig": { + "defaults": { + "decimals": null, + "unit": null + } + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 48 + }, + "id": 7, + "pluginVersion": "v11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "histogram_quantile(0.99, sum(rate(workqueue_queue_duration_seconds_bucket{job=\"$job\", namespace=\"$namespace\"}[5m])) by (instance, name, le))", + "legendFormat": "P99 {{name}}" + } + ], + "title": "Seconds for Items Stay in Queue (before being requested) P99", + "type": "timeseries" + }, + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "fieldConfig": { + "defaults": { + "decimals": null, + "unit": null + } + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 56 + }, + "id": 8, + "pluginVersion": "v11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(rate(workqueue_adds_total{job=\"$job\", namespace=\"$namespace\"}[2m])) by (instance, name)", + "legendFormat": "{{name}} {{instance}}" + } + ], + "title": "Work Queue Add Rate", + "type": "timeseries" + }, + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "fieldConfig": { + "defaults": { + "decimals": null, + "unit": null + } + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 64 + }, + "id": 9, + "pluginVersion": "v11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "rate(workqueue_unfinished_work_seconds{job=\"$job\", namespace=\"$namespace\"}[5m])", + "legendFormat": "{{name}} {{instance}}" + } + ], + "title": "Unfinished Seconds", + "type": "stat" + }, + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "fieldConfig": { + "defaults": { + "decimals": null, + "unit": null + } + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 72 + }, + "id": 10, + "pluginVersion": "v11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "histogram_quantile(0.50, sum(rate(workqueue_work_duration_seconds_bucket{job=\"$job\", namespace=\"$namespace\"}[5m])) by (instance, name, le))", + "legendFormat": "P50 {{name}}" + } + ], + "title": "Seconds Processing Items from WorkQueue - 50th Percentile", + "type": "timeseries" + }, + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "fieldConfig": { + "defaults": { + "decimals": null, + "unit": null + } + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 80 + }, + "id": 11, + "pluginVersion": "v11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "histogram_quantile(0.90, sum(rate(workqueue_work_duration_seconds_bucket{job=\"$job\", namespace=\"$namespace\"}[5m])) by (instance, name, le))", + "legendFormat": "P90 {{name}}" + } + ], + "title": "Seconds Processing Items from WorkQueue - 90th Percentile", + "type": "timeseries" + }, + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "fieldConfig": { + "defaults": { + "decimals": null, + "unit": null + } + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 88 + }, + "id": 12, + "pluginVersion": "v11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "histogram_quantile(0.99, sum(rate(workqueue_work_duration_seconds_bucket{job=\"$job\", namespace=\"$namespace\"}[5m])) by (instance, name, le))", + "legendFormat": "P99 {{name}}" + } + ], + "title": "Seconds Processing Items from WorkQueue - 99th Percentile", + "type": "timeseries" + }, + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "fieldConfig": { + "defaults": { + "decimals": null, + "unit": null + } + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 96 + }, + "id": 13, + "pluginVersion": "v11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(rate(workqueue_retries_total{job=\"$job\", namespace=\"$namespace\"}[5m])) by (instance, name)", + "legendFormat": "{{name}} {{instance}}" + } + ], + "title": "Work Queue Retries Rate", + "type": "timeseries" + } + ], + "schemaVersion": 39, + "templating": { + "list": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "name": "namespace", + "query": "label_values(controller_runtime_reconcile_total, namespace)", + "refresh": 2, + "type": "query" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "name": "job", + "query": "label_values(controller_runtime_reconcile_total{namespace=~\"$namespace\"}, job)", + "refresh": 2, + "type": "query" + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timezone": "utc", + "title": "Fleet / Controller-Runtime", + "uid": "fleet-controller-runtime" +} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/files/rancher/fleet/gitrepo.json b/charts/rancher-monitoring/103.2.2+up57.0.3/files/rancher/fleet/gitrepo.json new file mode 100644 index 00000000000..1a50c2937da --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/files/rancher/fleet/gitrepo.json @@ -0,0 +1,325 @@ +{ + "description": "GitRepo", + "graphTooltip": 1, + "panels": [ + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "fieldConfig": { + "defaults": { + "decimals": 0, + "unit": "percentunit" + } + }, + "gridPos": { + "h": 5, + "w": 7, + "x": 0, + "y": 0 + }, + "id": 1, + "pluginVersion": "v11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_gitrepo_ready_clusters{exported_namespace=\"$namespace\",name=~\"$name\"}) / sum(fleet_gitrepo_desired_ready_clusters{exported_namespace=\"$namespace\",name=~\"$name\"})" + } + ], + "title": "Ready Clusters", + "type": "stat" + }, + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "fieldConfig": { + "defaults": { + "decimals": 0, + "unit": null + } + }, + "gridPos": { + "h": 5, + "w": 17, + "x": 7, + "y": 0 + }, + "id": 2, + "pluginVersion": "v11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_gitrepo_desired_ready_clusters{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Desired Ready" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_gitrepo_ready_clusters{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Ready" + } + ], + "title": "Clusters", + "type": "stat" + }, + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "fieldConfig": { + "defaults": { + "decimals": 0, + "unit": null + } + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 8 + }, + "id": 3, + "pluginVersion": "v11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_gitrepo_desired_ready_clusters{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Desired Ready" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_gitrepo_ready_clusters{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Ready" + } + ], + "title": "Clusters", + "type": "timeseries" + }, + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "fieldConfig": { + "defaults": { + "decimals": 0, + "unit": "percentunit" + } + }, + "gridPos": { + "h": 5, + "w": 7, + "x": 0, + "y": 13 + }, + "id": 4, + "pluginVersion": "v11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_gitrepo_resources_ready{exported_namespace=\"$namespace\",name=~\"$name\"}) / sum(fleet_gitrepo_resources_desired_ready{exported_namespace=\"$namespace\",name=~\"$name\"})" + } + ], + "title": "Ready Resources", + "type": "stat" + }, + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "fieldConfig": { + "defaults": { + "decimals": 0, + "unit": null + } + }, + "gridPos": { + "h": 5, + "w": 17, + "x": 7, + "y": 13 + }, + "id": 5, + "pluginVersion": "v11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_gitrepo_resources_desired_ready{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Desired Ready" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_gitrepo_resources_ready{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Ready" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_gitrepo_resources_not_ready{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Not Ready" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_gitrepo_resources_missing{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Missing" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_gitrepo_resources_modified{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Modified" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_gitrepo_resources_unknown{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Unknown" + } + ], + "title": "Resources", + "type": "stat" + }, + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "fieldConfig": { + "defaults": { + "decimals": 0, + "unit": null + } + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 21 + }, + "id": 6, + "pluginVersion": "v11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_gitrepo_resources_desired_ready{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Desired Ready" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_gitrepo_resources_ready{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Ready" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_gitrepo_resources_not_ready{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Not Ready" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_gitrepo_resources_missing{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Missing" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_gitrepo_resources_modified{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Modified" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_gitrepo_resources_unknown{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Unknown" + } + ], + "title": "Resources", + "type": "timeseries" + } + ], + "schemaVersion": 39, + "templating": { + "list": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "name": "namespace", + "query": "label_values(fleet_gitrepo_desired_ready_clusters, exported_namespace)", + "refresh": 2, + "type": "query" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "includeAll": true, + "name": "name", + "query": "label_values(fleet_gitrepo_desired_ready_clusters{exported_namespace=~\"$namespace\"}, name)", + "refresh": 2, + "type": "query" + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timezone": "utc", + "title": "Fleet / GitRepo", + "uid": "fleet-gitrepo" +} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/files/rancher/home/rancher-default-home.json b/charts/rancher-monitoring/103.2.2+up57.0.3/files/rancher/home/rancher-default-home.json new file mode 100644 index 00000000000..3fce2075616 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/files/rancher/home/rancher-default-home.json @@ -0,0 +1,1290 @@ +{ + "annotations": { + "list": [] + }, + "editable": false, + "gnetId": null, + "graphTooltip": 0, + "id": null, + "links": [], + "panels": [ + { + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 1, + "title": "", + "type": "welcome" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": true, + "colors": [ + "rgba(50, 172, 45, 0.97)", + "rgba(237, 129, 40, 0.89)", + "rgba(245, 54, 54, 0.9)" + ], + "datasource": "Prometheus", + "decimals": 2, + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "format": "percent", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": true, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 5, + "w": 8, + "x": 0, + "y": 4 + }, + "height": "180px", + "id": 6, + "interval": null, + "isNew": true, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "(1 - (avg(irate({__name__=~\"node_cpu_seconds_total|windows_cpu_time_total\",mode=\"idle\"}[5m])))) * 100", + "format": "time_series", + "interval": "10s", + "intervalFactor": 1, + "refId": "A", + "step": 10 + } + ], + "thresholds": "65, 90", + "title": "CPU Utilization", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "0", + "value": "null" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": true, + "colors": [ + "rgba(50, 172, 45, 0.97)", + "rgba(237, 129, 40, 0.89)", + "rgba(245, 54, 54, 0.9)" + ], + "datasource": "Prometheus", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "format": "percent", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": true, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 5, + "w": 8, + "x": 8, + "y": 4 + }, + "height": "180px", + "id": 4, + "interval": null, + "isNew": true, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "(1 - sum({__name__=~\"node_memory_MemAvailable_bytes|windows_os_physical_memory_free_bytes\"}) / sum({__name__=~\"node_memory_MemTotal_bytes|windows_cs_physical_memory_bytes\"})) * 100", + "format": "time_series", + "interval": "10s", + "intervalFactor": 1, + "refId": "A", + "step": 10 + } + ], + "thresholds": "65, 90", + "title": "Memory Utilization", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "0", + "value": "null" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": true, + "colors": [ + "rgba(50, 172, 45, 0.97)", + "rgba(237, 129, 40, 0.89)", + "rgba(245, 54, 54, 0.9)" + ], + "datasource": "Prometheus", + "decimals": 2, + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "format": "percent", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": true, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 5, + "w": 8, + "x": 16, + "y": 4 + }, + "height": "180px", + "id": 7, + "interval": null, + "isNew": true, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "(1 - (((sum(node_filesystem_free_bytes{device!~\"rootfs|HarddiskVolume.+\"}) OR on() vector(0)) + (sum(windows_logical_disk_free_bytes{volume!~\"(HarddiskVolume.+|[A-Z]:.+)\"}) OR on() vector(0))) / ((sum(node_filesystem_size_bytes{device!~\"rootfs|HarddiskVolume.+\"}) OR on() vector(0)) + (sum(windows_logical_disk_size_bytes{volume!~\"(HarddiskVolume.+|[A-Z]:.+)\"}) OR on() vector(0))))) * 100", + "format": "time_series", + "interval": "10s", + "intervalFactor": 1, + "metric": "", + "refId": "A", + "step": 10 + } + ], + "thresholds": "65, 90", + "title": "Disk Utilization", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "0", + "value": "null" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(50, 172, 45, 0.97)", + "rgba(237, 129, 40, 0.89)", + "rgba(245, 54, 54, 0.9)" + ], + "datasource": "Prometheus", + "decimals": 2, + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 0, + "y": 9 + }, + "height": "1px", + "id": 11, + "interval": null, + "isNew": true, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": " cores", + "postfixFontSize": "30%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "sum(irate({__name__=~\"node_cpu_seconds_total|windows_cpu_time_total\",mode!=\"idle\"}[5m]))", + "format": "time_series", + "interval": "10s", + "intervalFactor": 1, + "refId": "A", + "step": 10 + } + ], + "thresholds": "", + "title": "CPU Used", + "type": "singlestat", + "valueFontSize": "50%", + "valueMaps": [ + { + "op": "=", + "text": "0", + "value": "null" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(50, 172, 45, 0.97)", + "rgba(237, 129, 40, 0.89)", + "rgba(245, 54, 54, 0.9)" + ], + "datasource": "Prometheus", + "decimals": 2, + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 4, + "y": 9 + }, + "height": "1px", + "id": 12, + "interval": null, + "isNew": true, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": " cores", + "postfixFontSize": "30%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "sum(kube_node_status_allocatable_cpu_cores{}) OR sum(kube_node_status_allocatable{resource=\"cpu\",unit=\"core\"})", + "interval": "10s", + "intervalFactor": 1, + "refId": "A", + "step": 10 + } + ], + "thresholds": "", + "title": "CPU Total", + "type": "singlestat", + "valueFontSize": "50%", + "valueMaps": [ + { + "op": "=", + "text": "0", + "value": "null" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(50, 172, 45, 0.97)", + "rgba(237, 129, 40, 0.89)", + "rgba(245, 54, 54, 0.9)" + ], + "datasource": "Prometheus", + "decimals": 2, + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "format": "bytes", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 8, + "y": 9 + }, + "height": "1px", + "id": 9, + "interval": null, + "isNew": true, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "20%", + "prefix": "", + "prefixFontSize": "20%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "sum({__name__=~\"node_memory_MemTotal_bytes|windows_cs_physical_memory_bytes\"}) - sum({__name__=~\"node_memory_MemAvailable_bytes|windows_os_physical_memory_free_bytes\"})", + "interval": "10s", + "intervalFactor": 1, + "refId": "A", + "step": 10 + } + ], + "thresholds": "", + "title": "Memory Used", + "type": "singlestat", + "valueFontSize": "50%", + "valueMaps": [ + { + "op": "=", + "text": "0", + "value": "null" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(50, 172, 45, 0.97)", + "rgba(237, 129, 40, 0.89)", + "rgba(245, 54, 54, 0.9)" + ], + "datasource": "Prometheus", + "decimals": 2, + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "format": "bytes", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 12, + "y": 9 + }, + "height": "1px", + "id": 10, + "interval": null, + "isNew": true, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "sum(kube_node_status_allocatable_memory_bytes{}) OR sum(kube_node_status_allocatable{resource=\"memory\", unit=\"byte\"})", + "interval": "10s", + "intervalFactor": 1, + "refId": "A", + "step": 10 + } + ], + "thresholds": "", + "title": "Memory Total", + "type": "singlestat", + "valueFontSize": "50%", + "valueMaps": [ + { + "op": "=", + "text": "0", + "value": "null" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(50, 172, 45, 0.97)", + "rgba(237, 129, 40, 0.89)", + "rgba(245, 54, 54, 0.9)" + ], + "datasource": "Prometheus", + "decimals": 2, + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "format": "bytes", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 16, + "y": 9 + }, + "height": "1px", + "id": 13, + "interval": null, + "isNew": true, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "(sum(node_filesystem_size_bytes{device!~\"rootfs|HarddiskVolume.+\"}) - sum(node_filesystem_free_bytes{device!~\"rootfs|HarddiskVolume.+\"}) OR on() vector(0)) + (sum(windows_logical_disk_size_bytes{volume!~\"(HarddiskVolume.+|[A-Z]:.+)\"}) - sum(windows_logical_disk_free_bytes{volume!~\"(HarddiskVolume.+|[A-Z]:.+)\"}) OR on() vector(0))", + "interval": "10s", + "intervalFactor": 1, + "refId": "A", + "step": 10 + } + ], + "thresholds": "", + "title": "Disk Used", + "type": "singlestat", + "valueFontSize": "50%", + "valueMaps": [ + { + "op": "=", + "text": "0", + "value": "null" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(50, 172, 45, 0.97)", + "rgba(237, 129, 40, 0.89)", + "rgba(245, 54, 54, 0.9)" + ], + "datasource": "Prometheus", + "decimals": 2, + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "format": "bytes", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 20, + "y": 9 + }, + "height": "1px", + "id": 14, + "interval": null, + "isNew": true, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "(sum(node_filesystem_size_bytes{device!~\"rootfs|HarddiskVolume.+\"}) OR on() vector(0)) + (sum(windows_logical_disk_size_bytes{volume!~\"(HarddiskVolume.+|[A-Z]:.+)\"}) OR on() vector(0))", + "interval": "10s", + "intervalFactor": 1, + "refId": "A", + "step": 10 + } + ], + "thresholds": "", + "title": "Disk Total", + "type": "singlestat", + "valueFontSize": "50%", + "valueMaps": [ + { + "op": "=", + "text": "0", + "value": "null" + } + ], + "valueName": "current" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 6, + "w": 8, + "x": 0, + "y": 12 + }, + "hiddenSeries": false, + "id": 2051, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "1 - (avg(irate({__name__=~\"node_cpu_seconds_total|windows_cpu_time_total\",mode=\"idle\"}[$__rate_interval])))", + "format": "time_series", + "hide": false, + "instant": false, + "intervalFactor": 1, + "legendFormat": "Cluster", + "refId": "A" + }, + { + "expr": "1 - avg(irate({__name__=~\"node_cpu_seconds_total|windows_cpu_time_total\", mode=\"idle\"}[$__rate_interval])) by (instance)", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{ instance }}", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "CPU Usage", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "percentunit", + "label": "", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": "", + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 6, + "w": 8, + "x": 8, + "y": 12 + }, + "hiddenSeries": false, + "id": 2052, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "100 * (1 - sum({__name__=~\"node_memory_MemAvailable_bytes|windows_os_physical_memory_free_bytes\"}) / sum({__name__=~\"node_memory_MemTotal_bytes|windows_cs_physical_memory_bytes\"}))", + "format": "time_series", + "hide": false, + "instant": false, + "intervalFactor": 1, + "legendFormat": "Cluster", + "refId": "A" + }, + { + "expr": "100 * (1- sum({__name__=~\"node_memory_MemAvailable_bytes|windows_os_physical_memory_free_bytes\"}) by (instance) / sum({__name__=~\"node_memory_MemTotal_bytes|windows_cs_physical_memory_bytes\"}) by (instance))", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{ instance }}", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Memory Usage", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "percent", + "label": "", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": "", + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 6, + "w": 8, + "x": 16, + "y": 12 + }, + "hiddenSeries": false, + "id": 2053, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "(1 - ((sum(node_filesystem_free_bytes{device!~\"rootfs|HarddiskVolume.+\"}) OR on() vector(0)) + (sum(windows_logical_disk_free_bytes{volume!~\"(HarddiskVolume.+|[A-Z]:.+)\"} OR on() vector(0)))) / ((sum(node_filesystem_size_bytes{device!~\"rootfs|HarddiskVolume.+\"}) OR on() vector(0)) + (sum(windows_logical_disk_size_bytes{volume!~\"(HarddiskVolume.+|[A-Z]:.+)\"}) OR on() vector(0)))) * 100", + "legendFormat": "Cluster", + "refId": "A" + }, + { + "expr": "(1 - (sum(node_filesystem_free_bytes{device!~\"rootfs|HarddiskVolume.+\"}) by (instance)) / sum(node_filesystem_size_bytes{device!~\"rootfs|HarddiskVolume.+\"}) by (instance)) * 100", + "hide": false, + "legendFormat": "{{ instance }}", + "refId": "B" + }, + { + "expr": "(1 - (sum(windows_logical_disk_free_bytes{volume!~\"(HarddiskVolume.+|[A-Z]:.+)\"}) by (instance)) / sum(windows_logical_disk_size_bytes{volume!~\"(HarddiskVolume.+|[A-Z]:.+)\"}) by (instance)) * 100", + "hide": false, + "legendFormat": "{{ instance }}", + "refId": "C" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Disk Usage", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": null, + "format": "percent", + "label": "", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": "", + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "folderId": 0, + "gridPos": { + "h": 15, + "w": 12, + "x": 0, + "y": 18 + }, + "headings": true, + "id": 3, + "limit": 30, + "links": [], + "query": "", + "recent": true, + "search": true, + "starred": false, + "tags": [], + "title": "Dashboards", + "type": "dashlist" + }, + { + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 18 + }, + "id": 2055, + "options": { + "content": "## About Rancher Monitoring\n\nRancher Monitoring is a Helm chart developed by Rancher that is powered by [Prometheus Operator](https://github.com/prometheus-operator/prometheus-operator). It is based on the upstream [kube-prometheus-stack](https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack) Helm chart maintained by the Prometheus community.\n\nBy default, the chart deploys Grafana alongside a set of Grafana dashboards curated by the [kube-prometheus](https://github.com/prometheus-operator/kube-prometheus) project.\n\nFor more information on how Rancher Monitoring differs from [kube-prometheus-stack](https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack), please view the CHANGELOG.md of the rancher-monitoring chart located in the [rancher/charts](https://github.com/rancher/charts) repository.\n\nFor more information about how to configure Rancher Monitoring, please view the [Rancher docs](https://rancher.com/docs/rancher/v2.x/en/).\n\n", + "mode": "markdown" + }, + "pluginVersion": "7.1.0", + "timeFrom": null, + "timeShift": null, + "title": "", + "type": "text" + } + ], + "schemaVersion": 26, + "style": "dark", + "tags": [], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": "Data Source", + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "hidden": true, + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ], + "type": "timepicker" + }, + "timezone": "browser", + "title": "Home", + "uid": "rancher-home-1", + "version": 5 +} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/files/rancher/k8s/rancher-etcd-nodes.json b/charts/rancher-monitoring/103.2.2+up57.0.3/files/rancher/k8s/rancher-etcd-nodes.json new file mode 100644 index 00000000000..8af4b81ce03 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/files/rancher/k8s/rancher-etcd-nodes.json @@ -0,0 +1,687 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "id": 32, + "links": [], + "panels": [ + { + "aliasColors": { + "{{instance}}": "#3797d5" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 0 + }, + "hiddenSeries": false, + "id": 2, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(etcd_network_client_grpc_received_bytes_total{job=\"kube-etcd\"}[$__rate_interval])) by (instance)", + "interval": "", + "legendFormat": "Client Traffic In ({{instance}})", + "refId": "A" + }, + { + "expr": "sum(rate(etcd_network_client_grpc_sent_bytes_total{job=\"kube-etcd\"}[$__rate_interval])) by (instance)", + "interval": "", + "legendFormat": "Client Traffic Out ({{instance}})", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "GRPC Client Traffic", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": null, + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "{{instance}}": "#3797d5" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Load[5m]({{instance}})" + }, + "properties": [] + } + ] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 0 + }, + "hiddenSeries": false, + "id": 3, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(etcd_mvcc_db_total_size_in_bytes) by (instance)", + "interval": "", + "legendFormat": "DB Size ({{instance}})", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "DB Size", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "{{instance}}": "#3797d5" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 0 + }, + "hiddenSeries": false, + "id": 4, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(grpc_server_started_total{grpc_service=\"etcdserverpb.Watch\",grpc_type=\"bidi_stream\"}) by (instance) - sum(grpc_server_handled_total{grpc_service=\"etcdserverpb.Watch\",grpc_type=\"bidi_stream\"}) by (instance)", + "interval": "", + "legendFormat": "Watch Streams ({{instance}})", + "refId": "A" + }, + { + "expr": "sum(grpc_server_started_total{grpc_service=\"etcdserverpb.Lease\",grpc_type=\"bidi_stream\"}) by (instance) - sum(grpc_server_handled_total{grpc_service=\"etcdserverpb.Lease\",grpc_type=\"bidi_stream\"}) by (instance)", + "interval": "", + "legendFormat": "Lease Watch Stream ({{instance}})", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Active Streams", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "{{instance}}": "#3797d5" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 7 + }, + "hiddenSeries": false, + "id": 5, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(etcd_server_proposals_committed_total[$__rate_interval])) by (instance)", + "interval": "", + "legendFormat": "Proposal Committed ({{instance}})", + "refId": "A" + }, + { + "expr": "sum(rate(etcd_server_proposals_applied_total[$__rate_interval])) by (instance)", + "interval": "", + "legendFormat": "Proposal Applied ({{instance}})", + "refId": "B" + }, + { + "expr": "sum(rate(etcd_server_proposals_failed_total[$__rate_interval])) by (instance)", + "interval": "", + "legendFormat": "Proposal Failed ({{instance}})", + "refId": "C" + }, + { + "expr": "sum(etcd_server_proposals_pending) by (instance)", + "interval": "", + "legendFormat": "Proposal Pending ({{instance}})", + "refId": "D" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Raft Proposals", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "{{instance}}": "#3797d5" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 7 + }, + "hiddenSeries": false, + "id": 6, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(grpc_server_started_total{grpc_type=\"unary\"}[$__rate_interval])) by (instance)", + "interval": "", + "legendFormat": "RPC Rate ({{instance}})", + "refId": "A" + }, + { + "expr": "sum(rate(grpc_server_handled_total{grpc_type=\"unary\",grpc_code!=\"OK\"}[$__rate_interval])) by (instance)", + "interval": "", + "legendFormat": "RPC Failure Rate ({{instance}})", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "RPC Rate", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 0, + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "decimals": null, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "{{instance}}": "#3797d5" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 7 + }, + "hiddenSeries": false, + "id": 7, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum(rate(etcd_disk_wal_fsync_duration_seconds_bucket[$__rate_interval])) by (instance, le))", + "interval": "", + "legendFormat": "WAL fsync ({{instance}})", + "refId": "A" + }, + { + "expr": "histogram_quantile(0.99, sum(rate(etcd_disk_backend_commit_duration_seconds_bucket[$__rate_interval])) by (instance, le))", + "interval": "", + "legendFormat": "DB fsync ({{instance}})", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Disk Sync Duration", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 2, + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + } + ], + "schemaVersion": 26, + "style": "dark", + "tags": [], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": "Data Source", + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ] + }, + "timezone": "", + "title": "Rancher / etcd (Nodes)", + "uid": "rancher-etcd-nodes-1", + "version": 5 +} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/files/rancher/k8s/rancher-etcd.json b/charts/rancher-monitoring/103.2.2+up57.0.3/files/rancher/k8s/rancher-etcd.json new file mode 100644 index 00000000000..0c058cafb9e --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/files/rancher/k8s/rancher-etcd.json @@ -0,0 +1,669 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "id": 33, + "links": [], + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 0 + }, + "hiddenSeries": false, + "id": 2, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(etcd_network_client_grpc_received_bytes_total{job=\"kube-etcd\"}[$__rate_interval]))", + "interval": "", + "legendFormat": "Client Traffic In", + "refId": "A" + }, + { + "expr": "sum(rate(etcd_network_client_grpc_sent_bytes_total{job=\"kube-etcd\"}[$__rate_interval]))", + "interval": "", + "legendFormat": "Client Traffic Out", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "GRPC Client Traffic", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": null, + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [ + { + "properties": [] + } + ] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 0 + }, + "hiddenSeries": false, + "id": 3, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(etcd_mvcc_db_total_size_in_bytes)", + "interval": "", + "legendFormat": "DB Size", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "DB Size", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 0 + }, + "hiddenSeries": false, + "id": 4, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(grpc_server_started_total{grpc_service=\"etcdserverpb.Watch\",grpc_type=\"bidi_stream\"}) - sum(grpc_server_handled_total{grpc_service=\"etcdserverpb.Watch\",grpc_type=\"bidi_stream\"})", + "interval": "", + "legendFormat": "Watch Streams", + "refId": "A" + }, + { + "expr": "sum(grpc_server_started_total{grpc_service=\"etcdserverpb.Lease\",grpc_type=\"bidi_stream\"}) - sum(grpc_server_handled_total{grpc_service=\"etcdserverpb.Lease\",grpc_type=\"bidi_stream\"})", + "interval": "", + "legendFormat": "Lease Watch Stream", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Active Streams", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 7 + }, + "hiddenSeries": false, + "id": 5, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(etcd_server_proposals_committed_total[$__rate_interval]))", + "interval": "", + "legendFormat": "Proposal Committed", + "refId": "A" + }, + { + "expr": "sum(rate(etcd_server_proposals_applied_total[$__rate_interval]))", + "interval": "", + "legendFormat": "Proposal Applied", + "refId": "B" + }, + { + "expr": "sum(rate(etcd_server_proposals_failed_total[$__rate_interval]))", + "interval": "", + "legendFormat": "Proposal Failed", + "refId": "C" + }, + { + "expr": "sum(etcd_server_proposals_pending)", + "interval": "", + "legendFormat": "Proposal Pending", + "refId": "D" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Raft Proposals", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 7 + }, + "hiddenSeries": false, + "id": 6, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(grpc_server_started_total{grpc_type=\"unary\"}[$__rate_interval]))", + "interval": "", + "legendFormat": "RPC Rate", + "refId": "A" + }, + { + "expr": "sum(rate(grpc_server_handled_total{grpc_type=\"unary\",grpc_code!=\"OK\"}[$__rate_interval]))", + "interval": "", + "legendFormat": "RPC Failure Rate", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "RPC Rate", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 0, + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "decimals": null, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 7 + }, + "hiddenSeries": false, + "id": 7, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum(rate(etcd_disk_wal_fsync_duration_seconds_bucket[$__rate_interval])) by (instance, le))", + "interval": "", + "legendFormat": "WAL fsync", + "refId": "A" + }, + { + "expr": "histogram_quantile(0.99, sum(rate(etcd_disk_backend_commit_duration_seconds_bucket[$__rate_interval])) by (instance, le))", + "interval": "", + "legendFormat": "DB fsync", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Disk Sync Duration", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 2, + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + } + ], + "schemaVersion": 26, + "style": "dark", + "tags": [], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": "Data Source", + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ] + }, + "timezone": "", + "title": "Rancher / etcd", + "uid": "rancher-etcd-1", + "version": 4 +} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/files/rancher/k8s/rancher-k8s-components-nodes.json b/charts/rancher-monitoring/103.2.2+up57.0.3/files/rancher/k8s/rancher-k8s-components-nodes.json new file mode 100644 index 00000000000..b31358eaaf2 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/files/rancher/k8s/rancher-k8s-components-nodes.json @@ -0,0 +1,527 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "id": 30, + "links": [], + "panels": [ + { + "aliasColors": { + "{{instance}}": "#3797d5" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 0 + }, + "hiddenSeries": false, + "id": 2, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(apiserver_request_total[$__rate_interval])) by (instance, code)", + "interval": "", + "legendFormat": "{{code}}({{instance}})", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "API Server Request Rate", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 0, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "{{instance}}": "#3797d5" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Load[5m]({{instance}})" + }, + "properties": [] + } + ] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 0 + }, + "hiddenSeries": false, + "id": 3, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(workqueue_depth{component=\"kube-controller-manager\", name=\"deployment\"}) by (instance, name)", + "interval": "", + "legendFormat": "Deployment Depth ({{instance}})", + "refId": "A" + }, + { + "expr": "sum(workqueue_depth{component=\"kube-controller-manager\", name=\"volumes\"}) by (instance, name)", + "interval": "", + "legendFormat": "Volumes Depth ({{instance}})", + "refId": "B" + }, + { + "expr": "sum(workqueue_depth{component=\"kube-controller-manager\", name=\"replicaset\"}) by (instance, name)", + "interval": "", + "legendFormat": "ReplicaSet Depth ({{instance}})", + "refId": "C" + }, + { + "expr": "sum(workqueue_depth{component=\"kube-controller-manager\", name=\"service\"}) by (instance, name)", + "interval": "", + "legendFormat": "Service Depth ({{instance}})", + "refId": "D" + }, + { + "expr": "sum(workqueue_depth{component=\"kube-controller-manager\", name=\"serviceaccount\"}) by (instance, name)", + "interval": "", + "legendFormat": "ServiceAccount Depth ({{instance}})", + "refId": "E" + }, + { + "expr": "sum(workqueue_depth{component=\"kube-controller-manager\", name=\"endpoint\"}) by (instance, name)", + "interval": "", + "legendFormat": "Endpoint Depth ({{instance}})", + "refId": "F" + }, + { + "expr": "sum(workqueue_depth{component=\"kube-controller-manager\", name=\"daemonset\"}) by (instance, name)", + "interval": "", + "legendFormat": "DaemonSet Depth ({{instance}})", + "refId": "G" + }, + { + "expr": "sum(workqueue_depth{component=\"kube-controller-manager\", name=\"statefulset\"}) by (instance, name)", + "interval": "", + "legendFormat": "StatefulSet Depth ({{instance}})", + "refId": "H" + }, + { + "expr": "sum(workqueue_depth{component=\"kube-controller-manager\", name=\"replicationmanager\"}) by (instance, name)", + "interval": "", + "legendFormat": "ReplicationManager Depth ({{instance}})", + "refId": "I" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Controller Manager Queue Depth", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "{{instance}}": "#3797d5" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 0 + }, + "hiddenSeries": false, + "id": 4, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(kube_pod_status_scheduled{condition=\"false\"})", + "interval": "", + "legendFormat": "Failed To Schedule", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Pod Scheduling Status", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "{{instance}}": "#3797d5" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 7 + }, + "hiddenSeries": false, + "id": 5, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(nginx_ingress_controller_nginx_process_connections{state=\"reading\"}) by (instance)", + "interval": "", + "legendFormat": "Reading ({{instance}})", + "refId": "A" + }, + { + "expr": "sum(nginx_ingress_controller_nginx_process_connections{state=\"waiting\"}) by (instance)", + "interval": "", + "legendFormat": "Waiting ({{instance}})", + "refId": "B" + }, + { + "expr": "sum(nginx_ingress_controller_nginx_process_connections{state=\"writing\"}) by (instance)", + "interval": "", + "legendFormat": "Writing ({{instance}})", + "refId": "C" + }, + { + "expr": "sum(ceil(increase(nginx_ingress_controller_nginx_process_connections_total{state=\"accepted\"}[$__rate_interval]))) by (instance)", + "interval": "", + "legendFormat": "Accepted ({{instance}})", + "refId": "D" + }, + { + "expr": "sum(ceil(increase(nginx_ingress_controller_nginx_process_connections_total{state=\"handled\"}[$__rate_interval]))) by (instance)", + "interval": "", + "legendFormat": "Handled ({{instance}})", + "refId": "E" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Ingress Controller Connections", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": null, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + } + ], + "schemaVersion": 26, + "style": "dark", + "tags": [], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": "Data Source", + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ] + }, + "timezone": "", + "title": "Rancher / Kubernetes Components (Nodes)", + "uid": "rancher-k8s-components-nodes-1", + "version": 5 +} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/files/rancher/k8s/rancher-k8s-components.json b/charts/rancher-monitoring/103.2.2+up57.0.3/files/rancher/k8s/rancher-k8s-components.json new file mode 100644 index 00000000000..44cf97f9fd6 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/files/rancher/k8s/rancher-k8s-components.json @@ -0,0 +1,519 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "id": 31, + "links": [], + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 0 + }, + "hiddenSeries": false, + "id": 2, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(apiserver_request_total[$__rate_interval])) by (code)", + "interval": "", + "legendFormat": "{{code}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "API Server Request Rate", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 0, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Load[5m]({{instance}})" + }, + "properties": [] + } + ] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 0 + }, + "hiddenSeries": false, + "id": 3, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(workqueue_depth{component=\"kube-controller-manager\", name=\"deployment\"}) by (name)", + "interval": "", + "legendFormat": "Deployment Depth", + "refId": "A" + }, + { + "expr": "sum(workqueue_depth{component=\"kube-controller-manager\", name=\"volumes\"}) by (name)", + "interval": "", + "legendFormat": "Volumes Depth", + "refId": "B" + }, + { + "expr": "sum(workqueue_depth{component=\"kube-controller-manager\", name=\"replicaset\"}) by (name)", + "interval": "", + "legendFormat": "Replicaset Depth", + "refId": "C" + }, + { + "expr": "sum(workqueue_depth{component=\"kube-controller-manager\", name=\"service\"}) by (name)", + "interval": "", + "legendFormat": "Service Depth", + "refId": "D" + }, + { + "expr": "sum(workqueue_depth{component=\"kube-controller-manager\", name=\"serviceaccount\"}) by (name)", + "interval": "", + "legendFormat": "ServiceAccount Depth", + "refId": "E" + }, + { + "expr": "sum(workqueue_depth{component=\"kube-controller-manager\", name=\"endpoint\"}) by (name)", + "interval": "", + "legendFormat": "Endpoint Depth", + "refId": "F" + }, + { + "expr": "sum(workqueue_depth{component=\"kube-controller-manager\", name=\"daemonset\"}) by (name)", + "interval": "", + "legendFormat": "DaemonSet Depth", + "refId": "G" + }, + { + "expr": "sum(workqueue_depth{component=\"kube-controller-manager\", name=\"statefulset\"}) by (name)", + "interval": "", + "legendFormat": "StatefulSet Depth", + "refId": "H" + }, + { + "expr": "sum(workqueue_depth{component=\"kube-controller-manager\", name=\"replicationmanager\"}) by (name)", + "interval": "", + "legendFormat": "ReplicationManager Depth", + "refId": "I" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Controller Manager Queue Depth", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 0 + }, + "hiddenSeries": false, + "id": 4, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(kube_pod_status_scheduled{condition=\"false\"})", + "interval": "", + "legendFormat": "Failed To Schedule", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Pod Scheduling Status", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 7 + }, + "hiddenSeries": false, + "id": 5, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(nginx_ingress_controller_nginx_process_connections{state=\"reading\"})", + "interval": "", + "legendFormat": "Reading", + "refId": "A" + }, + { + "expr": "sum(nginx_ingress_controller_nginx_process_connections{state=\"waiting\"})", + "interval": "", + "legendFormat": "Waiting", + "refId": "B" + }, + { + "expr": "sum(nginx_ingress_controller_nginx_process_connections{state=\"writing\"})", + "interval": "", + "legendFormat": "Writing", + "refId": "C" + }, + { + "expr": "sum(ceil(increase(nginx_ingress_controller_nginx_process_connections_total{state=\"accepted\"}[$__rate_interval])))", + "interval": "", + "legendFormat": "Accepted", + "refId": "D" + }, + { + "expr": "sum(ceil(increase(nginx_ingress_controller_nginx_process_connections_total{state=\"handled\"}[$__rate_interval])))", + "interval": "", + "legendFormat": "Handled", + "refId": "E" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Ingress Controller Connections", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": null, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + } + ], + "schemaVersion": 26, + "style": "dark", + "tags": [], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": "Data Source", + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ] + }, + "timezone": "", + "title": "Rancher / Kubernetes Components", + "uid": "rancher-k8s-components-1", + "version": 5 +} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/files/rancher/nodes/rancher-node-detail.json b/charts/rancher-monitoring/103.2.2+up57.0.3/files/rancher/nodes/rancher-node-detail.json new file mode 100644 index 00000000000..920fb94cf7d --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/files/rancher/nodes/rancher-node-detail.json @@ -0,0 +1,805 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "id": 28, + "links": [], + "panels": [ + { + "aliasColors": { + "{{mode}}": "#3797d5" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 0 + }, + "hiddenSeries": false, + "id": 2, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "avg(irate({__name__=~\"node_cpu_seconds_total|windows_cpu_time_total\", instance=\"$instance\"}[$__rate_interval])) by (mode)", + "interval": "", + "legendFormat": "{{mode}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "CPU Utilization", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "percentunit", + "label": null, + "logBase": 1, + "max": "1", + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Load[5m]" + }, + "properties": [] + } + ] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 0 + }, + "hiddenSeries": false, + "id": 3, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(node_load5{instance=~\"$instance\"} OR avg_over_time(windows_system_processor_queue_length{instance=~\"$instance\"}[5m]))", + "interval": "", + "legendFormat": "Load[5m]", + "refId": "A" + }, + { + "expr": "sum(node_load1{instance=~\"$instance\"} OR avg_over_time(windows_system_processor_queue_length{instance=~\"$instance\"}[1m]))", + "interval": "", + "legendFormat": "Load[1m]", + "refId": "B" + }, + { + "expr": "sum(node_load15{instance=~\"$instance\"} OR avg_over_time(windows_system_processor_queue_length{instance=~\"$instance\"}[15m]))", + "interval": "", + "legendFormat": "Load[15m]", + "refId": "C" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Load Average", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 0 + }, + "hiddenSeries": false, + "id": 4, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "1 - (node_memory_MemAvailable_bytes{instance=~\"$instance\"} OR windows_os_physical_memory_free_bytes{instance=~\"$instance\"}) / (node_memory_MemTotal_bytes{instance=~\"$instance\"} OR windows_cs_physical_memory_bytes{instance=~\"$instance\"})", + "interval": "", + "legendFormat": "Total", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Memory Utilization", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "percentunit", + "label": null, + "logBase": 1, + "max": "1", + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "{{device}}": "#3797d5" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 7 + }, + "hiddenSeries": false, + "id": 5, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "1 - (sum(node_filesystem_free_bytes{device!~\"rootfs|HarddiskVolume.+\", instance=~\"$instance\"} OR windows_logical_disk_free_bytes{volume!~\"(HarddiskVolume.+|[A-Z]:.+)\", instance=~\"$instance\"}) by (device) / sum(node_filesystem_size_bytes{device!~\"rootfs|HarddiskVolume.+\", instance=~\"$instance\"} OR windows_logical_disk_size_bytes{volume!~\"(HarddiskVolume.+|[A-Z]:.+)\", instance=~\"$instance\"}) by (device))", + "interval": "", + "legendFormat": "{{device}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Disk Utilization", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "percentunit", + "label": null, + "logBase": 1, + "max": "1", + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "{{device}}": "#3797d5" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 7 + }, + "hiddenSeries": false, + "id": 6, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(node_disk_read_bytes_total{instance=~\"$instance\"}[$__rate_interval]) OR rate(windows_logical_disk_read_bytes_total{instance=~\"$instance\"}[$__rate_interval])) by (device)", + "interval": "", + "legendFormat": "Read ({{device}})", + "refId": "A" + }, + { + "expr": "sum(rate(node_disk_written_bytes_total{instance=~\"$instance\"}[$__rate_interval]) OR rate(windows_logical_disk_write_bytes_total{instance=~\"$instance\"}[$__rate_interval])) by (device)", + "interval": "", + "legendFormat": "Write ({{device}})", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Disk I/O", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "{{device}}": "#3797d5" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 7 + }, + "hiddenSeries": false, + "id": 7, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(node_network_receive_errs_total{device!~\"lo|veth.*|docker.*|flannel.*|cali.*|cbr.*\", instance=~\"$instance\"}[$__rate_interval])) by (device) OR sum(rate(windows_net_packets_received_errors_total{nic!~'.*isatap.*|.*VPN.*|.*Pseudo.*|.*tunneling.*', instance=~\"$instance\"}[$__rate_interval])) by (device)", + "interval": "", + "legendFormat": "Receive Errors ({{device}})", + "refId": "A" + }, + { + "expr": "sum(rate(node_network_receive_packets_total{device!~\"lo|veth.*|docker.*|flannel.*|cali.*|cbr.*\", instance=~\"$instance\"}[$__rate_interval])) by (device) OR sum(rate(windows_net_packets_received_total_total{nic!~'.*isatap.*|.*VPN.*|.*Pseudo.*|.*tunneling.*', instance=~\"$instance\"}[$__rate_interval])) by (device)", + "interval": "", + "legendFormat": "Receive Total ({{device}})", + "refId": "B" + }, + { + "expr": "sum(rate(node_network_transmit_errs_total{device!~\"lo|veth.*|docker.*|flannel.*|cali.*|cbr.*\", instance=~\"$instance\"}[$__rate_interval])) by (device) OR sum(rate(windows_net_packets_outbound_errors_total{nic!~'.*isatap.*|.*VPN.*|.*Pseudo.*|.*tunneling.*', instance=~\"$instance\"}[$__rate_interval])) by (device)", + "interval": "", + "legendFormat": "Transmit Errors ({{device}})", + "refId": "C" + }, + { + "expr": "sum(rate(node_network_receive_drop_total{device!~\"lo|veth.*|docker.*|flannel.*|cali.*|cbr.*\", instance=~\"$instance\"}[$__rate_interval])) by (device) OR sum(rate(windows_net_packets_received_discarded_total{nic!~'.*isatap.*|.*VPN.*|.*Pseudo.*|.*tunneling.*', instance=~\"$instance\"}[$__rate_interval])) by (device)", + "interval": "", + "legendFormat": "Receive Dropped ({{device}})", + "refId": "D" + }, + { + "expr": "sum(rate(node_network_transmit_drop_total{device!~\"lo|veth.*|docker.*|flannel.*|cali.*|cbr.*\", instance=~\"$instance\"}[$__rate_interval])) by (device) OR sum(rate(windows_net_packets_outbound_discarded{nic!~'.*isatap.*|.*VPN.*|.*Pseudo.*|.*tunneling.*', instance=~\"$instance\"}[$__rate_interval])) by (device)", + "interval": "", + "legendFormat": "Transmit Dropped ({{device}})", + "refId": "E" + }, + { + "expr": "sum(rate(node_network_transmit_packets_total{device!~\"lo|veth.*|docker.*|flannel.*|cali.*|cbr.*\", instance=~\"$instance\"}[$__rate_interval])) by (device) OR sum(rate(windows_net_packets_sent_total{nic!~'.*isatap.*|.*VPN.*|.*Pseudo.*|.*tunneling.*', instance=~\"$instance\"}[$__rate_interval])) by (device)", + "interval": "", + "legendFormat": "Transmit Total ({{device}})", + "refId": "F" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Network Traffic", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "{{device}}": "#3797d5" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 14 + }, + "hiddenSeries": false, + "id": 8, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(node_network_transmit_bytes_total{device!~\"lo|veth.*|docker.*|flannel.*|cali.*|cbr.*\", instance=~\"$instance\"}[$__rate_interval]) OR rate(windows_net_packets_sent_total{nic!~'.*isatap.*|.*VPN.*|.*Pseudo.*|.*tunneling.*', instance=~\"$instance\"}[$__rate_interval])) by (device)", + "interval": "", + "legendFormat": "Transmit Total ({{device}})", + "refId": "A" + }, + { + "expr": "sum(rate(node_network_receive_bytes_total{device!~\"lo|veth.*|docker.*|flannel.*|cali.*|cbr.*\", instance=~\"$instance\"}[$__rate_interval]) OR rate(windows_net_packets_received_total_total{nic!~'.*isatap.*|.*VPN.*|.*Pseudo.*|.*tunneling.*', instance=~\"$instance\"}[$__rate_interval])) by (device)", + "interval": "", + "legendFormat": "Receive Total ({{device}})", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Network I/O", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + } + ], + "schemaVersion": 26, + "style": "dark", + "tags": [], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": "Data Source", + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": null, + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "instance", + "query": "label_values({__name__=~\"node_exporter_build_info|windows_exporter_build_info\"}, instance)", + "refresh": 2, + "regex": "", + "sort": 0, + "tagValuesQuery": "", + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ] + }, + "timezone": "", + "title": "Rancher / Node (Detail)", + "uid": "rancher-node-detail-1", + "version": 3 +} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/files/rancher/nodes/rancher-node.json b/charts/rancher-monitoring/103.2.2+up57.0.3/files/rancher/nodes/rancher-node.json new file mode 100644 index 00000000000..367df3cc9d6 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/files/rancher/nodes/rancher-node.json @@ -0,0 +1,792 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "id": 28, + "links": [], + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 0 + }, + "hiddenSeries": false, + "id": 2, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "1 - avg(irate({__name__=~\"node_cpu_seconds_total|windows_cpu_time_total\", instance=\"$instance\", mode=\"idle\"}[$__rate_interval]))", + "interval": "", + "legendFormat": "Total", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "CPU Utilization", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "percentunit", + "label": null, + "logBase": 1, + "max": "1", + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Load[5m]" + }, + "properties": [] + } + ] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 0 + }, + "hiddenSeries": false, + "id": 3, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(node_load5{instance=~\"$instance\"} OR avg_over_time(windows_system_processor_queue_length{instance=~\"$instance\"}[5m]))", + "interval": "", + "legendFormat": "Load[5m]", + "refId": "A" + }, + { + "expr": "sum(node_load1{instance=~\"$instance\"} OR avg_over_time(windows_system_processor_queue_length{instance=~\"$instance\"}[1m]))", + "interval": "", + "legendFormat": "Load[1m]", + "refId": "B" + }, + { + "expr": "sum(node_load15{instance=~\"$instance\"} OR avg_over_time(windows_system_processor_queue_length{instance=~\"$instance\"}[15m]))", + "interval": "", + "legendFormat": "Load[15m]", + "refId": "C" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Load Average", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 0 + }, + "hiddenSeries": false, + "id": 4, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "1 - sum(node_memory_MemAvailable_bytes{instance=~\"$instance\"} OR windows_os_physical_memory_free_bytes{instance=~\"$instance\"}) / sum(node_memory_MemTotal_bytes{instance=~\"$instance\"} OR windows_cs_physical_memory_bytes{instance=~\"$instance\"})", + "interval": "", + "legendFormat": "Total", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Memory Utilization", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "percentunit", + "label": null, + "logBase": 1, + "max": "1", + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 7 + }, + "hiddenSeries": false, + "id": 5, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "1 - (sum(node_filesystem_free_bytes{device!~\"rootfs|HarddiskVolume.+\", instance=~\"$instance\"} OR windows_logical_disk_free_bytes{volume!~\"(HarddiskVolume.+|[A-Z]:.+)\", instance=~\"$instance\"}) / sum(node_filesystem_size_bytes{device!~\"rootfs|HarddiskVolume.+\", instance=~\"$instance\"} OR windows_logical_disk_size_bytes{volume!~\"(HarddiskVolume.+|[A-Z]:.+)\", instance=~\"$instance\"}))", + "interval": "", + "legendFormat": "Total", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Disk Utilization", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "percentunit", + "label": null, + "logBase": 1, + "max": "1", + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 7 + }, + "hiddenSeries": false, + "id": 6, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(node_disk_read_bytes_total{instance=~\"$instance\"}[$__rate_interval]) OR rate(windows_logical_disk_read_bytes_total{instance=~\"$instance\"}[$__rate_interval]))", + "interval": "", + "legendFormat": "Read", + "refId": "A" + }, + { + "expr": "sum(rate(node_disk_written_bytes_total{instance=~\"$instance\"}[$__rate_interval]) OR rate(windows_logical_disk_write_bytes_total{instance=~\"$instance\"}[$__rate_interval]))", + "interval": "", + "legendFormat": "Write", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Disk I/O", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 7 + }, + "hiddenSeries": false, + "id": 7, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "(sum(rate(node_network_receive_errs_total{device!~\"lo|veth.*|docker.*|flannel.*|cali.*|cbr.*\", instance=~\"$instance\"}[$__rate_interval])) OR on() vector(0)) + (sum(rate(windows_net_packets_received_errors_total{nic!~'.*isatap.*|.*VPN.*|.*Pseudo.*|.*tunneling.*', instance=~\"$instance\"}[$__rate_interval])) OR on() vector(0))", + "interval": "", + "legendFormat": "Receive Errors", + "refId": "A" + }, + { + "expr": "(sum(rate(node_network_receive_packets_total{device!~\"lo|veth.*|docker.*|flannel.*|cali.*|cbr.*\", instance=~\"$instance\"}[$__rate_interval])) OR on() vector(0)) + (sum(rate(windows_net_packets_received_total_total{nic!~'.*isatap.*|.*VPN.*|.*Pseudo.*|.*tunneling.*', instance=~\"$instance\"}[$__rate_interval])) OR on() vector(0))", + "interval": "", + "legendFormat": "Receive Total", + "refId": "B" + }, + { + "expr": "(sum(rate(node_network_transmit_errs_total{device!~\"lo|veth.*|docker.*|flannel.*|cali.*|cbr.*\", instance=~\"$instance\"}[$__rate_interval])) OR on() vector(0)) + (sum(rate(windows_net_packets_outbound_errors_total{nic!~'.*isatap.*|.*VPN.*|.*Pseudo.*|.*tunneling.*', instance=~\"$instance\"}[$__rate_interval])) OR on() vector(0))", + "interval": "", + "legendFormat": "Transmit Errors", + "refId": "C" + }, + { + "expr": "(sum(rate(node_network_receive_drop_total{device!~\"lo|veth.*|docker.*|flannel.*|cali.*|cbr.*\", instance=~\"$instance\"}[$__rate_interval])) OR on() vector(0)) + (sum(rate(windows_net_packets_received_discarded_total{nic!~'.*isatap.*|.*VPN.*|.*Pseudo.*|.*tunneling.*', instance=~\"$instance\"}[$__rate_interval])) OR on() vector(0))", + "interval": "", + "legendFormat": "Receive Dropped", + "refId": "D" + }, + { + "expr": "(sum(rate(node_network_transmit_drop_total{device!~\"lo|veth.*|docker.*|flannel.*|cali.*|cbr.*\", instance=~\"$instance\"}[$__rate_interval])) OR on() vector(0)) + (sum(rate(windows_net_packets_outbound_discarded{nic!~'.*isatap.*|.*VPN.*|.*Pseudo.*|.*tunneling.*', instance=~\"$instance\"}[$__rate_interval])) OR on() vector(0))", + "interval": "", + "legendFormat": "Transmit Dropped", + "refId": "E" + }, + { + "expr": "(sum(rate(node_network_transmit_packets_total{device!~\"lo|veth.*|docker.*|flannel.*|cali.*|cbr.*\", instance=~\"$instance\"}[$__rate_interval])) OR on() vector(0)) + (sum(rate(windows_net_packets_sent_total{nic!~'.*isatap.*|.*VPN.*|.*Pseudo.*|.*tunneling.*', instance=~\"$instance\"}[$__rate_interval])) OR on() vector(0))", + "interval": "", + "legendFormat": "Transmit Total", + "refId": "F" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Network Traffic", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 14 + }, + "hiddenSeries": false, + "id": 8, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(node_network_transmit_bytes_total{device!~\"lo|veth.*|docker.*|flannel.*|cali.*|cbr.*\", instance=~\"$instance\"}[$__rate_interval]) OR rate(windows_net_packets_sent_total{nic!~'.*isatap.*|.*VPN.*|.*Pseudo.*|.*tunneling.*', instance=~\"$instance\"}[$__rate_interval]))", + "interval": "", + "legendFormat": "Transmit Total", + "refId": "A" + }, + { + "expr": "sum(rate(node_network_receive_bytes_total{device!~\"lo|veth.*|docker.*|flannel.*|cali.*|cbr.*\", instance=~\"$instance\"}[$__rate_interval]) OR rate(windows_net_packets_received_total_total{nic!~'.*isatap.*|.*VPN.*|.*Pseudo.*|.*tunneling.*', instance=~\"$instance\"}[$__rate_interval]))", + "interval": "", + "legendFormat": "Receive Total", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Network I/O", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + } + ], + "schemaVersion": 26, + "style": "dark", + "tags": [], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": "Data Source", + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": null, + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "instance", + "query": "label_values({__name__=~\"node_exporter_build_info|windows_exporter_build_info\"}, instance)", + "refresh": 2, + "regex": "", + "sort": 0, + "tagValuesQuery": "", + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ] + }, + "timezone": "", + "title": "Rancher / Node", + "uid": "rancher-node-1", + "version": 3 +} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/files/rancher/performance/performance-debugging.json b/charts/rancher-monitoring/103.2.2+up57.0.3/files/rancher/performance/performance-debugging.json new file mode 100644 index 00000000000..454bc393909 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/files/rancher/performance/performance-debugging.json @@ -0,0 +1,1652 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "target": { + "limit": 100, + "matchAny": false, + "tags": [], + "type": "dashboard" + }, + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "links": [], + "liveNow": false, + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "uid": "$datasource" + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 16, + "x": 0, + "y": 0 + }, + "hiddenSeries": false, + "id": 22, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "9.1.5", + "pointradius": 2, + "points": true, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "uid": "$datasource" + }, + "exemplar": true, + "expr": "topk(20,sum by (handler_name) (rate(lasso_controller_reconcile_time_seconds_sum[5m]))\n/\nsum by (handler_name) (rate(lasso_controller_reconcile_time_seconds_count[5m])))", + "interval": "", + "legendFormat": "{{handler_name}}", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Handler Average Execution Times Over Last 5 Minutes (Top 20)", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:1390", + "format": "short", + "label": "Execution Time in Seconds", + "logBase": 1, + "show": true + }, + { + "$$hashKey": "object:1391", + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "uid": "$datasource" + }, + "description": "", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 16, + "x": 0, + "y": 8 + }, + "hiddenSeries": false, + "id": 28, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "9.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "uid": "$datasource" + }, + "exemplar": true, + "expr": "topk(20,sum by (resource, method, code) (rate(steve_api_request_time_sum{resource!=\"subscribe\"}[5m]))\n/\nsum by (resource, method, code) (rate(steve_api_request_time_count{resource!=\"subscribe\"}[5m])))", + "interval": "", + "legendFormat": "{{resource}} {{method}} {{code}}", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Rancher API Average Request Times Over Last 5 Minutes (Top 20) (Subscribes Omitted)", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:178", + "format": "ms", + "label": "", + "logBase": 1, + "show": true + }, + { + "$$hashKey": "object:179", + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "uid": "$datasource" + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 16, + "x": 0, + "y": 16 + }, + "hiddenSeries": false, + "id": 30, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "9.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "uid": "$datasource" + }, + "exemplar": true, + "expr": "rate(steve_api_request_time_sum{resource=\"subscribe\"}[5m])\n/\nrate(steve_api_request_time_count{resource=\"subscribe\"}[5m])", + "interval": "", + "legendFormat": "{{resource}} {{method}} {{code}}", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Subscribe Average Request Times Over Last 5 Minutes", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:368", + "format": "ms", + "label": "", + "logBase": 1, + "show": true + }, + { + "$$hashKey": "object:369", + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "uid": "$datasource" + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 16, + "x": 0, + "y": 24 + }, + "hiddenSeries": false, + "id": 14, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "9.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "uid": "$datasource" + }, + "exemplar": true, + "expr": "topk(20,workqueue_depth)", + "interval": "", + "legendFormat": "{{name}}", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Lasso Controller Work Queue Depth (Top 20)", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:1553", + "format": "short", + "logBase": 1, + "show": true + }, + { + "$$hashKey": "object:1554", + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "uid": "$datasource" + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 13, + "w": 16, + "x": 0, + "y": 32 + }, + "hiddenSeries": false, + "id": 2, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": false, + "hideZero": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "9.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "uid": "$datasource" + }, + "exemplar": true, + "expr": "topk(20,sum by (id, resource, method, code) (steve_api_total_requests))", + "instant": false, + "interval": "", + "legendFormat": "{{id}} {{resource}} {{method}} {{code}}", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Number of Rancher Requests (Top 20)", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:290", + "format": "short", + "logBase": 1, + "show": true + }, + { + "$$hashKey": "object:291", + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "uid": "$datasource" + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 16, + "x": 0, + "y": 45 + }, + "hiddenSeries": false, + "id": 4, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "9.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "uid": "$datasource" + }, + "exemplar": true, + "expr": "topk(20,sum by (id, resource, method) (steve_api_total_requests{code!=\"200\",code!=\"201\"}))", + "interval": "", + "legendFormat": "{{id}} {{resource}} {{method}} {{code}}", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Number of Failed Rancher API Requests (Top 20)", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:428", + "format": "short", + "logBase": 1, + "show": true + }, + { + "$$hashKey": "object:429", + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "uid": "$datasource" + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 16, + "x": 0, + "y": 54 + }, + "hiddenSeries": false, + "id": 6, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "9.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "uid": "$datasource" + }, + "exemplar": true, + "expr": "topk(20,sum by (resource, method, code) (rate(k8s_proxy_store_request_time_sum[5m]))\n/\nsum by (resource, method, code) (rate(k8s_proxy_store_request_time_count[5m])))", + "interval": "", + "legendFormat": "{{resource}} {{method}} {{code}}", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "K8s Proxy Store Average Request Times Over Last 5 Minutes (Top 20)", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:662", + "format": "ms", + "label": "", + "logBase": 1, + "show": true + }, + { + "$$hashKey": "object:663", + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "uid": "$datasource" + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 16, + "x": 0, + "y": 62 + }, + "hiddenSeries": false, + "id": 8, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "9.1.5", + "pointradius": 2, + "points": true, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "uid": "$datasource" + }, + "exemplar": true, + "expr": "topk(20,sum by (resource, method, code) (rate(k8s_proxy_client_request_time_sum[5m]))\n/\nsum by (resource, method, code) (rate(k8s_proxy_client_request_time_count[5m])))", + "interval": "", + "legendFormat": "{{resource}} {{method}} {{code}}", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "K8s Proxy Client Average Request Times Over Last 5 Minutes (Top 20)", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:1710", + "format": "ms", + "label": "", + "logBase": 1, + "show": true + }, + { + "$$hashKey": "object:1711", + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "uid": "$datasource" + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 16, + "x": 0, + "y": 70 + }, + "hiddenSeries": false, + "id": 10, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "9.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "uid": "$datasource" + }, + "exemplar": true, + "expr": "topk(20,lasso_controller_total_cached_object)", + "interval": "", + "legendFormat": "{{kind}} {{version}} {{group}} {{pod}}", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Cached Objects by GroupVersionKind (Top 20)", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:744", + "format": "short", + "logBase": 1, + "show": true + }, + { + "$$hashKey": "object:745", + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "uid": "$datasource" + }, + "description": "", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 16, + "x": 0, + "y": 78 + }, + "hiddenSeries": false, + "id": 12, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "9.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "uid": "$datasource" + }, + "exemplar": true, + "expr": "topk(20,sum by (handler_name) (\nlasso_controller_total_handler_execution\n))", + "interval": "", + "legendFormat": "{{handler_name}}", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Lasso Handler Executions (Top 20)", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:824", + "format": "short", + "logBase": 1, + "show": true + }, + { + "$$hashKey": "object:825", + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "uid": "$datasource" + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 16, + "x": 0, + "y": 86 + }, + "hiddenSeries": false, + "id": 32, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "9.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "uid": "$datasource" + }, + "exemplar": true, + "expr": "topk(20, sum by (handler_name,controller_name) (\nincrease(lasso_controller_total_handler_execution[2m])\n))", + "interval": "", + "legendFormat": "{{controller_name}}.{{handler_name}}", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Handler Executions Over Last 2 Minutes (Top 20)", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "logBase": 1, + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "uid": "$datasource" + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 16, + "x": 0, + "y": 94 + }, + "hiddenSeries": false, + "id": 20, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "9.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "uid": "$datasource" + }, + "exemplar": true, + "expr": "topk(20,sum by (handler_name) (\nlasso_controller_total_handler_execution{has_error=\"true\"}\n))", + "interval": "", + "legendFormat": "{{handler_name}}", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Total Handler Executions with Error (Top 20)", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:1230", + "format": "short", + "logBase": 1, + "show": true + }, + { + "$$hashKey": "object:1231", + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "uid": "$datasource" + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 16, + "x": 0, + "y": 102 + }, + "hiddenSeries": false, + "id": 34, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "9.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "uid": "$datasource" + }, + "exemplar": true, + "expr": "topk(20,sum by (handler_name,controller_name) (\nincrease(lasso_controller_total_handler_execution{has_error=\"true\"}[2m])\n))", + "interval": "", + "legendFormat": "{{controller_name}}.{{handler_name}}", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Handler Executions Over Last 2 Minutes (Top 20)", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "logBase": 1, + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "uid": "$datasource" + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 16, + "x": 0, + "y": 110 + }, + "hiddenSeries": false, + "id": 16, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "9.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "uid": "$datasource" + }, + "exemplar": true, + "expr": "topk(20,session_server_total_transmit_bytes)", + "interval": "", + "legendFormat": "{{clientkey}} {{pod}}", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Data Transmitted by Remote Dialer Sessions (Top 20)", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:1953", + "format": "decbytes", + "label": "", + "logBase": 1, + "show": true + }, + { + "$$hashKey": "object:1954", + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "uid": "$datasource" + }, + "description": "", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 16, + "x": 0, + "y": 118 + }, + "hiddenSeries": false, + "id": 18, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "9.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "uid": "$datasource" + }, + "exemplar": true, + "expr": "session_server_total_transmit_error_bytes", + "interval": "", + "legendFormat": "{{clientkey}} {{pod}}", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Errors for Remote Dialer Sessions (Top 20)", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:2045", + "format": "ms", + "label": "Error Data", + "logBase": 1, + "show": true + }, + { + "$$hashKey": "object:2046", + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 16, + "x": 0, + "y": 126 + }, + "hiddenSeries": false, + "id": 26, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "9.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "uid": "$datasource" + }, + "editorMode": "code", + "exemplar": true, + "expr": "session_server_total_add_websocket_session - (session_server_total_remove_websocket_session or (0 * session_server_total_add_websocket_session))", + "interval": "", + "legendFormat": "{{clientkey}} {{pod}}", + "range": true, + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Remote Dialer Active Connections (Top 20)", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:2199", + "format": "short", + "logBase": 1, + "show": true + }, + { + "$$hashKey": "object:2200", + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 16, + "x": 0, + "y": 134 + }, + "hiddenSeries": false, + "id": 35, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "9.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "uid": "$datasource" + }, + "editorMode": "code", + "exemplar": true, + "expr": "rate(session_server_total_remove_connections[$__rate_interval])", + "interval": "", + "legendFormat": "{{clientkey}} {{pod}}", + "range": true, + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Remote Dialer Removed Connections Rate (Top 20)", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:2199", + "format": "short", + "logBase": 1, + "show": true + }, + { + "$$hashKey": "object:2200", + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 16, + "x": 0, + "y": 142 + }, + "hiddenSeries": false, + "id": 24, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "9.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "uid": "$datasource" + }, + "editorMode": "code", + "exemplar": true, + "expr": "rate(session_server_total_add_connections[$__rate_interval])", + "interval": "", + "legendFormat": "{{clientkey}} {{pod}}", + "range": true, + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Remote Dialer Added Connections Rate (Top 20)", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:2117", + "format": "short", + "logBase": 1, + "show": true + }, + { + "$$hashKey": "object:2118", + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + } + ], + "schemaVersion": 37, + "style": "dark", + "tags": [], + "templating": { + "list": [ + { + "current": { + "selected": false, + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "includeAll": false, + "label": "Data Source", + "multi": false, + "name": "datasource", + "options": [], + "query": "prometheus", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "type": "datasource" + } + ] + }, + "time": { + "from": "now-15m", + "to": "now" + }, + "timepicker": {}, + "timezone": "", + "title": "Rancher Performance Debugging", + "uid": "tfrfU0a7k", + "version": 1, + "weekStart": "" +} \ No newline at end of file diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/files/rancher/pods/rancher-pod-containers.json b/charts/rancher-monitoring/103.2.2+up57.0.3/files/rancher/pods/rancher-pod-containers.json new file mode 100644 index 00000000000..cf78a2204c9 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/files/rancher/pods/rancher-pod-containers.json @@ -0,0 +1,636 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "id": 28, + "iteration": 1618265214337, + "links": [], + "panels": [ + { + "aliasColors": { + "{{container}}": "#3797d5" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 0 + }, + "hiddenSeries": false, + "id": 2, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(container_cpu_cfs_throttled_seconds_total{container!=\"POD\",namespace=~\"$namespace\",pod=~\"$pod\", container!=\"\"}[$__rate_interval])) by (container)", + "interval": "", + "legendFormat": "CFS throttled ({{container}})", + "refId": "A" + }, + { + "expr": "sum(rate(container_cpu_system_seconds_total{container!=\"POD\",namespace=~\"$namespace\",pod=~\"$pod\",container!=\"\"}[$__rate_interval])) by (container) OR sum(rate(windows_container_cpu_usage_seconds_kernelmode{container!=\"POD\",namespace=~\"$namespace\",pod=~\"$pod\",container!=\"\"}[$__rate_interval])) by (container)", + "interval": "", + "legendFormat": "System ({{container}})", + "refId": "B" + }, + { + "expr": "sum(rate(container_cpu_usage_seconds_total{container!=\"POD\",namespace=~\"$namespace\",pod=~\"$pod\",container!=\"\"}[$__rate_interval])) by (container) OR sum(rate(windows_container_cpu_usage_seconds_total{container!=\"POD\",namespace=~\"$namespace\",pod=~\"$pod\",container!=\"\"}[$__rate_interval])) by (container)", + "interval": "", + "legendFormat": "Total ({{container}})", + "refId": "C" + }, + { + "expr": "sum(rate(container_cpu_user_seconds_total{container!=\"POD\",namespace=~\"$namespace\",pod=~\"$pod\",container!=\"\"}[$__rate_interval])) by (container) OR sum(rate(windows_container_cpu_usage_seconds_usermode{container!=\"POD\",namespace=~\"$namespace\",pod=~\"$pod\",container!=\"\"}[$__rate_interval])) by (container)", + "interval": "", + "legendFormat": "User ({{container}})", + "refId": "D" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "CPU Utilization", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": null, + "format": "cpu", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "{{container}}": "#3797d5" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 0 + }, + "hiddenSeries": false, + "id": 4, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(container_memory_working_set_bytes{container!=\"POD\",namespace=~\"$namespace\",pod=~\"$pod\", container!=\"\"} OR windows_container_memory_usage_commit_bytes{container!=\"POD\",namespace=~\"$namespace\",pod=~\"$pod\", container!=\"\"}) by (container)", + "interval": "", + "legendFormat": "({{container}})", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Memory Utilization", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "{{container}}": "#3797d5" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 0 + }, + "hiddenSeries": false, + "id": 7, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(container_network_receive_packets_total{namespace=~\"$namespace\",pod=~\"$pod\"}[$__rate_interval])) by (container) OR sum(irate(windows_container_network_receive_packets_total{namespace=~\"$namespace\",pod=~\"$pod\"}[$__rate_interval])) by (container)", + "interval": "", + "legendFormat": "Receive Total ({{container}})", + "refId": "A" + }, + { + "expr": "sum(irate(container_network_transmit_packets_total{namespace=~\"$namespace\",pod=~\"$pod\"}[$__rate_interval])) by (container) OR sum(irate(windows_container_network_transmit_packets_total{namespace=~\"$namespace\",pod=~\"$pod\"}[$__rate_interval])) by (container)", + "interval": "", + "legendFormat": "Transmit Total ({{container}})", + "refId": "B" + }, + { + "expr": "sum(irate(container_network_receive_packets_dropped_total{namespace=~\"$namespace\",pod=~\"$pod\"}[$__rate_interval])) by (container) OR sum(irate(windows_container_network_receive_packets_dropped_total{namespace=~\"$namespace\",pod=~\"$pod\"}[$__rate_interval])) by (container)", + "interval": "", + "legendFormat": "Receive Dropped ({{container}})", + "refId": "C" + }, + { + "expr": "sum(irate(container_network_receive_errors_total{namespace=~\"$namespace\",pod=~\"$pod\"}[$__rate_interval])) by (container)", + "interval": "", + "legendFormat": "Receive Errors ({{container}})", + "refId": "D" + }, + { + "expr": "sum(irate(container_network_transmit_errors_total{namespace=~\"$namespace\",pod=~\"$pod\"}[$__rate_interval])) by (container)", + "interval": "", + "legendFormat": "Transmit Errors ({{container}})", + "refId": "E" + }, + { + "expr": "sum(irate(container_network_transmit_packets_dropped_total{namespace=~\"$namespace\",pod=~\"$pod\"}[$__rate_interval])) by (container) OR sum(irate(windows_container_network_transmit_packets_dropped_total{namespace=~\"$namespace\",pod=~\"$pod\"}[$__rate_interval])) by (container)", + "interval": "", + "legendFormat": "Transmit Dropped ({{container}})", + "refId": "F" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Network Traffic", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "{{container}}": "#3797d5" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 7 + }, + "hiddenSeries": false, + "id": 8, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(container_network_receive_bytes_total{namespace=~\"$namespace\",pod=~\"$pod\"}[$__rate_interval])) by (container) OR sum(irate(windows_container_network_receive_bytes_total{namespace=~\"$namespace\",pod=~\"$pod\"}[$__rate_interval])) by (container)", + "interval": "", + "legendFormat": "Receive Total ({{container}})", + "refId": "A" + }, + { + "expr": "sum(irate(container_network_transmit_bytes_total{namespace=~\"$namespace\",pod=~\"$pod\"}[$__rate_interval])) by (container) OR sum(irate(windows_container_network_transmit_bytes_total{namespace=~\"$namespace\",pod=~\"$pod\"}[$__rate_interval])) by (container)", + "interval": "", + "legendFormat": "Transmit Total ({{container}})", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Network I/O", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "{{container}}": "#3797d5" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 7 + }, + "hiddenSeries": false, + "id": 6, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(container_fs_writes_bytes_total{namespace=~\"$namespace\",pod=~\"$pod\",container!=\"\"}[$__rate_interval])) by (container)", + "interval": "", + "legendFormat": "Write ({{container}})", + "refId": "A" + }, + { + "expr": "sum(rate(container_fs_reads_bytes_total{namespace=~\"$namespace\",pod=~\"$pod\",container!=\"\"}[$__rate_interval])) by (container)", + "interval": "", + "legendFormat": "Read ({{container}})", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Disk I/O", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + } + ], + "refresh": false, + "schemaVersion": 26, + "style": "dark", + "tags": [], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": "Data Source", + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": null, + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "namespace", + "query": "label_values(kube_pod_info{}, namespace)", + "refresh": 2, + "regex": "", + "sort": 0, + "tagValuesQuery": "", + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "pod", + "query": "label_values(kube_pod_info{namespace=\"$namespace\"}, pod)", + "refresh": 2, + "regex": "", + "sort": 0, + "tagValuesQuery": "", + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ] + }, + "timezone": "", + "title": "Rancher / Pod (Containers)", + "uid": "rancher-pod-containers-1", + "version": 8 +} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/files/rancher/pods/rancher-pod.json b/charts/rancher-monitoring/103.2.2+up57.0.3/files/rancher/pods/rancher-pod.json new file mode 100644 index 00000000000..4859eccc74b --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/files/rancher/pods/rancher-pod.json @@ -0,0 +1,636 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "id": 28, + "iteration": 1618265214337, + "links": [], + "panels": [ + { + "aliasColors": { + "": "#3797d5" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 0 + }, + "hiddenSeries": false, + "id": 2, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(container_cpu_cfs_throttled_seconds_total{container!=\"POD\",namespace=~\"$namespace\",pod=~\"$pod\", container!=\"\"}[$__rate_interval]))", + "interval": "", + "legendFormat": "CFS throttled", + "refId": "A" + }, + { + "expr": "sum(rate(container_cpu_system_seconds_total{container!=\"POD\",namespace=~\"$namespace\",pod=~\"$pod\",container!=\"\"}[$__rate_interval])) OR sum(rate(windows_container_cpu_usage_seconds_kernelmode{container!=\"POD\",namespace=~\"$namespace\",pod=~\"$pod\",container!=\"\"}[$__rate_interval]))", + "interval": "", + "legendFormat": "System", + "refId": "B" + }, + { + "expr": "sum(rate(container_cpu_usage_seconds_total{container!=\"POD\",namespace=~\"$namespace\",pod=~\"$pod\",container!=\"\"}[$__rate_interval])) OR sum(rate(windows_container_cpu_usage_seconds_total{container!=\"POD\",namespace=~\"$namespace\",pod=~\"$pod\",container!=\"\"}[$__rate_interval]))", + "interval": "", + "legendFormat": "Total", + "refId": "C" + }, + { + "expr": "sum(rate(container_cpu_user_seconds_total{container!=\"POD\",namespace=~\"$namespace\",pod=~\"$pod\",container!=\"\"}[$__rate_interval])) OR sum(rate(windows_container_cpu_usage_seconds_usermode{container!=\"POD\",namespace=~\"$namespace\",pod=~\"$pod\",container!=\"\"}[$__rate_interval]))", + "interval": "", + "legendFormat": "User", + "refId": "D" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "CPU Utilization", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": null, + "format": "cpu", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "": "#3797d5" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 0 + }, + "hiddenSeries": false, + "id": 4, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(container_memory_working_set_bytes{container!=\"POD\",namespace=~\"$namespace\",pod=~\"$pod\", container!=\"\"} OR windows_container_memory_usage_commit_bytes{container!=\"POD\",namespace=~\"$namespace\",pod=~\"$pod\", container!=\"\"})", + "interval": "", + "legendFormat": "Total", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Memory Utilization", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "": "#3797d5" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 0 + }, + "hiddenSeries": false, + "id": 7, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(container_network_receive_packets_total{namespace=~\"$namespace\",pod=~\"$pod\"}[$__rate_interval])) OR sum(irate(windows_container_network_receive_packets_total{namespace=~\"$namespace\",pod=~\"$pod\"}[$__rate_interval]))", + "interval": "", + "legendFormat": "Receive Total", + "refId": "A" + }, + { + "expr": "sum(irate(container_network_transmit_packets_total{namespace=~\"$namespace\",pod=~\"$pod\"}[$__rate_interval])) OR sum(irate(windows_container_network_transmit_packets_total{namespace=~\"$namespace\",pod=~\"$pod\"}[$__rate_interval]))", + "interval": "", + "legendFormat": "Transmit Total", + "refId": "B" + }, + { + "expr": "sum(irate(container_network_receive_packets_dropped_total{namespace=~\"$namespace\",pod=~\"$pod\"}[$__rate_interval])) OR sum(irate(windows_container_network_receive_packets_dropped_total{namespace=~\"$namespace\",pod=~\"$pod\"}[$__rate_interval]))", + "interval": "", + "legendFormat": "Receive Dropped", + "refId": "C" + }, + { + "expr": "sum(irate(container_network_receive_errors_total{namespace=~\"$namespace\",pod=~\"$pod\"}[$__rate_interval]))", + "interval": "", + "legendFormat": "Receive Errors", + "refId": "D" + }, + { + "expr": "sum(irate(container_network_transmit_errors_total{namespace=~\"$namespace\",pod=~\"$pod\"}[$__rate_interval]))", + "interval": "", + "legendFormat": "Transmit Errors", + "refId": "E" + }, + { + "expr": "sum(irate(container_network_transmit_packets_dropped_total{namespace=~\"$namespace\",pod=~\"$pod\"}[$__rate_interval])) OR sum(irate(windows_container_network_transmit_packets_dropped_total{namespace=~\"$namespace\",pod=~\"$pod\"}[$__rate_interval]))", + "interval": "", + "legendFormat": "Transmit Dropped", + "refId": "F" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Network Traffic", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "": "#3797d5" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 7 + }, + "hiddenSeries": false, + "id": 8, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(container_network_receive_bytes_total{namespace=~\"$namespace\",pod=~\"$pod\"}[$__rate_interval])) OR sum(irate(windows_container_network_receive_bytes_total{namespace=~\"$namespace\",pod=~\"$pod\"}[$__rate_interval]))", + "interval": "", + "legendFormat": "Receive Total", + "refId": "A" + }, + { + "expr": "sum(irate(container_network_transmit_bytes_total{namespace=~\"$namespace\",pod=~\"$pod\"}[$__rate_interval])) OR sum(irate(windows_container_network_transmit_bytes_total{namespace=~\"$namespace\",pod=~\"$pod\"}[$__rate_interval]))", + "interval": "", + "legendFormat": "Transmit Total", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Network I/O", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "": "#3797d5" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 7 + }, + "hiddenSeries": false, + "id": 6, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(container_fs_writes_bytes_total{namespace=~\"$namespace\",pod=~\"$pod\",container!=\"\"}[$__rate_interval]))", + "interval": "", + "legendFormat": "Write", + "refId": "A" + }, + { + "expr": "sum(rate(container_fs_reads_bytes_total{namespace=~\"$namespace\",pod=~\"$pod\",container!=\"\"}[$__rate_interval]))", + "interval": "", + "legendFormat": "Read", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Disk I/O", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + } + ], + "refresh": false, + "schemaVersion": 26, + "style": "dark", + "tags": [], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": "Data Source", + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": null, + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "namespace", + "query": "label_values(kube_pod_info{}, namespace)", + "refresh": 2, + "regex": "", + "sort": 0, + "tagValuesQuery": "", + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "pod", + "query": "label_values(kube_pod_info{namespace=\"$namespace\"}, pod)", + "refresh": 2, + "regex": "", + "sort": 0, + "tagValuesQuery": "", + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ] + }, + "timezone": "", + "title": "Rancher / Pod", + "uid": "rancher-pod-1", + "version": 8 +} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/files/rancher/workloads/rancher-workload-pods.json b/charts/rancher-monitoring/103.2.2+up57.0.3/files/rancher/workloads/rancher-workload-pods.json new file mode 100644 index 00000000000..92c0d24a6e7 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/files/rancher/workloads/rancher-workload-pods.json @@ -0,0 +1,652 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "id": 28, + "iteration": 1618265214337, + "links": [], + "panels": [ + { + "aliasColors": { + "{{pod}}": "#3797d5" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 0 + }, + "hiddenSeries": false, + "id": 2, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "(sum(rate(container_cpu_cfs_throttled_seconds_total{namespace=~\"$namespace\",container=\"\"}[$__rate_interval])) by (pod)) * on(pod) kube_pod_info{namespace=~\"$namespace\", created_by_kind=\"$kind\", created_by_name=\"$workload\"}", + "interval": "", + "legendFormat": "CFS throttled ({{pod}})", + "refId": "A" + }, + { + "expr": "(sum(rate(container_cpu_system_seconds_total{namespace=~\"$namespace\",container=\"\"}[$__rate_interval])) by (pod) OR sum(rate(windows_container_cpu_usage_seconds_kernelmode{namespace=~\"$namespace\"}[$__rate_interval])) by (pod)) * on(pod) kube_pod_info{namespace=~\"$namespace\", created_by_kind=\"$kind\", created_by_name=\"$workload\"}", + "interval": "", + "legendFormat": "System ({{pod}})", + "refId": "B" + }, + { + "expr": "(sum(rate(container_cpu_usage_seconds_total{namespace=~\"$namespace\",container=\"\"}[$__rate_interval])) by (pod) OR sum(rate(windows_container_cpu_usage_seconds_total{namespace=~\"$namespace\"}[$__rate_interval])) by (pod)) * on(pod) kube_pod_info{namespace=~\"$namespace\", created_by_kind=\"$kind\", created_by_name=\"$workload\"}", + "interval": "", + "legendFormat": "Total ({{pod}})", + "refId": "C" + }, + { + "expr": "(sum(rate(container_cpu_user_seconds_total{namespace=~\"$namespace\",container=\"\"}[$__rate_interval])) by (pod) OR sum(rate(windows_container_cpu_usage_seconds_usermode{namespace=~\"$namespace\"}[$__rate_interval])) by (pod)) * on(pod) kube_pod_info{namespace=~\"$namespace\", created_by_kind=\"$kind\", created_by_name=\"$workload\"}", + "interval": "", + "legendFormat": "User ({{pod}})", + "refId": "D" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "CPU Utilization", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": null, + "format": "cpu", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "{{pod}}": "#3797d5" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 0 + }, + "hiddenSeries": false, + "id": 4, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "(sum(container_memory_working_set_bytes{namespace=~\"$namespace\",container=\"\"} OR windows_container_memory_usage_commit_bytes{namespace=~\"$namespace\"}) by (pod)) * on(pod) kube_pod_info{namespace=~\"$namespace\", created_by_kind=\"$kind\", created_by_name=\"$workload\"}", + "interval": "", + "legendFormat": "({{pod}})", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Memory Utilization", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "{{pod}}": "#3797d5" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 0 + }, + "hiddenSeries": false, + "id": 7, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "(sum(irate(container_network_receive_packets_total{namespace=~\"$namespace\"}[$__rate_interval])) by (pod) OR sum(irate(windows_container_network_receive_packets_total{namespace=~\"$namespace\"}[$__rate_interval])) by (pod)) * on(pod) kube_pod_info{namespace=~\"$namespace\", created_by_kind=\"$kind\", created_by_name=\"$workload\"}", + "interval": "", + "legendFormat": "Receive Total ({{pod}})", + "refId": "A" + }, + { + "expr": "(sum(irate(container_network_transmit_packets_total{namespace=~\"$namespace\"}[$__rate_interval])) by (pod) OR sum(irate(windows_container_network_transmit_packets_total{namespace=~\"$namespace\"}[$__rate_interval])) by (pod)) * on(pod) kube_pod_info{namespace=~\"$namespace\", created_by_kind=\"$kind\", created_by_name=\"$workload\"}", + "interval": "", + "legendFormat": "Transmit Total ({{pod}})", + "refId": "B" + }, + { + "expr": "(sum(irate(container_network_receive_packets_dropped_total{namespace=~\"$namespace\"}[$__rate_interval])) by (pod) OR sum(irate(windows_container_network_receive_packets_dropped_total{namespace=~\"$namespace\"}[$__rate_interval])) by (pod)) * on(pod) kube_pod_info{namespace=~\"$namespace\", created_by_kind=\"$kind\", created_by_name=\"$workload\"}", + "interval": "", + "legendFormat": "Receive Dropped ({{pod}})", + "refId": "C" + }, + { + "expr": "(sum(irate(container_network_receive_errors_total{namespace=~\"$namespace\"}[$__rate_interval])) by (pod)) * on(pod) kube_pod_info{namespace=~\"$namespace\", created_by_kind=\"$kind\", created_by_name=\"$workload\"}", + "interval": "", + "legendFormat": "Receive Errors ({{pod}})", + "refId": "D" + }, + { + "expr": "(sum(irate(container_network_transmit_errors_total{namespace=~\"$namespace\"}[$__rate_interval])) by (pod)) * on(pod) kube_pod_info{namespace=~\"$namespace\", created_by_kind=\"$kind\", created_by_name=\"$workload\"}", + "interval": "", + "legendFormat": "Transmit Errors ({{pod}})", + "refId": "E" + }, + { + "expr": "(sum(irate(container_network_transmit_packets_dropped_total{namespace=~\"$namespace\"}[$__rate_interval])) by (pod) OR sum(irate(windows_container_network_transmit_packets_dropped_total{namespace=~\"$namespace\"}[$__rate_interval])) by (pod)) * on(pod) kube_pod_info{namespace=~\"$namespace\", created_by_kind=\"$kind\", created_by_name=\"$workload\"}", + "interval": "", + "legendFormat": "Transmit Dropped ({{pod}})", + "refId": "F" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Network Traffic", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "{{pod}}": "#3797d5" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 7 + }, + "hiddenSeries": false, + "id": 8, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "(sum(irate(container_network_receive_bytes_total{namespace=~\"$namespace\"}[$__rate_interval])) by (pod) OR sum(irate(windows_container_network_receive_bytes_total{namespace=~\"$namespace\"}[$__rate_interval])) by (pod)) * on(pod) kube_pod_info{namespace=~\"$namespace\", created_by_kind=\"$kind\", created_by_name=\"$workload\"}", + "interval": "", + "legendFormat": "Receive Total ({{pod}})", + "refId": "A" + }, + { + "expr": "(sum(irate(container_network_transmit_bytes_total{namespace=~\"$namespace\"}[$__rate_interval])) by (pod) OR sum(irate(windows_container_network_transmit_bytes_total{namespace=~\"$namespace\"}[$__rate_interval])) by (pod)) * on(pod) kube_pod_info{namespace=~\"$namespace\", created_by_kind=\"$kind\", created_by_name=\"$workload\"}", + "interval": "", + "legendFormat": "Transmit Total ({{pod}})", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Network I/O", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "{{pod}}": "#3797d5" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 7 + }, + "hiddenSeries": false, + "id": 6, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "(sum(rate(container_fs_writes_bytes_total{namespace=~\"$namespace\"}[$__rate_interval])) by (pod)) * on(pod) kube_pod_info{namespace=~\"$namespace\", created_by_kind=\"$kind\", created_by_name=\"$workload\"}", + "interval": "", + "legendFormat": "Write ({{pod}})", + "refId": "A" + }, + { + "expr": "(sum(rate(container_fs_reads_bytes_total{namespace=~\"$namespace\"}[$__rate_interval])) by (pod)) * on(pod) kube_pod_info{namespace=~\"$namespace\", created_by_kind=\"$kind\", created_by_name=\"$workload\"}", + "interval": "", + "legendFormat": "Read ({{pod}})", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Disk I/O", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + } + ], + "refresh": false, + "schemaVersion": 26, + "style": "dark", + "tags": [], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": "Data Source", + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": null, + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "namespace", + "query": "query_result(kube_pod_info{namespace!=\"\"} * on(pod) group_right(namespace, created_by_kind, created_by_name) count({__name__=~\"container_.*|windows_container_.*\", pod!=\"\"}) by (pod))", + "refresh": 2, + "regex": "/.*namespace=\"([^\"]*)\"/", + "sort": 0, + "tagValuesQuery": "", + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "kind", + "query": "query_result(kube_pod_info{namespace=\"$namespace\", created_by_kind!=\"\"} * on(pod) group_right(namespace, created_by_kind, created_by_name) count({__name__=~\"container_.*|windows_container_.*\", pod!=\"\"}) by (pod))", + "refresh": 2, + "regex": "/.*created_by_kind=\"([^\"]*)\"/", + "sort": 0, + "tagValuesQuery": "", + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "workload", + "query": "query_result(kube_pod_info{namespace=\"$namespace\", created_by_kind=\"$kind\", created_by_name!=\"\"} * on(pod) group_right(namespace, created_by_kind, created_by_name) count({__name__=~\"container_.*|windows_container_.*\", pod!=\"\"}) by (pod))", + "refresh": 2, + "regex": "/.*created_by_name=\"([^\"]*)\"/", + "sort": 0, + "tagValuesQuery": "", + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ] + }, + "timezone": "", + "title": "Rancher / Workload (Pods)", + "uid": "rancher-workload-pods-1", + "version": 8 +} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/files/rancher/workloads/rancher-workload.json b/charts/rancher-monitoring/103.2.2+up57.0.3/files/rancher/workloads/rancher-workload.json new file mode 100644 index 00000000000..9f5317c2f0f --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/files/rancher/workloads/rancher-workload.json @@ -0,0 +1,652 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "id": 28, + "iteration": 1618265214337, + "links": [], + "panels": [ + { + "aliasColors": { + "{{pod}}": "#3797d5" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 0 + }, + "hiddenSeries": false, + "id": 2, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum((sum(rate(container_cpu_cfs_throttled_seconds_total{namespace=~\"$namespace\"}[$__rate_interval])) by (pod)) * on(pod) kube_pod_info{namespace=~\"$namespace\", created_by_kind=\"$kind\", created_by_name=\"$workload\"})", + "interval": "", + "legendFormat": "CFS throttled", + "refId": "A" + }, + { + "expr": "sum((sum(rate(container_cpu_system_seconds_total{namespace=~\"$namespace\"}[$__rate_interval])) by (pod) OR sum(rate(windows_container_cpu_usage_seconds_kernelmode{namespace=~\"$namespace\"}[$__rate_interval])) by (pod)) * on(pod) kube_pod_info{namespace=~\"$namespace\", created_by_kind=\"$kind\", created_by_name=\"$workload\"})", + "interval": "", + "legendFormat": "System", + "refId": "B" + }, + { + "expr": "sum((sum(rate(container_cpu_usage_seconds_total{namespace=~\"$namespace\"}[$__rate_interval])) by (pod) OR sum(rate(windows_container_cpu_usage_seconds_total{namespace=~\"$namespace\"}[$__rate_interval])) by (pod)) * on(pod) kube_pod_info{namespace=~\"$namespace\", created_by_kind=\"$kind\", created_by_name=\"$workload\"})", + "interval": "", + "legendFormat": "Total", + "refId": "C" + }, + { + "expr": "sum((sum(rate(container_cpu_user_seconds_total{namespace=~\"$namespace\"}[$__rate_interval])) by (pod) OR sum(rate(windows_container_cpu_usage_seconds_usermode{namespace=~\"$namespace\"}[$__rate_interval])) by (pod)) * on(pod) kube_pod_info{namespace=~\"$namespace\", created_by_kind=\"$kind\", created_by_name=\"$workload\"})", + "interval": "", + "legendFormat": "User", + "refId": "D" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "CPU Utilization", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": null, + "format": "cpu", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "{{pod}}": "#3797d5" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 0 + }, + "hiddenSeries": false, + "id": 4, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum((sum(container_memory_working_set_bytes{namespace=~\"$namespace\"} OR windows_container_memory_usage_commit_bytes{namespace=~\"$namespace\"}) by (pod)) * on(pod) kube_pod_info{namespace=~\"$namespace\", created_by_kind=\"$kind\", created_by_name=\"$workload\"})", + "interval": "", + "legendFormat": "Total", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Memory Utilization", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "{{pod}}": "#3797d5" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 0 + }, + "hiddenSeries": false, + "id": 7, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum((sum(irate(container_network_receive_packets_total{namespace=~\"$namespace\"}[$__rate_interval])) by (pod) OR sum(irate(windows_container_network_receive_packets_total{namespace=~\"$namespace\"}[$__rate_interval])) by (pod)) * on(pod) kube_pod_info{namespace=~\"$namespace\", created_by_kind=\"$kind\", created_by_name=\"$workload\"})", + "interval": "", + "legendFormat": "Receive Total", + "refId": "A" + }, + { + "expr": "sum((sum(irate(container_network_transmit_packets_total{namespace=~\"$namespace\"}[$__rate_interval])) by (pod) OR sum(irate(windows_container_network_transmit_packets_total{namespace=~\"$namespace\"}[$__rate_interval])) by (pod)) * on(pod) kube_pod_info{namespace=~\"$namespace\", created_by_kind=\"$kind\", created_by_name=\"$workload\"})", + "interval": "", + "legendFormat": "Transmit Total", + "refId": "B" + }, + { + "expr": "sum((sum(irate(container_network_receive_packets_dropped_total{namespace=~\"$namespace\"}[$__rate_interval])) by (pod) OR sum(irate(windows_container_network_receive_packets_dropped_total{namespace=~\"$namespace\"}[$__rate_interval])) by (pod)) * on(pod) kube_pod_info{namespace=~\"$namespace\", created_by_kind=\"$kind\", created_by_name=\"$workload\"})", + "interval": "", + "legendFormat": "Receive Dropped", + "refId": "C" + }, + { + "expr": "sum((sum(irate(container_network_receive_errors_total{namespace=~\"$namespace\"}[$__rate_interval])) by (pod)) * on(pod) kube_pod_info{namespace=~\"$namespace\", created_by_kind=\"$kind\", created_by_name=\"$workload\"})", + "interval": "", + "legendFormat": "Receive Errors", + "refId": "D" + }, + { + "expr": "sum((sum(irate(container_network_transmit_errors_total{namespace=~\"$namespace\"}[$__rate_interval])) by (pod)) * on(pod) kube_pod_info{namespace=~\"$namespace\", created_by_kind=\"$kind\", created_by_name=\"$workload\"})", + "interval": "", + "legendFormat": "Transmit Errors", + "refId": "E" + }, + { + "expr": "sum((sum(irate(container_network_transmit_packets_dropped_total{namespace=~\"$namespace\"}[$__rate_interval])) by (pod) OR sum(irate(windows_container_network_transmit_packets_dropped_total{namespace=~\"$namespace\"}[$__rate_interval])) by (pod)) * on(pod) kube_pod_info{namespace=~\"$namespace\", created_by_kind=\"$kind\", created_by_name=\"$workload\"})", + "interval": "", + "legendFormat": "Transmit Dropped", + "refId": "F" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Network Traffic", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "{{pod}}": "#3797d5" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 7 + }, + "hiddenSeries": false, + "id": 8, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum((sum(irate(container_network_receive_bytes_total{namespace=~\"$namespace\"}[$__rate_interval])) by (pod) OR sum(irate(windows_container_network_receive_bytes_total{namespace=~\"$namespace\"}[$__rate_interval])) by (pod)) * on(pod) kube_pod_info{namespace=~\"$namespace\", created_by_kind=\"$kind\", created_by_name=\"$workload\"})", + "interval": "", + "legendFormat": "Receive Total", + "refId": "A" + }, + { + "expr": "sum((sum(irate(container_network_transmit_bytes_total{namespace=~\"$namespace\"}[$__rate_interval])) by (pod) OR sum(irate(windows_container_network_transmit_bytes_total{namespace=~\"$namespace\"}[$__rate_interval])) by (pod)) * on(pod) kube_pod_info{namespace=~\"$namespace\", created_by_kind=\"$kind\", created_by_name=\"$workload\"})", + "interval": "", + "legendFormat": "Transmit Total", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Network I/O", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "{{pod}}": "#3797d5" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 7 + }, + "hiddenSeries": false, + "id": 6, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum((sum(rate(container_fs_writes_bytes_total{namespace=~\"$namespace\"}[$__rate_interval])) by (pod)) * on(pod) kube_pod_info{namespace=~\"$namespace\", created_by_kind=\"$kind\", created_by_name=\"$workload\"})", + "interval": "", + "legendFormat": "Write", + "refId": "A" + }, + { + "expr": "sum((sum(rate(container_fs_reads_bytes_total{namespace=~\"$namespace\"}[$__rate_interval])) by (pod)) * on(pod) kube_pod_info{namespace=~\"$namespace\", created_by_kind=\"$kind\", created_by_name=\"$workload\"})", + "interval": "", + "legendFormat": "Read", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Disk I/O", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + } + ], + "refresh": false, + "schemaVersion": 26, + "style": "dark", + "tags": [], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": "Data Source", + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": null, + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "namespace", + "query": "query_result(kube_pod_info{namespace!=\"\"} * on(pod) group_right(namespace, created_by_kind, created_by_name) count({__name__=~\"container_.*|windows_container_.*\", pod!=\"\"}) by (pod))", + "refresh": 2, + "regex": "/.*namespace=\"([^\"]*)\"/", + "sort": 0, + "tagValuesQuery": "", + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "kind", + "query": "query_result(kube_pod_info{namespace=\"$namespace\", created_by_kind!=\"\"} * on(pod) group_right(namespace, created_by_kind, created_by_name) count({__name__=~\"container_.*|windows_container_.*\", pod!=\"\"}) by (pod))", + "refresh": 2, + "regex": "/.*created_by_kind=\"([^\"]*)\"/", + "sort": 0, + "tagValuesQuery": "", + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "workload", + "query": "query_result(kube_pod_info{namespace=\"$namespace\", created_by_kind=\"$kind\", created_by_name!=\"\"} * on(pod) group_right(namespace, created_by_kind, created_by_name) count({__name__=~\"container_.*|windows_container_.*\", pod!=\"\"}) by (pod))", + "refresh": 2, + "regex": "/.*created_by_name=\"([^\"]*)\"/", + "sort": 0, + "tagValuesQuery": "", + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ] + }, + "timezone": "", + "title": "Rancher / Workload", + "uid": "rancher-workload-1", + "version": 8 +} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/files/upgrade/scripts/delete-workloads-with-old-labels.sh b/charts/rancher-monitoring/103.2.2+up57.0.3/files/upgrade/scripts/delete-workloads-with-old-labels.sh new file mode 100644 index 00000000000..89431e7132b --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/files/upgrade/scripts/delete-workloads-with-old-labels.sh @@ -0,0 +1,14 @@ +#!/bin/bash + +set -e +set -x + +# node-exporter +kubectl delete daemonset -l app=prometheus-node-exporter,release=rancher-monitoring --ignore-not-found=true + +# prometheus-adapter +kubectl delete deployments -l app=prometheus-adapter,release=rancher-monitoring --ignore-not-found=true + +# kube-state-metrics +kubectl delete deployments -l app.kubernetes.io/instance=rancher-monitoring,app.kubernetes.io/name=kube-state-metrics --cascade=orphan --ignore-not-found=true +kubectl delete statefulsets -l app.kubernetes.io/instance=rancher-monitoring,app.kubernetes.io/name=kube-state-metrics --cascade=orphan --ignore-not-found=true diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/NOTES.txt b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/NOTES.txt new file mode 100644 index 00000000000..371f3ae3988 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/NOTES.txt @@ -0,0 +1,4 @@ +{{ $.Chart.Name }} has been installed. Check its status by running: + kubectl --namespace {{ template "kube-prometheus-stack.namespace" . }} get pods -l "release={{ $.Release.Name }}" + +Visit https://github.com/prometheus-operator/kube-prometheus for instructions on how to create & configure Alertmanager and Prometheus instances using the Operator. diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/_helpers.tpl b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/_helpers.tpl new file mode 100644 index 00000000000..2827e81196e --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/_helpers.tpl @@ -0,0 +1,467 @@ +# Rancher +{{- define "system_default_registry" -}} +{{- if .Values.global.cattle.systemDefaultRegistry -}} +{{- printf "%s/" .Values.global.cattle.systemDefaultRegistry -}} +{{- end -}} +{{- end -}} + +{{- define "monitoring_registry" -}} + {{- $temp_registry := (include "system_default_registry" .) -}} + {{- if $temp_registry -}} + {{- trimSuffix "/" $temp_registry -}} + {{- else -}} + {{- .Values.global.imageRegistry -}} + {{- end -}} +{{- end -}} + +{{/* +https://github.com/helm/helm/issues/4535#issuecomment-477778391 +Usage: {{ include "call-nested" (list . "SUBCHART_NAME" "TEMPLATE") }} +e.g. {{ include "call-nested" (list . "grafana" "grafana.fullname") }} +*/}} +{{- define "call-nested" }} +{{- $dot := index . 0 }} +{{- $subchart := index . 1 | splitList "." }} +{{- $template := index . 2 }} +{{- $values := $dot.Values }} +{{- range $subchart }} +{{- $values = index $values . }} +{{- end }} +{{- include $template (dict "Chart" (dict "Name" (last $subchart)) "Values" $values "Release" $dot.Release "Capabilities" $dot.Capabilities) }} +{{- end }} + +# Special Exporters +{{- define "exporter.kubeEtcd.enabled" -}} +{{- if or .Values.kubeEtcd.enabled .Values.rkeEtcd.enabled .Values.kubeAdmEtcd.enabled .Values.rke2Etcd.enabled -}} +"true" +{{- end -}} +{{- end }} + +{{- define "exporter.kubeControllerManager.enabled" -}} +{{- if or .Values.kubeControllerManager.enabled .Values.rkeControllerManager.enabled .Values.k3sServer.enabled .Values.kubeAdmControllerManager.enabled .Values.rke2ControllerManager.enabled -}} +"true" +{{- end -}} +{{- end }} + +{{- define "exporter.kubeScheduler.enabled" -}} +{{- if or .Values.kubeScheduler.enabled .Values.rkeScheduler.enabled .Values.k3sServer.enabled .Values.kubeAdmScheduler.enabled .Values.rke2Scheduler.enabled -}} +"true" +{{- end -}} +{{- end }} + +{{- define "exporter.kubeProxy.enabled" -}} +{{- if or .Values.kubeProxy.enabled .Values.rkeProxy.enabled .Values.k3sServer.enabled .Values.kubeAdmProxy.enabled .Values.rke2Proxy.enabled -}} +"true" +{{- end -}} +{{- end }} + +{{- define "exporter.kubelet.enabled" -}} +{{- if or .Values.kubelet.enabled .Values.hardenedKubelet.enabled .Values.k3sServer.enabled -}} +"true" +{{- end -}} +{{- end }} + +{{- define "exporter.kubeControllerManager.jobName" -}} +{{- if .Values.k3sServer.enabled -}} +k3s-server +{{- else -}} +kube-controller-manager +{{- end -}} +{{- end }} + +{{- define "exporter.kubeScheduler.jobName" -}} +{{- if .Values.k3sServer.enabled -}} +k3s-server +{{- else -}} +kube-scheduler +{{- end -}} +{{- end }} + +{{- define "exporter.kubeProxy.jobName" -}} +{{- if .Values.k3sServer.enabled -}} +k3s-server +{{- else -}} +kube-proxy +{{- end -}} +{{- end }} + +{{- define "exporter.kubelet.jobName" -}} +{{- if .Values.k3sServer.enabled -}} +k3s-server +{{- else -}} +kubelet +{{- end -}} +{{- end }} + +{{- define "kubelet.serviceMonitor.resourcePath" -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if not (eq .Values.kubelet.serviceMonitor.resourcePath "/metrics/resource/v1alpha1") -}} +{{ .Values.kubelet.serviceMonitor.resourcePath }} +{{- else if semverCompare ">=1.20.0-0" $kubeTargetVersion -}} +/metrics/resource +{{- else -}} +/metrics/resource/v1alpha1 +{{- end -}} +{{- end }} + +{{- define "rancher.serviceMonitor.selector" -}} +{{- if .Values.rancherMonitoring.selector }} +{{ .Values.rancherMonitoring.selector | toYaml }} +{{- else }} +{{- $rancherDeployment := (lookup "apps/v1" "Deployment" "cattle-system" "rancher") }} +{{- if $rancherDeployment }} +matchLabels: + app: rancher + chart: {{ index $rancherDeployment.metadata.labels "chart" }} + release: rancher +{{- end }} +{{- end }} +{{- end }} + +# Windows Support + +{{/* +Windows cluster will add default taint for linux nodes, +add below linux tolerations to workloads could be scheduled to those linux nodes +*/}} + +{{- define "linux-node-tolerations" -}} +- key: "cattle.io/os" + value: "linux" + effect: "NoSchedule" + operator: "Equal" +{{- end -}} + +{{- define "linux-node-selector" -}} +{{- if semverCompare "<1.14-0" .Capabilities.KubeVersion.GitVersion -}} +beta.kubernetes.io/os: linux +{{- else -}} +kubernetes.io/os: linux +{{- end -}} +{{- end -}} + +# Prometheus Operator + +{{/* vim: set filetype=mustache: */}} +{{/* Expand the name of the chart. This is suffixed with -alertmanager, which means subtract 13 from longest 63 available */}} +{{- define "kube-prometheus-stack.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 50 | trimSuffix "-" -}} +{{- end }} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +The components in this chart create additional resources that expand the longest created name strings. +The longest name that gets created adds and extra 37 characters, so truncation should be 63-35=26. +*/}} +{{- define "kube-prometheus-stack.fullname" -}} +{{- if .Values.fullnameOverride -}} +{{- .Values.fullnameOverride | trunc 26 | trimSuffix "-" -}} +{{- else -}} +{{- $name := default .Chart.Name .Values.nameOverride -}} +{{- if contains $name .Release.Name -}} +{{- .Release.Name | trunc 26 | trimSuffix "-" -}} +{{- else -}} +{{- printf "%s-%s" .Release.Name $name | trunc 26 | trimSuffix "-" -}} +{{- end -}} +{{- end -}} +{{- end -}} + +{{/* Fullname suffixed with -operator */}} +{{/* Adding 9 to 26 truncation of kube-prometheus-stack.fullname */}} +{{- define "kube-prometheus-stack.operator.fullname" -}} +{{- if .Values.prometheusOperator.fullnameOverride -}} +{{- .Values.prometheusOperator.fullnameOverride | trunc 35 | trimSuffix "-" -}} +{{- else -}} +{{- printf "%s-operator" (include "kube-prometheus-stack.fullname" .) -}} +{{- end }} +{{- end }} + +{{/* Prometheus custom resource instance name */}} +{{- define "kube-prometheus-stack.prometheus.crname" -}} +{{- if .Values.cleanPrometheusOperatorObjectNames }} +{{- include "kube-prometheus-stack.fullname" . }} +{{- else }} +{{- print (include "kube-prometheus-stack.fullname" .) "-prometheus" }} +{{- end }} +{{- end }} + +{{/* Prometheus apiVersion for networkpolicy */}} +{{- define "kube-prometheus-stack.prometheus.networkPolicy.apiVersion" -}} +{{- print "networking.k8s.io/v1" -}} +{{- end }} + +{{/* Alertmanager custom resource instance name */}} +{{- define "kube-prometheus-stack.alertmanager.crname" -}} +{{- if .Values.cleanPrometheusOperatorObjectNames }} +{{- include "kube-prometheus-stack.fullname" . }} +{{- else }} +{{- print (include "kube-prometheus-stack.fullname" .) "-alertmanager" -}} +{{- end }} +{{- end }} + +{{/* Fullname suffixed with thanos-ruler */}} +{{- define "kube-prometheus-stack.thanosRuler.fullname" -}} +{{- printf "%s-thanos-ruler" (include "kube-prometheus-stack.fullname" .) -}} +{{- end }} + +{{/* Shortened name suffixed with thanos-ruler */}} +{{- define "kube-prometheus-stack.thanosRuler.name" -}} +{{- default (printf "%s-thanos-ruler" (include "kube-prometheus-stack.name" .)) .Values.thanosRuler.name -}} +{{- end }} + + +{{/* Create chart name and version as used by the chart label. */}} +{{- define "kube-prometheus-stack.chartref" -}} +{{- replace "+" "_" .Chart.Version | printf "%s-%s" .Chart.Name -}} +{{- end }} + +{{/* Generate basic labels */}} +{{- define "kube-prometheus-stack.labels" }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +app.kubernetes.io/instance: {{ .Release.Name }} +app.kubernetes.io/version: "{{ replace "+" "_" .Chart.Version }}" +app.kubernetes.io/part-of: {{ template "kube-prometheus-stack.name" . }} +chart: {{ template "kube-prometheus-stack.chartref" . }} +release: {{ $.Release.Name | quote }} +heritage: {{ $.Release.Service | quote }} +{{- if .Values.commonLabels}} +{{ toYaml .Values.commonLabels }} +{{- end }} +{{- end }} + +{{/* Create the name of kube-prometheus-stack service account to use */}} +{{- define "kube-prometheus-stack.operator.serviceAccountName" -}} +{{- if .Values.prometheusOperator.serviceAccount.create -}} + {{ default (include "kube-prometheus-stack.operator.fullname" .) .Values.prometheusOperator.serviceAccount.name }} +{{- else -}} + {{ default "default" .Values.prometheusOperator.serviceAccount.name }} +{{- end -}} +{{- end -}} + +{{/* Create the name of kube-prometheus-stack service account to use */}} +{{- define "kube-prometheus-stack.operator.admissionWebhooks.serviceAccountName" -}} +{{- if .Values.prometheusOperator.serviceAccount.create -}} + {{ default (printf "%s-webhook" (include "kube-prometheus-stack.operator.fullname" .)) .Values.prometheusOperator.admissionWebhooks.deployment.serviceAccount.name }} +{{- else -}} + {{ default "default" .Values.prometheusOperator.admissionWebhooks.deployment.serviceAccount.name }} +{{- end -}} +{{- end -}} + +{{/* Create the name of prometheus service account to use */}} +{{- define "kube-prometheus-stack.prometheus.serviceAccountName" -}} +{{- if .Values.prometheus.serviceAccount.create -}} + {{ default (print (include "kube-prometheus-stack.fullname" .) "-prometheus") .Values.prometheus.serviceAccount.name }} +{{- else -}} + {{ default "default" .Values.prometheus.serviceAccount.name }} +{{- end -}} +{{- end -}} + +{{/* Create the name of alertmanager service account to use */}} +{{- define "kube-prometheus-stack.alertmanager.serviceAccountName" -}} +{{- if .Values.alertmanager.serviceAccount.create -}} + {{ default (print (include "kube-prometheus-stack.fullname" .) "-alertmanager") .Values.alertmanager.serviceAccount.name }} +{{- else -}} + {{ default "default" .Values.alertmanager.serviceAccount.name }} +{{- end -}} +{{- end -}} + +{{/* Create the name of thanosRuler service account to use */}} +{{- define "kube-prometheus-stack.thanosRuler.serviceAccountName" -}} +{{- if .Values.thanosRuler.serviceAccount.create -}} + {{ default (include "kube-prometheus-stack.thanosRuler.name" .) .Values.thanosRuler.serviceAccount.name }} +{{- else -}} + {{ default "default" .Values.thanosRuler.serviceAccount.name }} +{{- end -}} +{{- end -}} + +{{/* +Allow the release namespace to be overridden for multi-namespace deployments in combined charts +*/}} +{{- define "kube-prometheus-stack.namespace" -}} + {{- if .Values.namespaceOverride -}} + {{- .Values.namespaceOverride -}} + {{- else -}} + {{- .Release.Namespace -}} + {{- end -}} +{{- end -}} + +{{/* +Use the grafana namespace override for multi-namespace deployments in combined charts +*/}} +{{- define "kube-prometheus-stack-grafana.namespace" -}} + {{- if .Values.grafana.namespaceOverride -}} + {{- .Values.grafana.namespaceOverride -}} + {{- else -}} + {{- .Release.Namespace -}} + {{- end -}} +{{- end -}} + +{{/* +Allow kube-state-metrics job name to be overridden +*/}} +{{- define "kube-prometheus-stack-kube-state-metrics.name" -}} + {{- if index .Values "kube-state-metrics" "nameOverride" -}} + {{- index .Values "kube-state-metrics" "nameOverride" -}} + {{- else -}} + {{- print "kube-state-metrics" -}} + {{- end -}} +{{- end -}} + +{{/* +Use the kube-state-metrics namespace override for multi-namespace deployments in combined charts +*/}} +{{- define "kube-prometheus-stack-kube-state-metrics.namespace" -}} + {{- if index .Values "kube-state-metrics" "namespaceOverride" -}} + {{- index .Values "kube-state-metrics" "namespaceOverride" -}} + {{- else -}} + {{- .Release.Namespace -}} + {{- end -}} +{{- end -}} + +{{/* +Use the prometheus-node-exporter namespace override for multi-namespace deployments in combined charts +*/}} +{{- define "kube-prometheus-stack-prometheus-node-exporter.namespace" -}} + {{- if index .Values "prometheus-node-exporter" "namespaceOverride" -}} + {{- index .Values "prometheus-node-exporter" "namespaceOverride" -}} + {{- else -}} + {{- .Release.Namespace -}} + {{- end -}} +{{- end -}} + +{{/* Allow KubeVersion to be overridden. */}} +{{- define "kube-prometheus-stack.kubeVersion" -}} + {{- default .Capabilities.KubeVersion.Version .Values.kubeVersionOverride -}} +{{- end -}} + +{{/* Get Ingress API Version */}} +{{- define "kube-prometheus-stack.ingress.apiVersion" -}} + {{- if and (.Capabilities.APIVersions.Has "networking.k8s.io/v1") (semverCompare ">= 1.19-0" (include "kube-prometheus-stack.kubeVersion" .)) -}} + {{- print "networking.k8s.io/v1" -}} + {{- else if .Capabilities.APIVersions.Has "networking.k8s.io/v1beta1" -}} + {{- print "networking.k8s.io/v1beta1" -}} + {{- else -}} + {{- print "extensions/v1beta1" -}} + {{- end -}} +{{- end -}} + +{{/* Check Ingress stability */}} +{{- define "kube-prometheus-stack.ingress.isStable" -}} + {{- eq (include "kube-prometheus-stack.ingress.apiVersion" .) "networking.k8s.io/v1" -}} +{{- end -}} + +{{/* Check Ingress supports pathType */}} +{{/* pathType was added to networking.k8s.io/v1beta1 in Kubernetes 1.18 */}} +{{- define "kube-prometheus-stack.ingress.supportsPathType" -}} + {{- or (eq (include "kube-prometheus-stack.ingress.isStable" .) "true") (and (eq (include "kube-prometheus-stack.ingress.apiVersion" .) "networking.k8s.io/v1beta1") (semverCompare ">= 1.18-0" (include "kube-prometheus-stack.kubeVersion" .))) -}} +{{- end -}} + +{{/* Get Policy API Version */}} +{{- define "kube-prometheus-stack.pdb.apiVersion" -}} + {{- if and (.Capabilities.APIVersions.Has "policy/v1") (semverCompare ">= 1.21-0" (include "kube-prometheus-stack.kubeVersion" .)) -}} + {{- print "policy/v1" -}} + {{- else -}} + {{- print "policy/v1beta1" -}} + {{- end -}} + {{- end -}} + +{{/* Get value based on current Kubernetes version */}} +{{- define "kube-prometheus-stack.kubeVersionDefaultValue" -}} + {{- $values := index . 0 -}} + {{- $kubeVersion := index . 1 -}} + {{- $old := index . 2 -}} + {{- $new := index . 3 -}} + {{- $default := index . 4 -}} + {{- if kindIs "invalid" $default -}} + {{- if semverCompare $kubeVersion (include "kube-prometheus-stack.kubeVersion" $values) -}} + {{- print $new -}} + {{- else -}} + {{- print $old -}} + {{- end -}} + {{- else -}} + {{- print $default }} + {{- end -}} +{{- end -}} + +{{/* Get value for kube-controller-manager depending on insecure scraping availability */}} +{{- define "kube-prometheus-stack.kubeControllerManager.insecureScrape" -}} + {{- $values := index . 0 -}} + {{- $insecure := index . 1 -}} + {{- $secure := index . 2 -}} + {{- $userValue := index . 3 -}} + {{- include "kube-prometheus-stack.kubeVersionDefaultValue" (list $values ">= 1.22-0" $insecure $secure $userValue) -}} +{{- end -}} + +{{/* Get value for kube-scheduler depending on insecure scraping availability */}} +{{- define "kube-prometheus-stack.kubeScheduler.insecureScrape" -}} + {{- $values := index . 0 -}} + {{- $insecure := index . 1 -}} + {{- $secure := index . 2 -}} + {{- $userValue := index . 3 -}} + {{- include "kube-prometheus-stack.kubeVersionDefaultValue" (list $values ">= 1.23-0" $insecure $secure $userValue) -}} +{{- end -}} + +{{/* Sets default scrape limits for servicemonitor */}} +{{- define "servicemonitor.scrapeLimits" -}} +{{- with .sampleLimit }} +sampleLimit: {{ . }} +{{- end }} +{{- with .targetLimit }} +targetLimit: {{ . }} +{{- end }} +{{- with .labelLimit }} +labelLimit: {{ . }} +{{- end }} +{{- with .labelNameLengthLimit }} +labelNameLengthLimit: {{ . }} +{{- end }} +{{- with .labelValueLengthLimit }} +labelValueLengthLimit: {{ . }} +{{- end }} +{{- end -}} + +{{/* +To help compatibility with other charts which use global.imagePullSecrets. +Allow either an array of {name: pullSecret} maps (k8s-style), or an array of strings (more common helm-style). +global: + imagePullSecrets: + - name: pullSecret1 + - name: pullSecret2 + +or + +global: + imagePullSecrets: + - pullSecret1 + - pullSecret2 +*/}} +{{- define "kube-prometheus-stack.imagePullSecrets" -}} +{{- range .Values.global.imagePullSecrets }} + {{- if eq (typeOf .) "map[string]interface {}" }} +- {{ toYaml . | trim }} + {{- else }} +- name: {{ . }} + {{- end }} +{{- end }} +{{- end -}} + +{{- define "kube-prometheus-stack.operator.admission-webhook.dnsNames" }} +{{- $fullname := include "kube-prometheus-stack.operator.fullname" . }} +{{- $namespace := include "kube-prometheus-stack.namespace" . }} +{{- $fullname }} +{{ $fullname }}.{{ $namespace }}.svc +{{- if .Values.prometheusOperator.admissionWebhooks.deployment.enabled }} +{{ $fullname }}-webhook +{{ $fullname }}-webhook.{{ $namespace }}.svc +{{- end }} +{{- end }} + +{{- define "rke2-ingress-nginx.namespace" -}} + {{- if .Values.rke2IngressNginx.namespaceOverride -}} + {{- .Values.rke2IngressNginx.namespaceOverride -}} + {{- else -}} + {{- print "kube-system" -}} + {{- end -}} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/alertmanager/alertmanager.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/alertmanager/alertmanager.yaml new file mode 100644 index 00000000000..19044054acc --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/alertmanager/alertmanager.yaml @@ -0,0 +1,191 @@ +{{- if .Values.alertmanager.enabled }} +apiVersion: monitoring.coreos.com/v1 +kind: Alertmanager +metadata: + name: {{ template "kube-prometheus-stack.alertmanager.crname" . }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }}-alertmanager +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.alertmanager.annotations }} + annotations: +{{ toYaml .Values.alertmanager.annotations | indent 4 }} +{{- end }} +spec: +{{- if .Values.alertmanager.alertmanagerSpec.image }} + {{- $registry := include "monitoring_registry" . | default .Values.alertmanager.alertmanagerSpec.image.registry }} + {{- if and .Values.alertmanager.alertmanagerSpec.image.tag .Values.alertmanager.alertmanagerSpec.image.sha }} + image: "{{ $registry }}/{{ .Values.alertmanager.alertmanagerSpec.image.repository }}:{{ .Values.alertmanager.alertmanagerSpec.image.tag }}@sha256:{{ .Values.alertmanager.alertmanagerSpec.image.sha }}" + {{- else if .Values.alertmanager.alertmanagerSpec.image.sha }} + image: "{{ $registry }}/{{ .Values.alertmanager.alertmanagerSpec.image.repository }}@sha256:{{ .Values.alertmanager.alertmanagerSpec.image.sha }}" + {{- else if .Values.alertmanager.alertmanagerSpec.image.tag }} + image: "{{ $registry }}/{{ .Values.alertmanager.alertmanagerSpec.image.repository }}:{{ .Values.alertmanager.alertmanagerSpec.image.tag }}" + {{- else }} + image: "{{ $registry }}/{{ .Values.alertmanager.alertmanagerSpec.image.repository }}" + {{- end }} + version: {{ .Values.alertmanager.alertmanagerSpec.image.tag }} + {{- if .Values.alertmanager.alertmanagerSpec.image.sha }} + sha: {{ .Values.alertmanager.alertmanagerSpec.image.sha }} + {{- end }} +{{- end }} + replicas: {{ .Values.alertmanager.alertmanagerSpec.replicas }} + listenLocal: {{ .Values.alertmanager.alertmanagerSpec.listenLocal }} + serviceAccountName: {{ template "kube-prometheus-stack.alertmanager.serviceAccountName" . }} + automountServiceAccountToken: {{ .Values.alertmanager.alertmanagerSpec.automountServiceAccountToken }} +{{- if .Values.alertmanager.alertmanagerSpec.externalUrl }} + externalUrl: "{{ tpl .Values.alertmanager.alertmanagerSpec.externalUrl . }}" +{{- else if and .Values.alertmanager.ingress.enabled .Values.alertmanager.ingress.hosts }} + externalUrl: "http://{{ tpl (index .Values.alertmanager.ingress.hosts 0) . }}{{ .Values.alertmanager.alertmanagerSpec.routePrefix }}" +{{- else if not (or (kindIs "invalid" .Values.global.cattle.url) (kindIs "invalid" .Values.global.cattle.clusterId)) }} + externalUrl: "{{ .Values.global.cattle.url }}/k8s/clusters/{{ .Values.global.cattle.clusterId }}/api/v1/namespaces/{{ .Values.namespaceOverride }}/services/http:{{ template "kube-prometheus-stack.fullname" . }}-alertmanager:{{ .Values.alertmanager.service.port }}/proxy" +{{- else }} + externalUrl: http://{{ template "kube-prometheus-stack.fullname" . }}-alertmanager.{{ template "kube-prometheus-stack.namespace" . }}:{{ .Values.alertmanager.service.port }} +{{- end }} + nodeSelector: {{ include "linux-node-selector" . | nindent 4 }} +{{- if .Values.alertmanager.alertmanagerSpec.nodeSelector }} +{{ toYaml .Values.alertmanager.alertmanagerSpec.nodeSelector | indent 4 }} +{{- end }} + paused: {{ .Values.alertmanager.alertmanagerSpec.paused }} + logFormat: {{ .Values.alertmanager.alertmanagerSpec.logFormat | quote }} + logLevel: {{ .Values.alertmanager.alertmanagerSpec.logLevel | quote }} + retention: {{ .Values.alertmanager.alertmanagerSpec.retention | quote }} +{{- if .Values.alertmanager.alertmanagerSpec.secrets }} + secrets: +{{ toYaml .Values.alertmanager.alertmanagerSpec.secrets | indent 4 }} +{{- end }} +{{- if .Values.alertmanager.alertmanagerSpec.configSecret }} + configSecret: {{ .Values.alertmanager.alertmanagerSpec.configSecret }} +{{- end }} +{{- if .Values.alertmanager.alertmanagerSpec.configMaps }} + configMaps: +{{ toYaml .Values.alertmanager.alertmanagerSpec.configMaps | indent 4 }} +{{- end }} +{{- if .Values.alertmanager.alertmanagerSpec.alertmanagerConfigSelector }} + alertmanagerConfigSelector: +{{ tpl (toYaml .Values.alertmanager.alertmanagerSpec.alertmanagerConfigSelector | indent 4) . }} +{{ else }} + alertmanagerConfigSelector: {} +{{- end }} +{{- if .Values.alertmanager.alertmanagerSpec.alertmanagerConfigNamespaceSelector }} + alertmanagerConfigNamespaceSelector: +{{ tpl (toYaml .Values.alertmanager.alertmanagerSpec.alertmanagerConfigNamespaceSelector | indent 4) . }} +{{ else }} + alertmanagerConfigNamespaceSelector: {} +{{- end }} +{{- if .Values.alertmanager.alertmanagerSpec.web }} + web: +{{ toYaml .Values.alertmanager.alertmanagerSpec.web | indent 4 }} +{{- end }} +{{- if .Values.alertmanager.alertmanagerSpec.alertmanagerConfiguration }} + alertmanagerConfiguration: +{{ toYaml .Values.alertmanager.alertmanagerSpec.alertmanagerConfiguration | indent 4 }} +{{- end }} +{{- if .Values.alertmanager.alertmanagerSpec.alertmanagerConfigMatcherStrategy }} + alertmanagerConfigMatcherStrategy: +{{ toYaml .Values.alertmanager.alertmanagerSpec.alertmanagerConfigMatcherStrategy | indent 4 }} +{{- end }} +{{- if .Values.alertmanager.alertmanagerSpec.resources }} + resources: +{{ toYaml .Values.alertmanager.alertmanagerSpec.resources | indent 4 }} +{{- end }} +{{- if .Values.alertmanager.alertmanagerSpec.routePrefix }} + routePrefix: "{{ .Values.alertmanager.alertmanagerSpec.routePrefix }}" +{{- end }} +{{- if .Values.alertmanager.alertmanagerSpec.securityContext }} + securityContext: +{{ toYaml .Values.alertmanager.alertmanagerSpec.securityContext | indent 4 }} +{{- end }} +{{- if .Values.alertmanager.alertmanagerSpec.storage }} + storage: +{{ tpl (toYaml .Values.alertmanager.alertmanagerSpec.storage | indent 4) . }} +{{- end }} +{{- if .Values.alertmanager.alertmanagerSpec.podMetadata }} + podMetadata: +{{ toYaml .Values.alertmanager.alertmanagerSpec.podMetadata | indent 4 }} +{{- end }} +{{- if or .Values.alertmanager.alertmanagerSpec.podAntiAffinity .Values.alertmanager.alertmanagerSpec.affinity }} + affinity: +{{- end }} +{{- if .Values.alertmanager.alertmanagerSpec.affinity }} +{{ toYaml .Values.alertmanager.alertmanagerSpec.affinity | indent 4 }} +{{- end }} +{{- if eq .Values.alertmanager.alertmanagerSpec.podAntiAffinity "hard" }} + podAntiAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - topologyKey: {{ .Values.alertmanager.alertmanagerSpec.podAntiAffinityTopologyKey }} + labelSelector: + matchExpressions: + - {key: app.kubernetes.io/name, operator: In, values: [alertmanager]} + - {key: alertmanager, operator: In, values: [{{ template "kube-prometheus-stack.alertmanager.crname" . }}]} +{{- else if eq .Values.alertmanager.alertmanagerSpec.podAntiAffinity "soft" }} + podAntiAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 100 + podAffinityTerm: + topologyKey: {{ .Values.alertmanager.alertmanagerSpec.podAntiAffinityTopologyKey }} + labelSelector: + matchExpressions: + - {key: app.kubernetes.io/name, operator: In, values: [alertmanager]} + - {key: alertmanager, operator: In, values: [{{ template "kube-prometheus-stack.alertmanager.crname" . }}]} +{{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 4 }} +{{- if .Values.alertmanager.alertmanagerSpec.tolerations }} +{{ toYaml .Values.alertmanager.alertmanagerSpec.tolerations | indent 4 }} +{{- end }} +{{- if .Values.alertmanager.alertmanagerSpec.topologySpreadConstraints }} + topologySpreadConstraints: +{{ toYaml .Values.alertmanager.alertmanagerSpec.topologySpreadConstraints | indent 4 }} +{{- end }} +{{- if .Values.global.imagePullSecrets }} + imagePullSecrets: +{{ include "kube-prometheus-stack.imagePullSecrets" . | trim | indent 4 }} +{{- end }} +{{- if .Values.alertmanager.alertmanagerSpec.containers }} + containers: +{{ toYaml .Values.alertmanager.alertmanagerSpec.containers | indent 4 }} +{{- end }} +{{- if .Values.alertmanager.alertmanagerSpec.initContainers }} + initContainers: +{{ toYaml .Values.alertmanager.alertmanagerSpec.initContainers | indent 4 }} +{{- end }} +{{- if .Values.alertmanager.alertmanagerSpec.priorityClassName }} + priorityClassName: {{.Values.alertmanager.alertmanagerSpec.priorityClassName }} +{{- end }} +{{- if .Values.alertmanager.alertmanagerSpec.additionalPeers }} + additionalPeers: +{{ toYaml .Values.alertmanager.alertmanagerSpec.additionalPeers | indent 4 }} +{{- end }} +{{- if .Values.alertmanager.alertmanagerSpec.volumes }} + volumes: +{{ toYaml .Values.alertmanager.alertmanagerSpec.volumes | indent 4 }} +{{- end }} +{{- if .Values.alertmanager.alertmanagerSpec.volumeMounts }} + volumeMounts: +{{ toYaml .Values.alertmanager.alertmanagerSpec.volumeMounts | indent 4 }} +{{- end }} + portName: {{ .Values.alertmanager.alertmanagerSpec.portName }} +{{- if .Values.alertmanager.alertmanagerSpec.clusterAdvertiseAddress }} + clusterAdvertiseAddress: {{ .Values.alertmanager.alertmanagerSpec.clusterAdvertiseAddress }} +{{- end }} +{{- if .Values.alertmanager.alertmanagerSpec.clusterGossipInterval }} + clusterGossipInterval: {{ .Values.alertmanager.alertmanagerSpec.clusterGossipInterval }} +{{- end }} +{{- if .Values.alertmanager.alertmanagerSpec.clusterPeerTimeout }} + clusterPeerTimeout: {{ .Values.alertmanager.alertmanagerSpec.clusterPeerTimeout }} +{{- end }} +{{- if .Values.alertmanager.alertmanagerSpec.clusterPushpullInterval }} + clusterPushpullInterval: {{ .Values.alertmanager.alertmanagerSpec.clusterPushpullInterval }} +{{- end }} +{{- if .Values.alertmanager.alertmanagerSpec.forceEnableClusterMode }} + forceEnableClusterMode: {{ .Values.alertmanager.alertmanagerSpec.forceEnableClusterMode }} +{{- end }} +{{- if .Values.alertmanager.alertmanagerSpec.minReadySeconds }} + minReadySeconds: {{ .Values.alertmanager.alertmanagerSpec.minReadySeconds }} +{{- end }} +{{- with .Values.alertmanager.alertmanagerSpec.additionalConfig }} + {{- tpl (toYaml .) $ | nindent 2 }} +{{- end }} +{{- with .Values.alertmanager.alertmanagerSpec.additionalConfigString }} + {{- tpl . $ | nindent 2 }} +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/alertmanager/extrasecret.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/alertmanager/extrasecret.yaml new file mode 100644 index 00000000000..ecd8f47021c --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/alertmanager/extrasecret.yaml @@ -0,0 +1,20 @@ +{{- if .Values.alertmanager.extraSecret.data -}} +{{- $secretName := printf "alertmanager-%s-extra" (include "kube-prometheus-stack.fullname" . ) -}} +apiVersion: v1 +kind: Secret +metadata: + name: {{ default $secretName .Values.alertmanager.extraSecret.name }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} +{{- if .Values.alertmanager.extraSecret.annotations }} + annotations: +{{ toYaml .Values.alertmanager.extraSecret.annotations | indent 4 }} +{{- end }} + labels: + app: {{ template "kube-prometheus-stack.name" . }}-alertmanager + app.kubernetes.io/component: alertmanager +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +data: +{{- range $key, $val := .Values.alertmanager.extraSecret.data }} + {{ $key }}: {{ $val | b64enc | quote }} +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/alertmanager/ingress.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/alertmanager/ingress.yaml new file mode 100644 index 00000000000..be9f5aa279c --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/alertmanager/ingress.yaml @@ -0,0 +1,78 @@ +{{- if and .Values.alertmanager.enabled .Values.alertmanager.ingress.enabled }} +{{- $pathType := .Values.alertmanager.ingress.pathType | default "ImplementationSpecific" }} +{{- $serviceName := printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "alertmanager" }} +{{- $backendServiceName := .Values.alertmanager.ingress.serviceName | default (printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "alertmanager") }} +{{- $servicePort := .Values.alertmanager.ingress.servicePort | default .Values.alertmanager.service.port -}} +{{- $routePrefix := list .Values.alertmanager.alertmanagerSpec.routePrefix }} +{{- $paths := .Values.alertmanager.ingress.paths | default $routePrefix -}} +{{- $apiIsStable := eq (include "kube-prometheus-stack.ingress.isStable" .) "true" -}} +{{- $ingressSupportsPathType := eq (include "kube-prometheus-stack.ingress.supportsPathType" .) "true" -}} +apiVersion: {{ include "kube-prometheus-stack.ingress.apiVersion" . }} +kind: Ingress +metadata: + name: {{ $serviceName }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} +{{- if .Values.alertmanager.ingress.annotations }} + annotations: + {{- tpl (toYaml .Values.alertmanager.ingress.annotations) . | nindent 4 }} +{{- end }} + labels: + app: {{ template "kube-prometheus-stack.name" . }}-alertmanager +{{- if .Values.alertmanager.ingress.labels }} +{{ toYaml .Values.alertmanager.ingress.labels | indent 4 }} +{{- end }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +spec: + {{- if $apiIsStable }} + {{- if .Values.alertmanager.ingress.ingressClassName }} + ingressClassName: {{ .Values.alertmanager.ingress.ingressClassName }} + {{- end }} + {{- end }} + rules: + {{- if .Values.alertmanager.ingress.hosts }} + {{- range $host := .Values.alertmanager.ingress.hosts }} + - host: {{ tpl $host $ | quote }} + http: + paths: + {{- range $p := $paths }} + - path: {{ tpl $p $ }} + {{- if and $pathType $ingressSupportsPathType }} + pathType: {{ $pathType }} + {{- end }} + backend: + {{- if $apiIsStable }} + service: + name: {{ $backendServiceName }} + port: + number: {{ $servicePort }} + {{- else }} + serviceName: {{ $backendServiceName }} + servicePort: {{ $servicePort }} + {{- end }} + {{- end -}} + {{- end -}} + {{- else }} + - http: + paths: + {{- range $p := $paths }} + - path: {{ tpl $p $ }} + {{- if and $pathType $ingressSupportsPathType }} + pathType: {{ $pathType }} + {{- end }} + backend: + {{- if $apiIsStable }} + service: + name: {{ $backendServiceName }} + port: + number: {{ $servicePort }} + {{- else }} + serviceName: {{ $backendServiceName }} + servicePort: {{ $servicePort }} + {{- end }} + {{- end -}} + {{- end -}} + {{- if .Values.alertmanager.ingress.tls }} + tls: +{{ tpl (toYaml .Values.alertmanager.ingress.tls | indent 4) . }} + {{- end -}} +{{- end -}} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/alertmanager/ingressperreplica.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/alertmanager/ingressperreplica.yaml new file mode 100644 index 00000000000..b2e00a41629 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/alertmanager/ingressperreplica.yaml @@ -0,0 +1,67 @@ +{{- if and .Values.alertmanager.enabled .Values.alertmanager.servicePerReplica.enabled .Values.alertmanager.ingressPerReplica.enabled }} +{{- $pathType := .Values.alertmanager.ingressPerReplica.pathType | default "" }} +{{- $count := .Values.alertmanager.alertmanagerSpec.replicas | int -}} +{{- $servicePort := .Values.alertmanager.service.port -}} +{{- $ingressValues := .Values.alertmanager.ingressPerReplica -}} +{{- $apiIsStable := eq (include "kube-prometheus-stack.ingress.isStable" .) "true" -}} +{{- $ingressSupportsPathType := eq (include "kube-prometheus-stack.ingress.supportsPathType" .) "true" -}} +apiVersion: v1 +kind: List +metadata: + name: {{ include "kube-prometheus-stack.fullname" $ }}-alertmanager-ingressperreplica + namespace: {{ template "kube-prometheus-stack.namespace" . }} +items: +{{ range $i, $e := until $count }} + - kind: Ingress + apiVersion: {{ include "kube-prometheus-stack.ingress.apiVersion" $ }} + metadata: + name: {{ include "kube-prometheus-stack.fullname" $ }}-alertmanager-{{ $i }} + namespace: {{ template "kube-prometheus-stack.namespace" $ }} + labels: + app: {{ include "kube-prometheus-stack.name" $ }}-alertmanager + {{ include "kube-prometheus-stack.labels" $ | indent 8 }} + {{- if $ingressValues.labels }} +{{ toYaml $ingressValues.labels | indent 8 }} + {{- end }} + {{- if $ingressValues.annotations }} + annotations: + {{- tpl (toYaml $ingressValues.annotations) $ | nindent 8 }} + {{- end }} + spec: + {{- if $apiIsStable }} + {{- if $ingressValues.ingressClassName }} + ingressClassName: {{ $ingressValues.ingressClassName }} + {{- end }} + {{- end }} + rules: + - host: {{ $ingressValues.hostPrefix }}-{{ $i }}.{{ $ingressValues.hostDomain }} + http: + paths: + {{- range $p := $ingressValues.paths }} + - path: {{ tpl $p $ }} + {{- if and $pathType $ingressSupportsPathType }} + pathType: {{ $pathType }} + {{- end }} + backend: + {{- if $apiIsStable }} + service: + name: {{ include "kube-prometheus-stack.fullname" $ }}-alertmanager-{{ $i }} + port: + number: {{ $servicePort }} + {{- else }} + serviceName: {{ include "kube-prometheus-stack.fullname" $ }}-alertmanager-{{ $i }} + servicePort: {{ $servicePort }} + {{- end }} + {{- end -}} + {{- if or $ingressValues.tlsSecretName $ingressValues.tlsSecretPerReplica.enabled }} + tls: + - hosts: + - {{ $ingressValues.hostPrefix }}-{{ $i }}.{{ $ingressValues.hostDomain }} + {{- if $ingressValues.tlsSecretPerReplica.enabled }} + secretName: {{ $ingressValues.tlsSecretPerReplica.prefix }}-{{ $i }} + {{- else }} + secretName: {{ $ingressValues.tlsSecretName }} + {{- end }} + {{- end }} +{{- end -}} +{{- end -}} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/alertmanager/podDisruptionBudget.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/alertmanager/podDisruptionBudget.yaml new file mode 100644 index 00000000000..b1834031253 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/alertmanager/podDisruptionBudget.yaml @@ -0,0 +1,21 @@ +{{- if and .Values.alertmanager.enabled .Values.alertmanager.podDisruptionBudget.enabled }} +apiVersion: {{ include "kube-prometheus-stack.pdb.apiVersion" . }} +kind: PodDisruptionBudget +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-alertmanager + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }}-alertmanager +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +spec: + {{- if .Values.alertmanager.podDisruptionBudget.minAvailable }} + minAvailable: {{ .Values.alertmanager.podDisruptionBudget.minAvailable }} + {{- end }} + {{- if .Values.alertmanager.podDisruptionBudget.maxUnavailable }} + maxUnavailable: {{ .Values.alertmanager.podDisruptionBudget.maxUnavailable }} + {{- end }} + selector: + matchLabels: + app.kubernetes.io/name: alertmanager + alertmanager: {{ template "kube-prometheus-stack.alertmanager.crname" . }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/alertmanager/psp-role.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/alertmanager/psp-role.yaml new file mode 100644 index 00000000000..8810e93ded8 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/alertmanager/psp-role.yaml @@ -0,0 +1,23 @@ +{{- if and .Values.alertmanager.enabled (or .Values.global.cattle.psp.enabled (and .Values.global.rbac.create .Values.global.rbac.pspEnabled)) }} +{{- if .Capabilities.APIVersions.Has "policy/v1beta1/PodSecurityPolicy" }} +kind: Role +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-alertmanager + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }}-alertmanager +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +rules: +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if semverCompare "> 1.15.0-0" $kubeTargetVersion }} +- apiGroups: ['policy'] +{{- else }} +- apiGroups: ['extensions'] +{{- end }} + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - {{ template "kube-prometheus-stack.fullname" . }}-alertmanager +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/alertmanager/psp-rolebinding.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/alertmanager/psp-rolebinding.yaml new file mode 100644 index 00000000000..794f4ad1789 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/alertmanager/psp-rolebinding.yaml @@ -0,0 +1,20 @@ +{{- if and .Values.alertmanager.enabled (or .Values.global.cattle.psp.enabled (and .Values.global.rbac.create .Values.global.rbac.pspEnabled)) }} +{{- if .Capabilities.APIVersions.Has "policy/v1beta1/PodSecurityPolicy" }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-alertmanager + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }}-alertmanager +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: {{ template "kube-prometheus-stack.fullname" . }}-alertmanager +subjects: + - kind: ServiceAccount + name: {{ template "kube-prometheus-stack.alertmanager.serviceAccountName" . }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/alertmanager/psp.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/alertmanager/psp.yaml new file mode 100644 index 00000000000..07b616b5cb6 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/alertmanager/psp.yaml @@ -0,0 +1,47 @@ +{{- if and .Values.alertmanager.enabled (or .Values.global.cattle.psp.enabled (and .Values.global.rbac.create .Values.global.rbac.pspEnabled)) }} +{{- if .Capabilities.APIVersions.Has "policy/v1beta1/PodSecurityPolicy" }} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-alertmanager + labels: + app: {{ template "kube-prometheus-stack.name" . }}-alertmanager +{{- if .Values.global.rbac.pspAnnotations }} + annotations: +{{ toYaml .Values.global.rbac.pspAnnotations | indent 4 }} +{{- end }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +spec: + privileged: false + # Allow core volume types. + volumes: + - 'configMap' + - 'emptyDir' + - 'projected' + - 'secret' + - 'downwardAPI' + - 'persistentVolumeClaim' + hostNetwork: false + hostIPC: false + hostPID: false + runAsUser: + # Permits the container to run with root privileges as well. + rule: 'RunAsAny' + seLinux: + # This policy assumes the nodes are using AppArmor rather than SELinux. + rule: 'RunAsAny' + supplementalGroups: + rule: 'MustRunAs' + ranges: + # Allow adding the root group. + - min: 0 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + # Allow adding the root group. + - min: 0 + max: 65535 + readOnlyRootFilesystem: false +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/alertmanager/secret.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/alertmanager/secret.yaml new file mode 100644 index 00000000000..d2fe84a7bf5 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/alertmanager/secret.yaml @@ -0,0 +1,35 @@ +{{- if and (.Values.alertmanager.enabled) (not .Values.alertmanager.alertmanagerSpec.useExistingSecret) }} +{{/* This file is applied when the operation is helm install and the target secret does not exist. */}} +{{- $secretName := (printf "alertmanager-%s" (include "kube-prometheus-stack.alertmanager.crname" .)) }} +{{- if or (not (lookup "v1" "Secret" (include "kube-prometheus-stack.namespace" .) $secretName)) (eq .Values.alertmanager.secret.recreateIfExists true) }} +apiVersion: v1 +kind: Secret +metadata: + name: {{ $secretName }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + annotations: + "helm.sh/hook": pre-install, pre-upgrade + "helm.sh/hook-weight": "3" + "helm.sh/resource-policy": keep +{{- if .Values.alertmanager.secret.annotations }} +{{ toYaml .Values.alertmanager.secret.annotations | indent 4 }} +{{- end }} + labels: + app: {{ template "kube-prometheus-stack.name" . }}-alertmanager +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +data: +{{- if .Values.alertmanager.tplConfig }} +{{- if .Values.alertmanager.stringConfig }} + alertmanager.yaml: {{ tpl (.Values.alertmanager.stringConfig) . | b64enc | quote }} +{{- else if eq (typeOf .Values.alertmanager.config) "string" }} + alertmanager.yaml: {{ tpl (.Values.alertmanager.config) . | b64enc | quote }} +{{- else }} + alertmanager.yaml: {{ tpl (toYaml .Values.alertmanager.config) . | b64enc | quote }} +{{- end }} +{{- else }} + alertmanager.yaml: {{ toYaml .Values.alertmanager.config | b64enc | quote }} +{{- end }} +{{- range $key, $val := .Values.alertmanager.templateFiles }} + {{ $key }}: {{ $val | b64enc | quote }} +{{- end }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/alertmanager/service.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/alertmanager/service.yaml new file mode 100644 index 00000000000..373de328a5b --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/alertmanager/service.yaml @@ -0,0 +1,68 @@ +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if .Values.alertmanager.enabled }} +apiVersion: v1 +kind: Service +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-alertmanager + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }}-alertmanager + self-monitor: {{ .Values.alertmanager.serviceMonitor.selfMonitor | quote }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.alertmanager.service.labels }} +{{ toYaml .Values.alertmanager.service.labels | indent 4 }} +{{- end }} +{{- if .Values.alertmanager.service.annotations }} + annotations: +{{ toYaml .Values.alertmanager.service.annotations | indent 4 }} +{{- end }} +spec: +{{- if .Values.alertmanager.service.clusterIP }} + clusterIP: {{ .Values.alertmanager.service.clusterIP }} +{{- end }} +{{- if .Values.alertmanager.service.externalIPs }} + externalIPs: +{{ toYaml .Values.alertmanager.service.externalIPs | indent 4 }} +{{- end }} +{{- if .Values.alertmanager.service.loadBalancerIP }} + loadBalancerIP: {{ .Values.alertmanager.service.loadBalancerIP }} +{{- end }} +{{- if .Values.alertmanager.service.loadBalancerSourceRanges }} + loadBalancerSourceRanges: + {{- range $cidr := .Values.alertmanager.service.loadBalancerSourceRanges }} + - {{ $cidr }} + {{- end }} +{{- end }} +{{- if ne .Values.alertmanager.service.type "ClusterIP" }} + externalTrafficPolicy: {{ .Values.alertmanager.service.externalTrafficPolicy }} +{{- end }} + ports: + - name: {{ .Values.alertmanager.alertmanagerSpec.portName }} + {{- if eq .Values.alertmanager.service.type "NodePort" }} + nodePort: {{ .Values.alertmanager.service.nodePort }} + {{- end }} + port: {{ .Values.alertmanager.service.port }} + targetPort: {{ .Values.alertmanager.service.targetPort }} + protocol: TCP + - name: reloader-web + {{- if semverCompare ">=1.20.0-0" $kubeTargetVersion }} + appProtocol: http + {{- end }} + port: 8080 + targetPort: reloader-web +{{- if .Values.alertmanager.service.additionalPorts }} +{{ toYaml .Values.alertmanager.service.additionalPorts | indent 2 }} +{{- end }} + selector: + app.kubernetes.io/name: alertmanager + alertmanager: {{ template "kube-prometheus-stack.alertmanager.crname" . }} +{{- if .Values.alertmanager.service.sessionAffinity }} + sessionAffinity: {{ .Values.alertmanager.service.sessionAffinity }} +{{- end }} +{{- if eq .Values.alertmanager.service.sessionAffinity "ClientIP" }} + sessionAffinityConfig: + clientIP: + timeoutSeconds: {{ .Values.alertmanager.service.sessionAffinityConfig.clientIP.timeoutSeconds }} +{{- end }} + type: "{{ .Values.alertmanager.service.type }}" +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/alertmanager/serviceaccount.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/alertmanager/serviceaccount.yaml new file mode 100644 index 00000000000..745ced8bded --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/alertmanager/serviceaccount.yaml @@ -0,0 +1,21 @@ +{{- if and .Values.alertmanager.enabled .Values.alertmanager.serviceAccount.create }} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "kube-prometheus-stack.alertmanager.serviceAccountName" . }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }}-alertmanager + app.kubernetes.io/name: {{ template "kube-prometheus-stack.name" . }}-alertmanager + app.kubernetes.io/component: alertmanager +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.alertmanager.serviceAccount.annotations }} + annotations: +{{ toYaml .Values.alertmanager.serviceAccount.annotations | indent 4 }} +{{- end }} +automountServiceAccountToken: {{ .Values.alertmanager.serviceAccount.automountServiceAccountToken }} +{{- if .Values.global.imagePullSecrets }} +imagePullSecrets: +{{ include "kube-prometheus-stack.imagePullSecrets" . | trim | indent 2}} +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/alertmanager/servicemonitor.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/alertmanager/servicemonitor.yaml new file mode 100644 index 00000000000..6233690019a --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/alertmanager/servicemonitor.yaml @@ -0,0 +1,84 @@ +{{- if and .Values.alertmanager.enabled .Values.alertmanager.serviceMonitor.selfMonitor }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-alertmanager + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }}-alertmanager +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- with .Values.alertmanager.serviceMonitor.additionalLabels }} +{{- toYaml . | nindent 4 }} +{{- end }} +spec: + {{- include "servicemonitor.scrapeLimits" .Values.alertmanager.serviceMonitor | nindent 2 }} + selector: + matchLabels: + app: {{ template "kube-prometheus-stack.name" . }}-alertmanager + release: {{ $.Release.Name | quote }} + self-monitor: "true" + namespaceSelector: + matchNames: + - {{ printf "%s" (include "kube-prometheus-stack.namespace" .) | quote }} + endpoints: + - port: {{ .Values.alertmanager.alertmanagerSpec.portName }} + enableHttp2: {{ .Values.alertmanager.serviceMonitor.enableHttp2 }} + {{- if .Values.alertmanager.serviceMonitor.interval }} + interval: {{ .Values.alertmanager.serviceMonitor.interval }} + {{- end }} + {{- if .Values.alertmanager.serviceMonitor.proxyUrl }} + proxyUrl: {{ .Values.alertmanager.serviceMonitor.proxyUrl }} + {{- end }} + {{- if .Values.alertmanager.serviceMonitor.scheme }} + scheme: {{ .Values.alertmanager.serviceMonitor.scheme }} + {{- end }} + {{- if .Values.alertmanager.serviceMonitor.bearerTokenFile }} + bearerTokenFile: {{ .Values.alertmanager.serviceMonitor.bearerTokenFile }} + {{- end }} + {{- if .Values.alertmanager.serviceMonitor.tlsConfig }} + tlsConfig: {{- toYaml .Values.alertmanager.serviceMonitor.tlsConfig | nindent 6 }} + {{- end }} + path: "{{ trimSuffix "/" .Values.alertmanager.alertmanagerSpec.routePrefix }}/metrics" + metricRelabelings: + {{- if .Values.alertmanager.serviceMonitor.metricRelabelings }} + {{- tpl (toYaml .Values.alertmanager.serviceMonitor.metricRelabelings | nindent 6) . }} + {{- end }} + {{ if .Values.global.cattle.clusterId }} + - sourceLabels: [__address__] + targetLabel: cluster_id + replacement: {{ .Values.global.cattle.clusterId }} + {{- end }} + {{ if .Values.global.cattle.clusterName }} + - sourceLabels: [__address__] + targetLabel: cluster_name + replacement: {{ .Values.global.cattle.clusterName }} + {{- end }} + {{- if .Values.alertmanager.serviceMonitor.relabelings }} + relabelings: {{- toYaml .Values.alertmanager.serviceMonitor.relabelings | nindent 6 }} + {{- end }} + {{- range .Values.alertmanager.serviceMonitor.additionalEndpoints }} + - port: {{ .port }} + {{- if or $.Values.alertmanager.serviceMonitor.interval .interval }} + interval: {{ default $.Values.alertmanager.serviceMonitor.interval .interval }} + {{- end }} + {{- if or $.Values.alertmanager.serviceMonitor.proxyUrl .proxyUrl }} + proxyUrl: {{ default $.Values.alertmanager.serviceMonitor.proxyUrl .proxyUrl }} + {{- end }} + {{- if or $.Values.alertmanager.serviceMonitor.scheme .scheme }} + scheme: {{ default $.Values.alertmanager.serviceMonitor.scheme .scheme }} + {{- end }} + {{- if or $.Values.alertmanager.serviceMonitor.bearerTokenFile .bearerTokenFile }} + bearerTokenFile: {{ default $.Values.alertmanager.serviceMonitor.bearerTokenFile .bearerTokenFile }} + {{- end }} + {{- if or $.Values.alertmanager.serviceMonitor.tlsConfig .tlsConfig }} + tlsConfig: {{- default $.Values.alertmanager.serviceMonitor.tlsConfig .tlsConfig | toYaml | nindent 6 }} + {{- end }} + path: {{ .path }} + {{- if or $.Values.alertmanager.serviceMonitor.metricRelabelings .metricRelabelings }} + metricRelabelings: {{- tpl (default $.Values.alertmanager.serviceMonitor.metricRelabelings .metricRelabelings | toYaml | nindent 6) . }} + {{- end }} + {{- if or $.Values.alertmanager.serviceMonitor.relabelings .relabelings }} + relabelings: {{- default $.Values.alertmanager.serviceMonitor.relabelings .relabelings | toYaml | nindent 6 }} + {{- end }} + {{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/alertmanager/serviceperreplica.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/alertmanager/serviceperreplica.yaml new file mode 100644 index 00000000000..75a13bdf972 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/alertmanager/serviceperreplica.yaml @@ -0,0 +1,49 @@ +{{- if and .Values.alertmanager.enabled .Values.alertmanager.servicePerReplica.enabled }} +{{- $count := .Values.alertmanager.alertmanagerSpec.replicas | int -}} +{{- $serviceValues := .Values.alertmanager.servicePerReplica -}} +apiVersion: v1 +kind: List +metadata: + name: {{ include "kube-prometheus-stack.fullname" $ }}-alertmanager-serviceperreplica + namespace: {{ template "kube-prometheus-stack.namespace" . }} +items: +{{- range $i, $e := until $count }} + - apiVersion: v1 + kind: Service + metadata: + name: {{ include "kube-prometheus-stack.fullname" $ }}-alertmanager-{{ $i }} + namespace: {{ template "kube-prometheus-stack.namespace" $ }} + labels: + app: {{ include "kube-prometheus-stack.name" $ }}-alertmanager +{{ include "kube-prometheus-stack.labels" $ | indent 8 }} + {{- if $serviceValues.annotations }} + annotations: +{{ toYaml $serviceValues.annotations | indent 8 }} + {{- end }} + spec: + {{- if $serviceValues.clusterIP }} + clusterIP: {{ $serviceValues.clusterIP }} + {{- end }} + {{- if $serviceValues.loadBalancerSourceRanges }} + loadBalancerSourceRanges: + {{- range $cidr := $serviceValues.loadBalancerSourceRanges }} + - {{ $cidr }} + {{- end }} + {{- end }} + {{- if ne $serviceValues.type "ClusterIP" }} + externalTrafficPolicy: {{ $serviceValues.externalTrafficPolicy }} + {{- end }} + ports: + - name: {{ $.Values.alertmanager.alertmanagerSpec.portName }} + {{- if eq $serviceValues.type "NodePort" }} + nodePort: {{ $serviceValues.nodePort }} + {{- end }} + port: {{ $serviceValues.port }} + targetPort: {{ $serviceValues.targetPort }} + selector: + app.kubernetes.io/name: alertmanager + alertmanager: {{ template "kube-prometheus-stack.alertmanager.crname" $ }} + statefulset.kubernetes.io/pod-name: alertmanager-{{ include "kube-prometheus-stack.alertmanager.crname" $ }}-{{ $i }} + type: "{{ $serviceValues.type }}" +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/exporters/core-dns/service.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/exporters/core-dns/service.yaml new file mode 100644 index 00000000000..b8618f7558b --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/exporters/core-dns/service.yaml @@ -0,0 +1,24 @@ +{{- if and .Values.coreDns.enabled .Values.coreDns.service.enabled .Values.kubernetesServiceMonitors.enabled }} +apiVersion: v1 +kind: Service +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-coredns + labels: + app: {{ template "kube-prometheus-stack.name" . }}-coredns + jobLabel: coredns +{{ include "kube-prometheus-stack.labels" . | indent 4 }} + namespace: kube-system +spec: + clusterIP: None + ports: + - name: {{ .Values.coreDns.serviceMonitor.port }} + port: {{ .Values.coreDns.service.port }} + protocol: TCP + targetPort: {{ .Values.coreDns.service.targetPort }} + selector: + {{- if .Values.coreDns.service.selector }} +{{ toYaml .Values.coreDns.service.selector | indent 4 }} + {{- else}} + k8s-app: kube-dns + {{- end}} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/exporters/core-dns/servicemonitor.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/exporters/core-dns/servicemonitor.yaml new file mode 100644 index 00000000000..dc15a069374 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/exporters/core-dns/servicemonitor.yaml @@ -0,0 +1,58 @@ +{{- if and .Values.coreDns.enabled .Values.coreDns.serviceMonitor.enabled .Values.kubernetesServiceMonitors.enabled }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-coredns + {{- if .Values.prometheus.prometheusSpec.ignoreNamespaceSelectors }} + namespace: kube-system + {{- else }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + {{- end }} + labels: + app: {{ template "kube-prometheus-stack.name" . }}-coredns + {{- with .Values.coreDns.serviceMonitor.additionalLabels }} + {{- toYaml . | nindent 4 }} + {{- end }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +spec: + jobLabel: {{ .Values.coreDns.serviceMonitor.jobLabel }} + {{- include "servicemonitor.scrapeLimits" .Values.coreDns.serviceMonitor | nindent 2 }} + selector: + {{- if .Values.coreDns.serviceMonitor.selector }} + {{ tpl (toYaml .Values.coreDns.serviceMonitor.selector | nindent 4) . }} + {{- else }} + matchLabels: + app: {{ template "kube-prometheus-stack.name" . }}-coredns + release: {{ $.Release.Name | quote }} + {{- end }} + namespaceSelector: + matchNames: + - "kube-system" + endpoints: + - port: {{ .Values.coreDns.serviceMonitor.port }} + {{- if .Values.coreDns.serviceMonitor.interval}} + interval: {{ .Values.coreDns.serviceMonitor.interval }} + {{- end }} + {{- if .Values.coreDns.serviceMonitor.proxyUrl }} + proxyUrl: {{ .Values.coreDns.serviceMonitor.proxyUrl}} + {{- end }} + bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token + metricRelabelings: + {{- if .Values.coreDns.serviceMonitor.metricRelabelings }} + {{ tpl (toYaml .Values.coreDns.serviceMonitor.metricRelabelings | indent 4) . }} + {{- end }} + {{ if .Values.global.cattle.clusterId }} + - sourceLabels: [__address__] + targetLabel: cluster_id + replacement: {{ .Values.global.cattle.clusterId }} + {{- end }} + {{ if .Values.global.cattle.clusterName }} + - sourceLabels: [__address__] + targetLabel: cluster_name + replacement: {{ .Values.global.cattle.clusterName }} + {{- end }} +{{- if .Values.coreDns.serviceMonitor.relabelings }} + relabelings: +{{ tpl (toYaml .Values.coreDns.serviceMonitor.relabelings | indent 4) . }} +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/exporters/kube-api-server/servicemonitor.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/exporters/kube-api-server/servicemonitor.yaml new file mode 100644 index 00000000000..66e777632ea --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/exporters/kube-api-server/servicemonitor.yaml @@ -0,0 +1,57 @@ +{{- if and .Values.kubeApiServer.enabled .Values.kubernetesServiceMonitors.enabled }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-apiserver + {{- if .Values.prometheus.prometheusSpec.ignoreNamespaceSelectors }} + namespace: default + {{- else }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + {{- end }} + labels: + app: {{ template "kube-prometheus-stack.name" . }}-apiserver + {{- with .Values.kubeApiServer.serviceMonitor.additionalLabels }} + {{- toYaml . | nindent 4 }} + {{- end }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +spec: + {{- include "servicemonitor.scrapeLimits" .Values.kubeApiServer.serviceMonitor | nindent 2 }} + endpoints: + - bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token + {{- if .Values.kubeApiServer.serviceMonitor.interval }} + interval: {{ .Values.kubeApiServer.serviceMonitor.interval }} + {{- end }} + {{- if .Values.kubeApiServer.serviceMonitor.proxyUrl }} + proxyUrl: {{ .Values.kubeApiServer.serviceMonitor.proxyUrl }} + {{- end }} + port: https + scheme: https + metricRelabelings: + {{- if .Values.kubeApiServer.serviceMonitor.metricRelabelings }} +{{ tpl (toYaml .Values.kubeApiServer.serviceMonitor.metricRelabelings | indent 6) . }} + {{- end }} + {{ if .Values.global.cattle.clusterId }} + - sourceLabels: [__address__] + targetLabel: cluster_id + replacement: {{ .Values.global.cattle.clusterId }} + {{- end }} + {{ if .Values.global.cattle.clusterName}} + - sourceLabels: [__address__] + targetLabel: cluster_name + replacement: {{ .Values.global.cattle.clusterName }} + {{- end }} +{{- if .Values.kubeApiServer.serviceMonitor.relabelings }} + relabelings: +{{ tpl (toYaml .Values.kubeApiServer.serviceMonitor.relabelings | indent 6) . }} +{{- end }} + tlsConfig: + caFile: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt + serverName: {{ .Values.kubeApiServer.tlsConfig.serverName }} + insecureSkipVerify: {{ .Values.kubeApiServer.tlsConfig.insecureSkipVerify }} + jobLabel: {{ .Values.kubeApiServer.serviceMonitor.jobLabel }} + namespaceSelector: + matchNames: + - default + selector: +{{ toYaml .Values.kubeApiServer.serviceMonitor.selector | indent 4 }} +{{- end}} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/exporters/kube-controller-manager/endpoints.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/exporters/kube-controller-manager/endpoints.yaml new file mode 100644 index 00000000000..6a6afa6412f --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/exporters/kube-controller-manager/endpoints.yaml @@ -0,0 +1,22 @@ +{{- if and .Values.kubeControllerManager.enabled .Values.kubeControllerManager.endpoints .Values.kubernetesServiceMonitors.enabled }} +apiVersion: v1 +kind: Endpoints +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-kube-controller-manager + labels: + app: {{ template "kube-prometheus-stack.name" . }}-kube-controller-manager + k8s-app: kube-controller-manager +{{ include "kube-prometheus-stack.labels" . | indent 4 }} + namespace: kube-system +subsets: + - addresses: + {{- range .Values.kubeControllerManager.endpoints }} + - ip: {{ . }} + {{- end }} + ports: + - name: {{ .Values.kubeControllerManager.serviceMonitor.port }} + {{- $kubeControllerManagerDefaultInsecurePort := 10252 }} + {{- $kubeControllerManagerDefaultSecurePort := 10257 }} + port: {{ include "kube-prometheus-stack.kubeControllerManager.insecureScrape" (list . $kubeControllerManagerDefaultInsecurePort $kubeControllerManagerDefaultSecurePort .Values.kubeControllerManager.service.port) }} + protocol: TCP +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/exporters/kube-controller-manager/service.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/exporters/kube-controller-manager/service.yaml new file mode 100644 index 00000000000..43b1a976d5e --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/exporters/kube-controller-manager/service.yaml @@ -0,0 +1,29 @@ +{{- if and .Values.kubeControllerManager.enabled .Values.kubeControllerManager.service.enabled .Values.kubernetesServiceMonitors.enabled }} +apiVersion: v1 +kind: Service +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-kube-controller-manager + labels: + app: {{ template "kube-prometheus-stack.name" . }}-kube-controller-manager + jobLabel: kube-controller-manager +{{ include "kube-prometheus-stack.labels" . | indent 4 }} + namespace: kube-system +spec: + clusterIP: None + ports: + - name: {{ .Values.kubeControllerManager.serviceMonitor.port }} + {{- $kubeControllerManagerDefaultInsecurePort := 10252 }} + {{- $kubeControllerManagerDefaultSecurePort := 10257 }} + port: {{ include "kube-prometheus-stack.kubeControllerManager.insecureScrape" (list . $kubeControllerManagerDefaultInsecurePort $kubeControllerManagerDefaultSecurePort .Values.kubeControllerManager.service.port) }} + protocol: TCP + targetPort: {{ include "kube-prometheus-stack.kubeControllerManager.insecureScrape" (list . $kubeControllerManagerDefaultInsecurePort $kubeControllerManagerDefaultSecurePort .Values.kubeControllerManager.service.targetPort) }} +{{- if .Values.kubeControllerManager.endpoints }}{{- else }} + selector: + {{- if .Values.kubeControllerManager.service.selector }} +{{ toYaml .Values.kubeControllerManager.service.selector | indent 4 }} + {{- else}} + component: kube-controller-manager + {{- end}} +{{- end }} + type: ClusterIP +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/exporters/kube-controller-manager/servicemonitor.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/exporters/kube-controller-manager/servicemonitor.yaml new file mode 100644 index 00000000000..7ed3baa65f5 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/exporters/kube-controller-manager/servicemonitor.yaml @@ -0,0 +1,69 @@ +{{- if and .Values.kubeControllerManager.enabled .Values.kubeControllerManager.serviceMonitor.enabled .Values.kubernetesServiceMonitors.enabled }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-kube-controller-manager + {{- if .Values.prometheus.prometheusSpec.ignoreNamespaceSelectors }} + namespace: kube-system + {{- else }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + {{- end }} + labels: + app: {{ template "kube-prometheus-stack.name" . }}-kube-controller-manager + {{- with .Values.kubeControllerManager.serviceMonitor.additionalLabels }} + {{- toYaml . | nindent 4 }} + {{- end }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +spec: + jobLabel: {{ .Values.kubeControllerManager.serviceMonitor.jobLabel }} + {{- include "servicemonitor.scrapeLimits" .Values.kubeControllerManager.serviceMonitor | nindent 2 }} + selector: + {{- if .Values.kubeControllerManager.serviceMonitor.selector }} + {{ tpl (toYaml .Values.kubeControllerManager.serviceMonitor.selector | nindent 4) . }} + {{- else }} + matchLabels: + app: {{ template "kube-prometheus-stack.name" . }}-kube-controller-manager + release: {{ $.Release.Name | quote }} + {{- end }} + namespaceSelector: + matchNames: + - "kube-system" + endpoints: + - port: {{ .Values.kubeControllerManager.serviceMonitor.port }} + {{- if .Values.kubeControllerManager.serviceMonitor.interval }} + interval: {{ .Values.kubeControllerManager.serviceMonitor.interval }} + {{- end }} + bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token + {{- if .Values.kubeControllerManager.serviceMonitor.proxyUrl }} + proxyUrl: {{ .Values.kubeControllerManager.serviceMonitor.proxyUrl}} + {{- end }} + {{- if eq (include "kube-prometheus-stack.kubeControllerManager.insecureScrape" (list . false true .Values.kubeControllerManager.serviceMonitor.https )) "true" }} + scheme: https + tlsConfig: + caFile: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt + {{- if eq (include "kube-prometheus-stack.kubeControllerManager.insecureScrape" (list . nil true .Values.kubeControllerManager.serviceMonitor.insecureSkipVerify)) "true" }} + insecureSkipVerify: true + {{- end }} + {{- if .Values.kubeControllerManager.serviceMonitor.serverName }} + serverName: {{ .Values.kubeControllerManager.serviceMonitor.serverName }} + {{- end }} + {{- end }} + metricRelabelings: + {{- if.Values.kubeControllerManager.serviceMonitor.metricRelabelings }} + {{ tpl (toYaml .Values.kubeControllerManager.serviceMonitor.metricRelabelings | indent 4) . }} + {{- end }} + {{ if .Values.global.cattle.clusterId }} + - sourceLabels: [__address__] + targetLabel: cluster_id + replacement: {{ .Values.global.cattle.clusterId }} + {{- end }} + {{ if .Values.global.cattle.clusterName}} + - sourceLabels: [__address__] + targetLabel: cluster_name + replacement: {{ .Values.global.cattle.clusterName }} + {{- end }} +{{- if .Values.kubeControllerManager.serviceMonitor.relabelings }} + relabelings: +{{ tpl (toYaml .Values.kubeControllerManager.serviceMonitor.relabelings | indent 4) . }} +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/exporters/kube-dns/service.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/exporters/kube-dns/service.yaml new file mode 100644 index 00000000000..81b2c9930ce --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/exporters/kube-dns/service.yaml @@ -0,0 +1,28 @@ +{{- if and .Values.kubeDns.enabled .Values.kubernetesServiceMonitors.enabled }} +apiVersion: v1 +kind: Service +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-kube-dns + labels: + app: {{ template "kube-prometheus-stack.name" . }}-kube-dns + jobLabel: kube-dns +{{ include "kube-prometheus-stack.labels" . | indent 4 }} + namespace: kube-system +spec: + clusterIP: None + ports: + - name: http-metrics-dnsmasq + port: {{ .Values.kubeDns.service.dnsmasq.port }} + protocol: TCP + targetPort: {{ .Values.kubeDns.service.dnsmasq.targetPort }} + - name: http-metrics-skydns + port: {{ .Values.kubeDns.service.skydns.port }} + protocol: TCP + targetPort: {{ .Values.kubeDns.service.skydns.targetPort }} + selector: + {{- if .Values.kubeDns.service.selector }} +{{ toYaml .Values.kubeDns.service.selector | indent 4 }} + {{- else}} + k8s-app: kube-dns + {{- end}} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/exporters/kube-dns/servicemonitor.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/exporters/kube-dns/servicemonitor.yaml new file mode 100644 index 00000000000..9fa41b575f2 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/exporters/kube-dns/servicemonitor.yaml @@ -0,0 +1,71 @@ +{{- if and .Values.kubeDns.enabled .Values.kubernetesServiceMonitors.enabled }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-kube-dns + {{- if .Values.prometheus.prometheusSpec.ignoreNamespaceSelectors }} + namespace: kube-system + {{- else }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + {{- end }} + labels: + app: {{ template "kube-prometheus-stack.name" . }}-kube-dns + {{- with .Values.kubeDns.serviceMonitor.additionalLabels }} + {{- toYaml . | nindent 4 }} + {{- end }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +spec: + jobLabel: {{ .Values.kubeDns.serviceMonitor.jobLabel }} + {{- include "servicemonitor.scrapeLimits" .Values.kubeDns.serviceMonitor | nindent 2 }} + selector: + {{- if .Values.kubeDns.serviceMonitor.selector }} + {{ tpl (toYaml .Values.kubeDns.serviceMonitor.selector | nindent 4) . }} + {{- else }} + matchLabels: + app: {{ template "kube-prometheus-stack.name" . }}-kube-dns + release: {{ $.Release.Name | quote }} + {{- end }} + namespaceSelector: + matchNames: + - "kube-system" + endpoints: + - port: http-metrics-dnsmasq + {{- if .Values.kubeDns.serviceMonitor.interval }} + interval: {{ .Values.kubeDns.serviceMonitor.interval }} + {{- end }} + bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token + {{- if .Values.kubeDns.serviceMonitor.proxyUrl }} + proxyUrl: {{ .Values.kubeDns.serviceMonitor.proxyUrl}} + {{- end }} + metricRelabelings: + {{- if .Values.kubeDns.serviceMonitor.dnsmasqMetricRelabelings }} + {{ tpl (toYaml .Values.kubeDns.serviceMonitor.dnsmasqMetricRelabelings | indent 4) . }} + {{- end }} + {{ if .Values.global.cattle.clusterId }} + - sourceLabels: [__address__] + targetLabel: cluster_id + replacement: {{ .Values.global.cattle.clusterId }} + {{- end }} + {{ if .Values.global.cattle.clusterName}} + - sourceLabels: [__address__] + targetLabel: cluster_name + replacement: {{ .Values.global.cattle.clusterName }} + {{- end }} +{{- if .Values.kubeDns.serviceMonitor.dnsmasqRelabelings }} + relabelings: +{{ toYaml .Values.kubeDns.serviceMonitor.dnsmasqRelabelings | indent 4 }} +{{- end }} + - port: http-metrics-skydns + {{- if .Values.kubeDns.serviceMonitor.interval }} + interval: {{ .Values.kubeDns.serviceMonitor.interval }} + {{- end }} + bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token +{{- if .Values.kubeDns.serviceMonitor.metricRelabelings }} + metricRelabelings: +{{ tpl (toYaml .Values.kubeDns.serviceMonitor.metricRelabelings | indent 4) . }} +{{- end }} +{{- if .Values.kubeDns.serviceMonitor.relabelings }} + relabelings: +{{ tpl (toYaml .Values.kubeDns.serviceMonitor.relabelings | indent 4) . }} +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/exporters/kube-etcd/endpoints.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/exporters/kube-etcd/endpoints.yaml new file mode 100644 index 00000000000..e3664475774 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/exporters/kube-etcd/endpoints.yaml @@ -0,0 +1,20 @@ +{{- if and .Values.kubeEtcd.enabled .Values.kubeEtcd.endpoints .Values.kubernetesServiceMonitors.enabled }} +apiVersion: v1 +kind: Endpoints +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-kube-etcd + labels: + app: {{ template "kube-prometheus-stack.name" . }}-kube-etcd + k8s-app: etcd-server +{{ include "kube-prometheus-stack.labels" . | indent 4 }} + namespace: kube-system +subsets: + - addresses: + {{- range .Values.kubeEtcd.endpoints }} + - ip: {{ . }} + {{- end }} + ports: + - name: {{ .Values.kubeEtcd.serviceMonitor.port }} + port: {{ .Values.kubeEtcd.service.port }} + protocol: TCP +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/exporters/kube-etcd/service.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/exporters/kube-etcd/service.yaml new file mode 100644 index 00000000000..d07d4f35e31 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/exporters/kube-etcd/service.yaml @@ -0,0 +1,27 @@ +{{- if and .Values.kubeEtcd.enabled .Values.kubeEtcd.service.enabled .Values.kubernetesServiceMonitors.enabled }} +apiVersion: v1 +kind: Service +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-kube-etcd + labels: + app: {{ template "kube-prometheus-stack.name" . }}-kube-etcd + jobLabel: kube-etcd +{{ include "kube-prometheus-stack.labels" . | indent 4 }} + namespace: kube-system +spec: + clusterIP: None + ports: + - name: {{ .Values.kubeEtcd.serviceMonitor.port }} + port: {{ .Values.kubeEtcd.service.port }} + protocol: TCP + targetPort: {{ .Values.kubeEtcd.service.targetPort }} +{{- if .Values.kubeEtcd.endpoints }}{{- else }} + selector: + {{- if .Values.kubeEtcd.service.selector }} +{{ toYaml .Values.kubeEtcd.service.selector | indent 4 }} + {{- else}} + component: etcd + {{- end}} +{{- end }} + type: ClusterIP +{{- end -}} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/exporters/kube-etcd/servicemonitor.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/exporters/kube-etcd/servicemonitor.yaml new file mode 100644 index 00000000000..26fdbdbed39 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/exporters/kube-etcd/servicemonitor.yaml @@ -0,0 +1,75 @@ +{{- if and .Values.kubeEtcd.enabled .Values.kubeEtcd.serviceMonitor.enabled .Values.kubernetesServiceMonitors.enabled }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-kube-etcd + {{- if .Values.prometheus.prometheusSpec.ignoreNamespaceSelectors }} + namespace: kube-system + {{- else }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + {{- end }} + labels: + app: {{ template "kube-prometheus-stack.name" . }}-kube-etcd + {{- with .Values.kubeEtcd.serviceMonitor.additionalLabels }} + {{- toYaml . | nindent 4 }} + {{- end }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +spec: + jobLabel: {{ .Values.kubeEtcd.serviceMonitor.jobLabel }} + {{- include "servicemonitor.scrapeLimits" .Values.kubeEtcd.serviceMonitor | nindent 4 }} + selector: + {{- if .Values.kubeEtcd.serviceMonitor.selector }} + {{ tpl (toYaml .Values.kubeEtcd.serviceMonitor.selector | nindent 4) . }} + {{- else }} + matchLabels: + app: {{ template "kube-prometheus-stack.name" . }}-kube-etcd + release: {{ $.Release.Name | quote }} + {{- end }} + namespaceSelector: + matchNames: + - "kube-system" + endpoints: + - port: {{ .Values.kubeEtcd.serviceMonitor.port }} + {{- if .Values.kubeEtcd.serviceMonitor.interval }} + interval: {{ .Values.kubeEtcd.serviceMonitor.interval }} + {{- end }} + bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token + {{- if .Values.kubeEtcd.serviceMonitor.proxyUrl }} + proxyUrl: {{ .Values.kubeEtcd.serviceMonitor.proxyUrl}} + {{- end }} + {{- if eq .Values.kubeEtcd.serviceMonitor.scheme "https" }} + scheme: https + tlsConfig: + {{- if .Values.kubeEtcd.serviceMonitor.serverName }} + serverName: {{ .Values.kubeEtcd.serviceMonitor.serverName }} + {{- end }} + {{- if .Values.kubeEtcd.serviceMonitor.caFile }} + caFile: {{ .Values.kubeEtcd.serviceMonitor.caFile }} + {{- end }} + {{- if .Values.kubeEtcd.serviceMonitor.certFile }} + certFile: {{ .Values.kubeEtcd.serviceMonitor.certFile }} + {{- end }} + {{- if .Values.kubeEtcd.serviceMonitor.keyFile }} + keyFile: {{ .Values.kubeEtcd.serviceMonitor.keyFile }} + {{- end}} + insecureSkipVerify: {{ .Values.kubeEtcd.serviceMonitor.insecureSkipVerify }} + {{- end }} + metricRelabelings: + {{- if .Values.kubeEtcd.serviceMonitor.metricRelabelings }} + {{ tpl (toYaml .Values.kubeEtcd.serviceMonitor.metricRelabelings | indent 4) . }} + {{- end }} + {{ if .Values.global.cattle.clusterId }} + - sourceLabels: [__address__] + targetLabel: cluster_id + replacement: {{ .Values.global.cattle.clusterId }} + {{- end }} + {{ if .Values.global.cattle.clusterName}} + - sourceLabels: [__address__] + targetLabel: cluster_name + replacement: {{ .Values.global.cattle.clusterName }} + {{- end }} +{{- if .Values.kubeEtcd.serviceMonitor.relabelings }} + relabelings: +{{ tpl (toYaml .Values.kubeEtcd.serviceMonitor.relabelings | indent 4) . }} +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/exporters/kube-proxy/endpoints.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/exporters/kube-proxy/endpoints.yaml new file mode 100644 index 00000000000..8613e62425f --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/exporters/kube-proxy/endpoints.yaml @@ -0,0 +1,20 @@ +{{- if and .Values.kubeProxy.enabled .Values.kubeProxy.endpoints .Values.kubernetesServiceMonitors.enabled }} +apiVersion: v1 +kind: Endpoints +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-kube-proxy + labels: + app: {{ template "kube-prometheus-stack.name" . }}-kube-proxy + k8s-app: kube-proxy +{{ include "kube-prometheus-stack.labels" . | indent 4 }} + namespace: kube-system +subsets: + - addresses: + {{- range .Values.kubeProxy.endpoints }} + - ip: {{ . }} + {{- end }} + ports: + - name: {{ .Values.kubeProxy.serviceMonitor.port }} + port: {{ .Values.kubeProxy.service.port }} + protocol: TCP +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/exporters/kube-proxy/service.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/exporters/kube-proxy/service.yaml new file mode 100644 index 00000000000..8ccb2210d74 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/exporters/kube-proxy/service.yaml @@ -0,0 +1,27 @@ +{{- if and .Values.kubeProxy.enabled .Values.kubeProxy.service.enabled .Values.kubernetesServiceMonitors.enabled }} +apiVersion: v1 +kind: Service +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-kube-proxy + labels: + app: {{ template "kube-prometheus-stack.name" . }}-kube-proxy + jobLabel: kube-proxy +{{ include "kube-prometheus-stack.labels" . | indent 4 }} + namespace: kube-system +spec: + clusterIP: None + ports: + - name: {{ .Values.kubeProxy.serviceMonitor.port }} + port: {{ .Values.kubeProxy.service.port }} + protocol: TCP + targetPort: {{ .Values.kubeProxy.service.targetPort }} +{{- if .Values.kubeProxy.endpoints }}{{- else }} + selector: + {{- if .Values.kubeProxy.service.selector }} +{{ toYaml .Values.kubeProxy.service.selector | indent 4 }} + {{- else}} + k8s-app: kube-proxy + {{- end}} +{{- end }} + type: ClusterIP +{{- end -}} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/exporters/kube-proxy/servicemonitor.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/exporters/kube-proxy/servicemonitor.yaml new file mode 100644 index 00000000000..24b0ab20017 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/exporters/kube-proxy/servicemonitor.yaml @@ -0,0 +1,63 @@ +{{- if and .Values.kubeProxy.enabled .Values.kubeProxy.serviceMonitor.enabled .Values.kubernetesServiceMonitors.enabled }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-kube-proxy + {{- if .Values.prometheus.prometheusSpec.ignoreNamespaceSelectors }} + namespace: kube-system + {{- else }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + {{- end }} + labels: + app: {{ template "kube-prometheus-stack.name" . }}-kube-proxy + {{- with .Values.kubeProxy.serviceMonitor.additionalLabels }} + {{- toYaml . | nindent 4 }} + {{- end }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +spec: + jobLabel: {{ .Values.kubeProxy.serviceMonitor.jobLabel }} + {{- include "servicemonitor.scrapeLimits" .Values.kubeProxy.serviceMonitor | nindent 2 }} + selector: + {{- if .Values.kubeProxy.serviceMonitor.selector }} + {{ tpl (toYaml .Values.kubeProxy.serviceMonitor.selector | nindent 4) . }} + {{- else }} + matchLabels: + app: {{ template "kube-prometheus-stack.name" . }}-kube-proxy + release: {{ $.Release.Name | quote }} + {{- end }} + namespaceSelector: + matchNames: + - "kube-system" + endpoints: + - port: {{ .Values.kubeProxy.serviceMonitor.port }} + {{- if .Values.kubeProxy.serviceMonitor.interval }} + interval: {{ .Values.kubeProxy.serviceMonitor.interval }} + {{- end }} + bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token + {{- if .Values.kubeProxy.serviceMonitor.proxyUrl }} + proxyUrl: {{ .Values.kubeProxy.serviceMonitor.proxyUrl}} + {{- end }} + {{- if .Values.kubeProxy.serviceMonitor.https }} + scheme: https + tlsConfig: + caFile: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt + {{- end}} + metricRelabelings: + {{- if .Values.kubeProxy.serviceMonitor.metricRelabelings }} + {{ tpl (toYaml .Values.kubeProxy.serviceMonitor.metricRelabelings | indent 4) . }} + {{- end }} + {{ if .Values.global.cattle.clusterId }} + - sourceLabels: [__address__] + targetLabel: cluster_id + replacement: {{ .Values.global.cattle.clusterId }} + {{- end }} + {{ if .Values.global.cattle.clusterName}} + - sourceLabels: [__address__] + targetLabel: cluster_name + replacement: {{ .Values.global.cattle.clusterName }} + {{- end }} +{{- if .Values.kubeProxy.serviceMonitor.relabelings }} + relabelings: +{{ tpl (toYaml .Values.kubeProxy.serviceMonitor.relabelings | indent 4) . }} +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/exporters/kube-scheduler/endpoints.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/exporters/kube-scheduler/endpoints.yaml new file mode 100644 index 00000000000..6236b42f101 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/exporters/kube-scheduler/endpoints.yaml @@ -0,0 +1,22 @@ +{{- if and .Values.kubeScheduler.enabled .Values.kubeScheduler.endpoints .Values.kubernetesServiceMonitors.enabled }} +apiVersion: v1 +kind: Endpoints +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-kube-scheduler + labels: + app: {{ template "kube-prometheus-stack.name" . }}-kube-scheduler + k8s-app: kube-scheduler +{{ include "kube-prometheus-stack.labels" . | indent 4 }} + namespace: kube-system +subsets: + - addresses: + {{- range .Values.kubeScheduler.endpoints }} + - ip: {{ . }} + {{- end }} + ports: + - name: {{ .Values.kubeScheduler.serviceMonitor.port }} + {{- $kubeSchedulerDefaultInsecurePort := 10251 }} + {{- $kubeSchedulerDefaultSecurePort := 10259 }} + port: {{ include "kube-prometheus-stack.kubeScheduler.insecureScrape" (list . $kubeSchedulerDefaultInsecurePort $kubeSchedulerDefaultSecurePort .Values.kubeScheduler.service.port) }} + protocol: TCP +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/exporters/kube-scheduler/service.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/exporters/kube-scheduler/service.yaml new file mode 100644 index 00000000000..90b3a800a4c --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/exporters/kube-scheduler/service.yaml @@ -0,0 +1,29 @@ +{{- if and .Values.kubeScheduler.enabled .Values.kubeScheduler.service.enabled .Values.kubernetesServiceMonitors.enabled }} +apiVersion: v1 +kind: Service +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-kube-scheduler + labels: + app: {{ template "kube-prometheus-stack.name" . }}-kube-scheduler + jobLabel: kube-scheduler +{{ include "kube-prometheus-stack.labels" . | indent 4 }} + namespace: kube-system +spec: + clusterIP: None + ports: + - name: {{ .Values.kubeScheduler.serviceMonitor.port }} + {{- $kubeSchedulerDefaultInsecurePort := 10251 }} + {{- $kubeSchedulerDefaultSecurePort := 10259 }} + port: {{ include "kube-prometheus-stack.kubeScheduler.insecureScrape" (list . $kubeSchedulerDefaultInsecurePort $kubeSchedulerDefaultSecurePort .Values.kubeScheduler.service.port) }} + protocol: TCP + targetPort: {{ include "kube-prometheus-stack.kubeScheduler.insecureScrape" (list . $kubeSchedulerDefaultInsecurePort $kubeSchedulerDefaultSecurePort .Values.kubeScheduler.service.targetPort) }} +{{- if .Values.kubeScheduler.endpoints }}{{- else }} + selector: + {{- if .Values.kubeScheduler.service.selector }} +{{ toYaml .Values.kubeScheduler.service.selector | indent 4 }} + {{- else}} + component: kube-scheduler + {{- end}} +{{- end }} + type: ClusterIP +{{- end -}} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/exporters/kube-scheduler/servicemonitor.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/exporters/kube-scheduler/servicemonitor.yaml new file mode 100644 index 00000000000..b17c4f1d47e --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/exporters/kube-scheduler/servicemonitor.yaml @@ -0,0 +1,69 @@ +{{- if and .Values.kubeScheduler.enabled .Values.kubeScheduler.serviceMonitor.enabled .Values.kubernetesServiceMonitors.enabled }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-kube-scheduler + {{- if .Values.prometheus.prometheusSpec.ignoreNamespaceSelectors }} + namespace: kube-system + {{- else }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + {{- end }} + labels: + app: {{ template "kube-prometheus-stack.name" . }}-kube-scheduler + {{- with .Values.kubeScheduler.serviceMonitor.additionalLabels }} + {{- toYaml . | nindent 4 }} + {{- end }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +spec: + jobLabel: {{ .Values.kubeScheduler.serviceMonitor.jobLabel }} + {{- include "servicemonitor.scrapeLimits" .Values.kubeScheduler.serviceMonitor | nindent 2 }} + selector: + {{- if .Values.kubeScheduler.serviceMonitor.selector }} + {{ tpl (toYaml .Values.kubeScheduler.serviceMonitor.selector | nindent 4) . }} + {{- else }} + matchLabels: + app: {{ template "kube-prometheus-stack.name" . }}-kube-scheduler + release: {{ $.Release.Name | quote }} + {{- end }} + namespaceSelector: + matchNames: + - "kube-system" + endpoints: + - port: {{ .Values.kubeScheduler.serviceMonitor.port }} + {{- if .Values.kubeScheduler.serviceMonitor.interval }} + interval: {{ .Values.kubeScheduler.serviceMonitor.interval }} + {{- end }} + bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token + {{- if .Values.kubeScheduler.serviceMonitor.proxyUrl }} + proxyUrl: {{ .Values.kubeScheduler.serviceMonitor.proxyUrl}} + {{- end }} + {{- if eq (include "kube-prometheus-stack.kubeScheduler.insecureScrape" (list . false true .Values.kubeScheduler.serviceMonitor.https )) "true" }} + scheme: https + tlsConfig: + caFile: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt + {{- if eq (include "kube-prometheus-stack.kubeScheduler.insecureScrape" (list . nil true .Values.kubeScheduler.serviceMonitor.insecureSkipVerify)) "true" }} + insecureSkipVerify: true + {{- end }} + {{- if .Values.kubeScheduler.serviceMonitor.serverName }} + serverName: {{ .Values.kubeScheduler.serviceMonitor.serverName }} + {{- end}} + {{- end}} + metricRelabelings: + {{- if .Values.kubeScheduler.serviceMonitor.metricRelabelings }} + {{ tpl (toYaml .Values.kubeScheduler.serviceMonitor.metricRelabelings | indent 4) . }} + {{- end }} + {{ if .Values.global.cattle.clusterId }} + - sourceLabels: [__address__] + targetLabel: cluster_id + replacement: {{ .Values.global.cattle.clusterId }} + {{- end }} + {{ if .Values.global.cattle.clusterName}} + - sourceLabels: [__address__] + targetLabel: cluster_name + replacement: {{ .Values.global.cattle.clusterName }} + {{- end }} +{{- if .Values.kubeScheduler.serviceMonitor.relabelings }} + relabelings: +{{ tpl (toYaml .Values.kubeScheduler.serviceMonitor.relabelings | indent 4) . }} +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/exporters/kube-state-metrics/validate.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/exporters/kube-state-metrics/validate.yaml new file mode 100644 index 00000000000..9211b3d771f --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/exporters/kube-state-metrics/validate.yaml @@ -0,0 +1,7 @@ +{{- if .Values.kubeStateMetrics.enabled }} +{{- if not (kindIs "invalid" .Values.kubeStateMetrics.serviceMonitor) }} +{{- if .Values.kubeStateMetrics.serviceMonitor.namespaceOverride }} +{{- fail "kubeStateMetrics.serviceMonitor.namespaceOverride was removed. Please use kube-state-metrics.namespaceOverride instead." }} +{{- end }} +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/exporters/kubelet/servicemonitor.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/exporters/kubelet/servicemonitor.yaml new file mode 100644 index 00000000000..f570fbfdbca --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/exporters/kubelet/servicemonitor.yaml @@ -0,0 +1,246 @@ +{{- if (and (not .Values.kubelet.enabled) .Values.hardenedKubelet.enabled) }} +{{ required "Cannot set .Values.hardenedKubelet.enabled=true when .Values.kubelet.enabled=false" "" }} +{{- end }} +{{- if (and .Values.kubelet.enabled .Values.kubernetesServiceMonitors.enabled (not .Values.hardenedKubelet.enabled) (not .Values.k3sServer.enabled)) }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-kubelet + {{- if .Values.prometheus.prometheusSpec.ignoreNamespaceSelectors }} + namespace: {{ .Values.kubelet.namespace }} + {{- else }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + {{- end }} + labels: + app: {{ template "kube-prometheus-stack.name" . }}-kubelet + {{- with .Values.kubelet.serviceMonitor.additionalLabels }} + {{- toYaml . | nindent 4 }} + {{- end }} +{{- include "kube-prometheus-stack.labels" . | indent 4 }} +spec: + {{- include "servicemonitor.scrapeLimits" .Values.kubelet.serviceMonitor | nindent 2 }} + {{- with .Values.kubelet.serviceMonitor.attachMetadata }} + attachMetadata: + {{- toYaml . | nindent 4 }} + {{- end }} + endpoints: + {{- if .Values.kubelet.serviceMonitor.https }} + - port: https-metrics + scheme: https + {{- if .Values.kubelet.serviceMonitor.interval }} + interval: {{ .Values.kubelet.serviceMonitor.interval }} + {{- end }} + {{- if .Values.kubelet.serviceMonitor.proxyUrl }} + proxyUrl: {{ .Values.kubelet.serviceMonitor.proxyUrl }} + {{- end }} + {{- if .Values.kubelet.serviceMonitor.scrapeTimeout }} + scrapeTimeout: {{ .Values.kubelet.serviceMonitor.scrapeTimeout }} + {{- end }} + tlsConfig: + caFile: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt + insecureSkipVerify: true + bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token + honorLabels: {{ .Values.kubelet.serviceMonitor.honorLabels }} + honorTimestamps: {{ .Values.kubelet.serviceMonitor.honorTimestamps }} + metricRelabelings: + {{- if .Values.kubelet.serviceMonitor.metricRelabelings }} + {{- tpl (toYaml .Values.kubelet.serviceMonitor.metricRelabelings | nindent 6) . }} + {{- end }} + {{ if .Values.global.cattle.clusterId }} + - sourceLabels: [__address__] + targetLabel: cluster_id + replacement: {{ .Values.global.cattle.clusterId }} + {{- end }} + {{ if .Values.global.cattle.clusterName}} + - sourceLabels: [__address__] + targetLabel: cluster_name + replacement: {{ .Values.global.cattle.clusterName }} + {{- end }} +{{- if .Values.kubelet.serviceMonitor.relabelings }} + relabelings: +{{ tpl (toYaml .Values.kubelet.serviceMonitor.relabelings | indent 4) . }} +{{- end }} +{{- if .Values.kubelet.serviceMonitor.cAdvisor }} + - port: https-metrics + scheme: https + path: /metrics/cadvisor + {{- if .Values.kubelet.serviceMonitor.interval }} + interval: {{ .Values.kubelet.serviceMonitor.interval }} + {{- end }} + {{- if .Values.kubelet.serviceMonitor.proxyUrl }} + proxyUrl: {{ .Values.kubelet.serviceMonitor.proxyUrl }} + {{- end }} + {{- if .Values.kubelet.serviceMonitor.scrapeTimeout }} + scrapeTimeout: {{ .Values.kubelet.serviceMonitor.scrapeTimeout }} + {{- end }} + honorLabels: {{ .Values.kubelet.serviceMonitor.honorLabels }} + honorTimestamps: {{ .Values.kubelet.serviceMonitor.honorTimestamps }} + tlsConfig: + caFile: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt + insecureSkipVerify: true + bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token +{{- if .Values.kubelet.serviceMonitor.cAdvisorMetricRelabelings }} + metricRelabelings: +{{ tpl (toYaml .Values.kubelet.serviceMonitor.cAdvisorMetricRelabelings | indent 4) . }} +{{- end }} +{{- if .Values.kubelet.serviceMonitor.cAdvisorRelabelings }} + relabelings: +{{ tpl (toYaml .Values.kubelet.serviceMonitor.cAdvisorRelabelings | indent 4) . }} +{{- end }} +{{- end }} +{{- if .Values.kubelet.serviceMonitor.probes }} + - port: https-metrics + scheme: https + path: /metrics/probes + {{- if .Values.kubelet.serviceMonitor.interval }} + interval: {{ .Values.kubelet.serviceMonitor.interval }} + {{- end }} + {{- if .Values.kubelet.serviceMonitor.proxyUrl }} + proxyUrl: {{ .Values.kubelet.serviceMonitor.proxyUrl }} + {{- end }} + {{- if .Values.kubelet.serviceMonitor.scrapeTimeout }} + scrapeTimeout: {{ .Values.kubelet.serviceMonitor.scrapeTimeout }} + {{- end }} + honorLabels: {{ .Values.kubelet.serviceMonitor.honorLabels }} + honorTimestamps: {{ .Values.kubelet.serviceMonitor.honorTimestamps }} + tlsConfig: + caFile: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt + insecureSkipVerify: true + bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token +{{- if .Values.kubelet.serviceMonitor.probesMetricRelabelings }} + metricRelabelings: +{{ tpl (toYaml .Values.kubelet.serviceMonitor.probesMetricRelabelings | indent 4) . }} +{{- end }} +{{- if .Values.kubelet.serviceMonitor.probesRelabelings }} + relabelings: +{{ tpl (toYaml .Values.kubelet.serviceMonitor.probesRelabelings | indent 4) . }} +{{- end }} +{{- end }} +{{- if .Values.kubelet.serviceMonitor.resource }} + - port: https-metrics + scheme: https + path: {{ include "kubelet.serviceMonitor.resourcePath" . }} + {{- if .Values.kubelet.serviceMonitor.interval }} + interval: {{ .Values.kubelet.serviceMonitor.interval }} + {{- end }} + {{- if .Values.kubelet.serviceMonitor.proxyUrl }} + proxyUrl: {{ .Values.kubelet.serviceMonitor.proxyUrl }} + {{- end }} + {{- if .Values.kubelet.serviceMonitor.scrapeTimeout }} + scrapeTimeout: {{ .Values.kubelet.serviceMonitor.scrapeTimeout }} + {{- end }} + honorLabels: {{ .Values.kubelet.serviceMonitor.honorLabels }} + honorTimestamps: {{ .Values.kubelet.serviceMonitor.honorTimestamps }} + tlsConfig: + caFile: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt + insecureSkipVerify: true + bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token +{{- if .Values.kubelet.serviceMonitor.resourceMetricRelabelings }} + metricRelabelings: +{{ tpl (toYaml .Values.kubelet.serviceMonitor.resourceMetricRelabelings | indent 4) . }} +{{- end }} +{{- if .Values.kubelet.serviceMonitor.resourceRelabelings }} + relabelings: +{{ tpl (toYaml .Values.kubelet.serviceMonitor.resourceRelabelings | indent 4) . }} +{{- end }} +{{- end }} + {{- else }} + - port: http-metrics + {{- if .Values.kubelet.serviceMonitor.interval }} + interval: {{ .Values.kubelet.serviceMonitor.interval }} + {{- end }} + {{- if .Values.kubelet.serviceMonitor.proxyUrl }} + proxyUrl: {{ .Values.kubelet.serviceMonitor.proxyUrl }} + {{- end }} + {{- if .Values.kubelet.serviceMonitor.scrapeTimeout }} + scrapeTimeout: {{ .Values.kubelet.serviceMonitor.scrapeTimeout }} + {{- end }} + honorLabels: {{ .Values.kubelet.serviceMonitor.honorLabels }} + honorTimestamps: {{ .Values.kubelet.serviceMonitor.honorTimestamps }} +{{- if .Values.kubelet.serviceMonitor.metricRelabelings }} + metricRelabelings: +{{ tpl (toYaml .Values.kubelet.serviceMonitor.metricRelabelings | indent 4) . }} +{{- end }} +{{- if .Values.kubelet.serviceMonitor.relabelings }} + relabelings: +{{ tpl (toYaml .Values.kubelet.serviceMonitor.relabelings | indent 4) . }} +{{- end }} +{{- if .Values.kubelet.serviceMonitor.cAdvisor }} + - port: http-metrics + path: /metrics/cadvisor + {{- if .Values.kubelet.serviceMonitor.interval }} + interval: {{ .Values.kubelet.serviceMonitor.interval }} + {{- end }} + {{- if .Values.kubelet.serviceMonitor.proxyUrl }} + proxyUrl: {{ .Values.kubelet.serviceMonitor.proxyUrl }} + {{- end }} + {{- if .Values.kubelet.serviceMonitor.scrapeTimeout }} + scrapeTimeout: {{ .Values.kubelet.serviceMonitor.scrapeTimeout }} + {{- end }} + honorLabels: {{ .Values.kubelet.serviceMonitor.honorLabels }} + honorTimestamps: {{ .Values.kubelet.serviceMonitor.honorTimestamps }} +{{- if .Values.kubelet.serviceMonitor.cAdvisorMetricRelabelings }} + metricRelabelings: +{{ tpl (toYaml .Values.kubelet.serviceMonitor.cAdvisorMetricRelabelings | indent 4) . }} +{{- end }} +{{- if .Values.kubelet.serviceMonitor.cAdvisorRelabelings }} + relabelings: +{{ tpl (toYaml .Values.kubelet.serviceMonitor.cAdvisorRelabelings | indent 4) . }} +{{- end }} +{{- if .Values.kubelet.serviceMonitor.probes }} + - port: http-metrics + path: /metrics/probes + {{- if .Values.kubelet.serviceMonitor.interval }} + interval: {{ .Values.kubelet.serviceMonitor.interval }} + {{- end }} + {{- if .Values.kubelet.serviceMonitor.proxyUrl }} + proxyUrl: {{ .Values.kubelet.serviceMonitor.proxyUrl }} + {{- end }} + {{- if .Values.kubelet.serviceMonitor.scrapeTimeout }} + scrapeTimeout: {{ .Values.kubelet.serviceMonitor.scrapeTimeout }} + {{- end }} + honorLabels: {{ .Values.kubelet.serviceMonitor.honorLabels }} + honorTimestamps: {{ .Values.kubelet.serviceMonitor.honorTimestamps }} +{{- if .Values.kubelet.serviceMonitor.probesMetricRelabelings }} + metricRelabelings: +{{ tpl (toYaml .Values.kubelet.serviceMonitor.probesMetricRelabelings | indent 4) . }} +{{- end }} +{{- if .Values.kubelet.serviceMonitor.probesRelabelings }} + relabelings: +{{ tpl (toYaml .Values.kubelet.serviceMonitor.probesRelabelings | indent 4) . }} +{{- end }} +{{- end }} +{{- if .Values.kubelet.serviceMonitor.resource }} + - port: http-metrics + path: {{ include "kubelet.serviceMonitor.resourcePath" . }} + {{- if .Values.kubelet.serviceMonitor.interval }} + interval: {{ .Values.kubelet.serviceMonitor.interval }} + {{- end }} + {{- if .Values.kubelet.serviceMonitor.proxyUrl }} + proxyUrl: {{ .Values.kubelet.serviceMonitor.proxyUrl }} + {{- end }} + {{- if .Values.kubelet.serviceMonitor.scrapeTimeout }} + scrapeTimeout: {{ .Values.kubelet.serviceMonitor.scrapeTimeout }} + {{- end }} + honorLabels: {{ .Values.kubelet.serviceMonitor.honorLabels }} + honorTimestamps: {{ .Values.kubelet.serviceMonitor.honorTimestamps }} +{{- if .Values.kubelet.serviceMonitor.resourceMetricRelabelings }} + metricRelabelings: +{{ tpl (toYaml .Values.kubelet.serviceMonitor.resourceMetricRelabelings | indent 4) . }} +{{- end }} +{{- if .Values.kubelet.serviceMonitor.resourceRelabelings }} + relabelings: +{{ tpl (toYaml .Values.kubelet.serviceMonitor.resourceRelabelings | indent 4) . }} +{{- end }} +{{- end }} +{{- end }} + {{- end }} + jobLabel: k8s-app + namespaceSelector: + matchNames: + - {{ .Values.kubelet.namespace }} + selector: + matchLabels: + app.kubernetes.io/name: kubelet + k8s-app: kubelet +{{- end}} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/exporters/node-exporter/validate.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/exporters/node-exporter/validate.yaml new file mode 100644 index 00000000000..bdc73d61650 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/exporters/node-exporter/validate.yaml @@ -0,0 +1,3 @@ +{{- if (and (not .Values.nodeExporter.enabled) .Values.hardenedNodeExporter.enabled) }} +{{ required "Cannot set .Values.hardenedNodeExporter.enabled=true when .Values.nodeExporter.enabled=false" "" }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/extra-objects.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/extra-objects.yaml new file mode 100644 index 00000000000..567f7bf3297 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/extra-objects.yaml @@ -0,0 +1,4 @@ +{{ range .Values.extraManifests }} +--- +{{ tpl (toYaml .) $ }} +{{ end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/grafana/configmap-dashboards.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/grafana/configmap-dashboards.yaml new file mode 100644 index 00000000000..e719009ffea --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/grafana/configmap-dashboards.yaml @@ -0,0 +1,24 @@ +{{- if or (and .Values.grafana.enabled .Values.grafana.defaultDashboardsEnabled) .Values.grafana.forceDeployDashboards }} +{{- $files := .Files.Glob "dashboards-1.14/*.json" }} +{{- if $files }} +apiVersion: v1 +kind: ConfigMapList +items: +{{- range $path, $fileContents := $files }} +{{- $dashboardName := regexReplaceAll "(^.*/)(.*)\\.json$" $path "${2}" }} +- apiVersion: v1 + kind: ConfigMap + metadata: + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" $) $dashboardName | trunc 63 | trimSuffix "-" }} + namespace: {{ .Values.grafana.defaultDashboards.namespace }} + labels: + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: {{ ternary $.Values.grafana.sidecar.dashboards.labelValue "1" (not (empty $.Values.grafana.sidecar.dashboards.labelValue)) | quote }} + {{- end }} + app: {{ template "kube-prometheus-stack.name" $ }}-grafana +{{ include "kube-prometheus-stack.labels" $ | indent 6 }} + data: + {{ $dashboardName }}.json: {{ $.Files.Get $path | toJson }} +{{- end }} +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/grafana/configmaps-datasources.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/grafana/configmaps-datasources.yaml new file mode 100644 index 00000000000..718020d4f61 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/grafana/configmaps-datasources.yaml @@ -0,0 +1,81 @@ +{{- if or (and .Values.grafana.enabled .Values.grafana.sidecar.datasources.enabled) .Values.grafana.forceDeployDatasources }} +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-grafana-datasource + namespace: {{ default .Values.grafana.sidecar.datasources.searchNamespace (include "kube-prometheus-stack.namespace" .) }} +{{- if .Values.grafana.sidecar.datasources.annotations }} + annotations: + {{- toYaml .Values.grafana.sidecar.datasources.annotations | nindent 4 }} +{{- end }} + labels: + {{ $.Values.grafana.sidecar.datasources.label }}: {{ $.Values.grafana.sidecar.datasources.labelValue | quote }} + app: {{ template "kube-prometheus-stack.name" $ }}-grafana +{{ include "kube-prometheus-stack.labels" $ | indent 4 }} +data: + datasource.yaml: |- + apiVersion: 1 +{{- if .Values.grafana.deleteDatasources }} + deleteDatasources: +{{ tpl (toYaml .Values.grafana.deleteDatasources | indent 6) . }} +{{- end }} + datasources: +{{- $scrapeInterval := .Values.grafana.sidecar.datasources.defaultDatasourceScrapeInterval | default .Values.prometheus.prometheusSpec.scrapeInterval | default "30s" }} +{{- if .Values.grafana.sidecar.datasources.defaultDatasourceEnabled }} + - name: Prometheus + type: prometheus + uid: {{ .Values.grafana.sidecar.datasources.uid }} + {{- if .Values.grafana.sidecar.datasources.url }} + url: {{ .Values.grafana.sidecar.datasources.url }} + {{- else }} + url: http://{{ template "kube-prometheus-stack.fullname" . }}-prometheus.{{ template "kube-prometheus-stack.namespace" . }}:{{ .Values.prometheus.service.port }}/{{ trimPrefix "/" .Values.prometheus.prometheusSpec.routePrefix }} + {{- end }} + access: proxy + isDefault: {{ .Values.grafana.sidecar.datasources.isDefaultDatasource }} + jsonData: + httpMethod: {{ .Values.grafana.sidecar.datasources.httpMethod }} + timeInterval: {{ $scrapeInterval }} + {{- if .Values.grafana.sidecar.datasources.timeout }} + timeout: {{ .Values.grafana.sidecar.datasources.timeout }} + {{- end }} +{{- if .Values.grafana.sidecar.datasources.exemplarTraceIdDestinations }} + exemplarTraceIdDestinations: + - datasourceUid: {{ .Values.grafana.sidecar.datasources.exemplarTraceIdDestinations.datasourceUid }} + name: {{ .Values.grafana.sidecar.datasources.exemplarTraceIdDestinations.traceIdLabelName }} +{{- end }} +{{- if .Values.grafana.sidecar.datasources.createPrometheusReplicasDatasources }} +{{- range until (int .Values.prometheus.prometheusSpec.replicas) }} + - name: Prometheus-{{ . }} + type: prometheus + uid: {{ $.Values.grafana.sidecar.datasources.uid }}-replica-{{ . }} + url: http://prometheus-{{ template "kube-prometheus-stack.prometheus.crname" $ }}-{{ . }}.prometheus-operated:9090/{{ trimPrefix "/" $.Values.prometheus.prometheusSpec.routePrefix }} + access: proxy + isDefault: false + jsonData: + timeInterval: {{ $scrapeInterval }} +{{- if $.Values.grafana.sidecar.datasources.exemplarTraceIdDestinations }} + exemplarTraceIdDestinations: + - datasourceUid: {{ $.Values.grafana.sidecar.datasources.exemplarTraceIdDestinations.datasourceUid }} + name: {{ $.Values.grafana.sidecar.datasources.exemplarTraceIdDestinations.traceIdLabelName }} +{{- end }} +{{- end }} +{{- end }} +{{- if .Values.grafana.sidecar.datasources.alertmanager.enabled }} + - name: Alertmanager + type: alertmanager + uid: {{ .Values.grafana.sidecar.datasources.alertmanager.uid }} + {{- if .Values.grafana.sidecar.datasources.alertmanager.url }} + url: {{ .Values.grafana.sidecar.datasources.alertmanager.url }} + {{- else }} + url: http://{{ template "kube-prometheus-stack.fullname" . }}-alertmanager.{{ template "kube-prometheus-stack.namespace" . }}:{{ .Values.alertmanager.service.port }}/{{ trimPrefix "/" .Values.alertmanager.alertmanagerSpec.routePrefix }} + {{- end }} + access: proxy + jsonData: + handleGrafanaManagedAlerts: {{ .Values.grafana.sidecar.datasources.alertmanager.handleGrafanaManagedAlerts }} + implementation: {{ .Values.grafana.sidecar.datasources.alertmanager.implementation }} +{{- end }} +{{- end }} +{{- if .Values.grafana.additionalDataSources }} +{{ tpl (toYaml .Values.grafana.additionalDataSources | indent 4) . }} +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/grafana/dashboards-1.14/alertmanager-overview.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/grafana/dashboards-1.14/alertmanager-overview.yaml new file mode 100644 index 00000000000..dfc26d7ecd2 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/grafana/dashboards-1.14/alertmanager-overview.yaml @@ -0,0 +1,616 @@ +{{- /* +Generated from 'alertmanager-overview' from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/a8ba97a150c75be42010c75d10b720c55e182f1a/manifests/grafana-dashboardDefinitions.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (or .Values.grafana.enabled .Values.grafana.forceDeployDashboards) (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.grafana.defaultDashboardsEnabled }} +{{- if and .Values.alertmanager.enabled .Values.alertmanager.serviceMonitor.selfMonitor }} +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: {{ .Values.grafana.defaultDashboards.namespace }} + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" $) "alertmanager-overview" | trunc 63 | trimSuffix "-" }} + annotations: +{{ toYaml .Values.grafana.sidecar.dashboards.annotations | indent 4 }} + labels: + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: {{ ternary $.Values.grafana.sidecar.dashboards.labelValue "1" (not (empty $.Values.grafana.sidecar.dashboards.labelValue)) | quote }} + {{- end }} + app: {{ template "kube-prometheus-stack.name" $ }}-grafana +{{ include "kube-prometheus-stack.labels" $ | indent 4 }} +data: + alertmanager-overview.json: |- + { + "__inputs": [ + + ], + "__requires": [ + + ], + "annotations": { + "list": [ + + ] + }, + "editable": false, + "gnetId": null, + "graphTooltip": 1, + "hideControls": false, + "id": null, + "links": [ + + ], + "refresh": "30s", + "rows": [ + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "current set of alerts stored in the Alertmanager", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 2, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": false, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(alertmanager_alerts{namespace=~\"$namespace\",service=~\"$service\"}) by (namespace,service,instance)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Alerts", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "rate of successful and invalid alerts received by the Alertmanager", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 3, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": false, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(alertmanager_alerts_received_total{namespace=~\"$namespace\",service=~\"$service\"}[$__rate_interval])) by (namespace,service,instance)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}} Received", + "refId": "A" + }, + { + "expr": "sum(rate(alertmanager_alerts_invalid_total{namespace=~\"$namespace\",service=~\"$service\"}[$__rate_interval])) by (namespace,service,instance)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}} Invalid", + "refId": "B" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Alerts receive rate", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Alerts", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "rate of successful and invalid notifications sent by the Alertmanager", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 4, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": false, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": "integration", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(alertmanager_notifications_total{namespace=~\"$namespace\",service=~\"$service\", integration=\"$integration\"}[$__rate_interval])) by (integration,namespace,service,instance)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}} Total", + "refId": "A" + }, + { + "expr": "sum(rate(alertmanager_notifications_failed_total{namespace=~\"$namespace\",service=~\"$service\", integration=\"$integration\"}[$__rate_interval])) by (integration,namespace,service,instance)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}} Failed", + "refId": "B" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "$integration: Notifications Send Rate", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "latency of notifications sent by the Alertmanager", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 5, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": false, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": "integration", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.99,\n sum(rate(alertmanager_notification_latency_seconds_bucket{namespace=~\"$namespace\",service=~\"$service\", integration=\"$integration\"}[$__rate_interval])) by (le,namespace,service,instance)\n) \n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}} 99th Percentile", + "refId": "A" + }, + { + "expr": "histogram_quantile(0.50,\n sum(rate(alertmanager_notification_latency_seconds_bucket{namespace=~\"$namespace\",service=~\"$service\", integration=\"$integration\"}[$__rate_interval])) by (le,namespace,service,instance)\n) \n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}} Median", + "refId": "B" + }, + { + "expr": "sum(rate(alertmanager_notification_latency_seconds_sum{namespace=~\"$namespace\",service=~\"$service\", integration=\"$integration\"}[$__rate_interval])) by (namespace,service,instance)\n/\nsum(rate(alertmanager_notification_latency_seconds_count{namespace=~\"$namespace\",service=~\"$service\", integration=\"$integration\"}[$__rate_interval])) by (namespace,service,instance)\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}} Average", + "refId": "C" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "$integration: Notification Duration", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Notifications", + "titleSize": "h6", + "type": "row" + } + ], + "schemaVersion": 14, + "style": "dark", + "tags": [ + "alertmanager-mixin" + ], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": "Data Source", + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": null, + "current": { + "text": "", + "value": "" + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": false, + "label": "namespace", + "multi": false, + "name": "namespace", + "options": [ + + ], + "query": "label_values(alertmanager_alerts, namespace)", + "refresh": 2, + "regex": "", + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": { + "text": "", + "value": "" + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": false, + "label": "service", + "multi": false, + "name": "service", + "options": [ + + ], + "query": "label_values(alertmanager_alerts, service)", + "refresh": 2, + "regex": "", + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": { + "text": "all", + "value": "$__all" + }, + "datasource": "$datasource", + "hide": 2, + "includeAll": true, + "label": null, + "multi": false, + "name": "integration", + "options": [ + + ], + "query": "label_values(alertmanager_notifications_total{integration=~\".*\"}, integration)", + "refresh": 2, + "regex": "", + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "{{ .Values.grafana.defaultDashboardsTimezone }}", + "title": "Alertmanager / Overview", + "uid": "alertmanager-overview", + "version": 0 + } +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/grafana/dashboards-1.14/apiserver.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/grafana/dashboards-1.14/apiserver.yaml new file mode 100644 index 00000000000..bd1048b5678 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/grafana/dashboards-1.14/apiserver.yaml @@ -0,0 +1,1772 @@ +{{- /* +Generated from 'apiserver' from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/a8ba97a150c75be42010c75d10b720c55e182f1a/manifests/grafana-dashboardDefinitions.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (or .Values.grafana.enabled .Values.grafana.forceDeployDashboards) (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.grafana.defaultDashboardsEnabled .Values.kubeApiServer.enabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: {{ .Values.grafana.defaultDashboards.namespace }} + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" $) "apiserver" | trunc 63 | trimSuffix "-" }} + annotations: +{{ toYaml .Values.grafana.sidecar.dashboards.annotations | indent 4 }} + labels: + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: {{ ternary $.Values.grafana.sidecar.dashboards.labelValue "1" (not (empty $.Values.grafana.sidecar.dashboards.labelValue)) | quote }} + {{- end }} + app: {{ template "kube-prometheus-stack.name" $ }}-grafana +{{ include "kube-prometheus-stack.labels" $ | indent 4 }} +data: + apiserver.json: |- + { + "__inputs": [ + + ], + "__requires": [ + + ], + "annotations": { + "list": [ + + ] + }, + "editable": false, + "gnetId": null, + "graphTooltip": 0, + "hideControls": false, + "id": null, + "links": [ + + ], + "panels": [ + { + "content": "The SLO (service level objective) and other metrics displayed on this dashboard are for informational purposes only.", + "datasource": null, + "description": "The SLO (service level objective) and other metrics displayed on this dashboard are for informational purposes only.", + "gridPos": { + "h": 2, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 2, + "mode": "markdown", + "span": 12, + "title": "Notice", + "type": "text" + } + ], + "refresh": "10s", + "rows": [ + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": "$datasource", + "decimals": 3, + "description": "How many percent of requests (both read and write) in 30 days have been answered successfully and fast enough?", + "format": "percentunit", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + + }, + "id": 3, + "interval": "1m", + "legend": { + "alignAsTable": true, + "rightSide": true + }, + "links": [ + + ], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "span": 4, + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "apiserver_request:availability30d{verb=\"all\", cluster=\"$cluster\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "", + "refId": "A" + } + ], + "thresholds": "", + "title": "Availability (30d) > 99.000%", + "tooltip": { + "shared": false + }, + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "avg" + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "decimals": 3, + "description": "How much error budget is left looking at our 0.990% availability guarantees?", + "fill": 10, + "fillGradient": 0, + "gridPos": { + + }, + "id": 4, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 8, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "100 * (apiserver_request:availability30d{verb=\"all\", cluster=\"$cluster\"} - 0.990000)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "errorbudget", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "ErrorBudget (30d) > 99.000%", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "decimals": 3, + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "decimals": 3, + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "Dashboard Row", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": "$datasource", + "decimals": 3, + "description": "How many percent of read requests (LIST,GET) in 30 days have been answered successfully and fast enough?", + "format": "percentunit", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + + }, + "id": 5, + "interval": "1m", + "legend": { + "alignAsTable": true, + "rightSide": true + }, + "links": [ + + ], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "span": 3, + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "apiserver_request:availability30d{verb=\"read\", cluster=\"$cluster\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "", + "refId": "A" + } + ], + "thresholds": "", + "title": "Read Availability (30d)", + "tooltip": { + "shared": false + }, + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "avg" + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "How many read requests (LIST,GET) per second do the apiservers get by code?", + "fill": 10, + "fillGradient": 0, + "gridPos": { + + }, + "id": 6, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + { + "alias": "/2../i", + "color": "#56A64B" + }, + { + "alias": "/3../i", + "color": "#F2CC0C" + }, + { + "alias": "/4../i", + "color": "#3274D9" + }, + { + "alias": "/5../i", + "color": "#E02F44" + } + ], + "spaceLength": 10, + "span": 3, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum by (code) (code_resource:apiserver_request_total:rate5m{verb=\"read\", cluster=\"$cluster\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}} code {{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Read SLI - Requests", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "reqps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "reqps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "How many percent of read requests (LIST,GET) per second are returned with errors (5xx)?", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 7, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 3, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum by (resource) (code_resource:apiserver_request_total:rate5m{verb=\"read\",code=~\"5..\", cluster=\"$cluster\"}) / sum by (resource) (code_resource:apiserver_request_total:rate5m{verb=\"read\", cluster=\"$cluster\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}} resource {{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Read SLI - Errors", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "How many seconds is the 99th percentile for reading (LIST|GET) a given resource?", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 8, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 3, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "cluster_quantile:apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds:histogram_quantile{verb=\"read\", cluster=\"$cluster\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}} resource {{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Read SLI - Duration", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "Dashboard Row", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": "$datasource", + "decimals": 3, + "description": "How many percent of write requests (POST|PUT|PATCH|DELETE) in 30 days have been answered successfully and fast enough?", + "format": "percentunit", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + + }, + "id": 9, + "interval": "1m", + "legend": { + "alignAsTable": true, + "rightSide": true + }, + "links": [ + + ], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "span": 3, + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "apiserver_request:availability30d{verb=\"write\", cluster=\"$cluster\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "", + "refId": "A" + } + ], + "thresholds": "", + "title": "Write Availability (30d)", + "tooltip": { + "shared": false + }, + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "avg" + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "How many write requests (POST|PUT|PATCH|DELETE) per second do the apiservers get by code?", + "fill": 10, + "fillGradient": 0, + "gridPos": { + + }, + "id": 10, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + { + "alias": "/2../i", + "color": "#56A64B" + }, + { + "alias": "/3../i", + "color": "#F2CC0C" + }, + { + "alias": "/4../i", + "color": "#3274D9" + }, + { + "alias": "/5../i", + "color": "#E02F44" + } + ], + "spaceLength": 10, + "span": 3, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum by (code) (code_resource:apiserver_request_total:rate5m{verb=\"write\", cluster=\"$cluster\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}} code {{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Write SLI - Requests", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "reqps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "reqps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "How many percent of write requests (POST|PUT|PATCH|DELETE) per second are returned with errors (5xx)?", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 11, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 3, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum by (resource) (code_resource:apiserver_request_total:rate5m{verb=\"write\",code=~\"5..\", cluster=\"$cluster\"}) / sum by (resource) (code_resource:apiserver_request_total:rate5m{verb=\"write\", cluster=\"$cluster\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}} resource {{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Write SLI - Errors", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "How many seconds is the 99th percentile for writing (POST|PUT|PATCH|DELETE) a given resource?", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 12, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 3, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "cluster_quantile:apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds:histogram_quantile{verb=\"write\", cluster=\"$cluster\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}} resource {{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Write SLI - Duration", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "Dashboard Row", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 13, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": false, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(workqueue_adds_total{job=\"apiserver\", instance=~\"$instance\", cluster=\"$cluster\"}[$__rate_interval])) by (instance, name)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}} {{`{{`}}name{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Work Queue Add Rate", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 14, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": false, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(workqueue_depth{job=\"apiserver\", instance=~\"$instance\", cluster=\"$cluster\"}[$__rate_interval])) by (instance, name)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}} {{`{{`}}name{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Work Queue Depth", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 15, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum(rate(workqueue_queue_duration_seconds_bucket{job=\"apiserver\", instance=~\"$instance\", cluster=\"$cluster\"}[$__rate_interval])) by (instance, name, le))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}} {{`{{`}}name{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Work Queue Latency", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "Dashboard Row", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 16, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "process_resident_memory_bytes{job=\"apiserver\",instance=~\"$instance\", cluster=\"$cluster\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Memory", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 17, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(process_cpu_seconds_total{job=\"apiserver\",instance=~\"$instance\", cluster=\"$cluster\"}[$__rate_interval])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "CPU usage", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 18, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "go_goroutines{job=\"apiserver\",instance=~\"$instance\", cluster=\"$cluster\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Goroutines", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "Dashboard Row", + "titleSize": "h6", + "type": "row" + } + ], + "schemaVersion": 14, + "style": "dark", + "tags": [ + "kubernetes-mixin" + ], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": "Data Source", + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": null, + "current": { + + }, + "datasource": "$datasource", + "hide": {{ if .Values.grafana.sidecar.dashboards.multicluster.global.enabled }}0{{ else }}2{{ end }}, + "includeAll": false, + "label": "cluster", + "multi": false, + "name": "cluster", + "options": [ + + ], + "query": "label_values(up{job=\"apiserver\"}, cluster)", + "refresh": 2, + "regex": "", + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": { + + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": true, + "label": null, + "multi": false, + "name": "instance", + "options": [ + + ], + "query": "label_values(up{job=\"apiserver\", cluster=\"$cluster\"}, instance)", + "refresh": 2, + "regex": "", + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "{{ .Values.grafana.defaultDashboardsTimezone }}", + "title": "Kubernetes / API server", + "uid": "09ec8aa1e996d6ffcd6817bbaff4db1b", + "version": 0 + } +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/grafana/dashboards-1.14/cluster-total.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/grafana/dashboards-1.14/cluster-total.yaml new file mode 100644 index 00000000000..f4be0bbd45d --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/grafana/dashboards-1.14/cluster-total.yaml @@ -0,0 +1,1882 @@ +{{- /* +Generated from 'cluster-total' from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/a8ba97a150c75be42010c75d10b720c55e182f1a/manifests/grafana-dashboardDefinitions.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (or .Values.grafana.enabled .Values.grafana.forceDeployDashboards) (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.grafana.defaultDashboardsEnabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: {{ .Values.grafana.defaultDashboards.namespace }} + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" $) "cluster-total" | trunc 63 | trimSuffix "-" }} + annotations: +{{ toYaml .Values.grafana.sidecar.dashboards.annotations | indent 4 }} + labels: + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: {{ ternary $.Values.grafana.sidecar.dashboards.labelValue "1" (not (empty $.Values.grafana.sidecar.dashboards.labelValue)) | quote }} + {{- end }} + app: {{ template "kube-prometheus-stack.name" $ }}-grafana +{{ include "kube-prometheus-stack.labels" $ | indent 4 }} +data: + cluster-total.json: |- + { + "__inputs": [ + + ], + "__requires": [ + + ], + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "hideControls": false, + "id": null, + "links": [ + + ], + "panels": [ + { + "collapse": false, + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 2, + "panels": [ + + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Current Bandwidth", + "titleSize": "h6", + "type": "row" + }, + { + "aliasColors": { + + }, + "bars": true, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 1 + }, + "id": 3, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": false, + "linewidth": 1, + "links": [ + + ], + "minSpan": 24, + "nullPointMode": "null", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 24, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sort_desc(sum(irate(container_network_receive_bytes_total{cluster=\"$cluster\",namespace=~\".+\"}[$interval:$resolution])) by (namespace))", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}namespace{{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Current Rate of Bytes Received", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "series", + "name": null, + "show": false, + "values": [ + "current" + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": true, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 1 + }, + "id": 4, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": false, + "linewidth": 1, + "links": [ + + ], + "minSpan": 24, + "nullPointMode": "null", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 24, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sort_desc(sum(irate(container_network_transmit_bytes_total{cluster=\"$cluster\",namespace=~\".+\"}[$interval:$resolution])) by (namespace))", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}namespace{{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Current Rate of Bytes Transmitted", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "series", + "name": null, + "show": false, + "values": [ + "current" + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "columns": [ + { + "text": "Time", + "value": "Time" + }, + { + "text": "Value #A", + "value": "Value #A" + }, + { + "text": "Value #B", + "value": "Value #B" + }, + { + "text": "Value #C", + "value": "Value #C" + }, + { + "text": "Value #D", + "value": "Value #D" + }, + { + "text": "Value #E", + "value": "Value #E" + }, + { + "text": "Value #F", + "value": "Value #F" + }, + { + "text": "Value #G", + "value": "Value #G" + }, + { + "text": "Value #H", + "value": "Value #H" + }, + { + "text": "namespace", + "value": "namespace" + } + ], + "datasource": "$datasource", + "fill": 1, + "fontSize": "90%", + "gridPos": { + "h": 9, + "w": 24, + "x": 0, + "y": 10 + }, + "id": 5, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "minSpan": 24, + "nullPointMode": "null as zero", + "renderer": "flot", + "scroll": true, + "showHeader": true, + "sort": { + "col": 0, + "desc": false + }, + "spaceLength": 10, + "span": 24, + "styles": [ + { + "alias": "Time", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Time", + "thresholds": [ + + ], + "type": "hidden", + "unit": "short" + }, + { + "alias": "Current Bandwidth Received", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #A", + "thresholds": [ + + ], + "type": "number", + "unit": "Bps" + }, + { + "alias": "Current Bandwidth Transmitted", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #B", + "thresholds": [ + + ], + "type": "number", + "unit": "Bps" + }, + { + "alias": "Average Bandwidth Received", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #C", + "thresholds": [ + + ], + "type": "number", + "unit": "Bps" + }, + { + "alias": "Average Bandwidth Transmitted", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #D", + "thresholds": [ + + ], + "type": "number", + "unit": "Bps" + }, + { + "alias": "Rate of Received Packets", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #E", + "thresholds": [ + + ], + "type": "number", + "unit": "pps" + }, + { + "alias": "Rate of Transmitted Packets", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #F", + "thresholds": [ + + ], + "type": "number", + "unit": "pps" + }, + { + "alias": "Rate of Received Packets Dropped", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #G", + "thresholds": [ + + ], + "type": "number", + "unit": "pps" + }, + { + "alias": "Rate of Transmitted Packets Dropped", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #H", + "thresholds": [ + + ], + "type": "number", + "unit": "pps" + }, + { + "alias": "Namespace", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": true, + "linkTooltip": "Drill down", + "linkUrl": "d/8b7a8b326d7a6f1f04244066368c67af/kubernetes-networking-namespace-pods?orgId=1&refresh=30s&var-namespace=$__cell", + "pattern": "namespace", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + } + ], + "targets": [ + { + "expr": "sort_desc(sum(irate(container_network_receive_bytes_total{cluster=\"$cluster\",namespace=~\".+\"}[$interval:$resolution])) by (namespace))", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "A", + "step": 10 + }, + { + "expr": "sort_desc(sum(irate(container_network_transmit_bytes_total{cluster=\"$cluster\",namespace=~\".+\"}[$interval:$resolution])) by (namespace))", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "B", + "step": 10 + }, + { + "expr": "sort_desc(avg(irate(container_network_receive_bytes_total{cluster=\"$cluster\",namespace=~\".+\"}[$interval:$resolution])) by (namespace))", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "C", + "step": 10 + }, + { + "expr": "sort_desc(avg(irate(container_network_transmit_bytes_total{cluster=\"$cluster\",namespace=~\".+\"}[$interval:$resolution])) by (namespace))", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "D", + "step": 10 + }, + { + "expr": "sort_desc(sum(irate(container_network_receive_packets_total{cluster=\"$cluster\",namespace=~\".+\"}[$interval:$resolution])) by (namespace))", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "E", + "step": 10 + }, + { + "expr": "sort_desc(sum(irate(container_network_transmit_packets_total{cluster=\"$cluster\",namespace=~\".+\"}[$interval:$resolution])) by (namespace))", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "F", + "step": 10 + }, + { + "expr": "sort_desc(sum(irate(container_network_receive_packets_dropped_total{cluster=\"$cluster\",namespace=~\".+\"}[$interval:$resolution])) by (namespace))", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "G", + "step": 10 + }, + { + "expr": "sort_desc(sum(irate(container_network_transmit_packets_dropped_total{cluster=\"$cluster\",namespace=~\".+\"}[$interval:$resolution])) by (namespace))", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "H", + "step": 10 + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Current Status", + "type": "table" + }, + { + "collapse": true, + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 10 + }, + "id": 6, + "panels": [ + { + "aliasColors": { + + }, + "bars": true, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 11 + }, + "id": 7, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": false, + "linewidth": 1, + "links": [ + + ], + "minSpan": 24, + "nullPointMode": "null", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 24, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sort_desc(avg(irate(container_network_receive_bytes_total{cluster=\"$cluster\",namespace=~\".+\"}[$interval:$resolution])) by (namespace))", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}namespace{{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Average Rate of Bytes Received", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "series", + "name": null, + "show": false, + "values": [ + "current" + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": true, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 11 + }, + "id": 8, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": false, + "linewidth": 1, + "links": [ + + ], + "minSpan": 24, + "nullPointMode": "null", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 24, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sort_desc(avg(irate(container_network_transmit_bytes_total{cluster=\"$cluster\",namespace=~\".+\"}[$interval:$resolution])) by (namespace))", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}namespace{{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Average Rate of Bytes Transmitted", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "series", + "name": null, + "show": false, + "values": [ + "current" + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Average Bandwidth", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 11 + }, + "id": 9, + "panels": [ + + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Bandwidth History", + "titleSize": "h6", + "type": "row" + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 24, + "x": 0, + "y": 12 + }, + "id": 10, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "hideEmpty": true, + "hideZero": true, + "max": true, + "min": true, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [ + + ], + "minSpan": 24, + "nullPointMode": "connected", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 24, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sort_desc(sum(irate(container_network_receive_bytes_total{cluster=\"$cluster\",namespace=~\".+\"}[$interval:$resolution])) by (namespace))", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}namespace{{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Receive Bandwidth", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 24, + "x": 0, + "y": 21 + }, + "id": 11, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "hideEmpty": true, + "hideZero": true, + "max": true, + "min": true, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [ + + ], + "minSpan": 24, + "nullPointMode": "connected", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 24, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sort_desc(sum(irate(container_network_transmit_bytes_total{cluster=\"$cluster\",namespace=~\".+\"}[$interval:$resolution])) by (namespace))", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}namespace{{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Transmit Bandwidth", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "collapse": true, + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 30 + }, + "id": 12, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 24, + "x": 0, + "y": 31 + }, + "id": 13, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "hideEmpty": true, + "hideZero": true, + "max": true, + "min": true, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [ + + ], + "minSpan": 24, + "nullPointMode": "connected", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 24, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sort_desc(sum(irate(container_network_receive_packets_total{cluster=\"$cluster\",namespace=~\".+\"}[$interval:$resolution])) by (namespace))", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}namespace{{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Received Packets", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 24, + "x": 0, + "y": 40 + }, + "id": 14, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "hideEmpty": true, + "hideZero": true, + "max": true, + "min": true, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [ + + ], + "minSpan": 24, + "nullPointMode": "connected", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 24, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sort_desc(sum(irate(container_network_transmit_packets_total{cluster=\"$cluster\",namespace=~\".+\"}[$interval:$resolution])) by (namespace))", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}namespace{{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Transmitted Packets", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Packets", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": true, + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 31 + }, + "id": 15, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 24, + "x": 0, + "y": 50 + }, + "id": 16, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "hideEmpty": true, + "hideZero": true, + "max": true, + "min": true, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [ + + ], + "minSpan": 24, + "nullPointMode": "connected", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 24, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sort_desc(sum(irate(container_network_receive_packets_dropped_total{cluster=\"$cluster\",namespace=~\".+\"}[$interval:$resolution])) by (namespace))", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}namespace{{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Received Packets Dropped", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 24, + "x": 0, + "y": 59 + }, + "id": 17, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "hideEmpty": true, + "hideZero": true, + "max": true, + "min": true, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [ + + ], + "minSpan": 24, + "nullPointMode": "connected", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 24, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sort_desc(sum(irate(container_network_transmit_packets_dropped_total{cluster=\"$cluster\",namespace=~\".+\"}[$interval:$resolution])) by (namespace))", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}namespace{{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Transmitted Packets Dropped", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 24, + "x": 0, + "y": 59 + }, + "id": 18, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "hideEmpty": true, + "hideZero": true, + "max": true, + "min": true, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [ + { + "targetBlank": true, + "title": "What is TCP Retransmit?", + "url": "https://accedian.com/enterprises/blog/network-packet-loss-retransmissions-and-duplicate-acknowledgements/" + } + ], + "minSpan": 24, + "nullPointMode": "connected", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 24, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sort_desc(sum(rate(node_netstat_Tcp_RetransSegs{cluster=\"$cluster\"}[$interval:$resolution]) / rate(node_netstat_Tcp_OutSegs{cluster=\"$cluster\"}[$interval:$resolution])) by (instance))", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}instance{{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of TCP Retransmits out of all sent segments", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 24, + "x": 0, + "y": 59 + }, + "id": 19, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "hideEmpty": true, + "hideZero": true, + "max": true, + "min": true, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [ + { + "targetBlank": true, + "title": "Why monitor SYN retransmits?", + "url": "https://github.com/prometheus/node_exporter/issues/1023#issuecomment-408128365" + } + ], + "minSpan": 24, + "nullPointMode": "connected", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 24, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sort_desc(sum(rate(node_netstat_TcpExt_TCPSynRetrans{cluster=\"$cluster\"}[$interval:$resolution]) / rate(node_netstat_Tcp_RetransSegs{cluster=\"$cluster\"}[$interval:$resolution])) by (instance))", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}instance{{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of TCP SYN Retransmits out of all retransmits", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Errors", + "titleSize": "h6", + "type": "row" + } + ], + "refresh": "10s", + "rows": [ + + ], + "schemaVersion": 18, + "style": "dark", + "tags": [ + "kubernetes-mixin" + ], + "templating": { + "list": [ + { + "allValue": null, + "auto": false, + "auto_count": 30, + "auto_min": "10s", + "current": { + "text": "5m", + "value": "5m" + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "resolution", + "options": [ + { + "selected": false, + "text": "30s", + "value": "30s" + }, + { + "selected": true, + "text": "5m", + "value": "5m" + }, + { + "selected": false, + "text": "1h", + "value": "1h" + } + ], + "query": "30s,5m,1h", + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "interval", + "useTags": false + }, + { + "allValue": null, + "auto": false, + "auto_count": 30, + "auto_min": "10s", + "current": { + "text": "5m", + "value": "5m" + }, + "datasource": "$datasource", + "hide": 2, + "includeAll": false, + "label": null, + "multi": false, + "name": "interval", + "options": [ + { + "selected": true, + "text": "4h", + "value": "4h" + } + ], + "query": "4h", + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "interval", + "useTags": false + }, + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": "Data Source", + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": null, + "current": { + + }, + "datasource": "$datasource", + "hide": {{ if .Values.grafana.sidecar.dashboards.multicluster.global.enabled }}0{{ else }}2{{ end }}, + "includeAll": false, + "label": null, + "multi": false, + "name": "cluster", + "options": [ + + ], + "query": "label_values(up{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\"}, cluster)", + "refresh": 2, + "regex": "", + "sort": 0, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "{{ .Values.grafana.defaultDashboardsTimezone }}", + "title": "Kubernetes / Networking / Cluster", + "uid": "ff635a025bcfea7bc3dd4f508990a3e9", + "version": 0 + } +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/grafana/dashboards-1.14/controller-manager.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/grafana/dashboards-1.14/controller-manager.yaml new file mode 100644 index 00000000000..8d420d7a4fe --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/grafana/dashboards-1.14/controller-manager.yaml @@ -0,0 +1,1196 @@ +{{- /* +Generated from 'controller-manager' from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/a8ba97a150c75be42010c75d10b720c55e182f1a/manifests/grafana-dashboardDefinitions.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (or .Values.grafana.enabled .Values.grafana.forceDeployDashboards) (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.grafana.defaultDashboardsEnabled }} +{{- if (include "exporter.kubeControllerManager.enabled" .)}} +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: {{ .Values.grafana.defaultDashboards.namespace }} + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" $) "controller-manager" | trunc 63 | trimSuffix "-" }} + annotations: +{{ toYaml .Values.grafana.sidecar.dashboards.annotations | indent 4 }} + labels: + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: {{ ternary $.Values.grafana.sidecar.dashboards.labelValue "1" (not (empty $.Values.grafana.sidecar.dashboards.labelValue)) | quote }} + {{- end }} + app: {{ template "kube-prometheus-stack.name" $ }}-grafana +{{ include "kube-prometheus-stack.labels" $ | indent 4 }} +data: + controller-manager.json: |- + { + "__inputs": [ + + ], + "__requires": [ + + ], + "annotations": { + "list": [ + + ] + }, + "editable": false, + "gnetId": null, + "graphTooltip": 0, + "hideControls": false, + "id": null, + "links": [ + + ], + "refresh": "10s", + "rows": [ + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": "$datasource", + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + + }, + "id": 2, + "interval": "1m", + "legend": { + "alignAsTable": true, + "rightSide": true + }, + "links": [ + + ], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "span": 2, + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + {{- if .Values.k3sServer.enabled }} + "expr": "sum(up{cluster=\"$cluster\", job=\"{{ include "exporter.kubeControllerManager.jobName" . }}\", metrics_path=\"/metrics\"})", + {{- else }} + "expr": "sum(up{cluster=\"$cluster\", job=\"{{ include "exporter.kubeControllerManager.jobName" . }}\"})", + {{- end }} + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "", + "refId": "A" + } + ], + "thresholds": "", + "title": "Up", + "tooltip": { + "shared": false + }, + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "min" + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 3, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(workqueue_adds_total{cluster=\"$cluster\", job=\"{{ include "exporter.kubeControllerManager.jobName" . }}\", instance=~\"$instance\"}[$__rate_interval])) by (cluster, instance, name)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}cluster{{`}}`}} {{`{{`}}instance{{`}}`}} {{`{{`}}name{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Work Queue Add Rate", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "Dashboard Row", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 4, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(workqueue_depth{cluster=\"$cluster\", job=\"{{ include "exporter.kubeControllerManager.jobName" . }}\", instance=~\"$instance\"}[$__rate_interval])) by (cluster, instance, name)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}cluster{{`}}`}} {{`{{`}}instance{{`}}`}} {{`{{`}}name{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Work Queue Depth", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "Dashboard Row", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 5, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum(rate(workqueue_queue_duration_seconds_bucket{cluster=\"$cluster\", job=\"{{ include "exporter.kubeControllerManager.jobName" . }}\", instance=~\"$instance\"}[$__rate_interval])) by (cluster, instance, name, le))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}cluster{{`}}`}} {{`{{`}}instance{{`}}`}} {{`{{`}}name{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Work Queue Latency", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "Dashboard Row", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 6, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(rest_client_requests_total{job=\"{{ include "exporter.kubeControllerManager.jobName" . }}\", instance=~\"$instance\",code=~\"2..\"}[$__rate_interval]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "2xx", + "refId": "A" + }, + { + "expr": "sum(rate(rest_client_requests_total{job=\"{{ include "exporter.kubeControllerManager.jobName" . }}\", instance=~\"$instance\",code=~\"3..\"}[$__rate_interval]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "3xx", + "refId": "B" + }, + { + "expr": "sum(rate(rest_client_requests_total{job=\"{{ include "exporter.kubeControllerManager.jobName" . }}\", instance=~\"$instance\",code=~\"4..\"}[$__rate_interval]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "4xx", + "refId": "C" + }, + { + "expr": "sum(rate(rest_client_requests_total{job=\"{{ include "exporter.kubeControllerManager.jobName" . }}\", instance=~\"$instance\",code=~\"5..\"}[$__rate_interval]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "5xx", + "refId": "D" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Kube API Request Rate", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 7, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 8, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum(rate(rest_client_request_duration_seconds_bucket{cluster=\"$cluster\", job=\"{{ include "exporter.kubeControllerManager.jobName" . }}\", instance=~\"$instance\", verb=\"POST\"}[$__rate_interval])) by (verb, url, le))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}verb{{`}}`}} {{`{{`}}url{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Post Request Latency 99th Quantile", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "Dashboard Row", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 8, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum(rate(rest_client_request_duration_seconds_bucket{cluster=\"$cluster\", job=\"{{ include "exporter.kubeControllerManager.jobName" . }}\", instance=~\"$instance\", verb=\"GET\"}[$__rate_interval])) by (verb, url, le))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}verb{{`}}`}} {{`{{`}}url{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Get Request Latency 99th Quantile", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "Dashboard Row", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 9, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "process_resident_memory_bytes{cluster=\"$cluster\", job=\"{{ include "exporter.kubeControllerManager.jobName" . }}\",instance=~\"$instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Memory", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 10, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(process_cpu_seconds_total{cluster=\"$cluster\", job=\"{{ include "exporter.kubeControllerManager.jobName" . }}\",instance=~\"$instance\"}[$__rate_interval])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "CPU usage", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 11, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "go_goroutines{cluster=\"$cluster\", job=\"{{ include "exporter.kubeControllerManager.jobName" . }}\",instance=~\"$instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Goroutines", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "Dashboard Row", + "titleSize": "h6", + "type": "row" + } + ], + "schemaVersion": 14, + "style": "dark", + "tags": [ + "kubernetes-mixin" + ], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": "Data Source", + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": null, + "current": { + + }, + "datasource": "$datasource", + "hide": {{ if .Values.grafana.sidecar.dashboards.multicluster.global.enabled }}0{{ else }}2{{ end }}, + "includeAll": false, + "label": "cluster", + "multi": false, + "name": "cluster", + "options": [ + + ], + "query": "label_values(up{job=\"{{ include "exporter.kubeControllerManager.jobName" . }}\"}, cluster)", + "refresh": 2, + "regex": "", + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": { + + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": true, + "label": null, + "multi": false, + "name": "instance", + "options": [ + + ], + "query": "label_values(up{cluster=\"$cluster\", job=\"{{ include "exporter.kubeControllerManager.jobName" . }}\"}, instance)", + "refresh": 2, + "regex": "", + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "{{ .Values.grafana.defaultDashboardsTimezone }}", + "title": "Kubernetes / Controller Manager", + "uid": "72e0e05bef5099e5f049b05fdc429ed4", + "version": 0 + } +{{- end }} +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/grafana/dashboards-1.14/etcd.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/grafana/dashboards-1.14/etcd.yaml new file mode 100644 index 00000000000..0eeedc62990 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/grafana/dashboards-1.14/etcd.yaml @@ -0,0 +1,1229 @@ +{{- /* +Generated from 'etcd' from https://github.com/etcd-io/etcd.git +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (or .Values.grafana.enabled .Values.grafana.forceDeployDashboards) (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.grafana.defaultDashboardsEnabled }} +{{- if (include "exporter.kubeEtcd.enabled" .)}} +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: {{ .Values.grafana.defaultDashboards.namespace }} + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" $) "etcd" | trunc 63 | trimSuffix "-" }} + annotations: +{{ toYaml .Values.grafana.sidecar.dashboards.annotations | indent 4 }} + labels: + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: {{ ternary $.Values.grafana.sidecar.dashboards.labelValue "1" (not (empty $.Values.grafana.sidecar.dashboards.labelValue)) | quote }} + {{- end }} + app: {{ template "kube-prometheus-stack.name" $ }}-grafana +{{ include "kube-prometheus-stack.labels" $ | indent 4 }} +data: + etcd.json: |- + { + "annotations": { + "list": [] + }, + "description": "etcd sample Grafana dashboard with Prometheus", + "editable": true, + "gnetId": null, + "hideControls": false, + "links": [], + "refresh": "10s", + "rows": [ + { + "collapse": false, + "editable": true, + "height": "250px", + "panels": [ + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "datasource": "$datasource", + "editable": true, + "error": false, + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "id": 28, + "interval": null, + "isNew": true, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "span": 3, + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "targets": [ + { + "expr": "sum(etcd_server_has_leader{job=\"$cluster\"})", + "intervalFactor": 2, + "legendFormat": "", + "metric": "etcd_server_has_leader", + "refId": "A", + "step": 20 + } + ], + "thresholds": "", + "title": "Up", + "type": "singlestat", + "valueFontSize": "200%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "avg" + }, + { + "aliasColors": {}, + "bars": false, + "datasource": "$datasource", + "editable": true, + "error": false, + "fill": 0, + "id": 23, + "isNew": true, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "span": 5, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(grpc_server_started_total{job=\"$cluster\",grpc_type=\"unary\"}[$__rate_interval]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "RPC Rate", + "metric": "grpc_server_started_total", + "refId": "A", + "step": 2 + }, + { + "expr": "sum(rate(grpc_server_handled_total{job=\"$cluster\",grpc_type=\"unary\",grpc_code=~\"Unknown|FailedPrecondition|ResourceExhausted|Internal|Unavailable|DataLoss|DeadlineExceeded\"}[$__rate_interval]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "RPC Failed Rate", + "metric": "grpc_server_handled_total", + "refId": "B", + "step": 2 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "RPC Rate", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "datasource": "$datasource", + "editable": true, + "error": false, + "fill": 0, + "id": 41, + "isNew": true, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "span": 4, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(grpc_server_started_total{job=\"$cluster\",grpc_service=\"etcdserverpb.Watch\",grpc_type=\"bidi_stream\"}) - sum(grpc_server_handled_total{job=\"$cluster\",grpc_service=\"etcdserverpb.Watch\",grpc_type=\"bidi_stream\"})", + "intervalFactor": 2, + "legendFormat": "Watch Streams", + "metric": "grpc_server_handled_total", + "refId": "A", + "step": 4 + }, + { + "expr": "sum(grpc_server_started_total{job=\"$cluster\",grpc_service=\"etcdserverpb.Lease\",grpc_type=\"bidi_stream\"}) - sum(grpc_server_handled_total{job=\"$cluster\",grpc_service=\"etcdserverpb.Lease\",grpc_type=\"bidi_stream\"})", + "intervalFactor": 2, + "legendFormat": "Lease Streams", + "metric": "grpc_server_handled_total", + "refId": "B", + "step": 4 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Active Streams", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": "", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "showTitle": false, + "title": "Row" + }, + { + "collapse": false, + "editable": true, + "height": "250px", + "panels": [ + { + "aliasColors": {}, + "bars": false, + "datasource": "$datasource", + "decimals": null, + "editable": true, + "error": false, + "fill": 0, + "grid": {}, + "id": 1, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "etcd_mvcc_db_total_size_in_bytes{job=\"$cluster\"}", + "hide": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}} DB Size", + "metric": "", + "refId": "A", + "step": 4 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "DB Size", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "datasource": "$datasource", + "editable": true, + "error": false, + "fill": 0, + "grid": {}, + "id": 3, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 1, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "span": 4, + "stack": false, + "steppedLine": true, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum(rate(etcd_disk_wal_fsync_duration_seconds_bucket{job=\"$cluster\"}[$__rate_interval])) by (instance, le))", + "hide": false, + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}} WAL fsync", + "metric": "etcd_disk_wal_fsync_duration_seconds_bucket", + "refId": "A", + "step": 4 + }, + { + "expr": "histogram_quantile(0.99, sum(rate(etcd_disk_backend_commit_duration_seconds_bucket{job=\"$cluster\"}[$__rate_interval])) by (instance, le))", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}} DB fsync", + "metric": "etcd_disk_backend_commit_duration_seconds_bucket", + "refId": "B", + "step": 4 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Disk Sync Duration", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "s", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "datasource": "$datasource", + "editable": true, + "error": false, + "fill": 0, + "id": 29, + "isNew": true, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "process_resident_memory_bytes{job=\"$cluster\"}", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}} Resident Memory", + "metric": "process_resident_memory_bytes", + "refId": "A", + "step": 4 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Memory", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "title": "New row" + }, + { + "collapse": false, + "editable": true, + "height": "250px", + "panels": [ + { + "aliasColors": {}, + "bars": false, + "datasource": "$datasource", + "editable": true, + "error": false, + "fill": 5, + "id": 22, + "isNew": true, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "span": 3, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "rate(etcd_network_client_grpc_received_bytes_total{job=\"$cluster\"}[$__rate_interval])", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}} Client Traffic In", + "metric": "etcd_network_client_grpc_received_bytes_total", + "refId": "A", + "step": 4 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Client Traffic In", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "datasource": "$datasource", + "editable": true, + "error": false, + "fill": 5, + "id": 21, + "isNew": true, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "span": 3, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "rate(etcd_network_client_grpc_sent_bytes_total{job=\"$cluster\"}[$__rate_interval])", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}} Client Traffic Out", + "metric": "etcd_network_client_grpc_sent_bytes_total", + "refId": "A", + "step": 4 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Client Traffic Out", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "datasource": "$datasource", + "editable": true, + "error": false, + "fill": 0, + "id": 20, + "isNew": true, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "span": 3, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(etcd_network_peer_received_bytes_total{job=\"$cluster\"}[$__rate_interval])) by (instance)", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}} Peer Traffic In", + "metric": "etcd_network_peer_received_bytes_total", + "refId": "A", + "step": 4 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Peer Traffic In", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "datasource": "$datasource", + "decimals": null, + "editable": true, + "error": false, + "fill": 0, + "grid": {}, + "id": 16, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "span": 3, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(etcd_network_peer_sent_bytes_total{job=\"$cluster\"}[$__rate_interval])) by (instance)", + "hide": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}} Peer Traffic Out", + "metric": "etcd_network_peer_sent_bytes_total", + "refId": "A", + "step": 4 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Peer Traffic Out", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "Bps", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "title": "New row" + }, + { + "collapse": false, + "editable": true, + "height": "250px", + "panels": [ + { + "aliasColors": {}, + "bars": false, + "datasource": "$datasource", + "editable": true, + "error": false, + "fill": 0, + "id": 40, + "isNew": true, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(etcd_server_proposals_failed_total{job=\"$cluster\"}[$__rate_interval]))", + "intervalFactor": 2, + "legendFormat": "Proposal Failure Rate", + "metric": "etcd_server_proposals_failed_total", + "refId": "A", + "step": 2 + }, + { + "expr": "sum(etcd_server_proposals_pending{job=\"$cluster\"})", + "intervalFactor": 2, + "legendFormat": "Proposal Pending Total", + "metric": "etcd_server_proposals_pending", + "refId": "B", + "step": 2 + }, + { + "expr": "sum(rate(etcd_server_proposals_committed_total{job=\"$cluster\"}[$__rate_interval]))", + "intervalFactor": 2, + "legendFormat": "Proposal Commit Rate", + "metric": "etcd_server_proposals_committed_total", + "refId": "C", + "step": 2 + }, + { + "expr": "sum(rate(etcd_server_proposals_applied_total{job=\"$cluster\"}[$__rate_interval]))", + "intervalFactor": 2, + "legendFormat": "Proposal Apply Rate", + "refId": "D", + "step": 2 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Raft Proposals", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": "", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "datasource": "$datasource", + "decimals": 0, + "editable": true, + "error": false, + "fill": 0, + "id": 19, + "isNew": true, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "changes(etcd_server_leader_changes_seen_total{job=\"$cluster\"}[1d])", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}} Total Leader Elections Per Day", + "metric": "etcd_server_leader_changes_seen_total", + "refId": "A", + "step": 2 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Total Leader Elections Per Day", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "decimals": 0, + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 28 + }, + "hiddenSeries": false, + "id": 42, + "isNew": true, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.4.3", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum by (instance, le) (rate(etcd_network_peer_round_trip_time_seconds_bucket{job=\"$cluster\"}[$__rate_interval])))", + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}} Peer round trip time", + "metric": "etcd_network_peer_round_trip_time_seconds_bucket", + "refId": "A", + "step": 2 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Peer round trip time", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:925", + "decimals": null, + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "$$hashKey": "object:926", + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + } + ], + "title": "New row" + } + ], + "schemaVersion": 13, + "sharedCrosshair": false, + "style": "dark", + "tags": [ + "etcd-mixin" + ], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": "Data Source", + "name": "datasource", + "options": [], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": null, + "current": { + "text": "prod", + "value": "prod" + }, + "datasource": "$datasource", + "hide": {{ if (or .Values.grafana.sidecar.dashboards.multicluster.global.enabled .Values.grafana.sidecar.dashboards.multicluster.etcd.enabled) }}0{{ else }}2{{ end }}, + "includeAll": false, + "label": "cluster", + "multi": false, + "name": "cluster", + "options": [], + "query": "label_values(etcd_server_has_leader, job)", + "refresh": 2, + "regex": "", + "sort": 2, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-15m", + "to": "now" + }, + "timepicker": { + "now": true, + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "{{ .Values.grafana.defaultDashboardsTimezone }}", + "title": "etcd", + "uid": "c2f4e12cdf69feb95caa41a5a1b423d9", + "version": 215 + } +{{- end }} +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/grafana/dashboards-1.14/grafana-overview.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/grafana/dashboards-1.14/grafana-overview.yaml new file mode 100644 index 00000000000..d2609140cf0 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/grafana/dashboards-1.14/grafana-overview.yaml @@ -0,0 +1,635 @@ +{{- /* +Generated from 'grafana-overview' from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/a8ba97a150c75be42010c75d10b720c55e182f1a/manifests/grafana-dashboardDefinitions.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (or .Values.grafana.enabled .Values.grafana.forceDeployDashboards) (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.grafana.defaultDashboardsEnabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: {{ .Values.grafana.defaultDashboards.namespace }} + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" $) "grafana-overview" | trunc 63 | trimSuffix "-" }} + annotations: +{{ toYaml .Values.grafana.sidecar.dashboards.annotations | indent 4 }} + labels: + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: {{ ternary $.Values.grafana.sidecar.dashboards.labelValue "1" (not (empty $.Values.grafana.sidecar.dashboards.labelValue)) | quote }} + {{- end }} + app: {{ template "kube-prometheus-stack.name" $ }}-grafana +{{ include "kube-prometheus-stack.labels" $ | indent 4 }} +data: + grafana-overview.json: |- + { + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "target": { + "limit": 100, + "matchAny": false, + "tags": [ + + ], + "type": "dashboard" + }, + "type": "dashboard" + } + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "id": 3085, + "iteration": 1631554945276, + "links": [ + + ], + "panels": [ + { + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "mappings": [ + + ], + "noValue": "0", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [ + + ] + }, + "gridPos": { + "h": 5, + "w": 6, + "x": 0, + "y": 0 + }, + "id": 6, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "mean" + ], + "fields": "", + "values": false + }, + "text": { + + }, + "textMode": "auto" + }, + "pluginVersion": "8.1.3", + "targets": [ + { + "expr": "grafana_alerting_result_total{job=~\"$job\", instance=~\"$instance\", state=\"alerting\"}", + "instant": true, + "interval": "", + "legendFormat": "", + "refId": "A" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Firing Alerts", + "type": "stat" + }, + { + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "mappings": [ + + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [ + + ] + }, + "gridPos": { + "h": 5, + "w": 6, + "x": 6, + "y": 0 + }, + "id": 8, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "mean" + ], + "fields": "", + "values": false + }, + "text": { + + }, + "textMode": "auto" + }, + "pluginVersion": "8.1.3", + "targets": [ + { + "expr": "sum(grafana_stat_totals_dashboard{job=~\"$job\", instance=~\"$instance\"})", + "interval": "", + "legendFormat": "", + "refId": "A" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Dashboards", + "type": "stat" + }, + { + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": { + "align": null, + "displayMode": "auto" + }, + "mappings": [ + + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [ + + ] + }, + "gridPos": { + "h": 5, + "w": 12, + "x": 12, + "y": 0 + }, + "id": 10, + "options": { + "showHeader": true + }, + "pluginVersion": "8.1.3", + "targets": [ + { + "expr": "grafana_build_info{job=~\"$job\", instance=~\"$instance\"}", + "instant": true, + "interval": "", + "legendFormat": "", + "refId": "A" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Build Info", + "transformations": [ + { + "id": "labelsToFields", + "options": { + + } + }, + { + "id": "organize", + "options": { + "excludeByName": { + "Time": true, + "Value": true, + "branch": true, + "container": true, + "goversion": true, + "namespace": true, + "pod": true, + "revision": true + }, + "indexByName": { + "Time": 7, + "Value": 11, + "branch": 4, + "container": 8, + "edition": 2, + "goversion": 6, + "instance": 1, + "job": 0, + "namespace": 9, + "pod": 10, + "revision": 5, + "version": 3 + }, + "renameByName": { + + } + } + } + ], + "type": "table" + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "links": [ + + ] + }, + "overrides": [ + + ] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 5 + }, + "hiddenSeries": false, + "id": 2, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "8.1.3", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum by (status_code) (irate(grafana_http_request_duration_seconds_count{job=~\"$job\", instance=~\"$instance\"}[1m])) ", + "interval": "", + "legendFormat": "{{`{{`}}status_code{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeRegions": [ + + ], + "timeShift": null, + "title": "RPS", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "$$hashKey": "object:157", + "format": "reqps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "$$hashKey": "object:158", + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "links": [ + + ] + }, + "overrides": [ + + ] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 5 + }, + "hiddenSeries": false, + "id": 4, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "8.1.3", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "exemplar": true, + "expr": "histogram_quantile(0.99, sum(irate(grafana_http_request_duration_seconds_bucket{instance=~\"$instance\", job=~\"$job\"}[$__rate_interval])) by (le)) * 1", + "interval": "", + "legendFormat": "99th Percentile", + "refId": "A" + }, + { + "exemplar": true, + "expr": "histogram_quantile(0.50, sum(irate(grafana_http_request_duration_seconds_bucket{instance=~\"$instance\", job=~\"$job\"}[$__rate_interval])) by (le)) * 1", + "interval": "", + "legendFormat": "50th Percentile", + "refId": "B" + }, + { + "exemplar": true, + "expr": "sum(irate(grafana_http_request_duration_seconds_sum{instance=~\"$instance\", job=~\"$job\"}[$__rate_interval])) * 1 / sum(irate(grafana_http_request_duration_seconds_count{instance=~\"$instance\", job=~\"$job\"}[$__rate_interval]))", + "interval": "", + "legendFormat": "Average", + "refId": "C" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeRegions": [ + + ], + "timeShift": null, + "title": "Request Latency", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "$$hashKey": "object:210", + "format": "ms", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "$$hashKey": "object:211", + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + } + ], + "schemaVersion": 30, + "style": "dark", + "tags": [ + + ], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "description": null, + "error": null, + "hide": 0, + "includeAll": false, + "label": "Data Source", + "multi": false, + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "queryValue": "", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "type": "datasource" + }, + { + "allValue": ".*", + "current": { + "selected": false, + "text": [ + "default/grafana" + ], + "value": [ + "default/grafana" + ] + }, + "datasource": "$datasource", + "definition": "label_values(grafana_build_info, job)", + "description": null, + "error": null, + "hide": 0, + "includeAll": true, + "label": null, + "multi": true, + "name": "job", + "options": [ + + ], + "query": { + "query": "label_values(grafana_build_info, job)", + "refId": "Billing Admin-job-Variable-Query" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "tagValuesQuery": "", + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": ".*", + "current": { + "selected": false, + "text": "All", + "value": "$__all" + }, + "datasource": "$datasource", + "definition": "label_values(grafana_build_info, instance)", + "description": null, + "error": null, + "hide": 0, + "includeAll": true, + "label": null, + "multi": true, + "name": "instance", + "options": [ + + ], + "query": { + "query": "label_values(grafana_build_info, instance)", + "refId": "Billing Admin-instance-Variable-Query" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "tagValuesQuery": "", + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-6h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ] + }, + "timezone": "{{ .Values.grafana.defaultDashboardsTimezone }}", + "title": "Grafana Overview", + "uid": "6be0s85Mk", + "version": 2 + } +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/grafana/dashboards-1.14/k8s-coredns.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/grafana/dashboards-1.14/k8s-coredns.yaml new file mode 100644 index 00000000000..7ecca76f231 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/grafana/dashboards-1.14/k8s-coredns.yaml @@ -0,0 +1,1534 @@ +{{- /* +Generated from 'k8s-coredns' from ../files/dashboards/k8s-coredns.json +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (or .Values.grafana.enabled .Values.grafana.forceDeployDashboards) (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.grafana.defaultDashboardsEnabled .Values.coreDns.enabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: {{ .Values.grafana.defaultDashboards.namespace }} + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" $) "k8s-coredns" | trunc 63 | trimSuffix "-" }} + annotations: +{{ toYaml .Values.grafana.sidecar.dashboards.annotations | indent 4 }} + labels: + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: {{ ternary $.Values.grafana.sidecar.dashboards.labelValue "1" (not (empty $.Values.grafana.sidecar.dashboards.labelValue)) | quote }} + {{- end }} + app: {{ template "kube-prometheus-stack.name" $ }}-grafana +{{ include "kube-prometheus-stack.labels" $ | indent 4 }} +data: + k8s-coredns.json: |- + { + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "description": "A dashboard for the CoreDNS DNS server with updated metrics for version 1.7.0+. Based on the CoreDNS dashboard by buhay.", + "editable": true, + "gnetId": 12539, + "graphTooltip": 0, + "iteration": 1603798405693, + "links": [ + { + "icon": "external link", + "tags": [], + "targetBlank": true, + "title": "CoreDNS.io", + "type": "link", + "url": "https://coredns.io" + } + ], + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 0 + }, + "hiddenSeries": false, + "id": 2, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.2.0", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "total", + "yaxis": 2 + } + ], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(coredns_dns_request_count_total{job=\"coredns\",instance=~\"$instance\"}[5m])) by (proto) or\nsum(rate(coredns_dns_requests_total{job=\"coredns\",instance=~\"$instance\"}[5m])) by (proto)", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{"{{proto}}"}}", + "refId": "A", + "step": 60 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Requests (total)", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "pps", + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "pps", + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 0 + }, + "hiddenSeries": false, + "id": 4, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.2.0", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "total", + "yaxis": 2 + }, + { + "alias": "other", + "yaxis": 2 + } + ], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(coredns_dns_request_type_count_total{job=\"coredns\",instance=~\"$instance\"}[5m])) by (type) or \nsum(rate(coredns_dns_requests_total{job=\"coredns\",instance=~\"$instance\"}[5m])) by (type)", + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{"{{type}}"}}", + "refId": "A", + "step": 60 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Requests (by qtype)", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "pps", + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "pps", + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 0 + }, + "hiddenSeries": false, + "id": 6, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.2.0", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "total", + "yaxis": 2 + } + ], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(coredns_dns_request_count_total{job=\"coredns\",instance=~\"$instance\"}[5m])) by (zone) or\nsum(rate(coredns_dns_requests_total{job=\"coredns\",instance=~\"$instance\"}[5m])) by (zone)", + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{"{{zone}}"}}", + "refId": "A", + "step": 60 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Requests (by zone)", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "pps", + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "pps", + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 7 + }, + "hiddenSeries": false, + "id": 8, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.2.0", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "total", + "yaxis": 2 + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(coredns_dns_request_do_count_total{job=\"coredns\",instance=~\"$instance\"}[5m])) or\nsum(rate(coredns_dns_do_requests_total{job=\"coredns\",instance=~\"$instance\"}[5m]))", + "interval": "", + "intervalFactor": 2, + "legendFormat": "DO", + "refId": "A", + "step": 40 + }, + { + "expr": "sum(rate(coredns_dns_request_count_total{job=\"coredns\",instance=~\"$instance\"}[5m])) or\nsum(rate(coredns_dns_requests_total{job=\"coredns\",instance=~\"$instance\"}[5m]))", + "interval": "", + "intervalFactor": 2, + "legendFormat": "total", + "refId": "B", + "step": 40 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Requests (DO bit)", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "pps", + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "pps", + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 7, + "w": 6, + "x": 12, + "y": 7 + }, + "hiddenSeries": false, + "id": 10, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.2.0", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "tcp:90", + "yaxis": 2 + }, + { + "alias": "tcp:99 ", + "yaxis": 2 + }, + { + "alias": "tcp:50", + "yaxis": 2 + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum(rate(coredns_dns_request_size_bytes_bucket{job=\"coredns\",instance=~\"$instance\",proto=\"udp\"}[5m])) by (le,proto))", + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{"{{proto}}"}}:99 ", + "refId": "A", + "step": 60 + }, + { + "expr": "histogram_quantile(0.90, sum(rate(coredns_dns_request_size_bytes_bucket{job=\"coredns\",instance=~\"$instance\",proto=\"udp\"}[5m])) by (le,proto))", + "intervalFactor": 2, + "legendFormat": "{{"{{proto}}"}}:90", + "refId": "B", + "step": 60 + }, + { + "expr": "histogram_quantile(0.50, sum(rate(coredns_dns_request_size_bytes_bucket{job=\"coredns\",instance=~\"$instance\",proto=\"udp\"}[5m])) by (le,proto))", + "intervalFactor": 2, + "legendFormat": "{{"{{proto}}"}}:50", + "refId": "C", + "step": 60 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Requests (size, udp)", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 7, + "w": 6, + "x": 18, + "y": 7 + }, + "hiddenSeries": false, + "id": 12, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.2.0", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "tcp:90", + "yaxis": 1 + }, + { + "alias": "tcp:99 ", + "yaxis": 1 + }, + { + "alias": "tcp:50", + "yaxis": 1 + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum(rate(coredns_dns_request_size_bytes_bucket{job=\"coredns\",instance=~\"$instance\",proto=\"tcp\"}[5m])) by (le,proto))", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{"{{proto}}"}}:99 ", + "refId": "A", + "step": 60 + }, + { + "expr": "histogram_quantile(0.90, sum(rate(coredns_dns_request_size_bytes_bucket{job=\"coredns\",instance=~\"$instance\",proto=\"tcp\"}[5m])) by (le,proto))", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{"{{proto}}"}}:90", + "refId": "B", + "step": 60 + }, + { + "expr": "histogram_quantile(0.50, sum(rate(coredns_dns_request_size_bytes_bucket{job=\"coredns\",instance=~\"$instance\",proto=\"tcp\"}[5m])) by (le,proto))", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{"{{proto}}"}}:50", + "refId": "C", + "step": 60 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Requests (size,tcp)", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 14 + }, + "hiddenSeries": false, + "id": 14, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.2.0", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(coredns_dns_response_rcode_count_total{job=\"coredns\",instance=~\"$instance\"}[5m])) by (rcode) or\nsum(rate(coredns_dns_responses_total{job=\"coredns\",instance=~\"$instance\"}[5m])) by (rcode)", + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{"{{rcode}}"}}", + "refId": "A", + "step": 40 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Responses (by rcode)", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "pps", + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 14 + }, + "hiddenSeries": false, + "id": 32, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.2.0", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum(rate(coredns_dns_request_duration_seconds_bucket{job=\"coredns\",instance=~\"$instance\"}[5m])) by (le, job))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "99%", + "refId": "A", + "step": 40 + }, + { + "expr": "histogram_quantile(0.90, sum(rate(coredns_dns_request_duration_seconds_bucket{job=\"coredns\",instance=~\"$instance\"}[5m])) by (le))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "90%", + "refId": "B", + "step": 40 + }, + { + "expr": "histogram_quantile(0.50, sum(rate(coredns_dns_request_duration_seconds_bucket{job=\"coredns\",instance=~\"$instance\"}[5m])) by (le))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "50%", + "refId": "C", + "step": 40 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Responses (duration)", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "s", + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 21 + }, + "hiddenSeries": false, + "id": 18, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.2.0", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "udp:50%", + "yaxis": 1 + }, + { + "alias": "tcp:50%", + "yaxis": 2 + }, + { + "alias": "tcp:90%", + "yaxis": 2 + }, + { + "alias": "tcp:99%", + "yaxis": 2 + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum(rate(coredns_dns_response_size_bytes_bucket{job=\"coredns\",instance=~\"$instance\",proto=\"udp\"}[5m])) by (le,proto)) ", + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{"{{proto}}"}}:99%", + "refId": "A", + "step": 40 + }, + { + "expr": "histogram_quantile(0.90, sum(rate(coredns_dns_response_size_bytes_bucket{job=\"coredns\",instance=~\"$instance\",proto=\"udp\"}[5m])) by (le,proto)) ", + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{"{{proto}}"}}:90%", + "refId": "B", + "step": 40 + }, + { + "expr": "histogram_quantile(0.50, sum(rate(coredns_dns_response_size_bytes_bucket{job=\"coredns\",instance=~\"$instance\",proto=\"udp\"}[5m])) by (le,proto)) ", + "hide": false, + "intervalFactor": 2, + "legendFormat": "{{"{{proto}}"}}:50%", + "metric": "", + "refId": "C", + "step": 40 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Responses (size, udp)", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 21 + }, + "hiddenSeries": false, + "id": 20, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.2.0", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "udp:50%", + "yaxis": 1 + }, + { + "alias": "tcp:50%", + "yaxis": 1 + }, + { + "alias": "tcp:90%", + "yaxis": 1 + }, + { + "alias": "tcp:99%", + "yaxis": 1 + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum(rate(coredns_dns_response_size_bytes_bucket{job=\"coredns\",instance=~\"$instance\",proto=\"tcp\"}[5m])) by (le,proto)) ", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{"{{proto}}"}}:99%", + "refId": "A", + "step": 40 + }, + { + "expr": "histogram_quantile(0.90, sum(rate(coredns_dns_response_size_bytes_bucket{job=\"coredns\",instance=~\"$instance\",proto=\"tcp\"}[5m])) by (le,proto)) ", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{"{{proto}}"}}:90%", + "refId": "B", + "step": 40 + }, + { + "expr": "histogram_quantile(0.50, sum(rate(coredns_dns_response_size_bytes_bucket{job=\"coredns\",instance=~\"$instance\",proto=\"tcp\"}[5m])) by (le, proto)) ", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{"{{proto}}"}}:50%", + "metric": "", + "refId": "C", + "step": 40 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Responses (size, tcp)", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 28 + }, + "hiddenSeries": false, + "id": 22, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.2.0", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(coredns_cache_size{job=\"coredns\",instance=~\"$instance\"}) by (type) or\nsum(coredns_cache_entries{job=\"coredns\",instance=~\"$instance\"}) by (type)", + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{"{{type}}"}}", + "refId": "A", + "step": 40 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Cache (size)", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "decbytes", + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 28 + }, + "hiddenSeries": false, + "id": 24, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.2.0", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "misses", + "yaxis": 2 + } + ], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(coredns_cache_hits_total{job=\"coredns\",instance=~\"$instance\"}[5m])) by (type)", + "hide": false, + "intervalFactor": 2, + "legendFormat": "hits:{{"{{type}}"}}", + "refId": "A", + "step": 40 + }, + { + "expr": "sum(rate(coredns_cache_misses_total{job=\"coredns\",instance=~\"$instance\"}[5m])) by (type)", + "hide": false, + "intervalFactor": 2, + "legendFormat": "misses", + "refId": "B", + "step": 40 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Cache (hitrate)", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "pps", + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "pps", + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + } + ], + "refresh": "10s", + "schemaVersion": 26, + "style": "dark", + "tags": [ + "dns", + "coredns" + ], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "includeAll": false, + "label": "Data Source", + "multi": false, + "name": "datasource", + "options": [], + "query": "prometheus", + "queryValue": "", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "type": "datasource" + }, + { + "allValue": ".*", + "current": { + "selected": true, + "text": "All", + "value": "$__all" + }, + "datasource": "$datasource", + "definition": "label_values(up{job=\"coredns\"}, instance)", + "hide": 0, + "includeAll": true, + "label": "Instance", + "multi": false, + "name": "instance", + "options": [], + "query": "label_values(up{job=\"coredns\"}, instance)", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 3, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-3h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ] + }, + "timezone": "{{ .Values.grafana.defaultDashboardsTimezone }}", + "title": "CoreDNS", + "uid": "vkQ0UHxik", + "version": 2 + } +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/grafana/dashboards-1.14/k8s-resources-cluster.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/grafana/dashboards-1.14/k8s-resources-cluster.yaml new file mode 100644 index 00000000000..93ee57db931 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/grafana/dashboards-1.14/k8s-resources-cluster.yaml @@ -0,0 +1,3088 @@ +{{- /* +Generated from 'k8s-resources-cluster' from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/a8ba97a150c75be42010c75d10b720c55e182f1a/manifests/grafana-dashboardDefinitions.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (or .Values.grafana.enabled .Values.grafana.forceDeployDashboards) (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.grafana.defaultDashboardsEnabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: {{ .Values.grafana.defaultDashboards.namespace }} + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" $) "k8s-resources-cluster" | trunc 63 | trimSuffix "-" }} + annotations: +{{ toYaml .Values.grafana.sidecar.dashboards.annotations | indent 4 }} + labels: + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: {{ ternary $.Values.grafana.sidecar.dashboards.labelValue "1" (not (empty $.Values.grafana.sidecar.dashboards.labelValue)) | quote }} + {{- end }} + app: {{ template "kube-prometheus-stack.name" $ }}-grafana +{{ include "kube-prometheus-stack.labels" $ | indent 4 }} +data: + k8s-resources-cluster.json: |- + { + "annotations": { + "list": [ + + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "hideControls": false, + "links": [ + + ], + "refresh": "10s", + "rows": [ + { + "collapse": false, + "height": "100px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "format": "percentunit", + "id": 1, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 2, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "cluster:node_cpu:ratio_rate5m{cluster=\"$cluster\"}", + "format": "time_series", + "instant": true, + "intervalFactor": 2, + "refId": "A" + } + ], + "thresholds": "70,80", + "timeFrom": null, + "timeShift": null, + "title": "CPU Utilisation", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "singlestat", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "format": "percentunit", + "id": 2, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 2, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(namespace_cpu:kube_pod_container_resource_requests:sum{cluster=\"$cluster\"}) / sum(kube_node_status_allocatable{job=\"kube-state-metrics\",resource=\"cpu\",cluster=\"$cluster\"})", + "format": "time_series", + "instant": true, + "intervalFactor": 2, + "refId": "A" + } + ], + "thresholds": "70,80", + "timeFrom": null, + "timeShift": null, + "title": "CPU Requests Commitment", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "singlestat", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "format": "percentunit", + "id": 3, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 2, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(namespace_cpu:kube_pod_container_resource_limits:sum{cluster=\"$cluster\"}) / sum(kube_node_status_allocatable{job=\"kube-state-metrics\",resource=\"cpu\",cluster=\"$cluster\"})", + "format": "time_series", + "instant": true, + "intervalFactor": 2, + "refId": "A" + } + ], + "thresholds": "70,80", + "timeFrom": null, + "timeShift": null, + "title": "CPU Limits Commitment", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "singlestat", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "format": "percentunit", + "id": 4, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 2, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "1 - sum(:node_memory_MemAvailable_bytes:sum{cluster=\"$cluster\"}) / sum(node_memory_MemTotal_bytes{job=\"node-exporter\",cluster=\"$cluster\"})", + "format": "time_series", + "instant": true, + "intervalFactor": 2, + "refId": "A" + } + ], + "thresholds": "70,80", + "timeFrom": null, + "timeShift": null, + "title": "Memory Utilisation", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "singlestat", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "format": "percentunit", + "id": 5, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 2, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(namespace_memory:kube_pod_container_resource_requests:sum{cluster=\"$cluster\"}) / sum(kube_node_status_allocatable{job=\"kube-state-metrics\",resource=\"memory\",cluster=\"$cluster\"})", + "format": "time_series", + "instant": true, + "intervalFactor": 2, + "refId": "A" + } + ], + "thresholds": "70,80", + "timeFrom": null, + "timeShift": null, + "title": "Memory Requests Commitment", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "singlestat", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "format": "percentunit", + "id": 6, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 2, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(namespace_memory:kube_pod_container_resource_limits:sum{cluster=\"$cluster\"}) / sum(kube_node_status_allocatable{job=\"kube-state-metrics\",resource=\"memory\",cluster=\"$cluster\"})", + "format": "time_series", + "instant": true, + "intervalFactor": 2, + "refId": "A" + } + ], + "thresholds": "70,80", + "timeFrom": null, + "timeShift": null, + "title": "Memory Limits Commitment", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "singlestat", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "Headlines", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 7, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"$cluster\"}) by (namespace)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}namespace{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "CPU Usage", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "CPU", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 8, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "styles": [ + { + "alias": "Time", + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "pattern": "Time", + "type": "hidden" + }, + { + "alias": "Pods", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 0, + "link": true, + "linkTargetBlank": false, + "linkTooltip": "Drill down to pods", + "linkUrl": "d/85a562078cdf77779eaa1add43ccec1e/k8s-resources-namespace?var-datasource=$datasource&var-cluster=$cluster&var-namespace=$__cell_1", + "pattern": "Value #A", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "Workloads", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 0, + "link": true, + "linkTargetBlank": false, + "linkTooltip": "Drill down to workloads", + "linkUrl": "d/a87fb0d919ec0ea5f6543124e16c42a5/k8s-resources-workloads-namespace?var-datasource=$datasource&var-cluster=$cluster&var-namespace=$__cell_1", + "pattern": "Value #B", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "CPU Usage", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #C", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "CPU Requests", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #D", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "CPU Requests %", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #E", + "thresholds": [ + + ], + "type": "number", + "unit": "percentunit" + }, + { + "alias": "CPU Limits", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #F", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "CPU Limits %", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #G", + "thresholds": [ + + ], + "type": "number", + "unit": "percentunit" + }, + { + "alias": "Namespace", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": true, + "linkTargetBlank": false, + "linkTooltip": "Drill down to pods", + "linkUrl": "d/85a562078cdf77779eaa1add43ccec1e/k8s-resources-namespace?var-datasource=$datasource&var-cluster=$cluster&var-namespace=$__cell", + "pattern": "namespace", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "pattern": "/.*/", + "thresholds": [ + + ], + "type": "string", + "unit": "short" + } + ], + "targets": [ + { + "expr": "sum(kube_pod_owner{job=\"kube-state-metrics\", cluster=\"$cluster\"}) by (namespace)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "A", + "step": 10 + }, + { + "expr": "count(avg(namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\"}) by (workload, namespace)) by (namespace)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "B", + "step": 10 + }, + { + "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"$cluster\"}) by (namespace)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "C", + "step": 10 + }, + { + "expr": "sum(namespace_cpu:kube_pod_container_resource_requests:sum{cluster=\"$cluster\"}) by (namespace)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "D", + "step": 10 + }, + { + "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"$cluster\"}) by (namespace) / sum(namespace_cpu:kube_pod_container_resource_requests:sum{cluster=\"$cluster\"}) by (namespace)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "E", + "step": 10 + }, + { + "expr": "sum(namespace_cpu:kube_pod_container_resource_limits:sum{cluster=\"$cluster\"}) by (namespace)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "F", + "step": 10 + }, + { + "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"$cluster\"}) by (namespace) / sum(namespace_cpu:kube_pod_container_resource_limits:sum{cluster=\"$cluster\"}) by (namespace)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "G", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "CPU Quota", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "transform": "table", + "type": "table", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "CPU Quota", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 9, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(container_memory_rss{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", container!=\"\"}) by (namespace)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}namespace{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Memory Usage (w/o cache)", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Memory", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 10, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "styles": [ + { + "alias": "Time", + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "pattern": "Time", + "type": "hidden" + }, + { + "alias": "Pods", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 0, + "link": true, + "linkTargetBlank": false, + "linkTooltip": "Drill down to pods", + "linkUrl": "d/85a562078cdf77779eaa1add43ccec1e/k8s-resources-namespace?var-datasource=$datasource&var-cluster=$cluster&var-namespace=$__cell_1", + "pattern": "Value #A", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "Workloads", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 0, + "link": true, + "linkTargetBlank": false, + "linkTooltip": "Drill down to workloads", + "linkUrl": "d/a87fb0d919ec0ea5f6543124e16c42a5/k8s-resources-workloads-namespace?var-datasource=$datasource&var-cluster=$cluster&var-namespace=$__cell_1", + "pattern": "Value #B", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "Memory Usage", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #C", + "thresholds": [ + + ], + "type": "number", + "unit": "bytes" + }, + { + "alias": "Memory Requests", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #D", + "thresholds": [ + + ], + "type": "number", + "unit": "bytes" + }, + { + "alias": "Memory Requests %", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #E", + "thresholds": [ + + ], + "type": "number", + "unit": "percentunit" + }, + { + "alias": "Memory Limits", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #F", + "thresholds": [ + + ], + "type": "number", + "unit": "bytes" + }, + { + "alias": "Memory Limits %", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #G", + "thresholds": [ + + ], + "type": "number", + "unit": "percentunit" + }, + { + "alias": "Namespace", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": true, + "linkTargetBlank": false, + "linkTooltip": "Drill down to pods", + "linkUrl": "d/85a562078cdf77779eaa1add43ccec1e/k8s-resources-namespace?var-datasource=$datasource&var-cluster=$cluster&var-namespace=$__cell", + "pattern": "namespace", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "pattern": "/.*/", + "thresholds": [ + + ], + "type": "string", + "unit": "short" + } + ], + "targets": [ + { + "expr": "sum(kube_pod_owner{job=\"kube-state-metrics\", cluster=\"$cluster\"}) by (namespace)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "A", + "step": 10 + }, + { + "expr": "count(avg(namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\"}) by (workload, namespace)) by (namespace)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "B", + "step": 10 + }, + { + "expr": "sum(container_memory_rss{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", container!=\"\"}) by (namespace)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "C", + "step": 10 + }, + { + "expr": "sum(namespace_memory:kube_pod_container_resource_requests:sum{cluster=\"$cluster\"}) by (namespace)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "D", + "step": 10 + }, + { + "expr": "sum(container_memory_rss{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", container!=\"\"}) by (namespace) / sum(namespace_memory:kube_pod_container_resource_requests:sum{cluster=\"$cluster\"}) by (namespace)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "E", + "step": 10 + }, + { + "expr": "sum(namespace_memory:kube_pod_container_resource_limits:sum{cluster=\"$cluster\"}) by (namespace)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "F", + "step": 10 + }, + { + "expr": "sum(container_memory_rss{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", container!=\"\"}) by (namespace) / sum(namespace_memory:kube_pod_container_resource_limits:sum{cluster=\"$cluster\"}) by (namespace)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "G", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Requests by Namespace", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "transform": "table", + "type": "table", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Memory Requests", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 11, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "styles": [ + { + "alias": "Time", + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "pattern": "Time", + "type": "hidden" + }, + { + "alias": "Current Receive Bandwidth", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #A", + "thresholds": [ + + ], + "type": "number", + "unit": "Bps" + }, + { + "alias": "Current Transmit Bandwidth", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #B", + "thresholds": [ + + ], + "type": "number", + "unit": "Bps" + }, + { + "alias": "Rate of Received Packets", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #C", + "thresholds": [ + + ], + "type": "number", + "unit": "pps" + }, + { + "alias": "Rate of Transmitted Packets", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #D", + "thresholds": [ + + ], + "type": "number", + "unit": "pps" + }, + { + "alias": "Rate of Received Packets Dropped", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #E", + "thresholds": [ + + ], + "type": "number", + "unit": "pps" + }, + { + "alias": "Rate of Transmitted Packets Dropped", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #F", + "thresholds": [ + + ], + "type": "number", + "unit": "pps" + }, + { + "alias": "Namespace", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": true, + "linkTargetBlank": false, + "linkTooltip": "Drill down to pods", + "linkUrl": "d/85a562078cdf77779eaa1add43ccec1e/k8s-resources-namespace?var-datasource=$datasource&var-cluster=$cluster&var-namespace=$__cell", + "pattern": "namespace", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "pattern": "/.*/", + "thresholds": [ + + ], + "type": "string", + "unit": "short" + } + ], + "targets": [ + { + "expr": "sum(irate(container_network_receive_bytes_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=~\".+\"}[$__rate_interval])) by (namespace)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "A", + "step": 10 + }, + { + "expr": "sum(irate(container_network_transmit_bytes_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=~\".+\"}[$__rate_interval])) by (namespace)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "B", + "step": 10 + }, + { + "expr": "sum(irate(container_network_receive_packets_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=~\".+\"}[$__rate_interval])) by (namespace)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "C", + "step": 10 + }, + { + "expr": "sum(irate(container_network_transmit_packets_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=~\".+\"}[$__rate_interval])) by (namespace)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "D", + "step": 10 + }, + { + "expr": "sum(irate(container_network_receive_packets_dropped_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=~\".+\"}[$__rate_interval])) by (namespace)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "E", + "step": 10 + }, + { + "expr": "sum(irate(container_network_transmit_packets_dropped_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=~\".+\"}[$__rate_interval])) by (namespace)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "F", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Current Network Usage", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "transform": "table", + "type": "table", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Current Network Usage", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 12, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(container_network_receive_bytes_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=~\".+\"}[$__rate_interval])) by (namespace)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}namespace{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Receive Bandwidth", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 13, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(container_network_transmit_bytes_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=~\".+\"}[$__rate_interval])) by (namespace)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}namespace{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Transmit Bandwidth", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Bandwidth", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 14, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "avg(irate(container_network_receive_bytes_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=~\".+\"}[$__rate_interval])) by (namespace)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}namespace{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Average Container Bandwidth by Namespace: Received", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 15, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "avg(irate(container_network_transmit_bytes_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=~\".+\"}[$__rate_interval])) by (namespace)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}namespace{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Average Container Bandwidth by Namespace: Transmitted", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Average Container Bandwidth by Namespace", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 16, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(container_network_receive_packets_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=~\".+\"}[$__rate_interval])) by (namespace)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}namespace{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Received Packets", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 17, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(container_network_transmit_packets_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=~\".+\"}[$__rate_interval])) by (namespace)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}namespace{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Transmitted Packets", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Rate of Packets", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 18, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(container_network_receive_packets_dropped_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=~\".+\"}[$__rate_interval])) by (namespace)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}namespace{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Received Packets Dropped", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 19, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(container_network_transmit_packets_dropped_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=~\".+\"}[$__rate_interval])) by (namespace)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}namespace{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Transmitted Packets Dropped", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Rate of Packets Dropped", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "decimals": -1, + "fill": 10, + "id": 20, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "ceil(sum by(namespace) (rate(container_fs_reads_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", container!=\"\", device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\", cluster=\"$cluster\", namespace!=\"\"}[$__rate_interval]) + rate(container_fs_writes_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", container!=\"\", cluster=\"$cluster\", namespace!=\"\"}[$__rate_interval])))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}namespace{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "IOPS(Reads+Writes)", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 21, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum by(namespace) (rate(container_fs_reads_bytes_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", container!=\"\", device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\", cluster=\"$cluster\", namespace!=\"\"}[$__rate_interval]) + rate(container_fs_writes_bytes_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", container!=\"\", cluster=\"$cluster\", namespace!=\"\"}[$__rate_interval]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}namespace{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "ThroughPut(Read+Write)", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Storage IO", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 22, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "sort": { + "col": 4, + "desc": true + }, + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "styles": [ + { + "alias": "Time", + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "pattern": "Time", + "type": "hidden" + }, + { + "alias": "IOPS(Reads)", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": -1, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #A", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "IOPS(Writes)", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": -1, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #B", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "IOPS(Reads + Writes)", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": -1, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #C", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "Throughput(Read)", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #D", + "thresholds": [ + + ], + "type": "number", + "unit": "Bps" + }, + { + "alias": "Throughput(Write)", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #E", + "thresholds": [ + + ], + "type": "number", + "unit": "Bps" + }, + { + "alias": "Throughput(Read + Write)", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #F", + "thresholds": [ + + ], + "type": "number", + "unit": "Bps" + }, + { + "alias": "Namespace", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": true, + "linkTargetBlank": false, + "linkTooltip": "Drill down to pods", + "linkUrl": "d/85a562078cdf77779eaa1add43ccec1e/k8s-resources-namespace?var-datasource=$datasource&var-cluster=$cluster&var-namespace=$__cell", + "pattern": "namespace", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "pattern": "/.*/", + "thresholds": [ + + ], + "type": "string", + "unit": "short" + } + ], + "targets": [ + { + "expr": "sum by(namespace) (rate(container_fs_reads_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\", container!=\"\", cluster=\"$cluster\", namespace!=\"\"}[$__rate_interval]))", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "A", + "step": 10 + }, + { + "expr": "sum by(namespace) (rate(container_fs_writes_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\", container!=\"\", cluster=\"$cluster\", namespace!=\"\"}[$__rate_interval]))", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "B", + "step": 10 + }, + { + "expr": "sum by(namespace) (rate(container_fs_reads_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\", container!=\"\", cluster=\"$cluster\", namespace!=\"\"}[$__rate_interval]) + rate(container_fs_writes_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\", container!=\"\", cluster=\"$cluster\", namespace!=\"\"}[$__rate_interval]))", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "C", + "step": 10 + }, + { + "expr": "sum by(namespace) (rate(container_fs_reads_bytes_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\", container!=\"\", cluster=\"$cluster\", namespace!=\"\"}[$__rate_interval]))", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "D", + "step": 10 + }, + { + "expr": "sum by(namespace) (rate(container_fs_writes_bytes_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\", container!=\"\", cluster=\"$cluster\", namespace!=\"\"}[$__rate_interval]))", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "E", + "step": 10 + }, + { + "expr": "sum by(namespace) (rate(container_fs_reads_bytes_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\", container!=\"\", cluster=\"$cluster\", namespace!=\"\"}[$__rate_interval]) + rate(container_fs_writes_bytes_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\", container!=\"\", cluster=\"$cluster\", namespace!=\"\"}[$__rate_interval]))", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "F", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Current Storage IO", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "transform": "table", + "type": "table", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Storage IO - Distribution", + "titleSize": "h6" + } + ], + "schemaVersion": 14, + "style": "dark", + "tags": [ + "kubernetes-mixin" + ], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": "Data Source", + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": null, + "current": { + "text": "", + "value": "" + }, + "datasource": "$datasource", + "hide": {{ if .Values.grafana.sidecar.dashboards.multicluster.global.enabled }}0{{ else }}2{{ end }}, + "includeAll": false, + "label": null, + "multi": false, + "name": "cluster", + "options": [ + + ], + "query": "label_values(up{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\"}, cluster)", + "refresh": 2, + "regex": "", + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "{{ .Values.grafana.defaultDashboardsTimezone }}", + "title": "Kubernetes / Compute Resources / Cluster", + "uid": "efa86fd1d0c121a26444b636a3f509a8", + "version": 0 + } +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/grafana/dashboards-1.14/k8s-resources-multicluster.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/grafana/dashboards-1.14/k8s-resources-multicluster.yaml new file mode 100644 index 00000000000..9c295831a59 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/grafana/dashboards-1.14/k8s-resources-multicluster.yaml @@ -0,0 +1,24 @@ +{{- /* +Generated from 'k8s-resources-multicluster' from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/a8ba97a150c75be42010c75d10b720c55e182f1a/manifests/grafana-dashboardDefinitions.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (or .Values.grafana.enabled .Values.grafana.forceDeployDashboards) (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.grafana.defaultDashboardsEnabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: {{ template "kube-prometheus-stack-grafana.namespace" . }} + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" $) "k8s-resources-multicluster" | trunc 63 | trimSuffix "-" }} + annotations: +{{ toYaml .Values.grafana.sidecar.dashboards.annotations | indent 4 }} + labels: + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: {{ ternary $.Values.grafana.sidecar.dashboards.labelValue "1" (not (empty $.Values.grafana.sidecar.dashboards.labelValue)) | quote }} + {{- end }} + app: {{ template "kube-prometheus-stack.name" $ }}-grafana +{{ include "kube-prometheus-stack.labels" $ | indent 4 }} +data: + k8s-resources-multicluster.json: |- + {{`{"annotations":{"list":[]},"editable":`}}{{ .Values.grafana.defaultDashboardsEditable }}{{`,"gnetId":null,"graphTooltip":0,"hideControls":false,"links":[],"refresh":"10s","rows":[{"collapse":false,"height":"100px","panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":1,"format":"percentunit","id":1,"interval":"1m","legend":{"alignAsTable":true,"avg":false,"current":false,"max":false,"min":false,"rightSide":true,"show":true,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":2,"stack":false,"steppedLine":false,"targets":[{"expr":"cluster:node_cpu:ratio_rate5m","format":"time_series","instant":true,"refId":"A"}],"thresholds":"70,80","timeFrom":null,"timeShift":null,"title":"CPU Utilisation","tooltip":{"shared":false,"sort":2,"value_type":"individual"},"type":"singlestat","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"short","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":1,"format":"percentunit","id":2,"interval":"1m","legend":{"alignAsTable":true,"avg":false,"current":false,"max":false,"min":false,"rightSide":true,"show":true,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":2,"stack":false,"steppedLine":false,"targets":[{"expr":"sum(kube_pod_container_resource_requests{job=\"kube-state-metrics\", resource=\"cpu\"}) / sum(kube_node_status_allocatable{job=\"kube-state-metrics\", resource=\"cpu\"})","format":"time_series","instant":true,"refId":"A"}],"thresholds":"70,80","timeFrom":null,"timeShift":null,"title":"CPU Requests Commitment","tooltip":{"shared":false,"sort":2,"value_type":"individual"},"type":"singlestat","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"short","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":1,"format":"percentunit","id":3,"interval":"1m","legend":{"alignAsTable":true,"avg":false,"current":false,"max":false,"min":false,"rightSide":true,"show":true,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":2,"stack":false,"steppedLine":false,"targets":[{"expr":"sum(kube_pod_container_resource_limits{job=\"kube-state-metrics\", resource=\"cpu\"}) / sum(kube_node_status_allocatable{job=\"kube-state-metrics\", resource=\"cpu\"})","format":"time_series","instant":true,"refId":"A"}],"thresholds":"70,80","timeFrom":null,"timeShift":null,"title":"CPU Limits Commitment","tooltip":{"shared":false,"sort":2,"value_type":"individual"},"type":"singlestat","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"short","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":1,"format":"percentunit","id":4,"interval":"1m","legend":{"alignAsTable":true,"avg":false,"current":false,"max":false,"min":false,"rightSide":true,"show":true,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":2,"stack":false,"steppedLine":false,"targets":[{"expr":"1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(node_memory_MemTotal_bytes{job=\"node-exporter\"})","format":"time_series","instant":true,"refId":"A"}],"thresholds":"70,80","timeFrom":null,"timeShift":null,"title":"Memory Utilisation","tooltip":{"shared":false,"sort":2,"value_type":"individual"},"type":"singlestat","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"short","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":1,"format":"percentunit","id":5,"interval":"1m","legend":{"alignAsTable":true,"avg":false,"current":false,"max":false,"min":false,"rightSide":true,"show":true,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":2,"stack":false,"steppedLine":false,"targets":[{"expr":"sum(kube_pod_container_resource_requests{job=\"kube-state-metrics\", resource=\"memory\"}) / sum(kube_node_status_allocatable{job=\"kube-state-metrics\", resource=\"memory\"})","format":"time_series","instant":true,"refId":"A"}],"thresholds":"70,80","timeFrom":null,"timeShift":null,"title":"Memory Requests Commitment","tooltip":{"shared":false,"sort":2,"value_type":"individual"},"type":"singlestat","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"short","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":1,"format":"percentunit","id":6,"interval":"1m","legend":{"alignAsTable":true,"avg":false,"current":false,"max":false,"min":false,"rightSide":true,"show":true,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":2,"stack":false,"steppedLine":false,"targets":[{"expr":"sum(kube_pod_container_resource_limits{job=\"kube-state-metrics\", resource=\"memory\"}) / sum(kube_node_status_allocatable{job=\"kube-state-metrics\", resource=\"memory\"})","format":"time_series","instant":true,"refId":"A"}],"thresholds":"70,80","timeFrom":null,"timeShift":null,"title":"Memory Limits Commitment","tooltip":{"shared":false,"sort":2,"value_type":"individual"},"type":"singlestat","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"short","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":false,"title":"Headlines","titleSize":"h6"},{"collapse":false,"height":"250px","panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":0,"id":7,"interval":"1m","legend":{"alignAsTable":true,"avg":false,"current":false,"max":false,"min":false,"rightSide":true,"show":true,"total":false,"values":false},"lines":true,"linewidth":2,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":12,"stack":false,"steppedLine":false,"targets":[{"expr":"sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate) by (cluster)","format":"time_series","legendFormat":"{{cluster}}","legendLink":null}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"CPU Usage","tooltip":{"shared":false,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"short","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":true,"title":"CPU","titleSize":"h6"},{"collapse":false,"height":"250px","panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":1,"id":8,"interval":"1m","legend":{"alignAsTable":true,"avg":false,"current":false,"max":false,"min":false,"rightSide":true,"show":true,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":12,"stack":false,"steppedLine":false,"styles":[{"alias":"Time","dateFormat":"YYYY-MM-DD HH:mm:ss","pattern":"Time","type":"hidden"},{"alias":"CPU Usage","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #A","thresholds":[],"type":"number","unit":"short"},{"alias":"CPU Requests","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #B","thresholds":[],"type":"number","unit":"short"},{"alias":"CPU Requests %","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #C","thresholds":[],"type":"number","unit":"percentunit"},{"alias":"CPU Limits","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #D","thresholds":[],"type":"number","unit":"short"},{"alias":"CPU Limits %","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #E","thresholds":[],"type":"number","unit":"percentunit"},{"alias":"Cluster","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":true,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"/d/efa86fd1d0c121a26444b636a3f509a8/k8s-resources-cluster?var-datasource=$datasource&var-cluster=`}}{{ if .Values.grafana.sidecar.dashboards.enableNewTablePanelSyntax }}${__value.text}{{ else }}$__cell{{ end }}{{`","pattern":"cluster","thresholds":[],"type":"number","unit":"short"},{"alias":"","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"pattern":"/.*/","thresholds":[],"type":"string","unit":"short"}],"targets":[{"expr":"sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate) by (cluster)","format":"table","instant":true,"legendFormat":"","refId":"A"},{"expr":"sum(kube_pod_container_resource_requests{job=\"kube-state-metrics\", resource=\"cpu\"}) by (cluster)","format":"table","instant":true,"legendFormat":"","refId":"B"},{"expr":"sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate) by (cluster) / sum(kube_pod_container_resource_requests{job=\"kube-state-metrics\", resource=\"cpu\"}) by (cluster)","format":"table","instant":true,"legendFormat":"","refId":"C"},{"expr":"sum(kube_pod_container_resource_limits{job=\"kube-state-metrics\", resource=\"cpu\"}) by (cluster)","format":"table","instant":true,"legendFormat":"","refId":"D"},{"expr":"sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate) by (cluster) / sum(kube_pod_container_resource_limits{job=\"kube-state-metrics\", resource=\"cpu\"}) by (cluster)","format":"table","instant":true,"legendFormat":"","refId":"E"}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"CPU Quota","tooltip":{"shared":false,"sort":2,"value_type":"individual"},"transform":"table","type":"table","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"short","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":true,"title":"CPU Quota","titleSize":"h6"},{"collapse":false,"height":"250px","panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":0,"id":9,"interval":"1m","legend":{"alignAsTable":true,"avg":false,"current":false,"max":false,"min":false,"rightSide":true,"show":true,"total":false,"values":false},"lines":true,"linewidth":2,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":12,"stack":false,"steppedLine":false,"targets":[{"expr":"sum(container_memory_rss{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", container!=\"\"}) by (cluster)","format":"time_series","legendFormat":"{{cluster}}","legendLink":null}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Memory Usage (w/o cache)","tooltip":{"shared":false,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"bytes","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":true,"title":"Memory","titleSize":"h6"},{"collapse":false,"height":"250px","panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":1,"id":10,"interval":"1m","legend":{"alignAsTable":true,"avg":false,"current":false,"max":false,"min":false,"rightSide":true,"show":true,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":12,"stack":false,"steppedLine":false,"styles":[{"alias":"Time","dateFormat":"YYYY-MM-DD HH:mm:ss","pattern":"Time","type":"hidden"},{"alias":"Memory Usage","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #A","thresholds":[],"type":"number","unit":"bytes"},{"alias":"Memory Requests","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #B","thresholds":[],"type":"number","unit":"bytes"},{"alias":"Memory Requests %","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #C","thresholds":[],"type":"number","unit":"percentunit"},{"alias":"Memory Limits","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #D","thresholds":[],"type":"number","unit":"bytes"},{"alias":"Memory Limits %","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #E","thresholds":[],"type":"number","unit":"percentunit"},{"alias":"Cluster","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":true,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"/d/efa86fd1d0c121a26444b636a3f509a8/k8s-resources-cluster?var-datasource=$datasource&var-cluster=`}}{{ if .Values.grafana.sidecar.dashboards.enableNewTablePanelSyntax }}${__value.text}{{ else }}$__cell{{ end }}{{`","pattern":"cluster","thresholds":[],"type":"number","unit":"short"},{"alias":"","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"pattern":"/.*/","thresholds":[],"type":"string","unit":"short"}],"targets":[{"expr":"sum(container_memory_rss{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", container!=\"\"}) by (cluster)","format":"table","instant":true,"legendFormat":"","refId":"A"},{"expr":"sum(kube_pod_container_resource_requests{job=\"kube-state-metrics\", resource=\"memory\"}) by (cluster)","format":"table","instant":true,"legendFormat":"","refId":"B"},{"expr":"sum(container_memory_rss{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", container!=\"\"}) by (cluster) / sum(kube_pod_container_resource_requests{job=\"kube-state-metrics\", resource=\"memory\"}) by (cluster)","format":"table","instant":true,"legendFormat":"","refId":"C"},{"expr":"sum(kube_pod_container_resource_limits{job=\"kube-state-metrics\", resource=\"memory\"}) by (cluster)","format":"table","instant":true,"legendFormat":"","refId":"D"},{"expr":"sum(container_memory_rss{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", container!=\"\"}) by (cluster) / sum(kube_pod_container_resource_limits{job=\"kube-state-metrics\", resource=\"memory\"}) by (cluster)","format":"table","instant":true,"legendFormat":"","refId":"E"}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Requests by Cluster","tooltip":{"shared":false,"sort":2,"value_type":"individual"},"transform":"table","type":"table","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"short","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":true,"title":"Memory Requests","titleSize":"h6"}],"schemaVersion":14,"style":"dark","tags":["kubernetes-mixin"],"templating":{"list":[{"current":{"text":"default","value":"default"},"hide":0,"label":"Data source","name":"datasource","options":[],"query":"prometheus","refresh":1,"regex":"","type":"datasource"}]},"time":{"from":"now-1h","to":"now"},"timepicker":{"refresh_intervals":["5s","10s","30s","1m","5m","15m","30m","1h","2h","1d"],"time_options":["5m","15m","1h","6h","12h","24h","2d","7d","30d"]},"timezone": "`}}{{ .Values.grafana.defaultDashboardsTimezone }}{{`","title":"Kubernetes / Compute Resources / Multi-Cluster","uid":"b59e6c9f2fcbe2e16d77fc492374cc4f","version":0}`}} +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/grafana/dashboards-1.14/k8s-resources-namespace.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/grafana/dashboards-1.14/k8s-resources-namespace.yaml new file mode 100644 index 00000000000..1c32c9c02ef --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/grafana/dashboards-1.14/k8s-resources-namespace.yaml @@ -0,0 +1,2797 @@ +{{- /* +Generated from 'k8s-resources-namespace' from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/a8ba97a150c75be42010c75d10b720c55e182f1a/manifests/grafana-dashboardDefinitions.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (or .Values.grafana.enabled .Values.grafana.forceDeployDashboards) (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.grafana.defaultDashboardsEnabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: {{ .Values.grafana.defaultDashboards.namespace }} + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" $) "k8s-resources-namespace" | trunc 63 | trimSuffix "-" }} + annotations: +{{ toYaml .Values.grafana.sidecar.dashboards.annotations | indent 4 }} + labels: + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: {{ ternary $.Values.grafana.sidecar.dashboards.labelValue "1" (not (empty $.Values.grafana.sidecar.dashboards.labelValue)) | quote }} + {{- end }} + app: {{ template "kube-prometheus-stack.name" $ }}-grafana +{{ include "kube-prometheus-stack.labels" $ | indent 4 }} +data: + k8s-resources-namespace.json: |- + { + "annotations": { + "list": [ + + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "hideControls": false, + "links": [ + + ], + "refresh": "10s", + "rows": [ + { + "collapse": false, + "height": "100px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "format": "percentunit", + "id": 1, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 3, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"$cluster\", namespace=\"$namespace\"}) / sum(kube_pod_container_resource_requests{job=\"kube-state-metrics\", cluster=\"$cluster\", namespace=\"$namespace\", resource=\"cpu\"})", + "format": "time_series", + "instant": true, + "intervalFactor": 2, + "refId": "A" + } + ], + "thresholds": "70,80", + "timeFrom": null, + "timeShift": null, + "title": "CPU Utilisation (from requests)", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "singlestat", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "format": "percentunit", + "id": 2, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 3, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"$cluster\", namespace=\"$namespace\"}) / sum(kube_pod_container_resource_limits{job=\"kube-state-metrics\", cluster=\"$cluster\", namespace=\"$namespace\", resource=\"cpu\"})", + "format": "time_series", + "instant": true, + "intervalFactor": 2, + "refId": "A" + } + ], + "thresholds": "70,80", + "timeFrom": null, + "timeShift": null, + "title": "CPU Utilisation (from limits)", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "singlestat", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "format": "percentunit", + "id": 3, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 3, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(container_memory_working_set_bytes{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\",container!=\"\", image!=\"\"}) / sum(kube_pod_container_resource_requests{job=\"kube-state-metrics\", cluster=\"$cluster\", namespace=\"$namespace\", resource=\"memory\"})", + "format": "time_series", + "instant": true, + "intervalFactor": 2, + "refId": "A" + } + ], + "thresholds": "70,80", + "timeFrom": null, + "timeShift": null, + "title": "Memory Utilisation (from requests)", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "singlestat", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "format": "percentunit", + "id": 4, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 3, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(container_memory_working_set_bytes{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\",container!=\"\", image!=\"\"}) / sum(kube_pod_container_resource_limits{job=\"kube-state-metrics\", cluster=\"$cluster\", namespace=\"$namespace\", resource=\"memory\"})", + "format": "time_series", + "instant": true, + "intervalFactor": 2, + "refId": "A" + } + ], + "thresholds": "70,80", + "timeFrom": null, + "timeShift": null, + "title": "Memory Utilisation (from limits)", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "singlestat", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "Headlines", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 5, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "quota - requests", + "color": "#F2495C", + "dashes": true, + "fill": 0, + "hiddenSeries": true, + "hideTooltip": true, + "legend": true, + "linewidth": 2, + "stack": false + }, + { + "alias": "quota - limits", + "color": "#FF9830", + "dashes": true, + "fill": 0, + "hiddenSeries": true, + "hideTooltip": true, + "legend": true, + "linewidth": 2, + "stack": false + } + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "legendLink": null, + "step": 10 + }, + { + "expr": "scalar(kube_resourcequota{cluster=\"$cluster\", namespace=\"$namespace\", type=\"hard\",resource=\"requests.cpu\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "quota - requests", + "legendLink": null, + "step": 10 + }, + { + "expr": "scalar(kube_resourcequota{cluster=\"$cluster\", namespace=\"$namespace\", type=\"hard\",resource=\"limits.cpu\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "quota - limits", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "CPU Usage", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "CPU Usage", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 6, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "styles": [ + { + "alias": "Time", + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "pattern": "Time", + "type": "hidden" + }, + { + "alias": "CPU Usage", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #A", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "CPU Requests", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #B", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "CPU Requests %", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #C", + "thresholds": [ + + ], + "type": "number", + "unit": "percentunit" + }, + { + "alias": "CPU Limits", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #D", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "CPU Limits %", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #E", + "thresholds": [ + + ], + "type": "number", + "unit": "percentunit" + }, + { + "alias": "Pod", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": true, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "d/6581e46e4e5c7ba40a07646395ef7b23/k8s-resources-pod?var-datasource=$datasource&var-cluster=$cluster&var-namespace=$namespace&var-pod=$__cell", + "pattern": "pod", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "pattern": "/.*/", + "thresholds": [ + + ], + "type": "string", + "unit": "short" + } + ], + "targets": [ + { + "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "A", + "step": 10 + }, + { + "expr": "sum(cluster:namespace:pod_cpu:active:kube_pod_container_resource_requests{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "B", + "step": 10 + }, + { + "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod) / sum(cluster:namespace:pod_cpu:active:kube_pod_container_resource_requests{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "C", + "step": 10 + }, + { + "expr": "sum(cluster:namespace:pod_cpu:active:kube_pod_container_resource_limits{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "D", + "step": 10 + }, + { + "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod) / sum(cluster:namespace:pod_cpu:active:kube_pod_container_resource_limits{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "E", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "CPU Quota", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "transform": "table", + "type": "table", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "CPU Quota", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 7, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "quota - requests", + "color": "#F2495C", + "dashes": true, + "fill": 0, + "hiddenSeries": true, + "hideTooltip": true, + "legend": true, + "linewidth": 2, + "stack": false + }, + { + "alias": "quota - limits", + "color": "#FF9830", + "dashes": true, + "fill": 0, + "hiddenSeries": true, + "hideTooltip": true, + "legend": true, + "linewidth": 2, + "stack": false + } + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(container_memory_working_set_bytes{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\", container!=\"\", image!=\"\"}) by (pod)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "legendLink": null, + "step": 10 + }, + { + "expr": "scalar(kube_resourcequota{cluster=\"$cluster\", namespace=\"$namespace\", type=\"hard\",resource=\"requests.memory\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "quota - requests", + "legendLink": null, + "step": 10 + }, + { + "expr": "scalar(kube_resourcequota{cluster=\"$cluster\", namespace=\"$namespace\", type=\"hard\",resource=\"limits.memory\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "quota - limits", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Memory Usage (w/o cache)", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Memory Usage", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 8, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "styles": [ + { + "alias": "Time", + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "pattern": "Time", + "type": "hidden" + }, + { + "alias": "Memory Usage", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #A", + "thresholds": [ + + ], + "type": "number", + "unit": "bytes" + }, + { + "alias": "Memory Requests", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #B", + "thresholds": [ + + ], + "type": "number", + "unit": "bytes" + }, + { + "alias": "Memory Requests %", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #C", + "thresholds": [ + + ], + "type": "number", + "unit": "percentunit" + }, + { + "alias": "Memory Limits", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #D", + "thresholds": [ + + ], + "type": "number", + "unit": "bytes" + }, + { + "alias": "Memory Limits %", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #E", + "thresholds": [ + + ], + "type": "number", + "unit": "percentunit" + }, + { + "alias": "Memory Usage (RSS)", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #F", + "thresholds": [ + + ], + "type": "number", + "unit": "bytes" + }, + { + "alias": "Memory Usage (Cache)", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #G", + "thresholds": [ + + ], + "type": "number", + "unit": "bytes" + }, + { + "alias": "Memory Usage (Swap)", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #H", + "thresholds": [ + + ], + "type": "number", + "unit": "bytes" + }, + { + "alias": "Pod", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": true, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "d/6581e46e4e5c7ba40a07646395ef7b23/k8s-resources-pod?var-datasource=$datasource&var-cluster=$cluster&var-namespace=$namespace&var-pod=$__cell", + "pattern": "pod", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "pattern": "/.*/", + "thresholds": [ + + ], + "type": "string", + "unit": "short" + } + ], + "targets": [ + { + "expr": "sum(container_memory_working_set_bytes{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\",container!=\"\", image!=\"\"}) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "A", + "step": 10 + }, + { + "expr": "sum(cluster:namespace:pod_memory:active:kube_pod_container_resource_requests{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "B", + "step": 10 + }, + { + "expr": "sum(container_memory_working_set_bytes{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\",container!=\"\", image!=\"\"}) by (pod) / sum(cluster:namespace:pod_memory:active:kube_pod_container_resource_requests{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "C", + "step": 10 + }, + { + "expr": "sum(cluster:namespace:pod_memory:active:kube_pod_container_resource_limits{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "D", + "step": 10 + }, + { + "expr": "sum(container_memory_working_set_bytes{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\",container!=\"\", image!=\"\"}) by (pod) / sum(cluster:namespace:pod_memory:active:kube_pod_container_resource_limits{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "E", + "step": 10 + }, + { + "expr": "sum(container_memory_rss{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\",container!=\"\"}) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "F", + "step": 10 + }, + { + "expr": "sum(container_memory_cache{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\",container!=\"\"}) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "G", + "step": 10 + }, + { + "expr": "sum(container_memory_swap{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\",container!=\"\"}) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "H", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Memory Quota", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "transform": "table", + "type": "table", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Memory Quota", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 9, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "styles": [ + { + "alias": "Time", + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "pattern": "Time", + "type": "hidden" + }, + { + "alias": "Current Receive Bandwidth", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #A", + "thresholds": [ + + ], + "type": "number", + "unit": "Bps" + }, + { + "alias": "Current Transmit Bandwidth", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #B", + "thresholds": [ + + ], + "type": "number", + "unit": "Bps" + }, + { + "alias": "Rate of Received Packets", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #C", + "thresholds": [ + + ], + "type": "number", + "unit": "pps" + }, + { + "alias": "Rate of Transmitted Packets", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #D", + "thresholds": [ + + ], + "type": "number", + "unit": "pps" + }, + { + "alias": "Rate of Received Packets Dropped", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #E", + "thresholds": [ + + ], + "type": "number", + "unit": "pps" + }, + { + "alias": "Rate of Transmitted Packets Dropped", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #F", + "thresholds": [ + + ], + "type": "number", + "unit": "pps" + }, + { + "alias": "Pod", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": true, + "linkTargetBlank": false, + "linkTooltip": "Drill down to pods", + "linkUrl": "d/6581e46e4e5c7ba40a07646395ef7b23/k8s-resources-pod?var-datasource=$datasource&var-cluster=$cluster&var-namespace=$namespace&var-pod=$__cell", + "pattern": "pod", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "pattern": "/.*/", + "thresholds": [ + + ], + "type": "string", + "unit": "short" + } + ], + "targets": [ + { + "expr": "sum(irate(container_network_receive_bytes_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "A", + "step": 10 + }, + { + "expr": "sum(irate(container_network_transmit_bytes_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "B", + "step": 10 + }, + { + "expr": "sum(irate(container_network_receive_packets_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "C", + "step": 10 + }, + { + "expr": "sum(irate(container_network_transmit_packets_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "D", + "step": 10 + }, + { + "expr": "sum(irate(container_network_receive_packets_dropped_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "E", + "step": 10 + }, + { + "expr": "sum(irate(container_network_transmit_packets_dropped_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "F", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Current Network Usage", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "transform": "table", + "type": "table", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Current Network Usage", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 10, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(container_network_receive_bytes_total{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (pod)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Receive Bandwidth", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 11, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(container_network_transmit_bytes_total{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (pod)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Transmit Bandwidth", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Bandwidth", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 12, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(container_network_receive_packets_total{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (pod)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Received Packets", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 13, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(container_network_transmit_packets_total{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (pod)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Transmitted Packets", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Rate of Packets", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 14, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(container_network_receive_packets_dropped_total{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (pod)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Received Packets Dropped", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 15, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(container_network_transmit_packets_dropped_total{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (pod)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Transmitted Packets Dropped", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Rate of Packets Dropped", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "decimals": -1, + "fill": 10, + "id": 16, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "ceil(sum by(pod) (rate(container_fs_reads_total{container!=\"\", device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval]) + rate(container_fs_writes_total{container!=\"\", device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "IOPS(Reads+Writes)", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 17, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum by(pod) (rate(container_fs_reads_bytes_total{container!=\"\", device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval]) + rate(container_fs_writes_bytes_total{container!=\"\", device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "ThroughPut(Read+Write)", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Storage IO", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 18, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "sort": { + "col": 4, + "desc": true + }, + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "styles": [ + { + "alias": "Time", + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "pattern": "Time", + "type": "hidden" + }, + { + "alias": "IOPS(Reads)", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": -1, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #A", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "IOPS(Writes)", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": -1, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #B", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "IOPS(Reads + Writes)", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": -1, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #C", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "Throughput(Read)", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #D", + "thresholds": [ + + ], + "type": "number", + "unit": "Bps" + }, + { + "alias": "Throughput(Write)", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #E", + "thresholds": [ + + ], + "type": "number", + "unit": "Bps" + }, + { + "alias": "Throughput(Read + Write)", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #F", + "thresholds": [ + + ], + "type": "number", + "unit": "Bps" + }, + { + "alias": "Pod", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": true, + "linkTargetBlank": false, + "linkTooltip": "Drill down to pods", + "linkUrl": "d/6581e46e4e5c7ba40a07646395ef7b23/k8s-resources-pod?var-datasource=$datasource&var-cluster=$cluster&var-namespace=$namespace&var-pod=$__cell", + "pattern": "pod", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "pattern": "/.*/", + "thresholds": [ + + ], + "type": "string", + "unit": "short" + } + ], + "targets": [ + { + "expr": "sum by(pod) (rate(container_fs_reads_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\", container!=\"\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval]))", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "A", + "step": 10 + }, + { + "expr": "sum by(pod) (rate(container_fs_writes_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\", container!=\"\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval]))", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "B", + "step": 10 + }, + { + "expr": "sum by(pod) (rate(container_fs_reads_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\", container!=\"\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval]) + rate(container_fs_writes_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\", container!=\"\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval]))", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "C", + "step": 10 + }, + { + "expr": "sum by(pod) (rate(container_fs_reads_bytes_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\", container!=\"\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval]))", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "D", + "step": 10 + }, + { + "expr": "sum by(pod) (rate(container_fs_writes_bytes_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\", container!=\"\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval]))", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "E", + "step": 10 + }, + { + "expr": "sum by(pod) (rate(container_fs_reads_bytes_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\", container!=\"\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval]) + rate(container_fs_writes_bytes_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\", container!=\"\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval]))", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "F", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Current Storage IO", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "transform": "table", + "type": "table", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Storage IO - Distribution", + "titleSize": "h6" + } + ], + "schemaVersion": 14, + "style": "dark", + "tags": [ + "kubernetes-mixin" + ], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": "Data Source", + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": null, + "current": { + "text": "", + "value": "" + }, + "datasource": "$datasource", + "hide": {{ if .Values.grafana.sidecar.dashboards.multicluster.global.enabled }}0{{ else }}2{{ end }}, + "includeAll": false, + "label": null, + "multi": false, + "name": "cluster", + "options": [ + + ], + "query": "label_values(up{job=\"kube-state-metrics\"}, cluster)", + "refresh": 2, + "regex": "", + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": { + "text": "", + "value": "" + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "namespace", + "options": [ + + ], + "query": "label_values(kube_namespace_status_phase{job=\"kube-state-metrics\", cluster=\"$cluster\"}, namespace)", + "refresh": 2, + "regex": "", + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "{{ .Values.grafana.defaultDashboardsTimezone }}", + "title": "Kubernetes / Compute Resources / Namespace (Pods)", + "uid": "85a562078cdf77779eaa1add43ccec1e", + "version": 0 + } +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/grafana/dashboards-1.14/k8s-resources-node.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/grafana/dashboards-1.14/k8s-resources-node.yaml new file mode 100644 index 00000000000..e60a42d7473 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/grafana/dashboards-1.14/k8s-resources-node.yaml @@ -0,0 +1,1026 @@ +{{- /* +Generated from 'k8s-resources-node' from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/a8ba97a150c75be42010c75d10b720c55e182f1a/manifests/grafana-dashboardDefinitions.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (or .Values.grafana.enabled .Values.grafana.forceDeployDashboards) (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.grafana.defaultDashboardsEnabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: {{ .Values.grafana.defaultDashboards.namespace }} + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" $) "k8s-resources-node" | trunc 63 | trimSuffix "-" }} + annotations: +{{ toYaml .Values.grafana.sidecar.dashboards.annotations | indent 4 }} + labels: + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: {{ ternary $.Values.grafana.sidecar.dashboards.labelValue "1" (not (empty $.Values.grafana.sidecar.dashboards.labelValue)) | quote }} + {{- end }} + app: {{ template "kube-prometheus-stack.name" $ }}-grafana +{{ include "kube-prometheus-stack.labels" $ | indent 4 }} +data: + k8s-resources-node.json: |- + { + "annotations": { + "list": [ + + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "hideControls": false, + "links": [ + + ], + "refresh": "10s", + "rows": [ + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 1, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "max capacity", + "color": "#F2495C", + "dashes": true, + "fill": 0, + "hiddenSeries": true, + "hideTooltip": true, + "legend": true, + "linewidth": 2, + "stack": false + } + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(kube_node_status_capacity{cluster=\"$cluster\", node=~\"$node\", resource=\"cpu\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "max capacity", + "legendLink": null, + "step": 10 + }, + { + "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"$cluster\", node=~\"$node\"}) by (pod)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "CPU Usage", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "CPU Usage", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 2, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "styles": [ + { + "alias": "Time", + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "pattern": "Time", + "type": "hidden" + }, + { + "alias": "CPU Usage", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #A", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "CPU Requests", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #B", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "CPU Requests %", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #C", + "thresholds": [ + + ], + "type": "number", + "unit": "percentunit" + }, + { + "alias": "CPU Limits", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #D", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "CPU Limits %", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #E", + "thresholds": [ + + ], + "type": "number", + "unit": "percentunit" + }, + { + "alias": "Pod", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "pod", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "pattern": "/.*/", + "thresholds": [ + + ], + "type": "string", + "unit": "short" + } + ], + "targets": [ + { + "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"$cluster\", node=~\"$node\"}) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "A", + "step": 10 + }, + { + "expr": "sum(cluster:namespace:pod_cpu:active:kube_pod_container_resource_requests{cluster=\"$cluster\", node=~\"$node\"}) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "B", + "step": 10 + }, + { + "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"$cluster\", node=~\"$node\"}) by (pod) / sum(cluster:namespace:pod_cpu:active:kube_pod_container_resource_requests{cluster=\"$cluster\", node=~\"$node\"}) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "C", + "step": 10 + }, + { + "expr": "sum(cluster:namespace:pod_cpu:active:kube_pod_container_resource_limits{cluster=\"$cluster\", node=~\"$node\"}) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "D", + "step": 10 + }, + { + "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"$cluster\", node=~\"$node\"}) by (pod) / sum(cluster:namespace:pod_cpu:active:kube_pod_container_resource_limits{cluster=\"$cluster\", node=~\"$node\"}) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "E", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "CPU Quota", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "transform": "table", + "type": "table", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "CPU Quota", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 3, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "max capacity", + "color": "#F2495C", + "dashes": true, + "fill": 0, + "hiddenSeries": true, + "hideTooltip": true, + "legend": true, + "linewidth": 2, + "stack": false + } + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(kube_node_status_capacity{cluster=\"$cluster\", node=~\"$node\", resource=\"memory\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "max capacity", + "legendLink": null, + "step": 10 + }, + { + "expr": "sum(node_namespace_pod_container:container_memory_working_set_bytes{cluster=\"$cluster\", node=~\"$node\", container!=\"\"}) by (pod)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Memory Usage (w/o cache)", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Memory Usage", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 4, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "styles": [ + { + "alias": "Time", + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "pattern": "Time", + "type": "hidden" + }, + { + "alias": "Memory Usage", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #A", + "thresholds": [ + + ], + "type": "number", + "unit": "bytes" + }, + { + "alias": "Memory Requests", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #B", + "thresholds": [ + + ], + "type": "number", + "unit": "bytes" + }, + { + "alias": "Memory Requests %", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #C", + "thresholds": [ + + ], + "type": "number", + "unit": "percentunit" + }, + { + "alias": "Memory Limits", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #D", + "thresholds": [ + + ], + "type": "number", + "unit": "bytes" + }, + { + "alias": "Memory Limits %", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #E", + "thresholds": [ + + ], + "type": "number", + "unit": "percentunit" + }, + { + "alias": "Memory Usage (RSS)", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #F", + "thresholds": [ + + ], + "type": "number", + "unit": "bytes" + }, + { + "alias": "Memory Usage (Cache)", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #G", + "thresholds": [ + + ], + "type": "number", + "unit": "bytes" + }, + { + "alias": "Memory Usage (Swap)", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #H", + "thresholds": [ + + ], + "type": "number", + "unit": "bytes" + }, + { + "alias": "Pod", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "pod", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "pattern": "/.*/", + "thresholds": [ + + ], + "type": "string", + "unit": "short" + } + ], + "targets": [ + { + "expr": "sum(node_namespace_pod_container:container_memory_working_set_bytes{cluster=\"$cluster\", node=~\"$node\",container!=\"\"}) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "A", + "step": 10 + }, + { + "expr": "sum(cluster:namespace:pod_memory:active:kube_pod_container_resource_requests{cluster=\"$cluster\", node=~\"$node\"}) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "B", + "step": 10 + }, + { + "expr": "sum(node_namespace_pod_container:container_memory_working_set_bytes{cluster=\"$cluster\", node=~\"$node\",container!=\"\"}) by (pod) / sum(cluster:namespace:pod_memory:active:kube_pod_container_resource_requests{cluster=\"$cluster\", node=~\"$node\"}) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "C", + "step": 10 + }, + { + "expr": "sum(cluster:namespace:pod_memory:active:kube_pod_container_resource_limits{cluster=\"$cluster\", node=~\"$node\"}) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "D", + "step": 10 + }, + { + "expr": "sum(node_namespace_pod_container:container_memory_working_set_bytes{cluster=\"$cluster\", node=~\"$node\",container!=\"\"}) by (pod) / sum(cluster:namespace:pod_memory:active:kube_pod_container_resource_limits{cluster=\"$cluster\", node=~\"$node\"}) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "E", + "step": 10 + }, + { + "expr": "sum(node_namespace_pod_container:container_memory_rss{cluster=\"$cluster\", node=~\"$node\",container!=\"\"}) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "F", + "step": 10 + }, + { + "expr": "sum(node_namespace_pod_container:container_memory_cache{cluster=\"$cluster\", node=~\"$node\",container!=\"\"}) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "G", + "step": 10 + }, + { + "expr": "sum(node_namespace_pod_container:container_memory_swap{cluster=\"$cluster\", node=~\"$node\",container!=\"\"}) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "H", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Memory Quota", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "transform": "table", + "type": "table", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Memory Quota", + "titleSize": "h6" + } + ], + "schemaVersion": 14, + "style": "dark", + "tags": [ + "kubernetes-mixin" + ], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": "Data Source", + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": null, + "current": { + "text": "", + "value": "" + }, + "datasource": "$datasource", + "hide": {{ if .Values.grafana.sidecar.dashboards.multicluster.global.enabled }}0{{ else }}2{{ end }}, + "includeAll": false, + "label": null, + "multi": false, + "name": "cluster", + "options": [ + + ], + "query": "label_values(up{job=\"kube-state-metrics\"}, cluster)", + "refresh": 2, + "regex": "", + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": { + "text": "", + "value": "" + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": false, + "label": null, + "multi": true, + "name": "node", + "options": [ + + ], + "query": "label_values(kube_node_info{cluster=\"$cluster\"}, node)", + "refresh": 2, + "regex": "", + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "{{ .Values.grafana.defaultDashboardsTimezone }}", + "title": "Kubernetes / Compute Resources / Node (Pods)", + "uid": "200ac8fdbfbb74b39aff88118e4d1c2c", + "version": 0 + } +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/grafana/dashboards-1.14/k8s-resources-pod.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/grafana/dashboards-1.14/k8s-resources-pod.yaml new file mode 100644 index 00000000000..80fab51c003 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/grafana/dashboards-1.14/k8s-resources-pod.yaml @@ -0,0 +1,2469 @@ +{{- /* +Generated from 'k8s-resources-pod' from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/a8ba97a150c75be42010c75d10b720c55e182f1a/manifests/grafana-dashboardDefinitions.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (or .Values.grafana.enabled .Values.grafana.forceDeployDashboards) (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.grafana.defaultDashboardsEnabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: {{ .Values.grafana.defaultDashboards.namespace }} + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" $) "k8s-resources-pod" | trunc 63 | trimSuffix "-" }} + annotations: +{{ toYaml .Values.grafana.sidecar.dashboards.annotations | indent 4 }} + labels: + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: {{ ternary $.Values.grafana.sidecar.dashboards.labelValue "1" (not (empty $.Values.grafana.sidecar.dashboards.labelValue)) | quote }} + {{- end }} + app: {{ template "kube-prometheus-stack.name" $ }}-grafana +{{ include "kube-prometheus-stack.labels" $ | indent 4 }} +data: + k8s-resources-pod.json: |- + { + "annotations": { + "list": [ + + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "hideControls": false, + "links": [ + + ], + "refresh": "10s", + "rows": [ + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 1, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "requests", + "color": "#F2495C", + "fill": 0, + "hideTooltip": true, + "legend": true, + "linewidth": 2, + "stack": false + }, + { + "alias": "limits", + "color": "#FF9830", + "fill": 0, + "hideTooltip": true, + "legend": true, + "linewidth": 2, + "stack": false + } + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{namespace=\"$namespace\", pod=\"$pod\", cluster=\"$cluster\"}) by (container)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}container{{`}}`}}", + "legendLink": null, + "step": 10 + }, + { + "expr": "sum(\n kube_pod_container_resource_requests{job=\"kube-state-metrics\", cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\", resource=\"cpu\"}\n)\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "requests", + "legendLink": null, + "step": 10 + }, + { + "expr": "sum(\n kube_pod_container_resource_limits{job=\"kube-state-metrics\", cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\", resource=\"cpu\"}\n)\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "limits", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "CPU Usage", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "CPU Usage", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 2, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": true, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(increase(container_cpu_cfs_throttled_periods_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", namespace=\"$namespace\", pod=\"$pod\", container!=\"\", cluster=\"$cluster\"}[$__rate_interval])) by (container) /sum(increase(container_cpu_cfs_periods_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", namespace=\"$namespace\", pod=\"$pod\", container!=\"\", cluster=\"$cluster\"}[$__rate_interval])) by (container)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}container{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + { + "colorMode": "critical", + "fill": true, + "line": true, + "op": "gt", + "value": 0.25, + "yaxis": "left" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "CPU Throttling", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": 1, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "CPU Throttling", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 3, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "styles": [ + { + "alias": "Time", + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "pattern": "Time", + "type": "hidden" + }, + { + "alias": "CPU Usage", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #A", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "CPU Requests", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #B", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "CPU Requests %", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #C", + "thresholds": [ + + ], + "type": "number", + "unit": "percentunit" + }, + { + "alias": "CPU Limits", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #D", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "CPU Limits %", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #E", + "thresholds": [ + + ], + "type": "number", + "unit": "percentunit" + }, + { + "alias": "Container", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "container", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "pattern": "/.*/", + "thresholds": [ + + ], + "type": "string", + "unit": "short" + } + ], + "targets": [ + { + "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}) by (container)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "A", + "step": 10 + }, + { + "expr": "sum(cluster:namespace:pod_cpu:active:kube_pod_container_resource_requests{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}) by (container)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "B", + "step": 10 + }, + { + "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}) by (container) / sum(cluster:namespace:pod_cpu:active:kube_pod_container_resource_requests{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}) by (container)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "C", + "step": 10 + }, + { + "expr": "sum(cluster:namespace:pod_cpu:active:kube_pod_container_resource_limits{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}) by (container)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "D", + "step": 10 + }, + { + "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}) by (container) / sum(cluster:namespace:pod_cpu:active:kube_pod_container_resource_limits{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}) by (container)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "E", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "CPU Quota", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "transform": "table", + "type": "table", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "CPU Quota", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 4, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "requests", + "color": "#F2495C", + "dashes": true, + "fill": 0, + "hideTooltip": true, + "legend": true, + "linewidth": 2, + "stack": false + }, + { + "alias": "limits", + "color": "#FF9830", + "dashes": true, + "fill": 0, + "hideTooltip": true, + "legend": true, + "linewidth": 2, + "stack": false + } + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(container_memory_working_set_bytes{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\", container!=\"\", image!=\"\"}) by (container)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}container{{`}}`}}", + "legendLink": null, + "step": 10 + }, + { + "expr": "sum(\n kube_pod_container_resource_requests{job=\"kube-state-metrics\", cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\", resource=\"memory\"}\n)\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "requests", + "legendLink": null, + "step": 10 + }, + { + "expr": "sum(\n kube_pod_container_resource_limits{job=\"kube-state-metrics\", cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\", resource=\"memory\"}\n)\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "limits", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Memory Usage (WSS)", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Memory Usage", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 5, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "styles": [ + { + "alias": "Time", + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "pattern": "Time", + "type": "hidden" + }, + { + "alias": "Memory Usage (WSS)", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #A", + "thresholds": [ + + ], + "type": "number", + "unit": "bytes" + }, + { + "alias": "Memory Requests", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #B", + "thresholds": [ + + ], + "type": "number", + "unit": "bytes" + }, + { + "alias": "Memory Requests %", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #C", + "thresholds": [ + + ], + "type": "number", + "unit": "percentunit" + }, + { + "alias": "Memory Limits", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #D", + "thresholds": [ + + ], + "type": "number", + "unit": "bytes" + }, + { + "alias": "Memory Limits %", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #E", + "thresholds": [ + + ], + "type": "number", + "unit": "percentunit" + }, + { + "alias": "Memory Usage (RSS)", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #F", + "thresholds": [ + + ], + "type": "number", + "unit": "bytes" + }, + { + "alias": "Memory Usage (Cache)", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #G", + "thresholds": [ + + ], + "type": "number", + "unit": "bytes" + }, + { + "alias": "Memory Usage (Swap)", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #H", + "thresholds": [ + + ], + "type": "number", + "unit": "bytes" + }, + { + "alias": "Container", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "container", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "pattern": "/.*/", + "thresholds": [ + + ], + "type": "string", + "unit": "short" + } + ], + "targets": [ + { + "expr": "sum(container_memory_working_set_bytes{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\", container!=\"\", image!=\"\"}) by (container)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "A", + "step": 10 + }, + { + "expr": "sum(cluster:namespace:pod_memory:active:kube_pod_container_resource_requests{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}) by (container)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "B", + "step": 10 + }, + { + "expr": "sum(container_memory_working_set_bytes{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\", image!=\"\"}) by (container) / sum(cluster:namespace:pod_memory:active:kube_pod_container_resource_requests{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}) by (container)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "C", + "step": 10 + }, + { + "expr": "sum(cluster:namespace:pod_memory:active:kube_pod_container_resource_limits{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}) by (container)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "D", + "step": 10 + }, + { + "expr": "sum(container_memory_working_set_bytes{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\", container!=\"\", image!=\"\"}) by (container) / sum(cluster:namespace:pod_memory:active:kube_pod_container_resource_limits{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}) by (container)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "E", + "step": 10 + }, + { + "expr": "sum(container_memory_rss{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\", container != \"\", container != \"POD\"}) by (container)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "F", + "step": 10 + }, + { + "expr": "sum(container_memory_cache{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\", container != \"\", container != \"POD\"}) by (container)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "G", + "step": 10 + }, + { + "expr": "sum(container_memory_swap{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\", container != \"\", container != \"POD\"}) by (container)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "H", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Memory Quota", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "transform": "table", + "type": "table", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Memory Quota", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 6, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(container_network_receive_bytes_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\", pod=~\"$pod\"}[$__rate_interval])) by (pod)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Receive Bandwidth", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 7, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(container_network_transmit_bytes_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\", pod=~\"$pod\"}[$__rate_interval])) by (pod)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Transmit Bandwidth", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Bandwidth", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 8, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(container_network_receive_packets_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\", pod=~\"$pod\"}[$__rate_interval])) by (pod)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Received Packets", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 9, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(container_network_transmit_packets_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\", pod=~\"$pod\"}[$__rate_interval])) by (pod)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Transmitted Packets", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Rate of Packets", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 10, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(container_network_receive_packets_dropped_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\", pod=~\"$pod\"}[$__rate_interval])) by (pod)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Received Packets Dropped", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 11, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(container_network_transmit_packets_dropped_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\", pod=~\"$pod\"}[$__rate_interval])) by (pod)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Transmitted Packets Dropped", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Rate of Packets Dropped", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "decimals": -1, + "fill": 10, + "id": 12, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "ceil(sum by(pod) (rate(container_fs_reads_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\", container!=\"\", cluster=\"$cluster\", namespace=\"$namespace\", pod=~\"$pod\"}[$__rate_interval])))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Reads", + "legendLink": null, + "step": 10 + }, + { + "expr": "ceil(sum by(pod) (rate(container_fs_writes_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\", container!=\"\", cluster=\"$cluster\",namespace=\"$namespace\", pod=~\"$pod\"}[$__rate_interval])))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Writes", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "IOPS", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 13, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum by(pod) (rate(container_fs_reads_bytes_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\", container!=\"\", cluster=\"$cluster\", namespace=\"$namespace\", pod=~\"$pod\"}[$__rate_interval]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Reads", + "legendLink": null, + "step": 10 + }, + { + "expr": "sum by(pod) (rate(container_fs_writes_bytes_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\", container!=\"\", cluster=\"$cluster\", namespace=\"$namespace\", pod=~\"$pod\"}[$__rate_interval]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Writes", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "ThroughPut", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Storage IO - Distribution(Pod - Read & Writes)", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "decimals": -1, + "fill": 10, + "id": 14, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "ceil(sum by(container) (rate(container_fs_reads_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", container!=\"\", cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}[$__rate_interval]) + rate(container_fs_writes_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", container!=\"\", cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}[$__rate_interval])))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}container{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "IOPS(Reads+Writes)", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 15, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum by(container) (rate(container_fs_reads_bytes_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", container!=\"\", cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}[$__rate_interval]) + rate(container_fs_writes_bytes_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", container!=\"\", cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}[$__rate_interval]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}container{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "ThroughPut(Read+Write)", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Storage IO - Distribution(Containers)", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 16, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "sort": { + "col": 4, + "desc": true + }, + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "styles": [ + { + "alias": "Time", + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "pattern": "Time", + "type": "hidden" + }, + { + "alias": "IOPS(Reads)", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": -1, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #A", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "IOPS(Writes)", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": -1, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #B", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "IOPS(Reads + Writes)", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": -1, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #C", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "Throughput(Read)", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #D", + "thresholds": [ + + ], + "type": "number", + "unit": "Bps" + }, + { + "alias": "Throughput(Write)", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #E", + "thresholds": [ + + ], + "type": "number", + "unit": "Bps" + }, + { + "alias": "Throughput(Read + Write)", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #F", + "thresholds": [ + + ], + "type": "number", + "unit": "Bps" + }, + { + "alias": "Container", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "container", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "pattern": "/.*/", + "thresholds": [ + + ], + "type": "string", + "unit": "short" + } + ], + "targets": [ + { + "expr": "sum by(container) (rate(container_fs_reads_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\", container!=\"\", cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}[$__rate_interval]))", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "A", + "step": 10 + }, + { + "expr": "sum by(container) (rate(container_fs_writes_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\",device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\", container!=\"\", cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}[$__rate_interval]))", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "B", + "step": 10 + }, + { + "expr": "sum by(container) (rate(container_fs_reads_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\", container!=\"\", cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}[$__rate_interval]) + rate(container_fs_writes_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\", container!=\"\", cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}[$__rate_interval]))", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "C", + "step": 10 + }, + { + "expr": "sum by(container) (rate(container_fs_reads_bytes_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\", container!=\"\", cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}[$__rate_interval]))", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "D", + "step": 10 + }, + { + "expr": "sum by(container) (rate(container_fs_writes_bytes_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\", container!=\"\", cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}[$__rate_interval]))", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "E", + "step": 10 + }, + { + "expr": "sum by(container) (rate(container_fs_reads_bytes_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\", container!=\"\", cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}[$__rate_interval]) + rate(container_fs_writes_bytes_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\", container!=\"\", cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}[$__rate_interval]))", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "F", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Current Storage IO", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "transform": "table", + "type": "table", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Storage IO - Distribution", + "titleSize": "h6" + } + ], + "schemaVersion": 14, + "style": "dark", + "tags": [ + "kubernetes-mixin" + ], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": "Data Source", + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": null, + "current": { + "text": "", + "value": "" + }, + "datasource": "$datasource", + "hide": {{ if .Values.grafana.sidecar.dashboards.multicluster.global.enabled }}0{{ else }}2{{ end }}, + "includeAll": false, + "label": null, + "multi": false, + "name": "cluster", + "options": [ + + ], + "query": "label_values(up{job=\"kube-state-metrics\"}, cluster)", + "refresh": 2, + "regex": "", + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": { + "text": "", + "value": "" + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "namespace", + "options": [ + + ], + "query": "label_values(kube_namespace_status_phase{job=\"kube-state-metrics\", cluster=\"$cluster\"}, namespace)", + "refresh": 2, + "regex": "", + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": { + "text": "", + "value": "" + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "pod", + "options": [ + + ], + "query": "label_values(kube_pod_info{job=\"kube-state-metrics\", cluster=\"$cluster\", namespace=\"$namespace\"}, pod)", + "refresh": 2, + "regex": "", + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "{{ .Values.grafana.defaultDashboardsTimezone }}", + "title": "Kubernetes / Compute Resources / Pod", + "uid": "6581e46e4e5c7ba40a07646395ef7b23", + "version": 0 + } +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/grafana/dashboards-1.14/k8s-resources-windows-cluster.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/grafana/dashboards-1.14/k8s-resources-windows-cluster.yaml new file mode 100644 index 00000000000..d77170afd8a --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/grafana/dashboards-1.14/k8s-resources-windows-cluster.yaml @@ -0,0 +1,24 @@ +{{- /* +Generated from 'k8s-resources-windows-cluster' from https://github.com/kubernetes-monitoring/kubernetes-mixin.git +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (or .Values.grafana.enabled .Values.grafana.forceDeployDashboards) (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.grafana.defaultDashboardsEnabled .Values.windowsMonitoring.enabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: {{ template "kube-prometheus-stack-grafana.namespace" . }} + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" $) "k8s-resources-windows-cluster" | trunc 63 | trimSuffix "-" }} + annotations: +{{ toYaml .Values.grafana.sidecar.dashboards.annotations | indent 4 }} + labels: + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: {{ ternary $.Values.grafana.sidecar.dashboards.labelValue "1" (not (empty $.Values.grafana.sidecar.dashboards.labelValue)) | quote }} + {{- end }} + app: {{ template "kube-prometheus-stack.name" $ }}-grafana +{{ include "kube-prometheus-stack.labels" $ | indent 4 }} +data: + k8s-resources-windows-cluster.json: |- + {{`{"__inputs":[],"__requires":[],"annotations":{"list":[]},"editable":`}}{{ .Values.grafana.defaultDashboardsEditable }}{{`,"gnetId":null,"graphTooltip":0,"hideControls":false,"id":null,"links":[],"refresh":"","rows":[{"collapse":false,"height":"100px","panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":1,"format":"percentunit","id":2,"legend":{"avg":false,"current":false,"max":false,"min":false,"show":true,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":2,"stack":false,"steppedLine":false,"targets":[{"expr":"1 - avg(rate(windows_cpu_time_total{cluster=\"$cluster\", job=\"windows-exporter\", mode=\"idle\"}[1m]))","format":"time_series","instant":true,"refId":"A"}],"thresholds":"70,80","timeFrom":null,"timeShift":null,"title":"CPU Utilisation","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"type":"singlestat","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"short","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":1,"format":"percentunit","id":3,"legend":{"avg":false,"current":false,"max":false,"min":false,"show":true,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":2,"stack":false,"steppedLine":false,"targets":[{"expr":"sum(kube_pod_windows_container_resource_cpu_cores_request{cluster=\"$cluster\"}) / sum(node:windows_node_num_cpu:sum{cluster=\"$cluster\"})","format":"time_series","instant":true,"refId":"A"}],"thresholds":"70,80","timeFrom":null,"timeShift":null,"title":"CPU Requests Commitment","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"type":"singlestat","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"short","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":1,"format":"percentunit","id":4,"legend":{"avg":false,"current":false,"max":false,"min":false,"show":true,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":2,"stack":false,"steppedLine":false,"targets":[{"expr":"sum(kube_pod_windows_container_resource_cpu_cores_limit{cluster=\"$cluster\"}) / sum(node:windows_node_num_cpu:sum{cluster=\"$cluster\"})","format":"time_series","instant":true,"refId":"A"}],"thresholds":"70,80","timeFrom":null,"timeShift":null,"title":"CPU Limits Commitment","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"type":"singlestat","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"short","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":1,"format":"percentunit","id":5,"legend":{"avg":false,"current":false,"max":false,"min":false,"show":true,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":2,"stack":false,"steppedLine":false,"targets":[{"expr":"1 - sum(:windows_node_memory_MemFreeCached_bytes:sum{cluster=\"$cluster\"}) / sum(:windows_node_memory_MemTotal_bytes:sum{cluster=\"$cluster\"})","format":"time_series","instant":true,"refId":"A"}],"thresholds":"70,80","timeFrom":null,"timeShift":null,"title":"Memory Utilisation","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"type":"singlestat","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"short","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":1,"format":"percentunit","id":6,"legend":{"avg":false,"current":false,"max":false,"min":false,"show":true,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":2,"stack":false,"steppedLine":false,"targets":[{"expr":"sum(kube_pod_windows_container_resource_memory_request{cluster=\"$cluster\"}) / sum(:windows_node_memory_MemTotal_bytes:sum{cluster=\"$cluster\"})","format":"time_series","instant":true,"refId":"A"}],"thresholds":"70,80","timeFrom":null,"timeShift":null,"title":"Memory Requests Commitment","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"type":"singlestat","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"short","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":1,"format":"percentunit","id":7,"legend":{"avg":false,"current":false,"max":false,"min":false,"show":true,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":2,"stack":false,"steppedLine":false,"targets":[{"expr":"sum(kube_pod_windows_container_resource_memory_limit{cluster=\"$cluster\"}) / sum(:windows_node_memory_MemTotal_bytes:sum{cluster=\"$cluster\"})","format":"time_series","instant":true,"refId":"A"}],"thresholds":"70,80","timeFrom":null,"timeShift":null,"title":"Memory Limits Commitment","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"type":"singlestat","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"short","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":false,"title":"Headlines","titleSize":"h6"},{"collapse":false,"height":"250px","panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":10,"id":8,"legend":{"avg":false,"current":false,"max":false,"min":false,"show":true,"total":false,"values":false},"lines":true,"linewidth":0,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":12,"stack":true,"steppedLine":false,"targets":[{"expr":"sum(namespace_pod_container:windows_container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\"}) by (namespace)","format":"time_series","legendFormat":"{{namespace}}","legendLink":null}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"CPU Usage","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"short","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":true,"title":"CPU","titleSize":"h6"},{"collapse":false,"height":"250px","panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":1,"id":9,"legend":{"avg":false,"current":false,"max":false,"min":false,"show":true,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":12,"stack":false,"steppedLine":false,"styles":[{"alias":"Time","dateFormat":"YYYY-MM-DD HH:mm:ss","pattern":"Time","type":"hidden"},{"alias":"CPU Usage","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #A","thresholds":[],"type":"number","unit":"short"},{"alias":"CPU Requests","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #B","thresholds":[],"type":"number","unit":"short"},{"alias":"CPU Requests %","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #C","thresholds":[],"type":"number","unit":"percentunit"},{"alias":"CPU Limits","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #D","thresholds":[],"type":"number","unit":"short"},{"alias":"CPU Limits %","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #E","thresholds":[],"type":"number","unit":"percentunit"},{"alias":"Namespace","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":true,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"/d/490b402361724ab1d4c45666c1fa9b6f/k8s-resources-windows-namespace?var-datasource=$datasource&var-namespace=`}}{{ if .Values.grafana.sidecar.dashboards.enableNewTablePanelSyntax }}${__value.text}{{ else }}$__cell{{ end }}{{`","pattern":"namespace","thresholds":[],"type":"number","unit":"short"},{"alias":"","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"pattern":"/.*/","thresholds":[],"type":"string","unit":"short"}],"targets":[{"expr":"sum(namespace_pod_container:windows_container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\"}) by (namespace)","format":"table","instant":true,"legendFormat":"","refId":"A"},{"expr":"sum(kube_pod_windows_container_resource_cpu_cores_request{cluster=\"$cluster\"}) by (namespace)","format":"table","instant":true,"legendFormat":"","refId":"B"},{"expr":"sum(namespace_pod_container:windows_container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\"}) by (namespace) / sum(kube_pod_windows_container_resource_cpu_cores_request{cluster=\"$cluster\"}) by (namespace)","format":"table","instant":true,"legendFormat":"","refId":"C"},{"expr":"sum(kube_pod_windows_container_resource_cpu_cores_limit{cluster=\"$cluster\"}) by (namespace)","format":"table","instant":true,"legendFormat":"","refId":"D"},{"expr":"sum(namespace_pod_container:windows_container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\"}) by (namespace) / sum(kube_pod_windows_container_resource_cpu_cores_limit{cluster=\"$cluster\"}) by (namespace)","format":"table","instant":true,"legendFormat":"","refId":"E"}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"CPU Quota","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"transform":"table","type":"table","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"short","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":true,"title":"CPU Quota","titleSize":"h6"},{"collapse":false,"height":"250px","panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":10,"id":10,"legend":{"avg":false,"current":false,"max":false,"min":false,"show":true,"total":false,"values":false},"lines":true,"linewidth":0,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":12,"stack":true,"steppedLine":false,"targets":[{"expr":"sum(windows_container_private_working_set_usage{cluster=\"$cluster\"}) by (namespace)","format":"time_series","legendFormat":"{{namespace}}","legendLink":null}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Memory Usage (Private Working Set)","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"decbytes","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":true,"title":"Memory","titleSize":"h6"},{"collapse":false,"height":"250px","panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":1,"id":11,"legend":{"avg":false,"current":false,"max":false,"min":false,"show":true,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":12,"stack":false,"steppedLine":false,"styles":[{"alias":"Time","dateFormat":"YYYY-MM-DD HH:mm:ss","pattern":"Time","type":"hidden"},{"alias":"Memory Usage","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #A","thresholds":[],"type":"number","unit":"decbytes"},{"alias":"Memory Requests","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #B","thresholds":[],"type":"number","unit":"decbytes"},{"alias":"Memory Requests %","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #C","thresholds":[],"type":"number","unit":"percentunit"},{"alias":"Memory Limits","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #D","thresholds":[],"type":"number","unit":"decbytes"},{"alias":"Memory Limits %","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #E","thresholds":[],"type":"number","unit":"percentunit"},{"alias":"Namespace","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":true,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"/d/490b402361724ab1d4c45666c1fa9b6f/k8s-resources-windows-namespace?var-datasource=$datasource&var-namespace=`}}{{ if .Values.grafana.sidecar.dashboards.enableNewTablePanelSyntax }}${__value.text}{{ else }}$__cell{{ end }}{{`","pattern":"namespace","thresholds":[],"type":"number","unit":"short"},{"alias":"","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"pattern":"/.*/","thresholds":[],"type":"string","unit":"short"}],"targets":[{"expr":"sum(windows_container_private_working_set_usage{cluster=\"$cluster\"}) by (namespace)","format":"table","instant":true,"legendFormat":"","refId":"A"},{"expr":"sum(kube_pod_windows_container_resource_memory_request{cluster=\"$cluster\"}) by (namespace)","format":"table","instant":true,"legendFormat":"","refId":"B"},{"expr":"sum(windows_container_private_working_set_usage{cluster=\"$cluster\"}) by (namespace) / sum(kube_pod_windows_container_resource_memory_request{cluster=\"$cluster\"}) by (namespace)","format":"table","instant":true,"legendFormat":"","refId":"C"},{"expr":"sum(kube_pod_windows_container_resource_memory_limit{cluster=\"$cluster\"}) by (namespace)","format":"table","instant":true,"legendFormat":"","refId":"D"},{"expr":"sum(windows_container_private_working_set_usage{cluster=\"$cluster\"}) by (namespace) / sum(kube_pod_windows_container_resource_memory_limit{cluster=\"$cluster\"}) by (namespace)","format":"table","instant":true,"legendFormat":"","refId":"E"}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Requests by Namespace","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"transform":"table","type":"table","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"short","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":true,"title":"Memory Requests","titleSize":"h6"}],"schemaVersion":14,"style":"dark","tags":["kubernetes-mixin"],"templating":{"list":[{"current":{"text":"default","value":"default"},"hide":0,"label":null,"name":"datasource","options":[],"query":"prometheus","refresh":1,"regex":"","type":"datasource"},{"allValue":null,"current":{},"datasource":"$datasource","hide":`}}{{ if .Values.grafana.sidecar.dashboards.multicluster.global.enabled }}0{{ else }}2{{ end }}{{`,"includeAll":false,"label":"cluster","multi":false,"name":"cluster","options":[],"query":"label_values(up{job=\"windows-exporter\"}, cluster)","refresh":2,"regex":"","sort":1,"tagValuesQuery":"","tags":[],"tagsQuery":"","type":"query","useTags":false}]},"time":{"from":"now-6h","to":"now"},"timepicker":{"refresh_intervals":["5s","10s","30s","1m","5m","15m","30m","1h","2h","1d"],"time_options":["5m","15m","1h","6h","12h","24h","2d","7d","30d"]},"timezone": "`}}{{ .Values.grafana.defaultDashboardsTimezone }}{{`","title":"Kubernetes / Compute Resources / Cluster(Windows)","uid":"4d08557fd9391b100730f2494bccac68","version":0}`}} +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/grafana/dashboards-1.14/k8s-resources-windows-namespace.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/grafana/dashboards-1.14/k8s-resources-windows-namespace.yaml new file mode 100644 index 00000000000..13a1fc3abda --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/grafana/dashboards-1.14/k8s-resources-windows-namespace.yaml @@ -0,0 +1,24 @@ +{{- /* +Generated from 'k8s-resources-windows-namespace' from https://github.com/kubernetes-monitoring/kubernetes-mixin.git +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (or .Values.grafana.enabled .Values.grafana.forceDeployDashboards) (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.grafana.defaultDashboardsEnabled .Values.windowsMonitoring.enabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: {{ template "kube-prometheus-stack-grafana.namespace" . }} + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" $) "k8s-resources-windows-namespace" | trunc 63 | trimSuffix "-" }} + annotations: +{{ toYaml .Values.grafana.sidecar.dashboards.annotations | indent 4 }} + labels: + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: {{ ternary $.Values.grafana.sidecar.dashboards.labelValue "1" (not (empty $.Values.grafana.sidecar.dashboards.labelValue)) | quote }} + {{- end }} + app: {{ template "kube-prometheus-stack.name" $ }}-grafana +{{ include "kube-prometheus-stack.labels" $ | indent 4 }} +data: + k8s-resources-windows-namespace.json: |- + {{`{"__inputs":[],"__requires":[],"annotations":{"list":[]},"editable":`}}{{ .Values.grafana.defaultDashboardsEditable }}{{`,"gnetId":null,"graphTooltip":0,"hideControls":false,"id":null,"links":[],"refresh":"","rows":[{"collapse":false,"height":"250px","panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":10,"id":2,"legend":{"avg":false,"current":false,"max":false,"min":false,"show":true,"total":false,"values":false},"lines":true,"linewidth":0,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":12,"stack":true,"steppedLine":false,"targets":[{"expr":"sum(namespace_pod_container:windows_container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod)","format":"time_series","legendFormat":"{{pod}}","legendLink":null}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"CPU Usage","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"short","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":true,"title":"CPU Usage","titleSize":"h6"},{"collapse":false,"height":"250px","panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":1,"id":3,"legend":{"avg":false,"current":false,"max":false,"min":false,"show":true,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":12,"stack":false,"steppedLine":false,"styles":[{"alias":"Time","dateFormat":"YYYY-MM-DD HH:mm:ss","pattern":"Time","type":"hidden"},{"alias":"CPU Usage","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #A","thresholds":[],"type":"number","unit":"short"},{"alias":"CPU Requests","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #B","thresholds":[],"type":"number","unit":"short"},{"alias":"CPU Requests %","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #C","thresholds":[],"type":"number","unit":"percentunit"},{"alias":"CPU Limits","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #D","thresholds":[],"type":"number","unit":"short"},{"alias":"CPU Limits %","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #E","thresholds":[],"type":"number","unit":"percentunit"},{"alias":"Pod","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":true,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"/d/40597a704a610e936dc6ed374a7ce023/k8s-resources-windows-pod?var-datasource=$datasource&var-namespace=$namespace&var-pod=`}}{{ if .Values.grafana.sidecar.dashboards.enableNewTablePanelSyntax }}${__value.text}{{ else }}$__cell{{ end }}{{`","pattern":"pod","thresholds":[],"type":"number","unit":"short"},{"alias":"","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"pattern":"/.*/","thresholds":[],"type":"string","unit":"short"}],"targets":[{"expr":"sum(namespace_pod_container:windows_container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod)","format":"table","instant":true,"legendFormat":"","refId":"A"},{"expr":"sum(kube_pod_windows_container_resource_cpu_cores_request{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod)","format":"table","instant":true,"legendFormat":"","refId":"B"},{"expr":"sum(namespace_pod_container:windows_container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod) / sum(kube_pod_windows_container_resource_cpu_cores_request{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod)","format":"table","instant":true,"legendFormat":"","refId":"C"},{"expr":"sum(kube_pod_windows_container_resource_cpu_cores_limit{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod)","format":"table","instant":true,"legendFormat":"","refId":"D"},{"expr":"sum(namespace_pod_container:windows_container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod) / sum(kube_pod_windows_container_resource_cpu_cores_limit{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod)","format":"table","instant":true,"legendFormat":"","refId":"E"}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"CPU Quota","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"transform":"table","type":"table","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"short","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":true,"title":"CPU Quota","titleSize":"h6"},{"collapse":false,"height":"250px","panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":10,"id":4,"legend":{"avg":false,"current":false,"max":false,"min":false,"show":true,"total":false,"values":false},"lines":true,"linewidth":0,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":12,"stack":true,"steppedLine":false,"targets":[{"expr":"sum(windows_container_private_working_set_usage{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod)","format":"time_series","legendFormat":"{{pod}}","legendLink":null}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Memory Usage","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"decbytes","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":true,"title":"Memory Usage","titleSize":"h6"},{"collapse":false,"height":"250px","panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":1,"id":5,"legend":{"avg":false,"current":false,"max":false,"min":false,"show":true,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":12,"stack":false,"steppedLine":false,"styles":[{"alias":"Time","dateFormat":"YYYY-MM-DD HH:mm:ss","pattern":"Time","type":"hidden"},{"alias":"Memory Usage","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #A","thresholds":[],"type":"number","unit":"decbytes"},{"alias":"Memory Requests","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #B","thresholds":[],"type":"number","unit":"decbytes"},{"alias":"Memory Requests %","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #C","thresholds":[],"type":"number","unit":"percentunit"},{"alias":"Memory Limits","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #D","thresholds":[],"type":"number","unit":"decbytes"},{"alias":"Memory Limits %","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #E","thresholds":[],"type":"number","unit":"percentunit"},{"alias":"Pod","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":true,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"/d/40597a704a610e936dc6ed374a7ce023/k8s-resources-windows-pod?var-datasource=$datasource&var-namespace=$namespace&var-pod=`}}{{ if .Values.grafana.sidecar.dashboards.enableNewTablePanelSyntax }}${__value.text}{{ else }}$__cell{{ end }}{{`","pattern":"pod","thresholds":[],"type":"number","unit":"short"},{"alias":"","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"pattern":"/.*/","thresholds":[],"type":"string","unit":"short"}],"targets":[{"expr":"sum(windows_container_private_working_set_usage{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod)","format":"table","instant":true,"legendFormat":"","refId":"A"},{"expr":"sum(kube_pod_windows_container_resource_memory_request{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod)","format":"table","instant":true,"legendFormat":"","refId":"B"},{"expr":"sum(windows_container_private_working_set_usage{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod) / sum(kube_pod_windows_container_resource_memory_request{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod)","format":"table","instant":true,"legendFormat":"","refId":"C"},{"expr":"sum(kube_pod_windows_container_resource_memory_limit{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod)","format":"table","instant":true,"legendFormat":"","refId":"D"},{"expr":"sum(windows_container_private_working_set_usage{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod) / sum(kube_pod_windows_container_resource_memory_limit{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod)","format":"table","instant":true,"legendFormat":"","refId":"E"}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Memory Quota","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"transform":"table","type":"table","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"short","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":true,"title":"Memory Quota","titleSize":"h6"}],"schemaVersion":14,"style":"dark","tags":["kubernetes-mixin"],"templating":{"list":[{"current":{"selected":true,"text":"default","value":"default"},"hide":0,"label":null,"name":"datasource","options":[],"query":"prometheus","refresh":1,"regex":"","type":"datasource"},{"allValue":null,"current":{},"datasource":"$datasource","hide":`}}{{ if .Values.grafana.sidecar.dashboards.multicluster.global.enabled }}0{{ else }}2{{ end }}{{`,"includeAll":false,"label":"cluster","multi":false,"name":"cluster","options":[],"query":"label_values(up{job=\"windows-exporter\"}, cluster)","refresh":2,"regex":"","sort":1,"tagValuesQuery":"","tags":[],"tagsQuery":"","type":"query","useTags":false},{"allValue":null,"current":{},"datasource":"$datasource","hide":0,"includeAll":false,"label":"Namespace","multi":false,"name":"namespace","options":[],"query":"label_values(windows_pod_container_available{cluster=\"$cluster\"}, namespace)","refresh":2,"regex":"","sort":1,"tagValuesQuery":"","tags":[],"tagsQuery":"","type":"query","useTags":false}]},"time":{"from":"now-6h","to":"now"},"timepicker":{"refresh_intervals":["5s","10s","30s","1m","5m","15m","30m","1h","2h","1d"],"time_options":["5m","15m","1h","6h","12h","24h","2d","7d","30d"]},"timezone": "`}}{{ .Values.grafana.defaultDashboardsTimezone }}{{`","title":"Kubernetes / Compute Resources / Namespace(Windows)","uid":"490b402361724ab1d4c45666c1fa9b6f","version":0}`}} +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/grafana/dashboards-1.14/k8s-resources-windows-pod.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/grafana/dashboards-1.14/k8s-resources-windows-pod.yaml new file mode 100644 index 00000000000..6686e540531 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/grafana/dashboards-1.14/k8s-resources-windows-pod.yaml @@ -0,0 +1,24 @@ +{{- /* +Generated from 'k8s-resources-windows-pod' from https://github.com/kubernetes-monitoring/kubernetes-mixin.git +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (or .Values.grafana.enabled .Values.grafana.forceDeployDashboards) (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.grafana.defaultDashboardsEnabled .Values.windowsMonitoring.enabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: {{ template "kube-prometheus-stack-grafana.namespace" . }} + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" $) "k8s-resources-windows-pod" | trunc 63 | trimSuffix "-" }} + annotations: +{{ toYaml .Values.grafana.sidecar.dashboards.annotations | indent 4 }} + labels: + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: {{ ternary $.Values.grafana.sidecar.dashboards.labelValue "1" (not (empty $.Values.grafana.sidecar.dashboards.labelValue)) | quote }} + {{- end }} + app: {{ template "kube-prometheus-stack.name" $ }}-grafana +{{ include "kube-prometheus-stack.labels" $ | indent 4 }} +data: + k8s-resources-windows-pod.json: |- + {{`{"__inputs":[],"__requires":[],"annotations":{"list":[]},"editable":`}}{{ .Values.grafana.defaultDashboardsEditable }}{{`,"gnetId":null,"graphTooltip":0,"hideControls":false,"id":null,"links":[],"refresh":"","rows":[{"collapse":false,"height":"250px","panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":10,"id":2,"legend":{"avg":false,"current":false,"max":false,"min":false,"show":true,"total":false,"values":false},"lines":true,"linewidth":0,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":12,"stack":true,"steppedLine":false,"targets":[{"expr":"sum(namespace_pod_container:windows_container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}) by (container)","format":"time_series","legendFormat":"{{container}}","legendLink":null}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"CPU Usage","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"short","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":true,"title":"CPU Usage","titleSize":"h6"},{"collapse":false,"height":"250px","panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":1,"id":3,"legend":{"avg":false,"current":false,"max":false,"min":false,"show":true,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":12,"stack":false,"steppedLine":false,"styles":[{"alias":"Time","dateFormat":"YYYY-MM-DD HH:mm:ss","pattern":"Time","type":"hidden"},{"alias":"CPU Usage","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #A","thresholds":[],"type":"number","unit":"short"},{"alias":"CPU Requests","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #B","thresholds":[],"type":"number","unit":"short"},{"alias":"CPU Requests %","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #C","thresholds":[],"type":"number","unit":"percentunit"},{"alias":"CPU Limits","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #D","thresholds":[],"type":"number","unit":"short"},{"alias":"CPU Limits %","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #E","thresholds":[],"type":"number","unit":"percentunit"},{"alias":"Container","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"container","thresholds":[],"type":"number","unit":"short"},{"alias":"","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"pattern":"/.*/","thresholds":[],"type":"string","unit":"short"}],"targets":[{"expr":"sum(namespace_pod_container:windows_container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}) by (container)","format":"table","instant":true,"legendFormat":"","refId":"A"},{"expr":"sum(kube_pod_windows_container_resource_cpu_cores_request{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}) by (container)","format":"table","instant":true,"legendFormat":"","refId":"B"},{"expr":"sum(namespace_pod_container:windows_container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}) by (container) / sum(kube_pod_windows_container_resource_cpu_cores_request{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}) by (container)","format":"table","instant":true,"legendFormat":"","refId":"C"},{"expr":"sum(kube_pod_windows_container_resource_cpu_cores_limit{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}) by (container)","format":"table","instant":true,"legendFormat":"","refId":"D"},{"expr":"sum(namespace_pod_container:windows_container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}) by (container) / sum(kube_pod_windows_container_resource_cpu_cores_limit{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}) by (container)","format":"table","instant":true,"legendFormat":"","refId":"E"}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"CPU Quota","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"transform":"table","type":"table","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"short","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":true,"title":"CPU Quota","titleSize":"h6"},{"collapse":false,"height":"250px","panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":10,"id":4,"legend":{"avg":false,"current":false,"max":false,"min":false,"show":true,"total":false,"values":false},"lines":true,"linewidth":0,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":12,"stack":true,"steppedLine":false,"targets":[{"expr":"sum(windows_container_private_working_set_usage{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}) by (container)","format":"time_series","legendFormat":"{{container}}","legendLink":null}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Memory Usage","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"short","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":true,"title":"Memory Usage","titleSize":"h6"},{"collapse":false,"height":"250px","panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":1,"id":5,"legend":{"avg":false,"current":false,"max":false,"min":false,"show":true,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":12,"stack":false,"steppedLine":false,"styles":[{"alias":"Time","dateFormat":"YYYY-MM-DD HH:mm:ss","pattern":"Time","type":"hidden"},{"alias":"Memory Usage","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #A","thresholds":[],"type":"number","unit":"decbytes"},{"alias":"Memory Requests","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #B","thresholds":[],"type":"number","unit":"decbytes"},{"alias":"Memory Requests %","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #C","thresholds":[],"type":"number","unit":"percentunit"},{"alias":"Memory Limits","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #D","thresholds":[],"type":"number","unit":"decbytes"},{"alias":"Memory Limits %","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #E","thresholds":[],"type":"number","unit":"percentunit"},{"alias":"Container","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"container","thresholds":[],"type":"number","unit":"short"},{"alias":"","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"pattern":"/.*/","thresholds":[],"type":"string","unit":"short"}],"targets":[{"expr":"sum(windows_container_private_working_set_usage{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}) by (container)","format":"table","instant":true,"legendFormat":"","refId":"A"},{"expr":"sum(kube_pod_windows_container_resource_memory_request{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}) by (container)","format":"table","instant":true,"legendFormat":"","refId":"B"},{"expr":"sum(windows_container_private_working_set_usage{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}) by (container) / sum(kube_pod_windows_container_resource_memory_request{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}) by (container)","format":"table","instant":true,"legendFormat":"","refId":"C"},{"expr":"sum(kube_pod_windows_container_resource_memory_limit{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}) by (container)","format":"table","instant":true,"legendFormat":"","refId":"D"},{"expr":"sum(windows_container_private_working_set_usage{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}) by (container) / sum(kube_pod_windows_container_resource_memory_limit{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}) by (container)","format":"table","instant":true,"legendFormat":"","refId":"E"}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Memory Quota","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"transform":"table","type":"table","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"short","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":true,"title":"Memory Quota","titleSize":"h6"},{"collapse":false,"height":"250px","panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":1,"fillGradient":0,"id":6,"legend":{"alignAsTable":true,"avg":true,"current":true,"max":false,"min":false,"rightSide":true,"show":true,"sideWidth":null,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null","percentage":false,"pointradius":5,"points":false,"renderer":"flot","repeat":null,"seriesOverrides":[],"spaceLength":10,"span":12,"stack":false,"steppedLine":false,"targets":[{"expr":"sort_desc(sum by (container) (rate(windows_container_network_received_bytes_total{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}[1m])))","format":"time_series","intervalFactor":2,"legendFormat":"Received : {{ container }}","refId":"A"},{"expr":"sort_desc(sum by (container) (rate(windows_container_network_transmitted_bytes_total{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}[1m])))","format":"time_series","intervalFactor":2,"legendFormat":"Transmitted : {{ container }}","refId":"B"}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Network I/O","tooltip":{"shared":true,"sort":0,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"bytes","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"bytes","label":null,"logBase":1,"max":null,"min":0,"show":true}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":true,"title":"Network I/O","titleSize":"h6"}],"schemaVersion":14,"style":"dark","tags":["kubernetes-mixin"],"templating":{"list":[{"current":{"text":"default","value":"default"},"hide":0,"label":null,"name":"datasource","options":[],"query":"prometheus","refresh":1,"regex":"","type":"datasource"},{"allValue":null,"current":{},"datasource":"$datasource","hide":`}}{{ if .Values.grafana.sidecar.dashboards.multicluster.global.enabled }}0{{ else }}2{{ end }}{{`,"includeAll":false,"label":"cluster","multi":false,"name":"cluster","options":[],"query":"label_values(up{job=\"windows-exporter\"}, cluster)","refresh":2,"regex":"","sort":1,"tagValuesQuery":"","tags":[],"tagsQuery":"","type":"query","useTags":false},{"allValue":null,"current":{},"datasource":"$datasource","hide":0,"includeAll":false,"label":"Namespace","multi":false,"name":"namespace","options":[],"query":"label_values(windows_pod_container_available{cluster=\"$cluster\"}, namespace)","refresh":2,"regex":"","sort":1,"tagValuesQuery":"","tags":[],"tagsQuery":"","type":"query","useTags":false},{"allValue":null,"current":{},"datasource":"$datasource","hide":0,"includeAll":false,"label":"Pod","multi":false,"name":"pod","options":[],"query":"label_values(windows_pod_container_available{cluster=\"$cluster\",namespace=\"$namespace\"}, pod)","refresh":2,"regex":"","sort":1,"tagValuesQuery":"","tags":[],"tagsQuery":"","type":"query","useTags":false}]},"time":{"from":"now-6h","to":"now"},"timepicker":{"refresh_intervals":["5s","10s","30s","1m","5m","15m","30m","1h","2h","1d"],"time_options":["5m","15m","1h","6h","12h","24h","2d","7d","30d"]},"timezone": "`}}{{ .Values.grafana.defaultDashboardsTimezone }}{{`","title":"Kubernetes / Compute Resources / Pod(Windows)","uid":"40597a704a610e936dc6ed374a7ce023","version":0}`}} +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/grafana/dashboards-1.14/k8s-resources-workload.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/grafana/dashboards-1.14/k8s-resources-workload.yaml new file mode 100644 index 00000000000..e2a63ae2084 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/grafana/dashboards-1.14/k8s-resources-workload.yaml @@ -0,0 +1,2024 @@ +{{- /* +Generated from 'k8s-resources-workload' from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/a8ba97a150c75be42010c75d10b720c55e182f1a/manifests/grafana-dashboardDefinitions.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (or .Values.grafana.enabled .Values.grafana.forceDeployDashboards) (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.grafana.defaultDashboardsEnabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: {{ .Values.grafana.defaultDashboards.namespace }} + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" $) "k8s-resources-workload" | trunc 63 | trimSuffix "-" }} + annotations: +{{ toYaml .Values.grafana.sidecar.dashboards.annotations | indent 4 }} + labels: + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: {{ ternary $.Values.grafana.sidecar.dashboards.labelValue "1" (not (empty $.Values.grafana.sidecar.dashboards.labelValue)) | quote }} + {{- end }} + app: {{ template "kube-prometheus-stack.name" $ }}-grafana +{{ include "kube-prometheus-stack.labels" $ | indent 4 }} +data: + k8s-resources-workload.json: |- + { + "annotations": { + "list": [ + + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "hideControls": false, + "links": [ + + ], + "refresh": "10s", + "rows": [ + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 1, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(\n node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"$cluster\", namespace=\"$namespace\"}\n * on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n) by (pod)\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "CPU Usage", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "CPU Usage", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 2, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "styles": [ + { + "alias": "Time", + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "pattern": "Time", + "type": "hidden" + }, + { + "alias": "CPU Usage", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #A", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "CPU Requests", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #B", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "CPU Requests %", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #C", + "thresholds": [ + + ], + "type": "number", + "unit": "percentunit" + }, + { + "alias": "CPU Limits", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #D", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "CPU Limits %", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #E", + "thresholds": [ + + ], + "type": "number", + "unit": "percentunit" + }, + { + "alias": "Pod", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": true, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "d/6581e46e4e5c7ba40a07646395ef7b23/k8s-resources-pod?var-datasource=$datasource&var-cluster=$cluster&var-namespace=$namespace&var-pod=$__cell", + "pattern": "pod", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "pattern": "/.*/", + "thresholds": [ + + ], + "type": "string", + "unit": "short" + } + ], + "targets": [ + { + "expr": "sum(\n node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"$cluster\", namespace=\"$namespace\"}\n * on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n) by (pod)\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "A", + "step": 10 + }, + { + "expr": "sum(\n kube_pod_container_resource_requests{job=\"kube-state-metrics\", cluster=\"$cluster\", namespace=\"$namespace\", resource=\"cpu\"}\n * on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n) by (pod)\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "B", + "step": 10 + }, + { + "expr": "sum(\n node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"$cluster\", namespace=\"$namespace\"}\n * on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n) by (pod)\n/sum(\n kube_pod_container_resource_requests{job=\"kube-state-metrics\", cluster=\"$cluster\", namespace=\"$namespace\", resource=\"cpu\"}\n * on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n) by (pod)\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "C", + "step": 10 + }, + { + "expr": "sum(\n kube_pod_container_resource_limits{job=\"kube-state-metrics\", cluster=\"$cluster\", namespace=\"$namespace\", resource=\"cpu\"}\n * on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n) by (pod)\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "D", + "step": 10 + }, + { + "expr": "sum(\n node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"$cluster\", namespace=\"$namespace\"}\n * on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n) by (pod)\n/sum(\n kube_pod_container_resource_limits{job=\"kube-state-metrics\", cluster=\"$cluster\", namespace=\"$namespace\", resource=\"cpu\"}\n * on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n) by (pod)\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "E", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "CPU Quota", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "transform": "table", + "type": "table", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "CPU Quota", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 3, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(\n container_memory_working_set_bytes{cluster=\"$cluster\", namespace=\"$namespace\", container!=\"\", image!=\"\"}\n * on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n) by (pod)\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Memory Usage", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Memory Usage", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 4, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "styles": [ + { + "alias": "Time", + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "pattern": "Time", + "type": "hidden" + }, + { + "alias": "Memory Usage", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #A", + "thresholds": [ + + ], + "type": "number", + "unit": "bytes" + }, + { + "alias": "Memory Requests", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #B", + "thresholds": [ + + ], + "type": "number", + "unit": "bytes" + }, + { + "alias": "Memory Requests %", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #C", + "thresholds": [ + + ], + "type": "number", + "unit": "percentunit" + }, + { + "alias": "Memory Limits", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #D", + "thresholds": [ + + ], + "type": "number", + "unit": "bytes" + }, + { + "alias": "Memory Limits %", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #E", + "thresholds": [ + + ], + "type": "number", + "unit": "percentunit" + }, + { + "alias": "Pod", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": true, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "d/6581e46e4e5c7ba40a07646395ef7b23/k8s-resources-pod?var-datasource=$datasource&var-cluster=$cluster&var-namespace=$namespace&var-pod=$__cell", + "pattern": "pod", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "pattern": "/.*/", + "thresholds": [ + + ], + "type": "string", + "unit": "short" + } + ], + "targets": [ + { + "expr": "sum(\n container_memory_working_set_bytes{cluster=\"$cluster\", namespace=\"$namespace\", container!=\"\", image!=\"\"}\n * on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n) by (pod)\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "A", + "step": 10 + }, + { + "expr": "sum(\n kube_pod_container_resource_requests{job=\"kube-state-metrics\", cluster=\"$cluster\", namespace=\"$namespace\", resource=\"memory\"}\n * on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n) by (pod)\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "B", + "step": 10 + }, + { + "expr": "sum(\n container_memory_working_set_bytes{cluster=\"$cluster\", namespace=\"$namespace\", container!=\"\", image!=\"\"}\n * on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n) by (pod)\n/sum(\n kube_pod_container_resource_requests{job=\"kube-state-metrics\", cluster=\"$cluster\", namespace=\"$namespace\", resource=\"memory\"}\n * on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n) by (pod)\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "C", + "step": 10 + }, + { + "expr": "sum(\n kube_pod_container_resource_limits{job=\"kube-state-metrics\", cluster=\"$cluster\", namespace=\"$namespace\", resource=\"memory\"}\n * on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n) by (pod)\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "D", + "step": 10 + }, + { + "expr": "sum(\n container_memory_working_set_bytes{cluster=\"$cluster\", namespace=\"$namespace\", container!=\"\", image!=\"\"}\n * on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n) by (pod)\n/sum(\n kube_pod_container_resource_limits{job=\"kube-state-metrics\", cluster=\"$cluster\", namespace=\"$namespace\", resource=\"memory\"}\n * on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n) by (pod)\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "E", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Memory Quota", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "transform": "table", + "type": "table", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Memory Quota", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 5, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "styles": [ + { + "alias": "Time", + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "pattern": "Time", + "type": "hidden" + }, + { + "alias": "Current Receive Bandwidth", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #A", + "thresholds": [ + + ], + "type": "number", + "unit": "Bps" + }, + { + "alias": "Current Transmit Bandwidth", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #B", + "thresholds": [ + + ], + "type": "number", + "unit": "Bps" + }, + { + "alias": "Rate of Received Packets", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #C", + "thresholds": [ + + ], + "type": "number", + "unit": "pps" + }, + { + "alias": "Rate of Transmitted Packets", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #D", + "thresholds": [ + + ], + "type": "number", + "unit": "pps" + }, + { + "alias": "Rate of Received Packets Dropped", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #E", + "thresholds": [ + + ], + "type": "number", + "unit": "pps" + }, + { + "alias": "Rate of Transmitted Packets Dropped", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #F", + "thresholds": [ + + ], + "type": "number", + "unit": "pps" + }, + { + "alias": "Pod", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": true, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "d/6581e46e4e5c7ba40a07646395ef7b23/k8s-resources-pod?var-datasource=$datasource&var-cluster=$cluster&var-namespace=$namespace&var-pod=$__cell", + "pattern": "pod", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "pattern": "/.*/", + "thresholds": [ + + ], + "type": "string", + "unit": "short" + } + ], + "targets": [ + { + "expr": "(sum(irate(container_network_receive_bytes_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "A", + "step": 10 + }, + { + "expr": "(sum(irate(container_network_transmit_bytes_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "B", + "step": 10 + }, + { + "expr": "(sum(irate(container_network_receive_packets_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "C", + "step": 10 + }, + { + "expr": "(sum(irate(container_network_transmit_packets_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "D", + "step": 10 + }, + { + "expr": "(sum(irate(container_network_receive_packets_dropped_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "E", + "step": 10 + }, + { + "expr": "(sum(irate(container_network_transmit_packets_dropped_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "F", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Current Network Usage", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "transform": "table", + "type": "table", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Current Network Usage", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 6, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "(sum(irate(container_network_receive_bytes_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Receive Bandwidth", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 7, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "(sum(irate(container_network_transmit_bytes_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Transmit Bandwidth", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Bandwidth", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 8, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "(avg(irate(container_network_receive_bytes_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Average Container Bandwidth by Pod: Received", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 9, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "(avg(irate(container_network_transmit_bytes_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Average Container Bandwidth by Pod: Transmitted", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Average Container Bandwidth by Pod", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 10, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "(sum(irate(container_network_receive_packets_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Received Packets", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 11, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "(sum(irate(container_network_transmit_packets_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Transmitted Packets", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Rate of Packets", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 12, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "(sum(irate(container_network_receive_packets_dropped_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Received Packets Dropped", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 13, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "(sum(irate(container_network_transmit_packets_dropped_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Transmitted Packets Dropped", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Rate of Packets Dropped", + "titleSize": "h6" + } + ], + "schemaVersion": 14, + "style": "dark", + "tags": [ + "kubernetes-mixin" + ], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": "Data Source", + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": null, + "current": { + "text": "", + "value": "" + }, + "datasource": "$datasource", + "hide": {{ if .Values.grafana.sidecar.dashboards.multicluster.global.enabled }}0{{ else }}2{{ end }}, + "includeAll": false, + "label": null, + "multi": false, + "name": "cluster", + "options": [ + + ], + "query": "label_values(up{job=\"kube-state-metrics\"}, cluster)", + "refresh": 2, + "regex": "", + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": { + "text": "", + "value": "" + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "namespace", + "options": [ + + ], + "query": "label_values(kube_namespace_status_phase{job=\"kube-state-metrics\", cluster=\"$cluster\"}, namespace)", + "refresh": 2, + "regex": "", + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": { + "text": "", + "value": "" + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "type", + "options": [ + + ], + "query": "label_values(namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\"}, workload_type)", + "refresh": 2, + "regex": "", + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": { + "text": "", + "value": "" + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "workload", + "options": [ + + ], + "query": "label_values(namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}, workload)", + "refresh": 2, + "regex": "", + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "{{ .Values.grafana.defaultDashboardsTimezone }}", + "title": "Kubernetes / Compute Resources / Workload", + "uid": "a164a7f0339f99e89cea5cb47e9be617", + "version": 0 + } +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/grafana/dashboards-1.14/k8s-resources-workloads-namespace.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/grafana/dashboards-1.14/k8s-resources-workloads-namespace.yaml new file mode 100644 index 00000000000..95d758ea2d4 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/grafana/dashboards-1.14/k8s-resources-workloads-namespace.yaml @@ -0,0 +1,2189 @@ +{{- /* +Generated from 'k8s-resources-workloads-namespace' from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/a8ba97a150c75be42010c75d10b720c55e182f1a/manifests/grafana-dashboardDefinitions.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (or .Values.grafana.enabled .Values.grafana.forceDeployDashboards) (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.grafana.defaultDashboardsEnabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: {{ .Values.grafana.defaultDashboards.namespace }} + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" $) "k8s-resources-workloads-namespace" | trunc 63 | trimSuffix "-" }} + annotations: +{{ toYaml .Values.grafana.sidecar.dashboards.annotations | indent 4 }} + labels: + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: {{ ternary $.Values.grafana.sidecar.dashboards.labelValue "1" (not (empty $.Values.grafana.sidecar.dashboards.labelValue)) | quote }} + {{- end }} + app: {{ template "kube-prometheus-stack.name" $ }}-grafana +{{ include "kube-prometheus-stack.labels" $ | indent 4 }} +data: + k8s-resources-workloads-namespace.json: |- + { + "annotations": { + "list": [ + + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "hideControls": false, + "links": [ + + ], + "refresh": "10s", + "rows": [ + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 1, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "quota - requests", + "color": "#F2495C", + "dashes": true, + "fill": 0, + "hiddenSeries": true, + "hideTooltip": true, + "legend": true, + "linewidth": 2, + "stack": false + }, + { + "alias": "quota - limits", + "color": "#FF9830", + "dashes": true, + "fill": 0, + "hiddenSeries": true, + "hideTooltip": true, + "legend": true, + "linewidth": 2, + "stack": false + } + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(\n node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"$cluster\", namespace=\"$namespace\"}\n* on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}\n) by (workload, workload_type)\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}workload{{`}}`}} - {{`{{`}}workload_type{{`}}`}}", + "legendLink": null, + "step": 10 + }, + { + "expr": "scalar(kube_resourcequota{cluster=\"$cluster\", namespace=\"$namespace\", type=\"hard\",resource=\"requests.cpu\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "quota - requests", + "legendLink": null, + "step": 10 + }, + { + "expr": "scalar(kube_resourcequota{cluster=\"$cluster\", namespace=\"$namespace\", type=\"hard\",resource=\"limits.cpu\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "quota - limits", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "CPU Usage", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "CPU Usage", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 2, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "styles": [ + { + "alias": "Time", + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "pattern": "Time", + "type": "hidden" + }, + { + "alias": "Running Pods", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 0, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #A", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "CPU Usage", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #B", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "CPU Requests", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #C", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "CPU Requests %", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #D", + "thresholds": [ + + ], + "type": "number", + "unit": "percentunit" + }, + { + "alias": "CPU Limits", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #E", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "CPU Limits %", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #F", + "thresholds": [ + + ], + "type": "number", + "unit": "percentunit" + }, + { + "alias": "Workload", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": true, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "d/a164a7f0339f99e89cea5cb47e9be617/k8s-resources-workload?var-datasource=$datasource&var-cluster=$cluster&var-namespace=$namespace&var-workload=$__cell&var-type=$__cell_2", + "pattern": "workload", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "Workload Type", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "workload_type", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "pattern": "/.*/", + "thresholds": [ + + ], + "type": "string", + "unit": "short" + } + ], + "targets": [ + { + "expr": "count(namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}) by (workload, workload_type)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "A", + "step": 10 + }, + { + "expr": "sum(\n node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"$cluster\", namespace=\"$namespace\"}\n* on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}\n) by (workload, workload_type)\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "B", + "step": 10 + }, + { + "expr": "sum(\n kube_pod_container_resource_requests{job=\"kube-state-metrics\", cluster=\"$cluster\", namespace=\"$namespace\", resource=\"cpu\"}\n* on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}\n) by (workload, workload_type)\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "C", + "step": 10 + }, + { + "expr": "sum(\n node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"$cluster\", namespace=\"$namespace\"}\n* on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}\n) by (workload, workload_type)\n/sum(\n kube_pod_container_resource_requests{job=\"kube-state-metrics\", cluster=\"$cluster\", namespace=\"$namespace\", resource=\"cpu\"}\n* on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}\n) by (workload, workload_type)\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "D", + "step": 10 + }, + { + "expr": "sum(\n kube_pod_container_resource_limits{job=\"kube-state-metrics\", cluster=\"$cluster\", namespace=\"$namespace\", resource=\"cpu\"}\n* on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}\n) by (workload, workload_type)\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "E", + "step": 10 + }, + { + "expr": "sum(\n node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"$cluster\", namespace=\"$namespace\"}\n* on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}\n) by (workload, workload_type)\n/sum(\n kube_pod_container_resource_limits{job=\"kube-state-metrics\", cluster=\"$cluster\", namespace=\"$namespace\", resource=\"cpu\"}\n* on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}\n) by (workload, workload_type)\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "F", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "CPU Quota", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "transform": "table", + "type": "table", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "CPU Quota", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 3, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "quota - requests", + "color": "#F2495C", + "dashes": true, + "fill": 0, + "hiddenSeries": true, + "hideTooltip": true, + "legend": true, + "linewidth": 2, + "stack": false + }, + { + "alias": "quota - limits", + "color": "#FF9830", + "dashes": true, + "fill": 0, + "hiddenSeries": true, + "hideTooltip": true, + "legend": true, + "linewidth": 2, + "stack": false + } + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(\n container_memory_working_set_bytes{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\", container!=\"\", image!=\"\"}\n * on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}\n) by (workload, workload_type)\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}workload{{`}}`}} - {{`{{`}}workload_type{{`}}`}}", + "legendLink": null, + "step": 10 + }, + { + "expr": "scalar(kube_resourcequota{cluster=\"$cluster\", namespace=\"$namespace\", type=\"hard\",resource=\"requests.memory\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "quota - requests", + "legendLink": null, + "step": 10 + }, + { + "expr": "scalar(kube_resourcequota{cluster=\"$cluster\", namespace=\"$namespace\", type=\"hard\",resource=\"limits.memory\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "quota - limits", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Memory Usage", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Memory Usage", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 4, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "styles": [ + { + "alias": "Time", + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "pattern": "Time", + "type": "hidden" + }, + { + "alias": "Running Pods", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 0, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #A", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "Memory Usage", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #B", + "thresholds": [ + + ], + "type": "number", + "unit": "bytes" + }, + { + "alias": "Memory Requests", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #C", + "thresholds": [ + + ], + "type": "number", + "unit": "bytes" + }, + { + "alias": "Memory Requests %", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #D", + "thresholds": [ + + ], + "type": "number", + "unit": "percentunit" + }, + { + "alias": "Memory Limits", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #E", + "thresholds": [ + + ], + "type": "number", + "unit": "bytes" + }, + { + "alias": "Memory Limits %", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #F", + "thresholds": [ + + ], + "type": "number", + "unit": "percentunit" + }, + { + "alias": "Workload", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": true, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "d/a164a7f0339f99e89cea5cb47e9be617/k8s-resources-workload?var-datasource=$datasource&var-cluster=$cluster&var-namespace=$namespace&var-workload=$__cell&var-type=$__cell_2", + "pattern": "workload", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "Workload Type", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "workload_type", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "pattern": "/.*/", + "thresholds": [ + + ], + "type": "string", + "unit": "short" + } + ], + "targets": [ + { + "expr": "count(namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}) by (workload, workload_type)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "A", + "step": 10 + }, + { + "expr": "sum(\n container_memory_working_set_bytes{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\", container!=\"\", image!=\"\"}\n * on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}\n) by (workload, workload_type)\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "B", + "step": 10 + }, + { + "expr": "sum(\n kube_pod_container_resource_requests{job=\"kube-state-metrics\", cluster=\"$cluster\", namespace=\"$namespace\", resource=\"memory\"}\n* on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}\n) by (workload, workload_type)\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "C", + "step": 10 + }, + { + "expr": "sum(\n container_memory_working_set_bytes{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\", container!=\"\", image!=\"\"}\n * on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}\n) by (workload, workload_type)\n/sum(\n kube_pod_container_resource_requests{job=\"kube-state-metrics\", cluster=\"$cluster\", namespace=\"$namespace\", resource=\"memory\"}\n* on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}\n) by (workload, workload_type)\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "D", + "step": 10 + }, + { + "expr": "sum(\n kube_pod_container_resource_limits{job=\"kube-state-metrics\", cluster=\"$cluster\", namespace=\"$namespace\", resource=\"memory\"}\n* on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}\n) by (workload, workload_type)\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "E", + "step": 10 + }, + { + "expr": "sum(\n container_memory_working_set_bytes{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\", container!=\"\", image!=\"\"}\n * on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}\n) by (workload, workload_type)\n/sum(\n kube_pod_container_resource_limits{job=\"kube-state-metrics\", cluster=\"$cluster\", namespace=\"$namespace\", resource=\"memory\"}\n* on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}\n) by (workload, workload_type)\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "F", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Memory Quota", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "transform": "table", + "type": "table", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Memory Quota", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 5, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "styles": [ + { + "alias": "Time", + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "pattern": "Time", + "type": "hidden" + }, + { + "alias": "Current Receive Bandwidth", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #A", + "thresholds": [ + + ], + "type": "number", + "unit": "Bps" + }, + { + "alias": "Current Transmit Bandwidth", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #B", + "thresholds": [ + + ], + "type": "number", + "unit": "Bps" + }, + { + "alias": "Rate of Received Packets", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #C", + "thresholds": [ + + ], + "type": "number", + "unit": "pps" + }, + { + "alias": "Rate of Transmitted Packets", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #D", + "thresholds": [ + + ], + "type": "number", + "unit": "pps" + }, + { + "alias": "Rate of Received Packets Dropped", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #E", + "thresholds": [ + + ], + "type": "number", + "unit": "pps" + }, + { + "alias": "Rate of Transmitted Packets Dropped", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #F", + "thresholds": [ + + ], + "type": "number", + "unit": "pps" + }, + { + "alias": "Workload", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": true, + "linkTargetBlank": false, + "linkTooltip": "Drill down to pods", + "linkUrl": "d/a164a7f0339f99e89cea5cb47e9be617/k8s-resources-workload?var-datasource=$datasource&var-cluster=$cluster&var-namespace=$namespace&var-workload=$__cell&var-type=$type", + "pattern": "workload", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "Workload Type", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "workload_type", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "pattern": "/.*/", + "thresholds": [ + + ], + "type": "string", + "unit": "short" + } + ], + "targets": [ + { + "expr": "(sum(irate(container_network_receive_bytes_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}) by (workload))\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "A", + "step": 10 + }, + { + "expr": "(sum(irate(container_network_transmit_bytes_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}) by (workload))\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "B", + "step": 10 + }, + { + "expr": "(sum(irate(container_network_receive_packets_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}) by (workload))\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "C", + "step": 10 + }, + { + "expr": "(sum(irate(container_network_transmit_packets_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}) by (workload))\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "D", + "step": 10 + }, + { + "expr": "(sum(irate(container_network_receive_packets_dropped_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}) by (workload))\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "E", + "step": 10 + }, + { + "expr": "(sum(irate(container_network_transmit_packets_dropped_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}) by (workload))\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "F", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Current Network Usage", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "transform": "table", + "type": "table", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Current Network Usage", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 6, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "(sum(irate(container_network_receive_bytes_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=~\".+\", workload_type=\"$type\"}) by (workload))\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}workload{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Receive Bandwidth", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 7, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "(sum(irate(container_network_transmit_bytes_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=~\".+\", workload_type=\"$type\"}) by (workload))\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}workload{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Transmit Bandwidth", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Bandwidth", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 8, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "(avg(irate(container_network_receive_bytes_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=~\".+\", workload_type=\"$type\"}) by (workload))\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}workload{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Average Container Bandwidth by Workload: Received", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 9, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "(avg(irate(container_network_transmit_bytes_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=~\".+\", workload_type=\"$type\"}) by (workload))\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}workload{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Average Container Bandwidth by Workload: Transmitted", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Average Container Bandwidth by Workload", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 10, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "(sum(irate(container_network_receive_packets_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=~\".+\", workload_type=\"$type\"}) by (workload))\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}workload{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Received Packets", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 11, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "(sum(irate(container_network_transmit_packets_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=~\".+\", workload_type=\"$type\"}) by (workload))\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}workload{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Transmitted Packets", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Rate of Packets", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 12, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "(sum(irate(container_network_receive_packets_dropped_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=~\".+\", workload_type=\"$type\"}) by (workload))\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}workload{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Received Packets Dropped", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 13, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "(sum(irate(container_network_transmit_packets_dropped_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=~\".+\", workload_type=\"$type\"}) by (workload))\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}workload{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Transmitted Packets Dropped", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Rate of Packets Dropped", + "titleSize": "h6" + } + ], + "schemaVersion": 14, + "style": "dark", + "tags": [ + "kubernetes-mixin" + ], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": "Data Source", + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": null, + "current": { + "text": "", + "value": "" + }, + "datasource": "$datasource", + "hide": {{ if .Values.grafana.sidecar.dashboards.multicluster.global.enabled }}0{{ else }}2{{ end }}, + "includeAll": false, + "label": null, + "multi": false, + "name": "cluster", + "options": [ + + ], + "query": "label_values(up{job=\"kube-state-metrics\"}, cluster)", + "refresh": 2, + "regex": "", + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": { + "text": "", + "value": "" + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "namespace", + "options": [ + + ], + "query": "label_values(kube_pod_info{job=\"kube-state-metrics\", cluster=\"$cluster\"}, namespace)", + "refresh": 2, + "regex": "", + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "auto": false, + "auto_count": 30, + "auto_min": "10s", + "current": { + "text": "deployment", + "value": "deployment" + }, + "datasource": "$datasource", + "definition": "label_values(namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=~\".+\"}, workload_type)", + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "type", + "options": [ + + ], + "query": "label_values(namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=~\".+\"}, workload_type)", + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "{{ .Values.grafana.defaultDashboardsTimezone }}", + "title": "Kubernetes / Compute Resources / Namespace (Workloads)", + "uid": "a87fb0d919ec0ea5f6543124e16c42a5", + "version": 0 + } +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/grafana/dashboards-1.14/k8s-windows-cluster-rsrc-use.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/grafana/dashboards-1.14/k8s-windows-cluster-rsrc-use.yaml new file mode 100644 index 00000000000..d9ce9d738c9 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/grafana/dashboards-1.14/k8s-windows-cluster-rsrc-use.yaml @@ -0,0 +1,24 @@ +{{- /* +Generated from 'k8s-windows-cluster-rsrc-use' from https://github.com/kubernetes-monitoring/kubernetes-mixin.git +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (or .Values.grafana.enabled .Values.grafana.forceDeployDashboards) (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.grafana.defaultDashboardsEnabled .Values.windowsMonitoring.enabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: {{ template "kube-prometheus-stack-grafana.namespace" . }} + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" $) "k8s-windows-cluster-rsrc-use" | trunc 63 | trimSuffix "-" }} + annotations: +{{ toYaml .Values.grafana.sidecar.dashboards.annotations | indent 4 }} + labels: + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: {{ ternary $.Values.grafana.sidecar.dashboards.labelValue "1" (not (empty $.Values.grafana.sidecar.dashboards.labelValue)) | quote }} + {{- end }} + app: {{ template "kube-prometheus-stack.name" $ }}-grafana +{{ include "kube-prometheus-stack.labels" $ | indent 4 }} +data: + k8s-windows-cluster-rsrc-use.json: |- + {{`{"__inputs":[],"__requires":[],"annotations":{"list":[]},"editable":`}}{{ .Values.grafana.defaultDashboardsEditable }}{{`,"gnetId":null,"graphTooltip":0,"hideControls":false,"id":null,"links":[],"refresh":"","rows":[{"collapse":false,"height":"250px","panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":10,"id":2,"legend":{"avg":false,"current":false,"max":false,"min":false,"show":true,"total":false,"values":false},"lines":true,"linewidth":0,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":12,"stack":true,"steppedLine":false,"targets":[{"expr":"node:windows_node_cpu_utilisation:avg1m{cluster=\"$cluster\"} * node:windows_node_num_cpu:sum{cluster=\"$cluster\"} / scalar(sum(node:windows_node_num_cpu:sum{cluster=\"$cluster\"}))","format":"time_series","legendFormat":"{{instance}}","legendLink":"/d/96e7484b0bb53b74fbc2bcb7723cd40b/k8s-windows-node-rsrc-use"}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"CPU Utilisation","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"percentunit","label":null,"logBase":1,"max":1,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":true,"title":"CPU","titleSize":"h6"},{"collapse":false,"height":"250px","panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":10,"id":3,"legend":{"avg":false,"current":false,"max":false,"min":false,"show":true,"total":false,"values":false},"lines":true,"linewidth":0,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":6,"stack":true,"steppedLine":false,"targets":[{"expr":"node:windows_node_memory_utilisation:ratio{cluster=\"$cluster\"}","format":"time_series","legendFormat":"{{instance}}","legendLink":"/d/96e7484b0bb53b74fbc2bcb7723cd40b/k8s-windows-node-rsrc-use"}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Memory Utilisation","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"percentunit","label":null,"logBase":1,"max":1,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":10,"id":4,"legend":{"avg":false,"current":false,"max":false,"min":false,"show":true,"total":false,"values":false},"lines":true,"linewidth":0,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":6,"stack":true,"steppedLine":false,"targets":[{"expr":"node:windows_node_memory_swap_io_pages:irate{cluster=\"$cluster\"}","format":"time_series","legendFormat":"{{instance}}","legendLink":"/d/96e7484b0bb53b74fbc2bcb7723cd40b/k8s-windows-node-rsrc-use"}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Memory Saturation (Swap I/O Pages)","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"short","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":true,"title":"Memory","titleSize":"h6"},{"collapse":false,"height":"250px","panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":10,"id":5,"legend":{"avg":false,"current":false,"max":false,"min":false,"show":true,"total":false,"values":false},"lines":true,"linewidth":0,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":12,"stack":true,"steppedLine":false,"targets":[{"expr":"node:windows_node_disk_utilisation:avg_irate{cluster=\"$cluster\"} / scalar(node:windows_node:sum{cluster=\"$cluster\"})","format":"time_series","legendFormat":"{{instance}}","legendLink":"/d/96e7484b0bb53b74fbc2bcb7723cd40b/k8s-windows-node-rsrc-use"}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Disk IO Utilisation","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"percentunit","label":null,"logBase":1,"max":1,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":true,"title":"Disk","titleSize":"h6"},{"collapse":false,"height":"250px","panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":10,"id":6,"legend":{"avg":false,"current":false,"max":false,"min":false,"show":true,"total":false,"values":false},"lines":true,"linewidth":0,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":6,"stack":true,"steppedLine":false,"targets":[{"expr":"node:windows_node_net_utilisation:sum_irate{cluster=\"$cluster\"}","format":"time_series","legendFormat":"{{instance}}","legendLink":"/d/96e7484b0bb53b74fbc2bcb7723cd40b/k8s-windows-node-rsrc-use"}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Net Utilisation (Transmitted)","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"Bps","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":10,"id":7,"legend":{"avg":false,"current":false,"max":false,"min":false,"show":true,"total":false,"values":false},"lines":true,"linewidth":0,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":6,"stack":true,"steppedLine":false,"targets":[{"expr":"node:windows_node_net_saturation:sum_irate{cluster=\"$cluster\"}","format":"time_series","legendFormat":"{{instance}}","legendLink":"/d/96e7484b0bb53b74fbc2bcb7723cd40b/k8s-windows-node-rsrc-use"}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Net Saturation (Dropped)","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"Bps","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":true,"title":"Network","titleSize":"h6"},{"collapse":false,"height":"250px","panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":10,"id":8,"legend":{"avg":false,"current":false,"max":false,"min":false,"show":true,"total":false,"values":false},"lines":true,"linewidth":0,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":12,"stack":true,"steppedLine":false,"targets":[{"expr":"sum by (instance)(node:windows_node_filesystem_usage:{cluster=\"$cluster\"})\n","format":"time_series","legendFormat":"{{instance}}","legendLink":"/d/96e7484b0bb53b74fbc2bcb7723cd40b/k8s-windows-node-rsrc-use"}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Disk Capacity","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"percentunit","label":null,"logBase":1,"max":1,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":true,"title":"Storage","titleSize":"h6"}],"schemaVersion":14,"style":"dark","tags":["kubernetes-mixin"],"templating":{"list":[{"current":{"text":"default","value":"default"},"hide":0,"label":null,"name":"datasource","options":[],"query":"prometheus","refresh":1,"regex":"","type":"datasource"},{"allValue":null,"current":{},"datasource":"$datasource","hide":`}}{{ if .Values.grafana.sidecar.dashboards.multicluster.global.enabled }}0{{ else }}2{{ end }}{{`,"includeAll":false,"label":"cluster","multi":false,"name":"cluster","options":[],"query":"label_values(up{job=\"windows-exporter\"}, cluster)","refresh":2,"regex":"","sort":1,"tagValuesQuery":"","tags":[],"tagsQuery":"","type":"query","useTags":false}]},"time":{"from":"now-6h","to":"now"},"timepicker":{"refresh_intervals":["5s","10s","30s","1m","5m","15m","30m","1h","2h","1d"],"time_options":["5m","15m","1h","6h","12h","24h","2d","7d","30d"]},"timezone": "`}}{{ .Values.grafana.defaultDashboardsTimezone }}{{`","title":"Kubernetes / USE Method / Cluster(Windows)","uid":"53a43377ec9aaf2ff64dfc7a1f539334","version":0}`}} +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/grafana/dashboards-1.14/k8s-windows-node-rsrc-use.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/grafana/dashboards-1.14/k8s-windows-node-rsrc-use.yaml new file mode 100644 index 00000000000..a7608496a30 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/grafana/dashboards-1.14/k8s-windows-node-rsrc-use.yaml @@ -0,0 +1,24 @@ +{{- /* +Generated from 'k8s-windows-node-rsrc-use' from https://github.com/kubernetes-monitoring/kubernetes-mixin.git +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (or .Values.grafana.enabled .Values.grafana.forceDeployDashboards) (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.grafana.defaultDashboardsEnabled .Values.windowsMonitoring.enabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: {{ template "kube-prometheus-stack-grafana.namespace" . }} + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" $) "k8s-windows-node-rsrc-use" | trunc 63 | trimSuffix "-" }} + annotations: +{{ toYaml .Values.grafana.sidecar.dashboards.annotations | indent 4 }} + labels: + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: {{ ternary $.Values.grafana.sidecar.dashboards.labelValue "1" (not (empty $.Values.grafana.sidecar.dashboards.labelValue)) | quote }} + {{- end }} + app: {{ template "kube-prometheus-stack.name" $ }}-grafana +{{ include "kube-prometheus-stack.labels" $ | indent 4 }} +data: + k8s-windows-node-rsrc-use.json: |- + {{`{"__inputs":[],"__requires":[],"annotations":{"list":[]},"editable":`}}{{ .Values.grafana.defaultDashboardsEditable }}{{`,"gnetId":null,"graphTooltip":0,"hideControls":false,"id":null,"links":[],"refresh":"","rows":[{"collapse":false,"height":"250px","panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":1,"id":2,"legend":{"avg":false,"current":false,"max":false,"min":false,"show":true,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":6,"stack":false,"steppedLine":false,"targets":[{"expr":"node:windows_node_cpu_utilisation:avg1m{cluster=\"$cluster\", instance=\"$instance\"}","format":"time_series","legendFormat":"Utilisation","legendLink":null}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"CPU Utilisation","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"percentunit","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":1,"id":3,"legend":{"avg":false,"current":false,"max":false,"min":false,"show":true,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":6,"stack":false,"steppedLine":false,"targets":[{"expr":"sum by (core) (irate(windows_cpu_time_total{cluster=\"$cluster\", job=\"windows-exporter\", mode!=\"idle\", instance=\"$instance\"}[$__rate_interval]))","format":"time_series","legendFormat":"{{core}}","legendLink":null}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"CPU Usage Per Core","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"percentunit","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":true,"title":"CPU","titleSize":"h6"},{"collapse":false,"height":"250px","panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":1,"id":4,"legend":{"avg":false,"current":false,"max":false,"min":false,"show":true,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":4,"stack":false,"steppedLine":false,"targets":[{"expr":"node:windows_node_memory_utilisation:{cluster=\"$cluster\", instance=\"$instance\"}","format":"time_series","legendFormat":"Memory","legendLink":null}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Memory Utilisation %","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"percentunit","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":1,"fillGradient":0,"id":5,"legend":{"alignAsTable":false,"avg":false,"current":false,"max":false,"min":false,"rightSide":false,"show":true,"sideWidth":null,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null","percentage":false,"pointradius":5,"points":false,"renderer":"flot","repeat":null,"seriesOverrides":[],"spaceLength":10,"span":4,"stack":false,"steppedLine":false,"targets":[{"expr":"max(\n windows_os_visible_memory_bytes{cluster=\"$cluster\", job=\"windows-exporter\", instance=\"$instance\"}\n - windows_memory_available_bytes{cluster=\"$cluster\", job=\"windows-exporter\", instance=\"$instance\"}\n)\n","format":"time_series","intervalFactor":2,"legendFormat":"memory used","refId":"A"},{"expr":"max(node:windows_node_memory_totalCached_bytes:sum{cluster=\"$cluster\", instance=\"$instance\"})","format":"time_series","intervalFactor":2,"legendFormat":"memory cached","refId":"B"},{"expr":"max(windows_memory_available_bytes{cluster=\"$cluster\", job=\"windows-exporter\", instance=\"$instance\"})","format":"time_series","intervalFactor":2,"legendFormat":"memory free","refId":"C"}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Memory Usage","tooltip":{"shared":true,"sort":0,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"bytes","label":null,"logBase":1,"max":null,"min":null,"show":true},{"format":"bytes","label":null,"logBase":1,"max":null,"min":null,"show":true}]},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":1,"id":6,"legend":{"avg":false,"current":false,"max":false,"min":false,"show":true,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":4,"stack":false,"steppedLine":false,"targets":[{"expr":"node:windows_node_memory_swap_io_pages:irate{cluster=\"$cluster\", instance=\"$instance\"}","format":"time_series","legendFormat":"Swap IO","legendLink":null}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Memory Saturation (Swap I/O) Pages","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"short","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":true,"title":"Memory","titleSize":"h6"},{"collapse":false,"height":"250px","panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":1,"id":7,"legend":{"avg":false,"current":false,"max":false,"min":false,"show":true,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":6,"stack":false,"steppedLine":false,"targets":[{"expr":"node:windows_node_disk_utilisation:avg_irate{cluster=\"$cluster\", instance=\"$instance\"}","format":"time_series","legendFormat":"Utilisation","legendLink":null}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Disk IO Utilisation","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"percentunit","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":1,"fillGradient":0,"id":8,"legend":{"alignAsTable":false,"avg":false,"current":false,"max":false,"min":false,"rightSide":false,"show":true,"sideWidth":null,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null","percentage":false,"pointradius":5,"points":false,"renderer":"flot","repeat":null,"seriesOverrides":[{"alias":"read","yaxis":1},{"alias":"io time","yaxis":2}],"spaceLength":10,"span":6,"stack":false,"steppedLine":false,"targets":[{"expr":"max(rate(windows_logical_disk_read_bytes_total{cluster=\"$cluster\", job=\"windows-exporter\", instance=\"$instance\"}[2m]))","format":"time_series","intervalFactor":2,"legendFormat":"read","refId":"A"},{"expr":"max(rate(windows_logical_disk_write_bytes_total{cluster=\"$cluster\", job=\"windows-exporter\", instance=\"$instance\"}[2m]))","format":"time_series","intervalFactor":2,"legendFormat":"written","refId":"B"},{"expr":"max(rate(windows_logical_disk_read_seconds_total{cluster=\"$cluster\", job=\"windows-exporter\", instance=\"$instance\"}[2m]) + rate(windows_logical_disk_write_seconds_total{cluster=\"$cluster\", job=\"windows-exporter\", instance=\"$instance\"}[2m]))","format":"time_series","intervalFactor":2,"legendFormat":"io time","refId":"C"}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Disk I/O","tooltip":{"shared":true,"sort":0,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"bytes","label":null,"logBase":1,"max":null,"min":null,"show":true},{"format":"ms","label":null,"logBase":1,"max":null,"min":null,"show":true}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":true,"title":"Disk","titleSize":"h6"},{"collapse":false,"height":"250px","panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":1,"id":9,"legend":{"avg":false,"current":false,"max":false,"min":false,"show":true,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":6,"stack":false,"steppedLine":false,"targets":[{"expr":"node:windows_node_net_utilisation:sum_irate{cluster=\"$cluster\", instance=\"$instance\"}","format":"time_series","legendFormat":"Utilisation","legendLink":null}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Net Utilisation (Transmitted)","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"Bps","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":1,"id":10,"legend":{"avg":false,"current":false,"max":false,"min":false,"show":true,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":6,"stack":false,"steppedLine":false,"targets":[{"expr":"node:windows_node_net_saturation:sum_irate{cluster=\"$cluster\", instance=\"$instance\"}","format":"time_series","legendFormat":"Saturation","legendLink":null}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Net Saturation (Dropped)","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"Bps","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":true,"title":"Net","titleSize":"h6"},{"collapse":false,"height":"250px","panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":1,"id":11,"legend":{"avg":false,"current":false,"max":false,"min":false,"show":true,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":12,"stack":false,"steppedLine":false,"targets":[{"expr":"node:windows_node_filesystem_usage:{cluster=\"$cluster\", instance=\"$instance\"}\n","format":"time_series","legendFormat":"{{volume}}","legendLink":null}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Disk Utilisation","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"percentunit","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":true,"title":"Disk","titleSize":"h6"}],"schemaVersion":14,"style":"dark","tags":["kubernetes-mixin"],"templating":{"list":[{"current":{"text":"default","value":"default"},"hide":0,"label":null,"name":"datasource","options":[],"query":"prometheus","refresh":1,"regex":"","type":"datasource"},{"allValue":null,"current":{},"datasource":"$datasource","hide":`}}{{ if .Values.grafana.sidecar.dashboards.multicluster.global.enabled }}0{{ else }}2{{ end }}{{`,"includeAll":false,"label":"cluster","multi":false,"name":"cluster","options":[],"query":"label_values(up{job=\"windows-exporter\"}, cluster)","refresh":2,"regex":"","sort":1,"tagValuesQuery":"","tags":[],"tagsQuery":"","type":"query","useTags":false},{"allValue":null,"current":{},"datasource":"$datasource","hide":0,"includeAll":false,"label":"Instance","multi":false,"name":"instance","options":[],"query":"label_values(windows_system_system_up_time{cluster=\"$cluster\"}, instance)","refresh":2,"regex":"","sort":1,"tagValuesQuery":"","tags":[],"tagsQuery":"","type":"query","useTags":false}]},"time":{"from":"now-6h","to":"now"},"timepicker":{"refresh_intervals":["5s","10s","30s","1m","5m","15m","30m","1h","2h","1d"],"time_options":["5m","15m","1h","6h","12h","24h","2d","7d","30d"]},"timezone": "`}}{{ .Values.grafana.defaultDashboardsTimezone }}{{`","title":"Kubernetes / USE Method / Node(Windows)","uid":"96e7484b0bb53b74fbc2bcb7723cd40b","version":0}`}} +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/grafana/dashboards-1.14/kubelet.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/grafana/dashboards-1.14/kubelet.yaml new file mode 100644 index 00000000000..74a5303f8fd --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/grafana/dashboards-1.14/kubelet.yaml @@ -0,0 +1,2256 @@ +{{- /* +Generated from 'kubelet' from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/a8ba97a150c75be42010c75d10b720c55e182f1a/manifests/grafana-dashboardDefinitions.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (or .Values.grafana.enabled .Values.grafana.forceDeployDashboards) (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.grafana.defaultDashboardsEnabled }} +{{- if (include "exporter.kubelet.enabled" .) }} +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: {{ .Values.grafana.defaultDashboards.namespace }} + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" $) "kubelet" | trunc 63 | trimSuffix "-" }} + annotations: +{{ toYaml .Values.grafana.sidecar.dashboards.annotations | indent 4 }} + labels: + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: {{ ternary $.Values.grafana.sidecar.dashboards.labelValue "1" (not (empty $.Values.grafana.sidecar.dashboards.labelValue)) | quote }} + {{- end }} + app: {{ template "kube-prometheus-stack.name" $ }}-grafana +{{ include "kube-prometheus-stack.labels" $ | indent 4 }} +data: + kubelet.json: |- + { + "__inputs": [ + + ], + "__requires": [ + + ], + "annotations": { + "list": [ + + ] + }, + "editable": false, + "gnetId": null, + "graphTooltip": 0, + "hideControls": false, + "id": null, + "links": [ + + ], + "panels": [ + { + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "links": [ + + ], + "mappings": [ + + ], + "thresholds": { + "mode": "absolute", + "steps": [ + + ] + }, + "unit": "none" + } + }, + "gridPos": { + "h": 7, + "w": 4, + "x": 0, + "y": 0 + }, + "id": 2, + "links": [ + + ], + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "7", + "targets": [ + { + "expr": "sum(kubelet_node_name{cluster=\"$cluster\", job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "", + "refId": "A" + } + ], + "title": "Running Kubelets", + "transparent": false, + "type": "stat" + }, + { + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "links": [ + + ], + "mappings": [ + + ], + "thresholds": { + "mode": "absolute", + "steps": [ + + ] + }, + "unit": "none" + } + }, + "gridPos": { + "h": 7, + "w": 4, + "x": 4, + "y": 0 + }, + "id": 3, + "links": [ + + ], + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "7", + "targets": [ + { + "expr": "sum(kubelet_running_pods{cluster=\"$cluster\", job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics\", instance=~\"$instance\"}) OR sum(kubelet_running_pod_count{cluster=\"$cluster\", job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics\", instance=~\"$instance\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}}", + "refId": "A" + } + ], + "title": "Running Pods", + "transparent": false, + "type": "stat" + }, + { + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "links": [ + + ], + "mappings": [ + + ], + "thresholds": { + "mode": "absolute", + "steps": [ + + ] + }, + "unit": "none" + } + }, + "gridPos": { + "h": 7, + "w": 4, + "x": 8, + "y": 0 + }, + "id": 4, + "links": [ + + ], + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "7", + "targets": [ + { + "expr": "sum(kubelet_running_containers{cluster=\"$cluster\", job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics\", instance=~\"$instance\"}) OR sum(kubelet_running_container_count{cluster=\"$cluster\", job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics\", instance=~\"$instance\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}}", + "refId": "A" + } + ], + "title": "Running Containers", + "transparent": false, + "type": "stat" + }, + { + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "links": [ + + ], + "mappings": [ + + ], + "thresholds": { + "mode": "absolute", + "steps": [ + + ] + }, + "unit": "none" + } + }, + "gridPos": { + "h": 7, + "w": 4, + "x": 12, + "y": 0 + }, + "id": 5, + "links": [ + + ], + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "7", + "targets": [ + { + "expr": "sum(volume_manager_total_volumes{cluster=\"$cluster\", job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics\", instance=~\"$instance\", state=\"actual_state_of_world\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}}", + "refId": "A" + } + ], + "title": "Actual Volume Count", + "transparent": false, + "type": "stat" + }, + { + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "links": [ + + ], + "mappings": [ + + ], + "thresholds": { + "mode": "absolute", + "steps": [ + + ] + }, + "unit": "none" + } + }, + "gridPos": { + "h": 7, + "w": 4, + "x": 16, + "y": 0 + }, + "id": 6, + "links": [ + + ], + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "7", + "targets": [ + { + "expr": "sum(volume_manager_total_volumes{cluster=\"$cluster\", job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics\", instance=~\"$instance\",state=\"desired_state_of_world\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}}", + "refId": "A" + } + ], + "title": "Desired Volume Count", + "transparent": false, + "type": "stat" + }, + { + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "links": [ + + ], + "mappings": [ + + ], + "thresholds": { + "mode": "absolute", + "steps": [ + + ] + }, + "unit": "none" + } + }, + "gridPos": { + "h": 7, + "w": 4, + "x": 20, + "y": 0 + }, + "id": 7, + "links": [ + + ], + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "7", + "targets": [ + { + "expr": "sum(rate(kubelet_node_config_error{cluster=\"$cluster\", job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics\", instance=~\"$instance\"}[$__rate_interval]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}}", + "refId": "A" + } + ], + "title": "Config Error Count", + "transparent": false, + "type": "stat" + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 7 + }, + "id": 8, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(kubelet_runtime_operations_total{cluster=\"$cluster\",job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics\",instance=~\"$instance\"}[$__rate_interval])) by (operation_type, instance)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}} {{`{{`}}operation_type{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Operation Rate", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 7 + }, + "id": 9, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(kubelet_runtime_operations_errors_total{cluster=\"$cluster\",job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics\",instance=~\"$instance\"}[$__rate_interval])) by (instance, operation_type)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}} {{`{{`}}operation_type{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Operation Error Rate", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 24, + "x": 0, + "y": 14 + }, + "id": 10, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum(rate(kubelet_runtime_operations_duration_seconds_bucket{cluster=\"$cluster\",job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics\",instance=~\"$instance\"}[$__rate_interval])) by (instance, operation_type, le))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}} {{`{{`}}operation_type{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Operation duration 99th quantile", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 21 + }, + "id": 11, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(kubelet_pod_start_duration_seconds_count{cluster=\"$cluster\",job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics\",instance=~\"$instance\"}[$__rate_interval])) by (instance)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}} pod", + "refId": "A" + }, + { + "expr": "sum(rate(kubelet_pod_worker_duration_seconds_count{cluster=\"$cluster\",job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics\",instance=~\"$instance\"}[$__rate_interval])) by (instance)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}} worker", + "refId": "B" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Pod Start Rate", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 21 + }, + "id": 12, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum(rate(kubelet_pod_start_duration_seconds_bucket{cluster=\"$cluster\",job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics\",instance=~\"$instance\"}[$__rate_interval])) by (instance, le))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}} pod", + "refId": "A" + }, + { + "expr": "histogram_quantile(0.99, sum(rate(kubelet_pod_worker_duration_seconds_bucket{cluster=\"$cluster\",job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics\",instance=~\"$instance\"}[$__rate_interval])) by (instance, le))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}} worker", + "refId": "B" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Pod Start Duration", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 28 + }, + "id": 13, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(storage_operation_duration_seconds_count{cluster=\"$cluster\",job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics\",instance=~\"$instance\"}[$__rate_interval])) by (instance, operation_name, volume_plugin)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}} {{`{{`}}operation_name{{`}}`}} {{`{{`}}volume_plugin{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Storage Operation Rate", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 28 + }, + "id": 14, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(storage_operation_errors_total{cluster=\"$cluster\",job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics\",instance=~\"$instance\"}[$__rate_interval])) by (instance, operation_name, volume_plugin)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}} {{`{{`}}operation_name{{`}}`}} {{`{{`}}volume_plugin{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Storage Operation Error Rate", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 24, + "x": 0, + "y": 35 + }, + "id": 15, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum(rate(storage_operation_duration_seconds_bucket{cluster=\"$cluster\", job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics\", instance=~\"$instance\"}[$__rate_interval])) by (instance, operation_name, volume_plugin, le))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}} {{`{{`}}operation_name{{`}}`}} {{`{{`}}volume_plugin{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Storage Operation Duration 99th quantile", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 42 + }, + "id": 16, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(kubelet_cgroup_manager_duration_seconds_count{cluster=\"$cluster\", job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics\", instance=~\"$instance\"}[$__rate_interval])) by (instance, operation_type)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}operation_type{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Cgroup manager operation rate", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 42 + }, + "id": 17, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum(rate(kubelet_cgroup_manager_duration_seconds_bucket{cluster=\"$cluster\", job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics\", instance=~\"$instance\"}[$__rate_interval])) by (instance, operation_type, le))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}} {{`{{`}}operation_type{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Cgroup manager 99th quantile", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "Pod lifecycle event generator", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 49 + }, + "id": 18, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(kubelet_pleg_relist_duration_seconds_count{cluster=\"$cluster\", job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics\", instance=~\"$instance\"}[$__rate_interval])) by (instance)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "PLEG relist rate", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 49 + }, + "id": 19, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum(rate(kubelet_pleg_relist_interval_seconds_bucket{cluster=\"$cluster\",job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics\",instance=~\"$instance\"}[$__rate_interval])) by (instance, le))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "PLEG relist interval", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 24, + "x": 0, + "y": 56 + }, + "id": 20, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum(rate(kubelet_pleg_relist_duration_seconds_bucket{cluster=\"$cluster\",job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics\",instance=~\"$instance\"}[$__rate_interval])) by (instance, le))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "PLEG relist duration", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 24, + "x": 0, + "y": 63 + }, + "id": 21, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(rest_client_requests_total{cluster=\"$cluster\",job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics\", instance=~\"$instance\",code=~\"2..\"}[$__rate_interval]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "2xx", + "refId": "A" + }, + { + "expr": "sum(rate(rest_client_requests_total{cluster=\"$cluster\",job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics\", instance=~\"$instance\",code=~\"3..\"}[$__rate_interval]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "3xx", + "refId": "B" + }, + { + "expr": "sum(rate(rest_client_requests_total{cluster=\"$cluster\",job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics\", instance=~\"$instance\",code=~\"4..\"}[$__rate_interval]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "4xx", + "refId": "C" + }, + { + "expr": "sum(rate(rest_client_requests_total{cluster=\"$cluster\",job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics\", instance=~\"$instance\",code=~\"5..\"}[$__rate_interval]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "5xx", + "refId": "D" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "RPC Rate", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 24, + "x": 0, + "y": 70 + }, + "id": 22, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum(rate(rest_client_request_duration_seconds_bucket{cluster=\"$cluster\",job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics\", instance=~\"$instance\"}[$__rate_interval])) by (instance, verb, url, le))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}} {{`{{`}}verb{{`}}`}} {{`{{`}}url{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Request duration 99th quantile", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 77 + }, + "id": 23, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "process_resident_memory_bytes{cluster=\"$cluster\",job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics\",instance=~\"$instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Memory", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 77 + }, + "id": 24, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(process_cpu_seconds_total{cluster=\"$cluster\",job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics\",instance=~\"$instance\"}[$__rate_interval])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "CPU usage", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 77 + }, + "id": 25, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "go_goroutines{cluster=\"$cluster\",job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics\",instance=~\"$instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Goroutines", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "refresh": "10s", + "rows": [ + + ], + "schemaVersion": 14, + "style": "dark", + "tags": [ + "kubernetes-mixin" + ], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": "Data Source", + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": null, + "current": { + + }, + "datasource": "$datasource", + "hide": {{ if .Values.grafana.sidecar.dashboards.multicluster.global.enabled }}0{{ else }}2{{ end }}, + "includeAll": false, + "label": "cluster", + "multi": false, + "name": "cluster", + "options": [ + + ], + "query": "label_values(up{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics\"}, cluster)", + "refresh": 2, + "regex": "", + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": { + + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": true, + "label": "instance", + "multi": false, + "name": "instance", + "options": [ + + ], + "query": "label_values(up{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics\",cluster=\"$cluster\"}, instance)", + "refresh": 2, + "regex": "", + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "{{ .Values.grafana.defaultDashboardsTimezone }}", + "title": "Kubernetes / Kubelet", + "uid": "3138fa155d5915769fbded898ac09fd9", + "version": 0 + } +{{- end }} +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/grafana/dashboards-1.14/namespace-by-pod.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/grafana/dashboards-1.14/namespace-by-pod.yaml new file mode 100644 index 00000000000..f5c72844fb7 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/grafana/dashboards-1.14/namespace-by-pod.yaml @@ -0,0 +1,1464 @@ +{{- /* +Generated from 'namespace-by-pod' from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/a8ba97a150c75be42010c75d10b720c55e182f1a/manifests/grafana-dashboardDefinitions.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (or .Values.grafana.enabled .Values.grafana.forceDeployDashboards) (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.grafana.defaultDashboardsEnabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: {{ .Values.grafana.defaultDashboards.namespace }} + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" $) "namespace-by-pod" | trunc 63 | trimSuffix "-" }} + annotations: +{{ toYaml .Values.grafana.sidecar.dashboards.annotations | indent 4 }} + labels: + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: {{ ternary $.Values.grafana.sidecar.dashboards.labelValue "1" (not (empty $.Values.grafana.sidecar.dashboards.labelValue)) | quote }} + {{- end }} + app: {{ template "kube-prometheus-stack.name" $ }}-grafana +{{ include "kube-prometheus-stack.labels" $ | indent 4 }} +data: + namespace-by-pod.json: |- + { + "__inputs": [ + + ], + "__requires": [ + + ], + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "hideControls": false, + "id": null, + "links": [ + + ], + "panels": [ + { + "collapse": false, + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 2, + "panels": [ + + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Current Bandwidth", + "titleSize": "h6", + "type": "row" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": "$datasource", + "decimals": 0, + "format": "time_series", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 1 + }, + "height": 9, + "id": 3, + "interval": null, + "links": [ + + ], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "minSpan": 12, + "nullPointMode": "connected", + "nullText": null, + "options": { + "fieldOptions": { + "calcs": [ + "last" + ], + "defaults": { + "max": 10000000000, + "min": 0, + "title": "$namespace", + "unit": "Bps" + }, + "mappings": [ + + ], + "override": { + + }, + "thresholds": [ + { + "color": "dark-green", + "index": 0, + "value": null + }, + { + "color": "dark-yellow", + "index": 1, + "value": 5000000000 + }, + { + "color": "dark-red", + "index": 2, + "value": 7000000000 + } + ], + "values": false + } + }, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "span": 12, + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "sum(irate(container_network_receive_bytes_total{cluster=\"$cluster\",namespace=~\"$namespace\"}[$interval:$resolution]))", + "format": "time_series", + "instant": null, + "intervalFactor": 1, + "legendFormat": "", + "refId": "A" + } + ], + "thresholds": "", + "timeFrom": null, + "timeShift": null, + "title": "Current Rate of Bytes Received", + "type": "gauge", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": "$datasource", + "decimals": 0, + "format": "time_series", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 1 + }, + "height": 9, + "id": 4, + "interval": null, + "links": [ + + ], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "minSpan": 12, + "nullPointMode": "connected", + "nullText": null, + "options": { + "fieldOptions": { + "calcs": [ + "last" + ], + "defaults": { + "max": 10000000000, + "min": 0, + "title": "$namespace", + "unit": "Bps" + }, + "mappings": [ + + ], + "override": { + + }, + "thresholds": [ + { + "color": "dark-green", + "index": 0, + "value": null + }, + { + "color": "dark-yellow", + "index": 1, + "value": 5000000000 + }, + { + "color": "dark-red", + "index": 2, + "value": 7000000000 + } + ], + "values": false + } + }, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "span": 12, + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "sum(irate(container_network_transmit_bytes_total{cluster=\"$cluster\",namespace=~\"$namespace\"}[$interval:$resolution]))", + "format": "time_series", + "instant": null, + "intervalFactor": 1, + "legendFormat": "", + "refId": "A" + } + ], + "thresholds": "", + "timeFrom": null, + "timeShift": null, + "title": "Current Rate of Bytes Transmitted", + "type": "gauge", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + }, + { + "columns": [ + { + "text": "Time", + "value": "Time" + }, + { + "text": "Value #A", + "value": "Value #A" + }, + { + "text": "Value #B", + "value": "Value #B" + }, + { + "text": "Value #C", + "value": "Value #C" + }, + { + "text": "Value #D", + "value": "Value #D" + }, + { + "text": "Value #E", + "value": "Value #E" + }, + { + "text": "Value #F", + "value": "Value #F" + }, + { + "text": "pod", + "value": "pod" + } + ], + "datasource": "$datasource", + "fill": 1, + "fontSize": "100%", + "gridPos": { + "h": 9, + "w": 24, + "x": 0, + "y": 10 + }, + "id": 5, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "minSpan": 24, + "nullPointMode": "null as zero", + "renderer": "flot", + "scroll": true, + "showHeader": true, + "sort": { + "col": 0, + "desc": false + }, + "spaceLength": 10, + "span": 24, + "styles": [ + { + "alias": "Time", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Time", + "thresholds": [ + + ], + "type": "hidden", + "unit": "short" + }, + { + "alias": "Bandwidth Received", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #A", + "thresholds": [ + + ], + "type": "number", + "unit": "Bps" + }, + { + "alias": "Bandwidth Transmitted", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #B", + "thresholds": [ + + ], + "type": "number", + "unit": "Bps" + }, + { + "alias": "Rate of Received Packets", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #C", + "thresholds": [ + + ], + "type": "number", + "unit": "pps" + }, + { + "alias": "Rate of Transmitted Packets", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #D", + "thresholds": [ + + ], + "type": "number", + "unit": "pps" + }, + { + "alias": "Rate of Received Packets Dropped", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #E", + "thresholds": [ + + ], + "type": "number", + "unit": "pps" + }, + { + "alias": "Rate of Transmitted Packets Dropped", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #F", + "thresholds": [ + + ], + "type": "number", + "unit": "pps" + }, + { + "alias": "Pod", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": true, + "linkTooltip": "Drill down", + "linkUrl": "d/7a18067ce943a40ae25454675c19ff5c/kubernetes-networking-pod?orgId=1&refresh=30s&var-namespace=$namespace&var-pod=$__cell", + "pattern": "pod", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + } + ], + "targets": [ + { + "expr": "sum(irate(container_network_receive_bytes_total{cluster=\"$cluster\",namespace=~\"$namespace\"}[$interval:$resolution])) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "A", + "step": 10 + }, + { + "expr": "sum(irate(container_network_transmit_bytes_total{cluster=\"$cluster\",namespace=~\"$namespace\"}[$interval:$resolution])) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "B", + "step": 10 + }, + { + "expr": "sum(irate(container_network_receive_packets_total{cluster=\"$cluster\",namespace=~\"$namespace\"}[$interval:$resolution])) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "C", + "step": 10 + }, + { + "expr": "sum(irate(container_network_transmit_packets_total{cluster=\"$cluster\",namespace=~\"$namespace\"}[$interval:$resolution])) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "D", + "step": 10 + }, + { + "expr": "sum(irate(container_network_receive_packets_dropped_total{cluster=\"$cluster\",namespace=~\"$namespace\"}[$interval:$resolution])) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "E", + "step": 10 + }, + { + "expr": "sum(irate(container_network_transmit_packets_dropped_total{cluster=\"$cluster\",namespace=~\"$namespace\"}[$interval:$resolution])) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "F", + "step": 10 + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Current Status", + "type": "table" + }, + { + "collapse": false, + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 19 + }, + "id": 6, + "panels": [ + + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Bandwidth", + "titleSize": "h6", + "type": "row" + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 20 + }, + "id": 7, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [ + + ], + "minSpan": 12, + "nullPointMode": "connected", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(container_network_receive_bytes_total{cluster=\"$cluster\",namespace=~\"$namespace\"}[$interval:$resolution])) by (pod)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Receive Bandwidth", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 20 + }, + "id": 8, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [ + + ], + "minSpan": 12, + "nullPointMode": "connected", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(container_network_transmit_bytes_total{cluster=\"$cluster\",namespace=~\"$namespace\"}[$interval:$resolution])) by (pod)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Transmit Bandwidth", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "collapse": true, + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 29 + }, + "id": 9, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 30 + }, + "id": 10, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [ + + ], + "minSpan": 12, + "nullPointMode": "connected", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(container_network_receive_packets_total{cluster=\"$cluster\",namespace=~\"$namespace\"}[$interval:$resolution])) by (pod)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Received Packets", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 30 + }, + "id": 11, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [ + + ], + "minSpan": 12, + "nullPointMode": "connected", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(container_network_transmit_packets_total{cluster=\"$cluster\",namespace=~\"$namespace\"}[$interval:$resolution])) by (pod)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Transmitted Packets", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Packets", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": true, + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 30 + }, + "id": 12, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 40 + }, + "id": 13, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [ + + ], + "minSpan": 12, + "nullPointMode": "connected", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(container_network_receive_packets_dropped_total{cluster=\"$cluster\",namespace=~\"$namespace\"}[$interval:$resolution])) by (pod)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Received Packets Dropped", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 40 + }, + "id": 14, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [ + + ], + "minSpan": 12, + "nullPointMode": "connected", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(container_network_transmit_packets_dropped_total{cluster=\"$cluster\",namespace=~\"$namespace\"}[$interval:$resolution])) by (pod)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Transmitted Packets Dropped", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Errors", + "titleSize": "h6", + "type": "row" + } + ], + "refresh": "10s", + "rows": [ + + ], + "schemaVersion": 18, + "style": "dark", + "tags": [ + "kubernetes-mixin" + ], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": "Data Source", + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": null, + "current": { + + }, + "datasource": "$datasource", + "hide": {{ if .Values.grafana.sidecar.dashboards.multicluster.global.enabled }}0{{ else }}2{{ end }}, + "includeAll": false, + "label": null, + "multi": false, + "name": "cluster", + "options": [ + + ], + "query": "label_values(up{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\"}, cluster)", + "refresh": 2, + "regex": "", + "sort": 0, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": ".+", + "auto": false, + "auto_count": 30, + "auto_min": "10s", + "current": { + "text": "kube-system", + "value": "kube-system" + }, + "datasource": "$datasource", + "definition": "label_values(container_network_receive_packets_total{cluster=\"$cluster\"}, namespace)", + "hide": 0, + "includeAll": true, + "label": null, + "multi": false, + "name": "namespace", + "options": [ + + ], + "query": "label_values(container_network_receive_packets_total{cluster=\"$cluster\"}, namespace)", + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "auto": false, + "auto_count": 30, + "auto_min": "10s", + "current": { + "text": "5m", + "value": "5m" + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "resolution", + "options": [ + { + "selected": false, + "text": "30s", + "value": "30s" + }, + { + "selected": true, + "text": "5m", + "value": "5m" + }, + { + "selected": false, + "text": "1h", + "value": "1h" + } + ], + "query": "30s,5m,1h", + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "interval", + "useTags": false + }, + { + "allValue": null, + "auto": false, + "auto_count": 30, + "auto_min": "10s", + "current": { + "text": "5m", + "value": "5m" + }, + "datasource": "$datasource", + "hide": 2, + "includeAll": false, + "label": null, + "multi": false, + "name": "interval", + "options": [ + { + "selected": true, + "text": "4h", + "value": "4h" + } + ], + "query": "4h", + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "interval", + "useTags": false + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "{{ .Values.grafana.defaultDashboardsTimezone }}", + "title": "Kubernetes / Networking / Namespace (Pods)", + "uid": "8b7a8b326d7a6f1f04244066368c67af", + "version": 0 + } +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/grafana/dashboards-1.14/namespace-by-workload.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/grafana/dashboards-1.14/namespace-by-workload.yaml new file mode 100644 index 00000000000..801b09c2651 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/grafana/dashboards-1.14/namespace-by-workload.yaml @@ -0,0 +1,1736 @@ +{{- /* +Generated from 'namespace-by-workload' from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/a8ba97a150c75be42010c75d10b720c55e182f1a/manifests/grafana-dashboardDefinitions.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (or .Values.grafana.enabled .Values.grafana.forceDeployDashboards) (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.grafana.defaultDashboardsEnabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: {{ .Values.grafana.defaultDashboards.namespace }} + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" $) "namespace-by-workload" | trunc 63 | trimSuffix "-" }} + annotations: +{{ toYaml .Values.grafana.sidecar.dashboards.annotations | indent 4 }} + labels: + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: {{ ternary $.Values.grafana.sidecar.dashboards.labelValue "1" (not (empty $.Values.grafana.sidecar.dashboards.labelValue)) | quote }} + {{- end }} + app: {{ template "kube-prometheus-stack.name" $ }}-grafana +{{ include "kube-prometheus-stack.labels" $ | indent 4 }} +data: + namespace-by-workload.json: |- + { + "__inputs": [ + + ], + "__requires": [ + + ], + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "hideControls": false, + "id": null, + "links": [ + + ], + "panels": [ + { + "collapse": false, + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 2, + "panels": [ + + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Current Bandwidth", + "titleSize": "h6", + "type": "row" + }, + { + "aliasColors": { + + }, + "bars": true, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 1 + }, + "id": 3, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": false, + "linewidth": 1, + "links": [ + + ], + "minSpan": 24, + "nullPointMode": "null", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 24, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sort_desc(sum(irate(container_network_receive_bytes_total{cluster=\"$cluster\",namespace=\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=\"$namespace\", workload=~\".+\", workload_type=\"$type\"}) by (workload))\n", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}} workload {{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Current Rate of Bytes Received", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "series", + "name": null, + "show": false, + "values": [ + "current" + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": true, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 1 + }, + "id": 4, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": false, + "linewidth": 1, + "links": [ + + ], + "minSpan": 24, + "nullPointMode": "null", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 24, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sort_desc(sum(irate(container_network_transmit_bytes_total{cluster=\"$cluster\",namespace=\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=\"$namespace\", workload=~\".+\", workload_type=\"$type\"}) by (workload))\n", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}} workload {{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Current Rate of Bytes Transmitted", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "series", + "name": null, + "show": false, + "values": [ + "current" + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "columns": [ + { + "text": "Time", + "value": "Time" + }, + { + "text": "Value #A", + "value": "Value #A" + }, + { + "text": "Value #B", + "value": "Value #B" + }, + { + "text": "Value #C", + "value": "Value #C" + }, + { + "text": "Value #D", + "value": "Value #D" + }, + { + "text": "Value #E", + "value": "Value #E" + }, + { + "text": "Value #F", + "value": "Value #F" + }, + { + "text": "Value #G", + "value": "Value #G" + }, + { + "text": "Value #H", + "value": "Value #H" + }, + { + "text": "workload", + "value": "workload" + } + ], + "datasource": "$datasource", + "fill": 1, + "fontSize": "90%", + "gridPos": { + "h": 9, + "w": 24, + "x": 0, + "y": 10 + }, + "id": 5, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "minSpan": 24, + "nullPointMode": "null as zero", + "renderer": "flot", + "scroll": true, + "showHeader": true, + "sort": { + "col": 0, + "desc": false + }, + "spaceLength": 10, + "span": 24, + "styles": [ + { + "alias": "Time", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Time", + "thresholds": [ + + ], + "type": "hidden", + "unit": "short" + }, + { + "alias": "Current Bandwidth Received", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #A", + "thresholds": [ + + ], + "type": "number", + "unit": "Bps" + }, + { + "alias": "Current Bandwidth Transmitted", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #B", + "thresholds": [ + + ], + "type": "number", + "unit": "Bps" + }, + { + "alias": "Average Bandwidth Received", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #C", + "thresholds": [ + + ], + "type": "number", + "unit": "Bps" + }, + { + "alias": "Average Bandwidth Transmitted", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #D", + "thresholds": [ + + ], + "type": "number", + "unit": "Bps" + }, + { + "alias": "Rate of Received Packets", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #E", + "thresholds": [ + + ], + "type": "number", + "unit": "pps" + }, + { + "alias": "Rate of Transmitted Packets", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #F", + "thresholds": [ + + ], + "type": "number", + "unit": "pps" + }, + { + "alias": "Rate of Received Packets Dropped", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #G", + "thresholds": [ + + ], + "type": "number", + "unit": "pps" + }, + { + "alias": "Rate of Transmitted Packets Dropped", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #H", + "thresholds": [ + + ], + "type": "number", + "unit": "pps" + }, + { + "alias": "Workload", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": true, + "linkTooltip": "Drill down", + "linkUrl": "d/728bf77cc1166d2f3133bf25846876cc/kubernetes-networking-workload?orgId=1&refresh=30s&var-namespace=$namespace&var-type=$type&var-workload=$__cell", + "pattern": "workload", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + } + ], + "targets": [ + { + "expr": "sort_desc(sum(irate(container_network_receive_bytes_total{cluster=\"$cluster\",namespace=\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=\"$namespace\", workload=~\".+\", workload_type=\"$type\"}) by (workload))\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "A", + "step": 10 + }, + { + "expr": "sort_desc(sum(irate(container_network_transmit_bytes_total{cluster=\"$cluster\",namespace=\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=\"$namespace\", workload=~\".+\", workload_type=\"$type\"}) by (workload))\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "B", + "step": 10 + }, + { + "expr": "sort_desc(avg(irate(container_network_receive_bytes_total{cluster=\"$cluster\",namespace=\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=\"$namespace\", workload=~\".+\", workload_type=\"$type\"}) by (workload))\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "C", + "step": 10 + }, + { + "expr": "sort_desc(avg(irate(container_network_transmit_bytes_total{cluster=\"$cluster\",namespace=\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=\"$namespace\", workload=~\".+\", workload_type=\"$type\"}) by (workload))\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "D", + "step": 10 + }, + { + "expr": "sort_desc(sum(irate(container_network_receive_packets_total{cluster=\"$cluster\",namespace=\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=\"$namespace\", workload=~\".+\", workload_type=\"$type\"}) by (workload))\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "E", + "step": 10 + }, + { + "expr": "sort_desc(sum(irate(container_network_transmit_packets_total{cluster=\"$cluster\",namespace=\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=\"$namespace\", workload=~\".+\", workload_type=\"$type\"}) by (workload))\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "F", + "step": 10 + }, + { + "expr": "sort_desc(sum(irate(container_network_receive_packets_dropped_total{cluster=\"$cluster\",namespace=\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=\"$namespace\", workload=~\".+\", workload_type=\"$type\"}) by (workload))\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "G", + "step": 10 + }, + { + "expr": "sort_desc(sum(irate(container_network_transmit_packets_dropped_total{cluster=\"$cluster\",namespace=\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=\"$namespace\", workload=~\".+\", workload_type=\"$type\"}) by (workload))\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "H", + "step": 10 + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Current Status", + "type": "table" + }, + { + "collapse": true, + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 19 + }, + "id": 6, + "panels": [ + { + "aliasColors": { + + }, + "bars": true, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 20 + }, + "id": 7, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": false, + "linewidth": 1, + "links": [ + + ], + "minSpan": 24, + "nullPointMode": "null", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 24, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sort_desc(avg(irate(container_network_receive_bytes_total{cluster=\"$cluster\",namespace=\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=\"$namespace\", workload=~\".+\", workload_type=\"$type\"}) by (workload))\n", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}} workload {{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Average Rate of Bytes Received", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "series", + "name": null, + "show": false, + "values": [ + "current" + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": true, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 20 + }, + "id": 8, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": false, + "linewidth": 1, + "links": [ + + ], + "minSpan": 24, + "nullPointMode": "null", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 24, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sort_desc(avg(irate(container_network_transmit_bytes_total{cluster=\"$cluster\",namespace=\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=\"$namespace\", workload=~\".+\", workload_type=\"$type\"}) by (workload))\n", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}} workload {{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Average Rate of Bytes Transmitted", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "series", + "name": null, + "show": false, + "values": [ + "current" + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Average Bandwidth", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 29 + }, + "id": 9, + "panels": [ + + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Bandwidth HIstory", + "titleSize": "h6", + "type": "row" + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 38 + }, + "id": 10, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [ + + ], + "minSpan": 12, + "nullPointMode": "connected", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sort_desc(sum(irate(container_network_receive_bytes_total{cluster=\"$cluster\",namespace=\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=\"$namespace\", workload=~\".+\", workload_type=\"$type\"}) by (workload))\n", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}workload{{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Receive Bandwidth", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 38 + }, + "id": 11, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [ + + ], + "minSpan": 12, + "nullPointMode": "connected", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sort_desc(sum(irate(container_network_transmit_bytes_total{cluster=\"$cluster\",namespace=\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=\"$namespace\", workload=~\".+\", workload_type=\"$type\"}) by (workload))\n", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}workload{{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Transmit Bandwidth", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "collapse": true, + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 39 + }, + "id": 12, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 40 + }, + "id": 13, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [ + + ], + "minSpan": 12, + "nullPointMode": "connected", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sort_desc(sum(irate(container_network_receive_packets_total{cluster=\"$cluster\",namespace=\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=\"$namespace\", workload=~\".+\", workload_type=\"$type\"}) by (workload))\n", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}workload{{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Received Packets", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 40 + }, + "id": 14, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [ + + ], + "minSpan": 12, + "nullPointMode": "connected", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sort_desc(sum(irate(container_network_transmit_packets_total{cluster=\"$cluster\",namespace=\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=\"$namespace\", workload=~\".+\", workload_type=\"$type\"}) by (workload))\n", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}workload{{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Transmitted Packets", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Packets", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": true, + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 40 + }, + "id": 15, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 41 + }, + "id": 16, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [ + + ], + "minSpan": 12, + "nullPointMode": "connected", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sort_desc(sum(irate(container_network_receive_packets_dropped_total{cluster=\"$cluster\",namespace=\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=\"$namespace\", workload=~\".+\", workload_type=\"$type\"}) by (workload))\n", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}workload{{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Received Packets Dropped", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 41 + }, + "id": 17, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [ + + ], + "minSpan": 12, + "nullPointMode": "connected", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sort_desc(sum(irate(container_network_transmit_packets_dropped_total{cluster=\"$cluster\",namespace=\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=\"$namespace\", workload=~\".+\", workload_type=\"$type\"}) by (workload))\n", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}workload{{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Transmitted Packets Dropped", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Errors", + "titleSize": "h6", + "type": "row" + } + ], + "refresh": "10s", + "rows": [ + + ], + "schemaVersion": 18, + "style": "dark", + "tags": [ + "kubernetes-mixin" + ], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": "Data Source", + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": null, + "current": { + + }, + "datasource": "$datasource", + "hide": {{ if .Values.grafana.sidecar.dashboards.multicluster.global.enabled }}0{{ else }}2{{ end }}, + "includeAll": false, + "label": null, + "multi": false, + "name": "cluster", + "options": [ + + ], + "query": "label_values(up{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\"}, cluster)", + "refresh": 2, + "regex": "", + "sort": 0, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "auto": false, + "auto_count": 30, + "auto_min": "10s", + "current": { + "text": "kube-system", + "value": "kube-system" + }, + "datasource": "$datasource", + "definition": "label_values(container_network_receive_packets_total{cluster=\"$cluster\"}, namespace)", + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "namespace", + "options": [ + + ], + "query": "label_values(container_network_receive_packets_total{cluster=\"$cluster\"}, namespace)", + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "auto": false, + "auto_count": 30, + "auto_min": "10s", + "current": { + "text": "deployment", + "value": "deployment" + }, + "datasource": "$datasource", + "definition": "label_values(namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=\"$namespace\", workload=~\".+\"}, workload_type)", + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "type", + "options": [ + + ], + "query": "label_values(namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=\"$namespace\", workload=~\".+\"}, workload_type)", + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "auto": false, + "auto_count": 30, + "auto_min": "10s", + "current": { + "text": "5m", + "value": "5m" + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "resolution", + "options": [ + { + "selected": false, + "text": "30s", + "value": "30s" + }, + { + "selected": true, + "text": "5m", + "value": "5m" + }, + { + "selected": false, + "text": "1h", + "value": "1h" + } + ], + "query": "30s,5m,1h", + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "interval", + "useTags": false + }, + { + "allValue": null, + "auto": false, + "auto_count": 30, + "auto_min": "10s", + "current": { + "text": "5m", + "value": "5m" + }, + "datasource": "$datasource", + "hide": 2, + "includeAll": false, + "label": null, + "multi": false, + "name": "interval", + "options": [ + { + "selected": true, + "text": "4h", + "value": "4h" + } + ], + "query": "4h", + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "interval", + "useTags": false + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "{{ .Values.grafana.defaultDashboardsTimezone }}", + "title": "Kubernetes / Networking / Namespace (Workload)", + "uid": "bbb2a765a623ae38130206c7d94a160f", + "version": 0 + } +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/grafana/dashboards-1.14/node-cluster-rsrc-use.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/grafana/dashboards-1.14/node-cluster-rsrc-use.yaml new file mode 100644 index 00000000000..9869a3d3e0c --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/grafana/dashboards-1.14/node-cluster-rsrc-use.yaml @@ -0,0 +1,1063 @@ +{{- /* +Generated from 'node-cluster-rsrc-use' from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/a8ba97a150c75be42010c75d10b720c55e182f1a/manifests/grafana-dashboardDefinitions.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (or .Values.grafana.enabled .Values.grafana.forceDeployDashboards) (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.grafana.defaultDashboardsEnabled (or .Values.nodeExporter.enabled .Values.nodeExporter.forceDeployDashboards) }} +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: {{ .Values.grafana.defaultDashboards.namespace }} + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" $) "node-cluster-rsrc-use" | trunc 63 | trimSuffix "-" }} + annotations: +{{ toYaml .Values.grafana.sidecar.dashboards.annotations | indent 4 }} + labels: + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: {{ ternary $.Values.grafana.sidecar.dashboards.labelValue "1" (not (empty $.Values.grafana.sidecar.dashboards.labelValue)) | quote }} + {{- end }} + app: {{ template "kube-prometheus-stack.name" $ }}-grafana +{{ include "kube-prometheus-stack.labels" $ | indent 4 }} +data: + node-cluster-rsrc-use.json: |- + { + "__inputs": [ + + ], + "__requires": [ + + ], + "annotations": { + "list": [ + + ] + }, + "editable": false, + "gnetId": null, + "graphTooltip": 1, + "hideControls": false, + "id": null, + "links": [ + + ], + "refresh": "30s", + "rows": [ + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "fillGradient": 0, + "gridPos": { + + }, + "id": 2, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": false, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "((\n instance:node_cpu_utilisation:rate5m{job=\"node-exporter\", cluster=\"$cluster\"}\n *\n instance:node_num_cpu:sum{job=\"node-exporter\", cluster=\"$cluster\"}\n) != 0 )\n/ scalar(sum(instance:node_num_cpu:sum{job=\"node-exporter\", cluster=\"$cluster\"}))\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}} instance {{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "CPU Utilisation", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "fillGradient": 0, + "gridPos": { + + }, + "id": 3, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": false, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "(\n instance:node_load1_per_cpu:ratio{job=\"node-exporter\", cluster=\"$cluster\"}\n / scalar(count(instance:node_load1_per_cpu:ratio{job=\"node-exporter\", cluster=\"$cluster\"}))\n) != 0\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "CPU Saturation (Load1 per CPU)", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "CPU", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "fillGradient": 0, + "gridPos": { + + }, + "id": 4, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": false, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "(\n instance:node_memory_utilisation:ratio{job=\"node-exporter\", cluster=\"$cluster\"}\n / scalar(count(instance:node_memory_utilisation:ratio{job=\"node-exporter\", cluster=\"$cluster\"}))\n) != 0\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Memory Utilisation", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "fillGradient": 0, + "gridPos": { + + }, + "id": 5, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": false, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "instance:node_vmstat_pgmajfault:rate5m{job=\"node-exporter\", cluster=\"$cluster\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Memory Saturation (Major Page Faults)", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "rds", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "rds", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Memory", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "fillGradient": 0, + "gridPos": { + + }, + "id": 6, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": false, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + { + "alias": "/Receive/", + "stack": "A" + }, + { + "alias": "/Transmit/", + "stack": "B", + "transform": "negative-Y" + } + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "instance:node_network_receive_bytes_excluding_lo:rate5m{job=\"node-exporter\", cluster=\"$cluster\"} != 0", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}} Receive", + "refId": "A" + }, + { + "expr": "instance:node_network_transmit_bytes_excluding_lo:rate5m{job=\"node-exporter\", cluster=\"$cluster\"} != 0", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}} Transmit", + "refId": "B" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Network Utilisation (Bytes Receive/Transmit)", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "fillGradient": 0, + "gridPos": { + + }, + "id": 7, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": false, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + { + "alias": "/ Receive/", + "stack": "A" + }, + { + "alias": "/ Transmit/", + "stack": "B", + "transform": "negative-Y" + } + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "instance:node_network_receive_drop_excluding_lo:rate5m{job=\"node-exporter\", cluster=\"$cluster\"} != 0", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}} Receive", + "refId": "A" + }, + { + "expr": "instance:node_network_transmit_drop_excluding_lo:rate5m{job=\"node-exporter\", cluster=\"$cluster\"} != 0", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}} Transmit", + "refId": "B" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Network Saturation (Drops Receive/Transmit)", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Network", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "fillGradient": 0, + "gridPos": { + + }, + "id": 8, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": false, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "(\n instance_device:node_disk_io_time_seconds:rate5m{job=\"node-exporter\", cluster=\"$cluster\"}\n / scalar(count(instance_device:node_disk_io_time_seconds:rate5m{job=\"node-exporter\", cluster=\"$cluster\"}))\n) != 0\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}} {{`{{`}}device{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Disk IO Utilisation", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "fillGradient": 0, + "gridPos": { + + }, + "id": 9, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": false, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "(\n instance_device:node_disk_io_time_weighted_seconds:rate5m{job=\"node-exporter\", cluster=\"$cluster\"}\n / scalar(count(instance_device:node_disk_io_time_weighted_seconds:rate5m{job=\"node-exporter\", cluster=\"$cluster\"}))\n) != 0\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}} {{`{{`}}device{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Disk IO Saturation", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Disk IO", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "fillGradient": 0, + "gridPos": { + + }, + "id": 10, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": false, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum without (device) (\n max without (fstype, mountpoint) ((\n node_filesystem_size_bytes{job=\"node-exporter\", fstype!=\"\", mountpoint!=\"\", cluster=\"$cluster\"}\n -\n node_filesystem_avail_bytes{job=\"node-exporter\", fstype!=\"\", mountpoint!=\"\", cluster=\"$cluster\"}\n ) != 0)\n)\n/ scalar(sum(max without (fstype, mountpoint) (node_filesystem_size_bytes{job=\"node-exporter\", fstype!=\"\", mountpoint!=\"\", cluster=\"$cluster\"})))\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Disk Space Utilisation", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Disk Space", + "titleSize": "h6", + "type": "row" + } + ], + "schemaVersion": 14, + "style": "dark", + "tags": [ + "node-exporter-mixin" + ], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": "Data Source", + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": null, + "current": { + "text": "", + "value": "" + }, + "datasource": "$datasource", + "hide": {{ if .Values.grafana.sidecar.dashboards.multicluster.global.enabled }}0{{ else }}2{{ end }}, + "includeAll": false, + "label": null, + "multi": false, + "name": "cluster", + "options": [ + + ], + "query": "label_values(node_time_seconds, cluster)", + "refresh": 2, + "regex": "", + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "{{ .Values.grafana.defaultDashboardsTimezone }}", + "title": "Node Exporter / USE Method / Cluster", + "version": 0 + } +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/grafana/dashboards-1.14/node-rsrc-use.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/grafana/dashboards-1.14/node-rsrc-use.yaml new file mode 100644 index 00000000000..75e69afa22f --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/grafana/dashboards-1.14/node-rsrc-use.yaml @@ -0,0 +1,1089 @@ +{{- /* +Generated from 'node-rsrc-use' from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/a8ba97a150c75be42010c75d10b720c55e182f1a/manifests/grafana-dashboardDefinitions.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (or .Values.grafana.enabled .Values.grafana.forceDeployDashboards) (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.grafana.defaultDashboardsEnabled (or .Values.nodeExporter.enabled .Values.nodeExporter.forceDeployDashboards) }} +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: {{ .Values.grafana.defaultDashboards.namespace }} + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" $) "node-rsrc-use" | trunc 63 | trimSuffix "-" }} + annotations: +{{ toYaml .Values.grafana.sidecar.dashboards.annotations | indent 4 }} + labels: + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: {{ ternary $.Values.grafana.sidecar.dashboards.labelValue "1" (not (empty $.Values.grafana.sidecar.dashboards.labelValue)) | quote }} + {{- end }} + app: {{ template "kube-prometheus-stack.name" $ }}-grafana +{{ include "kube-prometheus-stack.labels" $ | indent 4 }} +data: + node-rsrc-use.json: |- + { + "__inputs": [ + + ], + "__requires": [ + + ], + "annotations": { + "list": [ + + ] + }, + "editable": false, + "gnetId": null, + "graphTooltip": 1, + "hideControls": false, + "id": null, + "links": [ + + ], + "refresh": "30s", + "rows": [ + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "fillGradient": 0, + "gridPos": { + + }, + "id": 2, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": false, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "instance:node_cpu_utilisation:rate5m{job=\"node-exporter\", instance=\"$instance\", cluster=\"$cluster\"} != 0", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Utilisation", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "CPU Utilisation", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "fillGradient": 0, + "gridPos": { + + }, + "id": 3, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": false, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "instance:node_load1_per_cpu:ratio{job=\"node-exporter\", instance=\"$instance\", cluster=\"$cluster\"} != 0", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Saturation", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "CPU Saturation (Load1 per CPU)", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "CPU", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "fillGradient": 0, + "gridPos": { + + }, + "id": 4, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": false, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "instance:node_memory_utilisation:ratio{job=\"node-exporter\", instance=\"$instance\", cluster=\"$cluster\"} != 0", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Utilisation", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Memory Utilisation", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "fillGradient": 0, + "gridPos": { + + }, + "id": 5, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": false, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "instance:node_vmstat_pgmajfault:rate5m{job=\"node-exporter\", instance=\"$instance\", cluster=\"$cluster\"} != 0", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Major page Faults", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Memory Saturation (Major Page Faults)", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "rds", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "rds", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Memory", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "fillGradient": 0, + "gridPos": { + + }, + "id": 6, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": false, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + { + "alias": "/Receive/", + "stack": "A" + }, + { + "alias": "/Transmit/", + "stack": "B", + "transform": "negative-Y" + } + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "instance:node_network_receive_bytes_excluding_lo:rate5m{job=\"node-exporter\", instance=\"$instance\", cluster=\"$cluster\"} != 0", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Receive", + "refId": "A" + }, + { + "expr": "instance:node_network_transmit_bytes_excluding_lo:rate5m{job=\"node-exporter\", instance=\"$instance\", cluster=\"$cluster\"} != 0", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Transmit", + "refId": "B" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Network Utilisation (Bytes Receive/Transmit)", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "fillGradient": 0, + "gridPos": { + + }, + "id": 7, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": false, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + { + "alias": "/ Receive/", + "stack": "A" + }, + { + "alias": "/ Transmit/", + "stack": "B", + "transform": "negative-Y" + } + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "instance:node_network_receive_drop_excluding_lo:rate5m{job=\"node-exporter\", instance=\"$instance\", cluster=\"$cluster\"} != 0", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Receive", + "refId": "A" + }, + { + "expr": "instance:node_network_transmit_drop_excluding_lo:rate5m{job=\"node-exporter\", instance=\"$instance\", cluster=\"$cluster\"} != 0", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Transmit", + "refId": "B" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Network Saturation (Drops Receive/Transmit)", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Network", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "fillGradient": 0, + "gridPos": { + + }, + "id": 8, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": false, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "instance_device:node_disk_io_time_seconds:rate5m{job=\"node-exporter\", instance=\"$instance\", cluster=\"$cluster\"} != 0", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}device{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Disk IO Utilisation", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "fillGradient": 0, + "gridPos": { + + }, + "id": 9, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": false, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "instance_device:node_disk_io_time_weighted_seconds:rate5m{job=\"node-exporter\", instance=\"$instance\", cluster=\"$cluster\"} != 0", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}device{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Disk IO Saturation", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Disk IO", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "fillGradient": 0, + "gridPos": { + + }, + "id": 10, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": false, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sort_desc(1 -\n (\n max without (mountpoint, fstype) (node_filesystem_avail_bytes{job=\"node-exporter\", fstype!=\"\", instance=\"$instance\", cluster=\"$cluster\"})\n /\n max without (mountpoint, fstype) (node_filesystem_size_bytes{job=\"node-exporter\", fstype!=\"\", instance=\"$instance\", cluster=\"$cluster\"})\n ) != 0\n)\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}device{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Disk Space Utilisation", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Disk Space", + "titleSize": "h6", + "type": "row" + } + ], + "schemaVersion": 14, + "style": "dark", + "tags": [ + "node-exporter-mixin" + ], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": "Data Source", + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": null, + "current": { + "text": "", + "value": "" + }, + "datasource": "$datasource", + "hide": {{ if .Values.grafana.sidecar.dashboards.multicluster.global.enabled }}0{{ else }}2{{ end }}, + "includeAll": false, + "label": null, + "multi": false, + "name": "cluster", + "options": [ + + ], + "query": "label_values(node_time_seconds, cluster)", + "refresh": 2, + "regex": "", + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": { + + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "instance", + "options": [ + + ], + "query": "label_values(node_exporter_build_info{job=\"node-exporter\", cluster=\"$cluster\"}, instance)", + "refresh": 2, + "regex": "", + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "{{ .Values.grafana.defaultDashboardsTimezone }}", + "title": "Node Exporter / USE Method / Node", + "version": 0 + } +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/grafana/dashboards-1.14/nodes-darwin.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/grafana/dashboards-1.14/nodes-darwin.yaml new file mode 100644 index 00000000000..fe118753248 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/grafana/dashboards-1.14/nodes-darwin.yaml @@ -0,0 +1,1073 @@ +{{- /* +Generated from 'nodes-darwin' from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/a8ba97a150c75be42010c75d10b720c55e182f1a/manifests/grafana-dashboardDefinitions.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (or .Values.grafana.enabled .Values.grafana.forceDeployDashboards) (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.grafana.defaultDashboardsEnabled (and (or .Values.nodeExporter.enabled .Values.nodeExporter.forceDeployDashboards) .Values.nodeExporter.operatingSystems.darwin.enabled) }} +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: {{ .Values.grafana.defaultDashboards.namespace }} + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" $) "nodes-darwin" | trunc 63 | trimSuffix "-" }} + annotations: +{{ toYaml .Values.grafana.sidecar.dashboards.annotations | indent 4 }} + labels: + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: {{ ternary $.Values.grafana.sidecar.dashboards.labelValue "1" (not (empty $.Values.grafana.sidecar.dashboards.labelValue)) | quote }} + {{- end }} + app: {{ template "kube-prometheus-stack.name" $ }}-grafana +{{ include "kube-prometheus-stack.labels" $ | indent 4 }} +data: + nodes-darwin.json: |- + { + "__inputs": [ + + ], + "__requires": [ + + ], + "annotations": { + "list": [ + + ] + }, + "editable": false, + "gnetId": null, + "graphTooltip": 1, + "hideControls": false, + "id": null, + "links": [ + + ], + "refresh": "30s", + "rows": [ + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 2, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "(\n (1 - sum without (mode) (rate(node_cpu_seconds_total{job=\"node-exporter\", mode=~\"idle|iowait|steal\", instance=\"$instance\"}[$__rate_interval])))\n/ ignoring(cpu) group_left\n count without (cpu, mode) (node_cpu_seconds_total{job=\"node-exporter\", mode=\"idle\", instance=\"$instance\"})\n)\n", + "format": "time_series", + "intervalFactor": 5, + "legendFormat": "{{`{{`}}cpu{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "CPU Usage", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": 1, + "min": 0, + "show": true + }, + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": 1, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 0, + "fillGradient": 0, + "gridPos": { + + }, + "id": 3, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "node_load1{job=\"node-exporter\", instance=\"$instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "1m load average", + "refId": "A" + }, + { + "expr": "node_load5{job=\"node-exporter\", instance=\"$instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "5m load average", + "refId": "B" + }, + { + "expr": "node_load15{job=\"node-exporter\", instance=\"$instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "15m load average", + "refId": "C" + }, + { + "expr": "count(node_cpu_seconds_total{job=\"node-exporter\", instance=\"$instance\", mode=\"idle\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "logical cores", + "refId": "D" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Load Average", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "CPU", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 4, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 9, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "node_memory_total_bytes{job=\"node-exporter\", instance=\"$instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Physical Memory", + "refId": "A" + }, + { + "expr": "(\n node_memory_internal_bytes{job=\"node-exporter\", instance=\"$instance\"} -\n node_memory_purgeable_bytes{job=\"node-exporter\", instance=\"$instance\"} +\n node_memory_wired_bytes{job=\"node-exporter\", instance=\"$instance\"} +\n node_memory_compressed_bytes{job=\"node-exporter\", instance=\"$instance\"}\n)\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Memory Used", + "refId": "B" + }, + { + "expr": "(\n node_memory_internal_bytes{job=\"node-exporter\", instance=\"$instance\"} -\n node_memory_purgeable_bytes{job=\"node-exporter\", instance=\"$instance\"}\n)\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "App Memory", + "refId": "C" + }, + { + "expr": "node_memory_wired_bytes{job=\"node-exporter\", instance=\"$instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Wired Memory", + "refId": "D" + }, + { + "expr": "node_memory_compressed_bytes{job=\"node-exporter\", instance=\"$instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Compressed", + "refId": "E" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Memory Usage", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "max": 100, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "rgba(50, 172, 45, 0.97)" + }, + { + "color": "rgba(237, 129, 40, 0.89)", + "value": 80 + }, + { + "color": "rgba(245, 54, 54, 0.9)", + "value": 90 + } + ] + }, + "unit": "percent" + } + }, + "gridPos": { + + }, + "id": 5, + "span": 3, + "targets": [ + { + "expr": "(\n (\n avg(node_memory_internal_bytes{job=\"node-exporter\", instance=\"$instance\"}) -\n avg(node_memory_purgeable_bytes{job=\"node-exporter\", instance=\"$instance\"}) +\n avg(node_memory_wired_bytes{job=\"node-exporter\", instance=\"$instance\"}) +\n avg(node_memory_compressed_bytes{job=\"node-exporter\", instance=\"$instance\"})\n ) /\n avg(node_memory_total_bytes{job=\"node-exporter\", instance=\"$instance\"})\n)\n*\n100\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "" + } + ], + "title": "Memory Usage", + "transparent": false, + "type": "gauge" + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Memory", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 0, + "fillGradient": 0, + "gridPos": { + + }, + "id": 6, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + { + "alias": "/ read| written/", + "yaxis": 1 + }, + { + "alias": "/ io time/", + "yaxis": 2 + } + ], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(node_disk_read_bytes_total{job=\"node-exporter\", instance=\"$instance\", device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\"}[$__rate_interval])", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}device{{`}}`}} read", + "refId": "A" + }, + { + "expr": "rate(node_disk_written_bytes_total{job=\"node-exporter\", instance=\"$instance\", device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\"}[$__rate_interval])", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}device{{`}}`}} written", + "refId": "B" + }, + { + "expr": "rate(node_disk_io_time_seconds_total{job=\"node-exporter\", instance=\"$instance\", device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\"}[$__rate_interval])", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}device{{`}}`}} io time", + "refId": "C" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Disk I/O", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": { + + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "yellow", + "value": 0.8 + }, + { + "color": "red", + "value": 0.9 + } + ] + }, + "unit": "decbytes" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Mounted on" + }, + "properties": [ + { + "id": "custom.width", + "value": 260 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Size" + }, + "properties": [ + { + "id": "custom.width", + "value": 93 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Used" + }, + "properties": [ + { + "id": "custom.width", + "value": 72 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Available" + }, + "properties": [ + { + "id": "custom.width", + "value": 88 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Used, %" + }, + "properties": [ + { + "id": "unit", + "value": "percentunit" + }, + { + "id": "custom.displayMode", + "value": "gradient-gauge" + }, + { + "id": "max", + "value": 1 + }, + { + "id": "min", + "value": 0 + } + ] + } + ] + }, + "gridPos": { + + }, + "id": 7, + "span": 6, + "targets": [ + { + "expr": "max by (mountpoint) (node_filesystem_size_bytes{job=\"node-exporter\", instance=\"$instance\", fstype!=\"\", mountpoint!=\"\"})\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "" + }, + { + "expr": "max by (mountpoint) (node_filesystem_avail_bytes{job=\"node-exporter\", instance=\"$instance\", fstype!=\"\", mountpoint!=\"\"})\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "" + } + ], + "title": "Disk Space Usage", + "transformations": [ + { + "id": "groupBy", + "options": { + "fields": { + "Value #A": { + "aggregations": [ + "lastNotNull" + ], + "operation": "aggregate" + }, + "Value #B": { + "aggregations": [ + "lastNotNull" + ], + "operation": "aggregate" + }, + "mountpoint": { + "aggregations": [ + + ], + "operation": "groupby" + } + } + } + }, + { + "id": "merge", + "options": { + + } + }, + { + "id": "calculateField", + "options": { + "alias": "Used", + "binary": { + "left": "Value #A (lastNotNull)", + "operator": "-", + "reducer": "sum", + "right": "Value #B (lastNotNull)" + }, + "mode": "binary", + "reduce": { + "reducer": "sum" + } + } + }, + { + "id": "calculateField", + "options": { + "alias": "Used, %", + "binary": { + "left": "Used", + "operator": "/", + "reducer": "sum", + "right": "Value #A (lastNotNull)" + }, + "mode": "binary", + "reduce": { + "reducer": "sum" + } + } + }, + { + "id": "organize", + "options": { + "excludeByName": { + + }, + "indexByName": { + + }, + "renameByName": { + "Value #A (lastNotNull)": "Size", + "Value #B (lastNotNull)": "Available", + "mountpoint": "Mounted on" + } + } + }, + { + "id": "sortBy", + "options": { + "fields": { + + }, + "sort": [ + { + "field": "Mounted on" + } + ] + } + } + ], + "transparent": false, + "type": "table" + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Disk", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "Network received (bits/s)", + "fill": 0, + "fillGradient": 0, + "gridPos": { + + }, + "id": 8, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(node_network_receive_bytes_total{job=\"node-exporter\", instance=\"$instance\", device!=\"lo\"}[$__rate_interval]) * 8", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}device{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Network Received", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "Network transmitted (bits/s)", + "fill": 0, + "fillGradient": 0, + "gridPos": { + + }, + "id": 9, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(node_network_transmit_bytes_total{job=\"node-exporter\", instance=\"$instance\", device!=\"lo\"}[$__rate_interval]) * 8", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}device{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Network Transmitted", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Network", + "titleSize": "h6", + "type": "row" + } + ], + "schemaVersion": 14, + "style": "dark", + "tags": [ + "node-exporter-mixin" + ], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": "Data Source", + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": null, + "current": { + + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": false, + "label": "Instance", + "multi": false, + "name": "instance", + "options": [ + + ], + "query": "label_values(node_uname_info{job=\"node-exporter\", sysname=\"Darwin\"}, instance)", + "refresh": 2, + "regex": "", + "sort": 0, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "{{ .Values.grafana.defaultDashboardsTimezone }}", + "title": "Node Exporter / MacOS", + "version": 0 + } +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/grafana/dashboards-1.14/nodes.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/grafana/dashboards-1.14/nodes.yaml new file mode 100644 index 00000000000..0da40a7b995 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/grafana/dashboards-1.14/nodes.yaml @@ -0,0 +1,1066 @@ +{{- /* +Generated from 'nodes' from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/a8ba97a150c75be42010c75d10b720c55e182f1a/manifests/grafana-dashboardDefinitions.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (or .Values.grafana.enabled .Values.grafana.forceDeployDashboards) (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.grafana.defaultDashboardsEnabled (and (or .Values.nodeExporter.enabled .Values.nodeExporter.forceDeployDashboards) .Values.nodeExporter.operatingSystems.linux.enabled) }} +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: {{ .Values.grafana.defaultDashboards.namespace }} + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" $) "nodes" | trunc 63 | trimSuffix "-" }} + annotations: +{{ toYaml .Values.grafana.sidecar.dashboards.annotations | indent 4 }} + labels: + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: {{ ternary $.Values.grafana.sidecar.dashboards.labelValue "1" (not (empty $.Values.grafana.sidecar.dashboards.labelValue)) | quote }} + {{- end }} + app: {{ template "kube-prometheus-stack.name" $ }}-grafana +{{ include "kube-prometheus-stack.labels" $ | indent 4 }} +data: + nodes.json: |- + { + "__inputs": [ + + ], + "__requires": [ + + ], + "annotations": { + "list": [ + + ] + }, + "editable": false, + "gnetId": null, + "graphTooltip": 1, + "hideControls": false, + "id": null, + "links": [ + + ], + "refresh": "30s", + "rows": [ + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 2, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "(\n (1 - sum without (mode) (rate(node_cpu_seconds_total{job=\"node-exporter\", mode=~\"idle|iowait|steal\", instance=\"$instance\"}[$__rate_interval])))\n/ ignoring(cpu) group_left\n count without (cpu, mode) (node_cpu_seconds_total{job=\"node-exporter\", mode=\"idle\", instance=\"$instance\"})\n)\n", + "format": "time_series", + "intervalFactor": 5, + "legendFormat": "{{`{{`}}cpu{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "CPU Usage", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": 1, + "min": 0, + "show": true + }, + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": 1, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 0, + "fillGradient": 0, + "gridPos": { + + }, + "id": 3, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "node_load1{job=\"node-exporter\", instance=\"$instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "1m load average", + "refId": "A" + }, + { + "expr": "node_load5{job=\"node-exporter\", instance=\"$instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "5m load average", + "refId": "B" + }, + { + "expr": "node_load15{job=\"node-exporter\", instance=\"$instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "15m load average", + "refId": "C" + }, + { + "expr": "count(node_cpu_seconds_total{job=\"node-exporter\", instance=\"$instance\", mode=\"idle\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "logical cores", + "refId": "D" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Load Average", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "CPU", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 4, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 9, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "(\n node_memory_MemTotal_bytes{job=\"node-exporter\", instance=\"$instance\"}\n-\n node_memory_MemFree_bytes{job=\"node-exporter\", instance=\"$instance\"}\n-\n node_memory_Buffers_bytes{job=\"node-exporter\", instance=\"$instance\"}\n-\n node_memory_Cached_bytes{job=\"node-exporter\", instance=\"$instance\"}\n)\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "memory used", + "refId": "A" + }, + { + "expr": "node_memory_Buffers_bytes{job=\"node-exporter\", instance=\"$instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "memory buffers", + "refId": "B" + }, + { + "expr": "node_memory_Cached_bytes{job=\"node-exporter\", instance=\"$instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "memory cached", + "refId": "C" + }, + { + "expr": "node_memory_MemFree_bytes{job=\"node-exporter\", instance=\"$instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "memory free", + "refId": "D" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Memory Usage", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "max": 100, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "rgba(50, 172, 45, 0.97)" + }, + { + "color": "rgba(237, 129, 40, 0.89)", + "value": 80 + }, + { + "color": "rgba(245, 54, 54, 0.9)", + "value": 90 + } + ] + }, + "unit": "percent" + } + }, + "gridPos": { + + }, + "id": 5, + "span": 3, + "targets": [ + { + "expr": "100 -\n(\n avg(node_memory_MemAvailable_bytes{job=\"node-exporter\", instance=\"$instance\"}) /\n avg(node_memory_MemTotal_bytes{job=\"node-exporter\", instance=\"$instance\"})\n* 100\n)\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "" + } + ], + "title": "Memory Usage", + "transparent": false, + "type": "gauge" + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Memory", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 0, + "fillGradient": 0, + "gridPos": { + + }, + "id": 6, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + { + "alias": "/ read| written/", + "yaxis": 1 + }, + { + "alias": "/ io time/", + "yaxis": 2 + } + ], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(node_disk_read_bytes_total{job=\"node-exporter\", instance=\"$instance\", device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\"}[$__rate_interval])", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}device{{`}}`}} read", + "refId": "A" + }, + { + "expr": "rate(node_disk_written_bytes_total{job=\"node-exporter\", instance=\"$instance\", device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\"}[$__rate_interval])", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}device{{`}}`}} written", + "refId": "B" + }, + { + "expr": "rate(node_disk_io_time_seconds_total{job=\"node-exporter\", instance=\"$instance\", device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\"}[$__rate_interval])", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}device{{`}}`}} io time", + "refId": "C" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Disk I/O", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": { + + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "yellow", + "value": 0.8 + }, + { + "color": "red", + "value": 0.9 + } + ] + }, + "unit": "decbytes" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Mounted on" + }, + "properties": [ + { + "id": "custom.width", + "value": 260 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Size" + }, + "properties": [ + { + "id": "custom.width", + "value": 93 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Used" + }, + "properties": [ + { + "id": "custom.width", + "value": 72 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Available" + }, + "properties": [ + { + "id": "custom.width", + "value": 88 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Used, %" + }, + "properties": [ + { + "id": "unit", + "value": "percentunit" + }, + { + "id": "custom.displayMode", + "value": "gradient-gauge" + }, + { + "id": "max", + "value": 1 + }, + { + "id": "min", + "value": 0 + } + ] + } + ] + }, + "gridPos": { + + }, + "id": 7, + "span": 6, + "targets": [ + { + "expr": "max by (mountpoint) (node_filesystem_size_bytes{job=\"node-exporter\", instance=\"$instance\", fstype!=\"\", mountpoint!=\"\"})\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "" + }, + { + "expr": "max by (mountpoint) (node_filesystem_avail_bytes{job=\"node-exporter\", instance=\"$instance\", fstype!=\"\", mountpoint!=\"\"})\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "" + } + ], + "title": "Disk Space Usage", + "transformations": [ + { + "id": "groupBy", + "options": { + "fields": { + "Value #A": { + "aggregations": [ + "lastNotNull" + ], + "operation": "aggregate" + }, + "Value #B": { + "aggregations": [ + "lastNotNull" + ], + "operation": "aggregate" + }, + "mountpoint": { + "aggregations": [ + + ], + "operation": "groupby" + } + } + } + }, + { + "id": "merge", + "options": { + + } + }, + { + "id": "calculateField", + "options": { + "alias": "Used", + "binary": { + "left": "Value #A (lastNotNull)", + "operator": "-", + "reducer": "sum", + "right": "Value #B (lastNotNull)" + }, + "mode": "binary", + "reduce": { + "reducer": "sum" + } + } + }, + { + "id": "calculateField", + "options": { + "alias": "Used, %", + "binary": { + "left": "Used", + "operator": "/", + "reducer": "sum", + "right": "Value #A (lastNotNull)" + }, + "mode": "binary", + "reduce": { + "reducer": "sum" + } + } + }, + { + "id": "organize", + "options": { + "excludeByName": { + + }, + "indexByName": { + + }, + "renameByName": { + "Value #A (lastNotNull)": "Size", + "Value #B (lastNotNull)": "Available", + "mountpoint": "Mounted on" + } + } + }, + { + "id": "sortBy", + "options": { + "fields": { + + }, + "sort": [ + { + "field": "Mounted on" + } + ] + } + } + ], + "transparent": false, + "type": "table" + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Disk", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "Network received (bits/s)", + "fill": 0, + "fillGradient": 0, + "gridPos": { + + }, + "id": 8, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(node_network_receive_bytes_total{job=\"node-exporter\", instance=\"$instance\", device!=\"lo\"}[$__rate_interval]) * 8", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}device{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Network Received", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "Network transmitted (bits/s)", + "fill": 0, + "fillGradient": 0, + "gridPos": { + + }, + "id": 9, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(node_network_transmit_bytes_total{job=\"node-exporter\", instance=\"$instance\", device!=\"lo\"}[$__rate_interval]) * 8", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}device{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Network Transmitted", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Network", + "titleSize": "h6", + "type": "row" + } + ], + "schemaVersion": 14, + "style": "dark", + "tags": [ + "node-exporter-mixin" + ], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": "Data Source", + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": null, + "current": { + + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": false, + "label": "Instance", + "multi": false, + "name": "instance", + "options": [ + + ], + "query": "label_values(node_uname_info{job=\"node-exporter\", sysname!=\"Darwin\"}, instance)", + "refresh": 2, + "regex": "", + "sort": 0, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "{{ .Values.grafana.defaultDashboardsTimezone }}", + "title": "Node Exporter / Nodes", + "version": 0 + } +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/grafana/dashboards-1.14/persistentvolumesusage.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/grafana/dashboards-1.14/persistentvolumesusage.yaml new file mode 100644 index 00000000000..4d1e33208b4 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/grafana/dashboards-1.14/persistentvolumesusage.yaml @@ -0,0 +1,587 @@ +{{- /* +Generated from 'persistentvolumesusage' from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/a8ba97a150c75be42010c75d10b720c55e182f1a/manifests/grafana-dashboardDefinitions.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (or .Values.grafana.enabled .Values.grafana.forceDeployDashboards) (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.grafana.defaultDashboardsEnabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: {{ .Values.grafana.defaultDashboards.namespace }} + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" $) "persistentvolumesusage" | trunc 63 | trimSuffix "-" }} + annotations: +{{ toYaml .Values.grafana.sidecar.dashboards.annotations | indent 4 }} + labels: + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: {{ ternary $.Values.grafana.sidecar.dashboards.labelValue "1" (not (empty $.Values.grafana.sidecar.dashboards.labelValue)) | quote }} + {{- end }} + app: {{ template "kube-prometheus-stack.name" $ }}-grafana +{{ include "kube-prometheus-stack.labels" $ | indent 4 }} +data: + persistentvolumesusage.json: |- + { + "__inputs": [ + + ], + "__requires": [ + + ], + "annotations": { + "list": [ + + ] + }, + "editable": false, + "gnetId": null, + "graphTooltip": 0, + "hideControls": false, + "id": null, + "links": [ + + ], + "refresh": "10s", + "rows": [ + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 2, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 9, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "(\n sum without(instance, node) (topk(1, (kubelet_volume_stats_capacity_bytes{cluster=\"$cluster\", job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics\", namespace=\"$namespace\", persistentvolumeclaim=\"$volume\"})))\n -\n sum without(instance, node) (topk(1, (kubelet_volume_stats_available_bytes{cluster=\"$cluster\", job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics\", namespace=\"$namespace\", persistentvolumeclaim=\"$volume\"})))\n)\n", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Used Space", + "refId": "A" + }, + { + "expr": "sum without(instance, node) (topk(1, (kubelet_volume_stats_available_bytes{cluster=\"$cluster\", job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics\", namespace=\"$namespace\", persistentvolumeclaim=\"$volume\"})))\n", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Free Space", + "refId": "B" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Volume Space Usage", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(50, 172, 45, 0.97)", + "rgba(237, 129, 40, 0.89)", + "rgba(245, 54, 54, 0.9)" + ], + "datasource": "$datasource", + "format": "percent", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": true, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + + }, + "id": 3, + "interval": "1m", + "legend": { + "alignAsTable": true, + "rightSide": true + }, + "links": [ + + ], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "span": 3, + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "max without(instance,node) (\n(\n topk(1, kubelet_volume_stats_capacity_bytes{cluster=\"$cluster\", job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics\", namespace=\"$namespace\", persistentvolumeclaim=\"$volume\"})\n -\n topk(1, kubelet_volume_stats_available_bytes{cluster=\"$cluster\", job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics\", namespace=\"$namespace\", persistentvolumeclaim=\"$volume\"})\n)\n/\ntopk(1, kubelet_volume_stats_capacity_bytes{cluster=\"$cluster\", job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics\", namespace=\"$namespace\", persistentvolumeclaim=\"$volume\"})\n* 100)\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "", + "refId": "A" + } + ], + "thresholds": "80, 90", + "title": "Volume Space Usage", + "tooltip": { + "shared": false + }, + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "Dashboard Row", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 4, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 9, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum without(instance, node) (topk(1, (kubelet_volume_stats_inodes_used{cluster=\"$cluster\", job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics\", namespace=\"$namespace\", persistentvolumeclaim=\"$volume\"})))\n", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Used inodes", + "refId": "A" + }, + { + "expr": "(\n sum without(instance, node) (topk(1, (kubelet_volume_stats_inodes{cluster=\"$cluster\", job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics\", namespace=\"$namespace\", persistentvolumeclaim=\"$volume\"})))\n -\n sum without(instance, node) (topk(1, (kubelet_volume_stats_inodes_used{cluster=\"$cluster\", job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics\", namespace=\"$namespace\", persistentvolumeclaim=\"$volume\"})))\n)\n", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": " Free inodes", + "refId": "B" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Volume inodes Usage", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(50, 172, 45, 0.97)", + "rgba(237, 129, 40, 0.89)", + "rgba(245, 54, 54, 0.9)" + ], + "datasource": "$datasource", + "format": "percent", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": true, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + + }, + "id": 5, + "interval": "1m", + "legend": { + "alignAsTable": true, + "rightSide": true + }, + "links": [ + + ], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "span": 3, + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "max without(instance,node) (\ntopk(1, kubelet_volume_stats_inodes_used{cluster=\"$cluster\", job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics\", namespace=\"$namespace\", persistentvolumeclaim=\"$volume\"})\n/\ntopk(1, kubelet_volume_stats_inodes{cluster=\"$cluster\", job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics\", namespace=\"$namespace\", persistentvolumeclaim=\"$volume\"})\n* 100)\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "", + "refId": "A" + } + ], + "thresholds": "80, 90", + "title": "Volume inodes Usage", + "tooltip": { + "shared": false + }, + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "Dashboard Row", + "titleSize": "h6", + "type": "row" + } + ], + "schemaVersion": 14, + "style": "dark", + "tags": [ + "kubernetes-mixin" + ], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": "Data Source", + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": null, + "current": { + + }, + "datasource": "$datasource", + "hide": {{ if .Values.grafana.sidecar.dashboards.multicluster.global.enabled }}0{{ else }}2{{ end }}, + "includeAll": false, + "label": "cluster", + "multi": false, + "name": "cluster", + "options": [ + + ], + "query": "label_values(kubelet_volume_stats_capacity_bytes{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics\"}, cluster)", + "refresh": 2, + "regex": "", + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": { + + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": false, + "label": "Namespace", + "multi": false, + "name": "namespace", + "options": [ + + ], + "query": "label_values(kubelet_volume_stats_capacity_bytes{cluster=\"$cluster\", job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics\"}, namespace)", + "refresh": 2, + "regex": "", + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": { + + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": false, + "label": "PersistentVolumeClaim", + "multi": false, + "name": "volume", + "options": [ + + ], + "query": "label_values(kubelet_volume_stats_capacity_bytes{cluster=\"$cluster\", job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics\", namespace=\"$namespace\"}, persistentvolumeclaim)", + "refresh": 2, + "regex": "", + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-7d", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "{{ .Values.grafana.defaultDashboardsTimezone }}", + "title": "Kubernetes / Persistent Volumes", + "uid": "919b92a8e8041bd567af9edab12c840c", + "version": 0 + } +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/grafana/dashboards-1.14/pod-total.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/grafana/dashboards-1.14/pod-total.yaml new file mode 100644 index 00000000000..9a7e7d0603c --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/grafana/dashboards-1.14/pod-total.yaml @@ -0,0 +1,1228 @@ +{{- /* +Generated from 'pod-total' from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/a8ba97a150c75be42010c75d10b720c55e182f1a/manifests/grafana-dashboardDefinitions.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (or .Values.grafana.enabled .Values.grafana.forceDeployDashboards) (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.grafana.defaultDashboardsEnabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: {{ .Values.grafana.defaultDashboards.namespace }} + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" $) "pod-total" | trunc 63 | trimSuffix "-" }} + annotations: +{{ toYaml .Values.grafana.sidecar.dashboards.annotations | indent 4 }} + labels: + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: {{ ternary $.Values.grafana.sidecar.dashboards.labelValue "1" (not (empty $.Values.grafana.sidecar.dashboards.labelValue)) | quote }} + {{- end }} + app: {{ template "kube-prometheus-stack.name" $ }}-grafana +{{ include "kube-prometheus-stack.labels" $ | indent 4 }} +data: + pod-total.json: |- + { + "__inputs": [ + + ], + "__requires": [ + + ], + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "hideControls": false, + "id": null, + "links": [ + + ], + "panels": [ + { + "collapse": false, + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 2, + "panels": [ + + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Current Bandwidth", + "titleSize": "h6", + "type": "row" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": "$datasource", + "decimals": 0, + "format": "time_series", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 1 + }, + "height": 9, + "id": 3, + "interval": null, + "links": [ + + ], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "minSpan": 12, + "nullPointMode": "connected", + "nullText": null, + "options": { + "fieldOptions": { + "calcs": [ + "last" + ], + "defaults": { + "max": 10000000000, + "min": 0, + "title": "$namespace: $pod", + "unit": "Bps" + }, + "mappings": [ + + ], + "override": { + + }, + "thresholds": [ + { + "color": "dark-green", + "index": 0, + "value": null + }, + { + "color": "dark-yellow", + "index": 1, + "value": 5000000000 + }, + { + "color": "dark-red", + "index": 2, + "value": 7000000000 + } + ], + "values": false + } + }, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "span": 12, + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "sum(irate(container_network_receive_bytes_total{cluster=\"$cluster\",namespace=~\"$namespace\", pod=~\"$pod\"}[$interval:$resolution]))", + "format": "time_series", + "instant": null, + "intervalFactor": 1, + "legendFormat": "", + "refId": "A" + } + ], + "thresholds": "", + "timeFrom": null, + "timeShift": null, + "title": "Current Rate of Bytes Received", + "type": "gauge", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": "$datasource", + "decimals": 0, + "format": "time_series", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 1 + }, + "height": 9, + "id": 4, + "interval": null, + "links": [ + + ], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "minSpan": 12, + "nullPointMode": "connected", + "nullText": null, + "options": { + "fieldOptions": { + "calcs": [ + "last" + ], + "defaults": { + "max": 10000000000, + "min": 0, + "title": "$namespace: $pod", + "unit": "Bps" + }, + "mappings": [ + + ], + "override": { + + }, + "thresholds": [ + { + "color": "dark-green", + "index": 0, + "value": null + }, + { + "color": "dark-yellow", + "index": 1, + "value": 5000000000 + }, + { + "color": "dark-red", + "index": 2, + "value": 7000000000 + } + ], + "values": false + } + }, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "span": 12, + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "sum(irate(container_network_transmit_bytes_total{cluster=\"$cluster\",namespace=~\"$namespace\", pod=~\"$pod\"}[$interval:$resolution]))", + "format": "time_series", + "instant": null, + "intervalFactor": 1, + "legendFormat": "", + "refId": "A" + } + ], + "thresholds": "", + "timeFrom": null, + "timeShift": null, + "title": "Current Rate of Bytes Transmitted", + "type": "gauge", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + }, + { + "collapse": false, + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 10 + }, + "id": 5, + "panels": [ + + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Bandwidth", + "titleSize": "h6", + "type": "row" + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 11 + }, + "id": 6, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [ + + ], + "minSpan": 12, + "nullPointMode": "connected", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(container_network_receive_bytes_total{cluster=\"$cluster\",namespace=~\"$namespace\", pod=~\"$pod\"}[$interval:$resolution])) by (pod)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Receive Bandwidth", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 11 + }, + "id": 7, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [ + + ], + "minSpan": 12, + "nullPointMode": "connected", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(container_network_transmit_bytes_total{cluster=\"$cluster\",namespace=~\"$namespace\", pod=~\"$pod\"}[$interval:$resolution])) by (pod)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Transmit Bandwidth", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "collapse": true, + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 20 + }, + "id": 8, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 21 + }, + "id": 9, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [ + + ], + "minSpan": 12, + "nullPointMode": "connected", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(container_network_receive_packets_total{cluster=\"$cluster\",namespace=~\"$namespace\", pod=~\"$pod\"}[$interval:$resolution])) by (pod)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Received Packets", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 21 + }, + "id": 10, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [ + + ], + "minSpan": 12, + "nullPointMode": "connected", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(container_network_transmit_packets_total{cluster=\"$cluster\",namespace=~\"$namespace\", pod=~\"$pod\"}[$interval:$resolution])) by (pod)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Transmitted Packets", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Packets", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": true, + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 21 + }, + "id": 11, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 32 + }, + "id": 12, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [ + + ], + "minSpan": 12, + "nullPointMode": "connected", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(container_network_receive_packets_dropped_total{cluster=\"$cluster\",namespace=~\"$namespace\", pod=~\"$pod\"}[$interval:$resolution])) by (pod)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Received Packets Dropped", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 32 + }, + "id": 13, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [ + + ], + "minSpan": 12, + "nullPointMode": "connected", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(container_network_transmit_packets_dropped_total{cluster=\"$cluster\",namespace=~\"$namespace\", pod=~\"$pod\"}[$interval:$resolution])) by (pod)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Transmitted Packets Dropped", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Errors", + "titleSize": "h6", + "type": "row" + } + ], + "refresh": "10s", + "rows": [ + + ], + "schemaVersion": 18, + "style": "dark", + "tags": [ + "kubernetes-mixin" + ], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": "Data Source", + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": null, + "current": { + + }, + "datasource": "$datasource", + "hide": {{ if .Values.grafana.sidecar.dashboards.multicluster.global.enabled }}0{{ else }}2{{ end }}, + "includeAll": false, + "label": null, + "multi": false, + "name": "cluster", + "options": [ + + ], + "query": "label_values(up{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\"}, cluster)", + "refresh": 2, + "regex": "", + "sort": 0, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": ".+", + "auto": false, + "auto_count": 30, + "auto_min": "10s", + "current": { + "text": "kube-system", + "value": "kube-system" + }, + "datasource": "$datasource", + "definition": "label_values(container_network_receive_packets_total{cluster=\"$cluster\"}, namespace)", + "hide": 0, + "includeAll": true, + "label": null, + "multi": false, + "name": "namespace", + "options": [ + + ], + "query": "label_values(container_network_receive_packets_total{cluster=\"$cluster\"}, namespace)", + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": ".+", + "auto": false, + "auto_count": 30, + "auto_min": "10s", + "current": { + "text": "", + "value": "" + }, + "datasource": "$datasource", + "definition": "label_values(container_network_receive_packets_total{cluster=\"$cluster\",namespace=~\"$namespace\"}, pod)", + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "pod", + "options": [ + + ], + "query": "label_values(container_network_receive_packets_total{cluster=\"$cluster\",namespace=~\"$namespace\"}, pod)", + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "auto": false, + "auto_count": 30, + "auto_min": "10s", + "current": { + "text": "5m", + "value": "5m" + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "resolution", + "options": [ + { + "selected": false, + "text": "30s", + "value": "30s" + }, + { + "selected": true, + "text": "5m", + "value": "5m" + }, + { + "selected": false, + "text": "1h", + "value": "1h" + } + ], + "query": "30s,5m,1h", + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "interval", + "useTags": false + }, + { + "allValue": null, + "auto": false, + "auto_count": 30, + "auto_min": "10s", + "current": { + "text": "5m", + "value": "5m" + }, + "datasource": "$datasource", + "hide": 2, + "includeAll": false, + "label": null, + "multi": false, + "name": "interval", + "options": [ + { + "selected": true, + "text": "4h", + "value": "4h" + } + ], + "query": "4h", + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "interval", + "useTags": false + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "{{ .Values.grafana.defaultDashboardsTimezone }}", + "title": "Kubernetes / Networking / Pod", + "uid": "7a18067ce943a40ae25454675c19ff5c", + "version": 0 + } +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/grafana/dashboards-1.14/prometheus-remote-write.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/grafana/dashboards-1.14/prometheus-remote-write.yaml new file mode 100644 index 00000000000..5c11900e698 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/grafana/dashboards-1.14/prometheus-remote-write.yaml @@ -0,0 +1,1674 @@ +{{- /* +Generated from 'prometheus-remote-write' from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/a8ba97a150c75be42010c75d10b720c55e182f1a/manifests/grafana-dashboardDefinitions.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (or .Values.grafana.enabled .Values.grafana.forceDeployDashboards) (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.grafana.defaultDashboardsEnabled .Values.prometheus.prometheusSpec.remoteWriteDashboards }} +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: {{ .Values.grafana.defaultDashboards.namespace }} + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" $) "prometheus-remote-write" | trunc 63 | trimSuffix "-" }} + annotations: +{{ toYaml .Values.grafana.sidecar.dashboards.annotations | indent 4 }} + labels: + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: {{ ternary $.Values.grafana.sidecar.dashboards.labelValue "1" (not (empty $.Values.grafana.sidecar.dashboards.labelValue)) | quote }} + {{- end }} + app: {{ template "kube-prometheus-stack.name" $ }}-grafana +{{ include "kube-prometheus-stack.labels" $ | indent 4 }} +data: + prometheus-remote-write.json: |- + { + "__inputs": [ + + ], + "__requires": [ + + ], + "annotations": { + "list": [ + + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "hideControls": false, + "id": null, + "links": [ + + ], + "refresh": "60s", + "rows": [ + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 2, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "(\n prometheus_remote_storage_highest_timestamp_in_seconds{cluster=~\"$cluster\", instance=~\"$instance\"} \n- \n ignoring(remote_name, url) group_right(instance) (prometheus_remote_storage_queue_highest_sent_timestamp_seconds{cluster=~\"$cluster\", instance=~\"$instance\"} != 0)\n)\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}cluster{{`}}`}}:{{`{{`}}instance{{`}}`}} {{`{{`}}remote_name{{`}}`}}:{{`{{`}}url{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Highest Timestamp In vs. Highest Timestamp Sent", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 3, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "clamp_min(\n rate(prometheus_remote_storage_highest_timestamp_in_seconds{cluster=~\"$cluster\", instance=~\"$instance\"}[5m]) \n- \n ignoring (remote_name, url) group_right(instance) rate(prometheus_remote_storage_queue_highest_sent_timestamp_seconds{cluster=~\"$cluster\", instance=~\"$instance\"}[5m])\n, 0)\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}cluster{{`}}`}}:{{`{{`}}instance{{`}}`}} {{`{{`}}remote_name{{`}}`}}:{{`{{`}}url{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate[5m]", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Timestamps", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 4, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(\n prometheus_remote_storage_samples_in_total{cluster=~\"$cluster\", instance=~\"$instance\"}[5m])\n- \n ignoring(remote_name, url) group_right(instance) (rate(prometheus_remote_storage_succeeded_samples_total{cluster=~\"$cluster\", instance=~\"$instance\"}[5m]) or rate(prometheus_remote_storage_samples_total{cluster=~\"$cluster\", instance=~\"$instance\"}[5m]))\n- \n (rate(prometheus_remote_storage_dropped_samples_total{cluster=~\"$cluster\", instance=~\"$instance\"}[5m]) or rate(prometheus_remote_storage_samples_dropped_total{cluster=~\"$cluster\", instance=~\"$instance\"}[5m]))\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}cluster{{`}}`}}:{{`{{`}}instance{{`}}`}} {{`{{`}}remote_name{{`}}`}}:{{`{{`}}url{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate, in vs. succeeded or dropped [5m]", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Samples", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 5, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "minSpan": 6, + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "prometheus_remote_storage_shards{cluster=~\"$cluster\", instance=~\"$instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}cluster{{`}}`}}:{{`{{`}}instance{{`}}`}} {{`{{`}}remote_name{{`}}`}}:{{`{{`}}url{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Current Shards", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 6, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "prometheus_remote_storage_shards_max{cluster=~\"$cluster\", instance=~\"$instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}cluster{{`}}`}}:{{`{{`}}instance{{`}}`}} {{`{{`}}remote_name{{`}}`}}:{{`{{`}}url{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Max Shards", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 7, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "prometheus_remote_storage_shards_min{cluster=~\"$cluster\", instance=~\"$instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}cluster{{`}}`}}:{{`{{`}}instance{{`}}`}} {{`{{`}}remote_name{{`}}`}}:{{`{{`}}url{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Min Shards", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 8, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "prometheus_remote_storage_shards_desired{cluster=~\"$cluster\", instance=~\"$instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}cluster{{`}}`}}:{{`{{`}}instance{{`}}`}} {{`{{`}}remote_name{{`}}`}}:{{`{{`}}url{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Desired Shards", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Shards", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 9, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "prometheus_remote_storage_shard_capacity{cluster=~\"$cluster\", instance=~\"$instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}cluster{{`}}`}}:{{`{{`}}instance{{`}}`}} {{`{{`}}remote_name{{`}}`}}:{{`{{`}}url{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Shard Capacity", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 10, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "prometheus_remote_storage_pending_samples{cluster=~\"$cluster\", instance=~\"$instance\"} or prometheus_remote_storage_samples_pending{cluster=~\"$cluster\", instance=~\"$instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}cluster{{`}}`}}:{{`{{`}}instance{{`}}`}} {{`{{`}}remote_name{{`}}`}}:{{`{{`}}url{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Pending Samples", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Shard Details", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 11, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "prometheus_tsdb_wal_segment_current{cluster=~\"$cluster\", instance=~\"$instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}cluster{{`}}`}}:{{`{{`}}instance{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "TSDB Current Segment", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 12, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "prometheus_wal_watcher_current_segment{cluster=~\"$cluster\", instance=~\"$instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}cluster{{`}}`}}:{{`{{`}}instance{{`}}`}} {{`{{`}}consumer{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Remote Write Current Segment", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Segments", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 13, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 3, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(prometheus_remote_storage_dropped_samples_total{cluster=~\"$cluster\", instance=~\"$instance\"}[5m]) or rate(prometheus_remote_storage_samples_dropped_total{cluster=~\"$cluster\", instance=~\"$instance\"}[5m])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}cluster{{`}}`}}:{{`{{`}}instance{{`}}`}} {{`{{`}}remote_name{{`}}`}}:{{`{{`}}url{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Dropped Samples", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 14, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 3, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(prometheus_remote_storage_failed_samples_total{cluster=~\"$cluster\", instance=~\"$instance\"}[5m]) or rate(prometheus_remote_storage_samples_failed_total{cluster=~\"$cluster\", instance=~\"$instance\"}[5m])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}cluster{{`}}`}}:{{`{{`}}instance{{`}}`}} {{`{{`}}remote_name{{`}}`}}:{{`{{`}}url{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Failed Samples", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 15, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 3, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(prometheus_remote_storage_retried_samples_total{cluster=~\"$cluster\", instance=~\"$instance\"}[5m]) or rate(prometheus_remote_storage_samples_retried_total{cluster=~\"$cluster\", instance=~\"$instance\"}[5m])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}cluster{{`}}`}}:{{`{{`}}instance{{`}}`}} {{`{{`}}remote_name{{`}}`}}:{{`{{`}}url{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Retried Samples", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 16, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 3, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(prometheus_remote_storage_enqueue_retries_total{cluster=~\"$cluster\", instance=~\"$instance\"}[5m])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}cluster{{`}}`}}:{{`{{`}}instance{{`}}`}} {{`{{`}}remote_name{{`}}`}}:{{`{{`}}url{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Enqueue Retries", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Misc. Rates", + "titleSize": "h6", + "type": "row" + } + ], + "schemaVersion": 14, + "style": "dark", + "tags": [ + "prometheus-mixin" + ], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": "Data Source", + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": null, + "current": { + "text": { + "selected": true, + "text": "All", + "value": "$__all" + }, + "value": { + "selected": true, + "text": "All", + "value": "$__all" + } + }, + "datasource": "$datasource", + "hide": {{ if .Values.grafana.sidecar.dashboards.multicluster.global.enabled }}0{{ else }}2{{ end }}, + "includeAll": true, + "label": null, + "multi": false, + "name": "cluster", + "options": [ + + ], + "query": "label_values(kube_pod_container_info{image=~\".*prometheus.*\"}, cluster)", + "refresh": 2, + "regex": "", + "sort": 0, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": { + "text": { + "selected": true, + "text": "All", + "value": "$__all" + }, + "value": { + "selected": true, + "text": "All", + "value": "$__all" + } + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": true, + "label": null, + "multi": false, + "name": "instance", + "options": [ + + ], + "query": "label_values(prometheus_build_info{cluster=~\"$cluster\"}, instance)", + "refresh": 2, + "regex": "", + "sort": 0, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": { + + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": true, + "label": null, + "multi": false, + "name": "url", + "options": [ + + ], + "query": "label_values(prometheus_remote_storage_shards{cluster=~\"$cluster\", instance=~\"$instance\"}, url)", + "refresh": 2, + "regex": "", + "sort": 0, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-6h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "{{ .Values.grafana.defaultDashboardsTimezone }}", + "title": "Prometheus / Remote Write", + "version": 0 + } +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/grafana/dashboards-1.14/prometheus.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/grafana/dashboards-1.14/prometheus.yaml new file mode 100644 index 00000000000..27f7c44e2c4 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/grafana/dashboards-1.14/prometheus.yaml @@ -0,0 +1,1235 @@ +{{- /* +Generated from 'prometheus' from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/a8ba97a150c75be42010c75d10b720c55e182f1a/manifests/grafana-dashboardDefinitions.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (or .Values.grafana.enabled .Values.grafana.forceDeployDashboards) (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.grafana.defaultDashboardsEnabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: {{ .Values.grafana.defaultDashboards.namespace }} + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" $) "prometheus" | trunc 63 | trimSuffix "-" }} + annotations: +{{ toYaml .Values.grafana.sidecar.dashboards.annotations | indent 4 }} + labels: + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: {{ ternary $.Values.grafana.sidecar.dashboards.labelValue "1" (not (empty $.Values.grafana.sidecar.dashboards.labelValue)) | quote }} + {{- end }} + app: {{ template "kube-prometheus-stack.name" $ }}-grafana +{{ include "kube-prometheus-stack.labels" $ | indent 4 }} +data: + prometheus.json: |- + { + "annotations": { + "list": [ + + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "hideControls": false, + "links": [ + + ], + "refresh": "60s", + "rows": [ + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 1, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "styles": [ + { + "alias": "Time", + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "pattern": "Time", + "type": "hidden" + }, + { + "alias": "Count", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #A", + "thresholds": [ + + ], + "type": "hidden", + "unit": "short" + }, + { + "alias": "Uptime", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #B", + "thresholds": [ + + ], + "type": "number", + "unit": "s" + }, + { + "alias": "Instance", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "instance", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "Job", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "job", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "Version", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "version", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "pattern": "/.*/", + "thresholds": [ + + ], + "type": "string", + "unit": "short" + } + ], + "targets": [ + { + "expr": "count by (job, instance, version) (prometheus_build_info{job=~\"$job\", instance=~\"$instance\"})", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "A", + "step": 10 + }, + { + "expr": "max by (job, instance) (time() - process_start_time_seconds{job=~\"$job\", instance=~\"$instance\"})", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "B", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Prometheus Stats", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "transform": "table", + "type": "table", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Prometheus Stats", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 2, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(prometheus_target_sync_length_seconds_sum{job=~\"$job\",instance=~\"$instance\"}[5m])) by (scrape_job) * 1e3", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}scrape_job{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Target Sync", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "ms", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 3, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(prometheus_sd_discovered_targets{job=~\"$job\",instance=~\"$instance\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Targets", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Targets", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Discovery", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 4, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(prometheus_target_interval_length_seconds_sum{job=~\"$job\",instance=~\"$instance\"}[5m]) / rate(prometheus_target_interval_length_seconds_count{job=~\"$job\",instance=~\"$instance\"}[5m]) * 1e3", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}interval{{`}}`}} configured", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Average Scrape Interval Duration", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "ms", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 5, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 4, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum by (job) (rate(prometheus_target_scrapes_exceeded_body_size_limit_total[1m]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "exceeded body size limit: {{`{{`}}job{{`}}`}}", + "legendLink": null, + "step": 10 + }, + { + "expr": "sum by (job) (rate(prometheus_target_scrapes_exceeded_sample_limit_total[1m]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "exceeded sample limit: {{`{{`}}job{{`}}`}}", + "legendLink": null, + "step": 10 + }, + { + "expr": "sum by (job) (rate(prometheus_target_scrapes_sample_duplicate_timestamp_total[1m]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "duplicate timestamp: {{`{{`}}job{{`}}`}}", + "legendLink": null, + "step": 10 + }, + { + "expr": "sum by (job) (rate(prometheus_target_scrapes_sample_out_of_bounds_total[1m]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "out of bounds: {{`{{`}}job{{`}}`}}", + "legendLink": null, + "step": 10 + }, + { + "expr": "sum by (job) (rate(prometheus_target_scrapes_sample_out_of_order_total[1m]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "out of order: {{`{{`}}job{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Scrape failures", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 6, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 4, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "rate(prometheus_tsdb_head_samples_appended_total{job=~\"$job\",instance=~\"$instance\"}[5m])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}job{{`}}`}} {{`{{`}}instance{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Appended Samples", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Retrieval", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 7, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "prometheus_tsdb_head_series{job=~\"$job\",instance=~\"$instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}job{{`}}`}} {{`{{`}}instance{{`}}`}} head series", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Head Series", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 8, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "prometheus_tsdb_head_chunks{job=~\"$job\",instance=~\"$instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}job{{`}}`}} {{`{{`}}instance{{`}}`}} head chunks", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Head Chunks", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Storage", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 9, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "rate(prometheus_engine_query_duration_seconds_count{job=~\"$job\",instance=~\"$instance\",slice=\"inner_eval\"}[5m])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}job{{`}}`}} {{`{{`}}instance{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Query Rate", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 10, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "max by (slice) (prometheus_engine_query_duration_seconds{quantile=\"0.9\",job=~\"$job\",instance=~\"$instance\"}) * 1e3", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}slice{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Stage Duration", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "ms", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Query", + "titleSize": "h6" + } + ], + "schemaVersion": 14, + "style": "dark", + "tags": [ + "prometheus-mixin" + ], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": "Data Source", + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": ".+", + "current": { + "selected": true, + "text": "All", + "value": "$__all" + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": true, + "label": "job", + "multi": true, + "name": "job", + "options": [ + + ], + "query": "label_values(prometheus_build_info{job=\"prometheus-k8s\",namespace=\"monitoring\"}, job)", + "refresh": 1, + "regex": "", + "sort": 2, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": ".+", + "current": { + "selected": true, + "text": "All", + "value": "$__all" + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": true, + "label": "instance", + "multi": true, + "name": "instance", + "options": [ + + ], + "query": "label_values(prometheus_build_info{job=~\"$job\"}, instance)", + "refresh": 1, + "regex": "", + "sort": 2, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "{{ .Values.grafana.defaultDashboardsTimezone }}", + "title": "Prometheus / Overview", + "uid": "", + "version": 0 + } +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/grafana/dashboards-1.14/proxy.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/grafana/dashboards-1.14/proxy.yaml new file mode 100644 index 00000000000..410812451ef --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/grafana/dashboards-1.14/proxy.yaml @@ -0,0 +1,1276 @@ +{{- /* +Generated from 'proxy' from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/a8ba97a150c75be42010c75d10b720c55e182f1a/manifests/grafana-dashboardDefinitions.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (or .Values.grafana.enabled .Values.grafana.forceDeployDashboards) (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.grafana.defaultDashboardsEnabled }} +{{- if (include "exporter.kubeProxy.enabled" .)}} +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: {{ .Values.grafana.defaultDashboards.namespace }} + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" $) "proxy" | trunc 63 | trimSuffix "-" }} + annotations: +{{ toYaml .Values.grafana.sidecar.dashboards.annotations | indent 4 }} + labels: + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: {{ ternary $.Values.grafana.sidecar.dashboards.labelValue "1" (not (empty $.Values.grafana.sidecar.dashboards.labelValue)) | quote }} + {{- end }} + app: {{ template "kube-prometheus-stack.name" $ }}-grafana +{{ include "kube-prometheus-stack.labels" $ | indent 4 }} +data: + proxy.json: |- + { + "__inputs": [ + + ], + "__requires": [ + + ], + "annotations": { + "list": [ + + ] + }, + "editable": false, + "gnetId": null, + "graphTooltip": 0, + "hideControls": false, + "id": null, + "links": [ + + ], + "refresh": "10s", + "rows": [ + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": "$datasource", + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + + }, + "id": 2, + "interval": "1m", + "legend": { + "alignAsTable": true, + "rightSide": true + }, + "links": [ + + ], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "span": 2, + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + {{- if .Values.k3sServer.enabled }} + "expr": "sum(up{cluster=\"$cluster\", job=\"{{ include "exporter.kubeProxy.jobName" . }}\", metrics_path=\"/metrics\"})", + {{- else }} + "expr": "sum(up{cluster=\"$cluster\", job=\"{{ include "exporter.kubeProxy.jobName" . }}\"})", + {{- end }} + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "", + "refId": "A" + } + ], + "thresholds": "", + "title": "Up", + "tooltip": { + "shared": false + }, + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "min" + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 3, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 5, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(kubeproxy_sync_proxy_rules_duration_seconds_count{cluster=\"$cluster\", job=\"{{ include "exporter.kubeProxy.jobName" . }}\", instance=~\"$instance\"}[$__rate_interval]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "rate", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rules Sync Rate", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 4, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 5, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.99,rate(kubeproxy_sync_proxy_rules_duration_seconds_bucket{cluster=\"$cluster\", job=\"{{ include "exporter.kubeProxy.jobName" . }}\", instance=~\"$instance\"}[$__rate_interval]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rule Sync Latency 99th Quantile", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "Dashboard Row", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 5, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(kubeproxy_network_programming_duration_seconds_count{cluster=\"$cluster\", job=\"{{ include "exporter.kubeProxy.jobName" . }}\", instance=~\"$instance\"}[$__rate_interval]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "rate", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Network Programming Rate", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 6, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum(rate(kubeproxy_network_programming_duration_seconds_bucket{cluster=\"$cluster\", job=\"{{ include "exporter.kubeProxy.jobName" . }}\", instance=~\"$instance\"}[$__rate_interval])) by (instance, le))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Network Programming Latency 99th Quantile", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "Dashboard Row", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 7, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(rest_client_requests_total{cluster=\"$cluster\", job=\"{{ include "exporter.kubeProxy.jobName" . }}\", instance=~\"$instance\",code=~\"2..\"}[$__rate_interval]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "2xx", + "refId": "A" + }, + { + "expr": "sum(rate(rest_client_requests_total{cluster=\"$cluster\", job=\"{{ include "exporter.kubeProxy.jobName" . }}\", instance=~\"$instance\",code=~\"3..\"}[$__rate_interval]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "3xx", + "refId": "B" + }, + { + "expr": "sum(rate(rest_client_requests_total{cluster=\"$cluster\", job=\"{{ include "exporter.kubeProxy.jobName" . }}\", instance=~\"$instance\",code=~\"4..\"}[$__rate_interval]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "4xx", + "refId": "C" + }, + { + "expr": "sum(rate(rest_client_requests_total{cluster=\"$cluster\", job=\"{{ include "exporter.kubeProxy.jobName" . }}\", instance=~\"$instance\",code=~\"5..\"}[$__rate_interval]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "5xx", + "refId": "D" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Kube API Request Rate", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 8, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 8, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum(rate(rest_client_request_duration_seconds_bucket{cluster=\"$cluster\", job=\"{{ include "exporter.kubeProxy.jobName" . }}\",instance=~\"$instance\",verb=\"POST\"}[$__rate_interval])) by (verb, url, le))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}verb{{`}}`}} {{`{{`}}url{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Post Request Latency 99th Quantile", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "Dashboard Row", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 9, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum(rate(rest_client_request_duration_seconds_bucket{cluster=\"$cluster\", job=\"{{ include "exporter.kubeProxy.jobName" . }}\", instance=~\"$instance\", verb=\"GET\"}[$__rate_interval])) by (verb, url, le))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}verb{{`}}`}} {{`{{`}}url{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Get Request Latency 99th Quantile", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "Dashboard Row", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 10, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "process_resident_memory_bytes{cluster=\"$cluster\", job=\"{{ include "exporter.kubeProxy.jobName" . }}\",instance=~\"$instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Memory", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 11, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(process_cpu_seconds_total{cluster=\"$cluster\", job=\"{{ include "exporter.kubeProxy.jobName" . }}\",instance=~\"$instance\"}[$__rate_interval])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "CPU usage", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 12, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "go_goroutines{cluster=\"$cluster\", job=\"{{ include "exporter.kubeProxy.jobName" . }}\",instance=~\"$instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Goroutines", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "Dashboard Row", + "titleSize": "h6", + "type": "row" + } + ], + "schemaVersion": 14, + "style": "dark", + "tags": [ + "kubernetes-mixin" + ], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": "Data Source", + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": null, + "current": { + + }, + "datasource": "$datasource", + "hide": {{ if .Values.grafana.sidecar.dashboards.multicluster.global.enabled }}0{{ else }}2{{ end }}, + "includeAll": false, + "label": "cluster", + "multi": false, + "name": "cluster", + "options": [ + + ], + "query": "label_values(up{job=\"{{ include "exporter.kubeProxy.jobName" . }}\"}, cluster)", + "refresh": 2, + "regex": "", + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": { + + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": true, + "label": null, + "multi": false, + "name": "instance", + "options": [ + + ], + "query": "label_values(up{job=\"{{ include "exporter.kubeProxy.jobName" . }}\", cluster=\"$cluster\", job=\"{{ include "exporter.kubeProxy.jobName" . }}\"}, instance)", + "refresh": 2, + "regex": "", + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "{{ .Values.grafana.defaultDashboardsTimezone }}", + "title": "Kubernetes / Proxy", + "uid": "632e265de029684c40b21cb76bca4f94", + "version": 0 + } +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/grafana/dashboards-1.14/scheduler.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/grafana/dashboards-1.14/scheduler.yaml new file mode 100644 index 00000000000..ee0cf08b2f2 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/grafana/dashboards-1.14/scheduler.yaml @@ -0,0 +1,1118 @@ +{{- /* +Generated from 'scheduler' from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/a8ba97a150c75be42010c75d10b720c55e182f1a/manifests/grafana-dashboardDefinitions.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (or .Values.grafana.enabled .Values.grafana.forceDeployDashboards) (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.grafana.defaultDashboardsEnabled }} +{{- if (include "exporter.kubeScheduler.enabled" .)}} +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: {{ .Values.grafana.defaultDashboards.namespace }} + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" $) "scheduler" | trunc 63 | trimSuffix "-" }} + annotations: +{{ toYaml .Values.grafana.sidecar.dashboards.annotations | indent 4 }} + labels: + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: {{ ternary $.Values.grafana.sidecar.dashboards.labelValue "1" (not (empty $.Values.grafana.sidecar.dashboards.labelValue)) | quote }} + {{- end }} + app: {{ template "kube-prometheus-stack.name" $ }}-grafana +{{ include "kube-prometheus-stack.labels" $ | indent 4 }} +data: + scheduler.json: |- + { + "__inputs": [ + + ], + "__requires": [ + + ], + "annotations": { + "list": [ + + ] + }, + "editable": false, + "gnetId": null, + "graphTooltip": 0, + "hideControls": false, + "id": null, + "links": [ + + ], + "refresh": "10s", + "rows": [ + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": "$datasource", + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + + }, + "id": 2, + "interval": "1m", + "legend": { + "alignAsTable": true, + "rightSide": true + }, + "links": [ + + ], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "span": 2, + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + {{- if .Values.k3sServer.enabled }} + "expr": "sum(up{cluster=\"$cluster\", job=\"{{ include "exporter.kubeScheduler.jobName" . }}\", metrics_path=\"/metrics\"})", + {{- else }} + "expr": "sum(up{cluster=\"$cluster\", job=\"{{ include "exporter.kubeScheduler.jobName" . }}\"})", + {{- end }} + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "", + "refId": "A" + } + ], + "thresholds": "", + "title": "Up", + "tooltip": { + "shared": false + }, + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "min" + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 3, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 5, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(scheduler_e2e_scheduling_duration_seconds_count{cluster=\"$cluster\", job=\"{{ include "exporter.kubeScheduler.jobName" . }}\", instance=~\"$instance\"}[$__rate_interval])) by (cluster, instance)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}cluster{{`}}`}} {{`{{`}}instance{{`}}`}} e2e", + "refId": "A" + }, + { + "expr": "sum(rate(scheduler_binding_duration_seconds_count{cluster=\"$cluster\", job=\"{{ include "exporter.kubeScheduler.jobName" . }}\", instance=~\"$instance\"}[$__rate_interval])) by (cluster, instance)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}cluster{{`}}`}} {{`{{`}}instance{{`}}`}} binding", + "refId": "B" + }, + { + "expr": "sum(rate(scheduler_scheduling_algorithm_duration_seconds_count{cluster=\"$cluster\", job=\"{{ include "exporter.kubeScheduler.jobName" . }}\", instance=~\"$instance\"}[$__rate_interval])) by (cluster, instance)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}cluster{{`}}`}} {{`{{`}}instance{{`}}`}} scheduling algorithm", + "refId": "C" + }, + { + "expr": "sum(rate(scheduler_volume_scheduling_duration_seconds_count{cluster=\"$cluster\", job=\"{{ include "exporter.kubeScheduler.jobName" . }}\", instance=~\"$instance\"}[$__rate_interval])) by (cluster, instance)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}cluster{{`}}`}} {{`{{`}}instance{{`}}`}} volume", + "refId": "D" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Scheduling Rate", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 4, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 5, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum(rate(scheduler_e2e_scheduling_duration_seconds_bucket{cluster=\"$cluster\", job=\"{{ include "exporter.kubeScheduler.jobName" . }}\",instance=~\"$instance\"}[$__rate_interval])) by (cluster, instance, le))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}cluster{{`}}`}} {{`{{`}}instance{{`}}`}} e2e", + "refId": "A" + }, + { + "expr": "histogram_quantile(0.99, sum(rate(scheduler_binding_duration_seconds_bucket{cluster=\"$cluster\", job=\"{{ include "exporter.kubeScheduler.jobName" . }}\",instance=~\"$instance\"}[$__rate_interval])) by (cluster, instance, le))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}cluster{{`}}`}} {{`{{`}}instance{{`}}`}} binding", + "refId": "B" + }, + { + "expr": "histogram_quantile(0.99, sum(rate(scheduler_scheduling_algorithm_duration_seconds_bucket{cluster=\"$cluster\", job=\"{{ include "exporter.kubeScheduler.jobName" . }}\",instance=~\"$instance\"}[$__rate_interval])) by (cluster, instance, le))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}cluster{{`}}`}} {{`{{`}}instance{{`}}`}} scheduling algorithm", + "refId": "C" + }, + { + "expr": "histogram_quantile(0.99, sum(rate(scheduler_volume_scheduling_duration_seconds_bucket{cluster=\"$cluster\", job=\"{{ include "exporter.kubeScheduler.jobName" . }}\",instance=~\"$instance\"}[$__rate_interval])) by (cluster, instance, le))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}cluster{{`}}`}} {{`{{`}}instance{{`}}`}} volume", + "refId": "D" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Scheduling latency 99th Quantile", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "Dashboard Row", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 5, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(rest_client_requests_total{cluster=\"$cluster\", job=\"{{ include "exporter.kubeScheduler.jobName" . }}\", instance=~\"$instance\",code=~\"2..\"}[$__rate_interval]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "2xx", + "refId": "A" + }, + { + "expr": "sum(rate(rest_client_requests_total{cluster=\"$cluster\", job=\"{{ include "exporter.kubeScheduler.jobName" . }}\", instance=~\"$instance\",code=~\"3..\"}[$__rate_interval]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "3xx", + "refId": "B" + }, + { + "expr": "sum(rate(rest_client_requests_total{cluster=\"$cluster\", job=\"{{ include "exporter.kubeScheduler.jobName" . }}\", instance=~\"$instance\",code=~\"4..\"}[$__rate_interval]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "4xx", + "refId": "C" + }, + { + "expr": "sum(rate(rest_client_requests_total{cluster=\"$cluster\", job=\"{{ include "exporter.kubeScheduler.jobName" . }}\", instance=~\"$instance\",code=~\"5..\"}[$__rate_interval]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "5xx", + "refId": "D" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Kube API Request Rate", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 6, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 8, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum(rate(rest_client_request_duration_seconds_bucket{cluster=\"$cluster\", job=\"{{ include "exporter.kubeScheduler.jobName" . }}\", instance=~\"$instance\", verb=\"POST\"}[$__rate_interval])) by (verb, url, le))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}verb{{`}}`}} {{`{{`}}url{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Post Request Latency 99th Quantile", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "Dashboard Row", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 7, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum(rate(rest_client_request_duration_seconds_bucket{cluster=\"$cluster\", job=\"{{ include "exporter.kubeScheduler.jobName" . }}\", instance=~\"$instance\", verb=\"GET\"}[$__rate_interval])) by (verb, url, le))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}verb{{`}}`}} {{`{{`}}url{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Get Request Latency 99th Quantile", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "Dashboard Row", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 8, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "process_resident_memory_bytes{cluster=\"$cluster\", job=\"{{ include "exporter.kubeScheduler.jobName" . }}\", instance=~\"$instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Memory", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 9, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(process_cpu_seconds_total{cluster=\"$cluster\", job=\"{{ include "exporter.kubeScheduler.jobName" . }}\", instance=~\"$instance\"}[$__rate_interval])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "CPU usage", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 10, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "go_goroutines{cluster=\"$cluster\", job=\"{{ include "exporter.kubeScheduler.jobName" . }}\",instance=~\"$instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Goroutines", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "Dashboard Row", + "titleSize": "h6", + "type": "row" + } + ], + "schemaVersion": 14, + "style": "dark", + "tags": [ + "kubernetes-mixin" + ], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": "Data Source", + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": null, + "current": { + + }, + "datasource": "$datasource", + "hide": {{ if .Values.grafana.sidecar.dashboards.multicluster.global.enabled }}0{{ else }}2{{ end }}, + "includeAll": false, + "label": "cluster", + "multi": false, + "name": "cluster", + "options": [ + + ], + "query": "label_values(up{job=\"{{ include "exporter.kubeScheduler.jobName" . }}\"}, cluster)", + "refresh": 2, + "regex": "", + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": { + + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": true, + "label": null, + "multi": false, + "name": "instance", + "options": [ + + ], + "query": "label_values(up{job=\"{{ include "exporter.kubeScheduler.jobName" . }}\", cluster=\"$cluster\"}, instance)", + "refresh": 2, + "regex": "", + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "{{ .Values.grafana.defaultDashboardsTimezone }}", + "title": "Kubernetes / Scheduler", + "uid": "2e6b6a3b4bddf1427b3a55aa1311c656", + "version": 0 + } +{{- end }} +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/grafana/dashboards-1.14/workload-total.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/grafana/dashboards-1.14/workload-total.yaml new file mode 100644 index 00000000000..5aafccdebe2 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/grafana/dashboards-1.14/workload-total.yaml @@ -0,0 +1,1438 @@ +{{- /* +Generated from 'workload-total' from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/a8ba97a150c75be42010c75d10b720c55e182f1a/manifests/grafana-dashboardDefinitions.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (or .Values.grafana.enabled .Values.grafana.forceDeployDashboards) (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.grafana.defaultDashboardsEnabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: {{ .Values.grafana.defaultDashboards.namespace }} + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" $) "workload-total" | trunc 63 | trimSuffix "-" }} + annotations: +{{ toYaml .Values.grafana.sidecar.dashboards.annotations | indent 4 }} + labels: + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: {{ ternary $.Values.grafana.sidecar.dashboards.labelValue "1" (not (empty $.Values.grafana.sidecar.dashboards.labelValue)) | quote }} + {{- end }} + app: {{ template "kube-prometheus-stack.name" $ }}-grafana +{{ include "kube-prometheus-stack.labels" $ | indent 4 }} +data: + workload-total.json: |- + { + "__inputs": [ + + ], + "__requires": [ + + ], + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "hideControls": false, + "id": null, + "links": [ + + ], + "panels": [ + { + "collapse": false, + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 2, + "panels": [ + + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Current Bandwidth", + "titleSize": "h6", + "type": "row" + }, + { + "aliasColors": { + + }, + "bars": true, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 1 + }, + "id": 3, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": false, + "linewidth": 1, + "links": [ + + ], + "minSpan": 24, + "nullPointMode": "null", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 24, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sort_desc(sum(irate(container_network_receive_bytes_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\",namespace=~\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=~\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}} pod {{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Current Rate of Bytes Received", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "series", + "name": null, + "show": false, + "values": [ + "current" + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": true, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 1 + }, + "id": 4, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": false, + "linewidth": 1, + "links": [ + + ], + "minSpan": 24, + "nullPointMode": "null", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 24, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sort_desc(sum(irate(container_network_transmit_bytes_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\",namespace=~\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=~\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}} pod {{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Current Rate of Bytes Transmitted", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "series", + "name": null, + "show": false, + "values": [ + "current" + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "collapse": true, + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 10 + }, + "id": 5, + "panels": [ + { + "aliasColors": { + + }, + "bars": true, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 11 + }, + "id": 6, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": false, + "linewidth": 1, + "links": [ + + ], + "minSpan": 24, + "nullPointMode": "null", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 24, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sort_desc(avg(irate(container_network_receive_bytes_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\",namespace=~\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=~\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}} pod {{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Average Rate of Bytes Received", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "series", + "name": null, + "show": false, + "values": [ + "current" + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": true, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 11 + }, + "id": 7, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": false, + "linewidth": 1, + "links": [ + + ], + "minSpan": 24, + "nullPointMode": "null", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 24, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sort_desc(avg(irate(container_network_transmit_bytes_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\",namespace=~\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=~\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}} pod {{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Average Rate of Bytes Transmitted", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "series", + "name": null, + "show": false, + "values": [ + "current" + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Average Bandwidth", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 11 + }, + "id": 8, + "panels": [ + + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Bandwidth HIstory", + "titleSize": "h6", + "type": "row" + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 12 + }, + "id": 9, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [ + + ], + "minSpan": 12, + "nullPointMode": "connected", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sort_desc(sum(irate(container_network_receive_bytes_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\",namespace=~\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=~\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Receive Bandwidth", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 12 + }, + "id": 10, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [ + + ], + "minSpan": 12, + "nullPointMode": "connected", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sort_desc(sum(irate(container_network_transmit_bytes_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\",namespace=~\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=~\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Transmit Bandwidth", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "collapse": true, + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 21 + }, + "id": 11, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 22 + }, + "id": 12, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [ + + ], + "minSpan": 12, + "nullPointMode": "connected", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sort_desc(sum(irate(container_network_receive_packets_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\",namespace=~\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=~\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Received Packets", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 22 + }, + "id": 13, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [ + + ], + "minSpan": 12, + "nullPointMode": "connected", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sort_desc(sum(irate(container_network_transmit_packets_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\",namespace=~\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=~\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Transmitted Packets", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Packets", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": true, + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 22 + }, + "id": 14, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 23 + }, + "id": 15, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [ + + ], + "minSpan": 12, + "nullPointMode": "connected", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sort_desc(sum(irate(container_network_receive_packets_dropped_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\",namespace=~\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=~\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Received Packets Dropped", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 23 + }, + "id": 16, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [ + + ], + "minSpan": 12, + "nullPointMode": "connected", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sort_desc(sum(irate(container_network_transmit_packets_dropped_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\",namespace=~\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=~\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Transmitted Packets Dropped", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Errors", + "titleSize": "h6", + "type": "row" + } + ], + "refresh": "10s", + "rows": [ + + ], + "schemaVersion": 18, + "style": "dark", + "tags": [ + "kubernetes-mixin" + ], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": "Data Source", + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": null, + "current": { + + }, + "datasource": "$datasource", + "hide": {{ if .Values.grafana.sidecar.dashboards.multicluster.global.enabled }}0{{ else }}2{{ end }}, + "includeAll": false, + "label": null, + "multi": false, + "name": "cluster", + "options": [ + + ], + "query": "label_values(kube_pod_info{job=\"kube-state-metrics\"}, cluster)", + "refresh": 2, + "regex": "", + "sort": 0, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": ".+", + "auto": false, + "auto_count": 30, + "auto_min": "10s", + "current": { + "text": "kube-system", + "value": "kube-system" + }, + "datasource": "$datasource", + "definition": "label_values(container_network_receive_packets_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\"}, namespace)", + "hide": 0, + "includeAll": true, + "label": null, + "multi": false, + "name": "namespace", + "options": [ + + ], + "query": "label_values(container_network_receive_packets_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\"}, namespace)", + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "auto": false, + "auto_count": 30, + "auto_min": "10s", + "current": { + "text": "", + "value": "" + }, + "datasource": "$datasource", + "definition": "label_values(namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=~\"$namespace\"}, workload)", + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "workload", + "options": [ + + ], + "query": "label_values(namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=~\"$namespace\"}, workload)", + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "auto": false, + "auto_count": 30, + "auto_min": "10s", + "current": { + "text": "deployment", + "value": "deployment" + }, + "datasource": "$datasource", + "definition": "label_values(namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=~\"$namespace\", workload=~\"$workload\"}, workload_type)", + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "type", + "options": [ + + ], + "query": "label_values(namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=~\"$namespace\", workload=~\"$workload\"}, workload_type)", + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "auto": false, + "auto_count": 30, + "auto_min": "10s", + "current": { + "text": "5m", + "value": "5m" + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "resolution", + "options": [ + { + "selected": false, + "text": "30s", + "value": "30s" + }, + { + "selected": true, + "text": "5m", + "value": "5m" + }, + { + "selected": false, + "text": "1h", + "value": "1h" + } + ], + "query": "30s,5m,1h", + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "interval", + "useTags": false + }, + { + "allValue": null, + "auto": false, + "auto_count": 30, + "auto_min": "10s", + "current": { + "text": "5m", + "value": "5m" + }, + "datasource": "$datasource", + "hide": 2, + "includeAll": false, + "label": null, + "multi": false, + "name": "interval", + "options": [ + { + "selected": true, + "text": "4h", + "value": "4h" + } + ], + "query": "4h", + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "interval", + "useTags": false + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "{{ .Values.grafana.defaultDashboardsTimezone }}", + "title": "Kubernetes / Networking / Workload", + "uid": "728bf77cc1166d2f3133bf25846876cc", + "version": 0 + } +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/grafana/namespaces.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/grafana/namespaces.yaml new file mode 100644 index 00000000000..39ed210ed46 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/grafana/namespaces.yaml @@ -0,0 +1,13 @@ +{{- if and .Values.grafana.enabled .Values.grafana.defaultDashboardsEnabled (not .Values.grafana.defaultDashboards.useExistingNamespace) }} +apiVersion: v1 +kind: Namespace +metadata: + name: {{ .Values.grafana.defaultDashboards.namespace }} + labels: + name: {{ .Values.grafana.defaultDashboards.namespace }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} + annotations: +{{- if not .Values.grafana.defaultDashboards.cleanupOnUninstall }} + helm.sh/resource-policy: "keep" +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus-operator/_prometheus-operator.tpl b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus-operator/_prometheus-operator.tpl new file mode 100644 index 00000000000..6ae9dc72e6c --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus-operator/_prometheus-operator.tpl @@ -0,0 +1,7 @@ +{{/* Generate basic labels for prometheus-operator */}} +{{- define "kube-prometheus-stack.prometheus-operator.labels" }} +{{- include "kube-prometheus-stack.labels" . }} +app: {{ template "kube-prometheus-stack.name" . }}-operator +app.kubernetes.io/name: {{ template "kube-prometheus-stack.name" . }}-prometheus-operator +app.kubernetes.io/component: prometheus-operator +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus-operator/admission-webhooks/_prometheus-operator-webhook.tpl b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus-operator/admission-webhooks/_prometheus-operator-webhook.tpl new file mode 100644 index 00000000000..f419caf54be --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus-operator/admission-webhooks/_prometheus-operator-webhook.tpl @@ -0,0 +1,6 @@ +{{/* Generate basic labels for prometheus-operator-webhook */}} +{{- define "kube-prometheus-stack.prometheus-operator-webhook.labels" }} +{{- include "kube-prometheus-stack.labels" . }} +app.kubernetes.io/name: {{ template "kube-prometheus-stack.name" . }}-prometheus-operator +app.kubernetes.io/component: prometheus-operator-webhook +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus-operator/admission-webhooks/deployment/deployment.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus-operator/admission-webhooks/deployment/deployment.yaml new file mode 100644 index 00000000000..054eac4a770 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus-operator/admission-webhooks/deployment/deployment.yaml @@ -0,0 +1,143 @@ +{{- if and .Values.prometheusOperator.enabled .Values.prometheusOperator.admissionWebhooks.deployment.enabled }} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ template "kube-prometheus-stack.operator.fullname" . }}-webhook + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }}-operator-webhook + {{- include "kube-prometheus-stack.prometheus-operator-webhook.labels" . | nindent 4 }} +{{- if .Values.prometheusOperator.admissionWebhooks.deployment.labels }} +{{ toYaml .Values.prometheusOperator.admissionWebhooks.deployment.labels | indent 4 }} +{{- end }} +{{- if .Values.prometheusOperator.admissionWebhooks.deployment.annotations }} + annotations: +{{ toYaml .Values.prometheusOperator.admissionWebhooks.deployment.annotations | indent 4 }} +{{- end }} +spec: + replicas: {{ .Values.prometheusOperator.admissionWebhooks.deployment.replicas }} + revisionHistoryLimit: {{ .Values.prometheusOperator.admissionWebhooks.deployment.revisionHistoryLimit }} + {{- with .Values.prometheusOperator.admissionWebhooks.deployment.strategy }} + strategy: + {{- toYaml . | nindent 4 }} + {{- end }} + selector: + matchLabels: + app: {{ template "kube-prometheus-stack.name" . }}-operator-webhook + release: {{ $.Release.Name | quote }} + template: + metadata: + labels: + app: {{ template "kube-prometheus-stack.name" . }}-operator-webhook + {{- include "kube-prometheus-stack.prometheus-operator-webhook.labels" . | nindent 8 }} +{{- if .Values.prometheusOperator.admissionWebhooks.deployment.podLabels }} +{{ toYaml .Values.prometheusOperator.admissionWebhooks.deployment.podLabels | indent 8 }} +{{- end }} +{{- if .Values.prometheusOperator.admissionWebhooks.deployment.podAnnotations }} + annotations: +{{ toYaml .Values.prometheusOperator.admissionWebhooks.deployment.podAnnotations | indent 8 }} +{{- end }} + spec: + {{- if .Values.prometheusOperator.admissionWebhooks.deployment.priorityClassName }} + priorityClassName: {{ .Values.prometheusOperator.admissionWebhooks.deployment.priorityClassName }} + {{- end }} + {{- if .Values.global.imagePullSecrets }} + imagePullSecrets: + {{- include "kube-prometheus-stack.imagePullSecrets" . | indent 8 }} + {{- end }} + containers: + - name: prometheus-operator-admission-webhook + {{- $operatorRegistry := .Values.global.imageRegistry | default .Values.prometheusOperator.admissionWebhooks.deployment.image.registry -}} + {{- if .Values.prometheusOperator.admissionWebhooks.deployment.image.sha }} + image: "{{ $operatorRegistry }}/{{ .Values.prometheusOperator.admissionWebhooks.deployment.image.repository }}:{{ .Values.prometheusOperator.admissionWebhooks.deployment.image.tag | default .Chart.AppVersion }}@sha256:{{ .Values.prometheusOperator.admissionWebhooks.deployment.image.sha }}" + {{- else }} + image: "{{ $operatorRegistry }}/{{ .Values.prometheusOperator.admissionWebhooks.deployment.image.repository }}:{{ .Values.prometheusOperator.admissionWebhooks.deployment.image.tag | default .Chart.AppVersion }}" + {{- end }} + imagePullPolicy: "{{ .Values.prometheusOperator.admissionWebhooks.deployment.image.pullPolicy }}" + args: + {{- if .Values.prometheusOperator.admissionWebhooks.deployment.logFormat }} + - --log-format={{ .Values.prometheusOperator.admissionWebhooks.deployment.logFormat }} + {{- end }} + {{- if .Values.prometheusOperator.admissionWebhooks.deployment.logLevel }} + - --log-level={{ .Values.prometheusOperator.admissionWebhooks.deployment.logLevel }} + {{- end }} + {{- if .Values.prometheusOperator.admissionWebhooks.deployment.tls.enabled }} + - "--web.enable-tls=true" + - "--web.cert-file=/cert/{{ if .Values.prometheusOperator.admissionWebhooks.certManager.enabled }}tls.crt{{ else }}cert{{ end }}" + - "--web.key-file=/cert/{{ if .Values.prometheusOperator.admissionWebhooks.certManager.enabled }}tls.key{{ else }}key{{ end }}" + - "--web.listen-address=:{{ .Values.prometheusOperator.admissionWebhooks.deployment.tls.internalPort }}" + - "--web.tls-min-version={{ .Values.prometheusOperator.admissionWebhooks.deployment.tls.tlsMinVersion }}" + ports: + - containerPort: {{ .Values.prometheusOperator.admissionWebhooks.deployment.tls.internalPort }} + name: https + {{- else }} + ports: + - containerPort: 8080 + name: http + {{- end }} + {{- if .Values.prometheusOperator.admissionWebhooks.deployment.readinessProbe.enabled }} + readinessProbe: + httpGet: + path: /healthz + port: {{ .Values.prometheusOperator.admissionWebhooks.deployment.tls.enabled | ternary "https" "http" }} + scheme: {{ .Values.prometheusOperator.admissionWebhooks.deployment.tls.enabled | ternary "HTTPS" "HTTP" }} + initialDelaySeconds: {{ .Values.prometheusOperator.admissionWebhooks.deployment.readinessProbe.initialDelaySeconds }} + periodSeconds: {{ .Values.prometheusOperator.admissionWebhooks.deployment.readinessProbe.periodSeconds }} + timeoutSeconds: {{ .Values.prometheusOperator.admissionWebhooks.deployment.readinessProbe.timeoutSeconds }} + successThreshold: {{ .Values.prometheusOperator.admissionWebhooks.deployment.readinessProbe.successThreshold }} + failureThreshold: {{ .Values.prometheusOperator.admissionWebhooks.deployment.readinessProbe.failureThreshold }} + {{- end }} + {{- if .Values.prometheusOperator.admissionWebhooks.deployment.livenessProbe.enabled }} + livenessProbe: + httpGet: + path: /healthz + port: {{ .Values.prometheusOperator.admissionWebhooks.deployment.tls.enabled | ternary "https" "http" }} + scheme: {{ .Values.prometheusOperator.admissionWebhooks.deployment.tls.enabled | ternary "HTTPS" "HTTP" }} + initialDelaySeconds: {{ .Values.prometheusOperator.admissionWebhooks.deployment.livenessProbe.initialDelaySeconds }} + periodSeconds: {{ .Values.prometheusOperator.admissionWebhooks.deployment.livenessProbe.periodSeconds }} + timeoutSeconds: {{ .Values.prometheusOperator.admissionWebhooks.deployment.livenessProbe.timeoutSeconds }} + successThreshold: {{ .Values.prometheusOperator.admissionWebhooks.deployment.livenessProbe.successThreshold }} + failureThreshold: {{ .Values.prometheusOperator.admissionWebhooks.deployment.livenessProbe.failureThreshold }} + {{- end }} + resources: +{{ toYaml .Values.prometheusOperator.admissionWebhooks.deployment.resources | indent 12 }} + securityContext: +{{ toYaml .Values.prometheusOperator.admissionWebhooks.deployment.containerSecurityContext | indent 12 }} +{{- if .Values.prometheusOperator.admissionWebhooks.deployment.tls.enabled }} + volumeMounts: + - name: tls-secret + mountPath: /cert + readOnly: true + volumes: + - name: tls-secret + secret: + defaultMode: 420 + secretName: {{ template "kube-prometheus-stack.fullname" . }}-admission +{{- end }} + {{- with .Values.prometheusOperator.admissionWebhooks.deployment.dnsConfig }} + dnsConfig: +{{ toYaml . | indent 8 }} + {{- end }} +{{- if .Values.prometheusOperator.admissionWebhooks.deployment.securityContext }} + securityContext: +{{ toYaml .Values.prometheusOperator.admissionWebhooks.deployment.securityContext | indent 8 }} +{{- end }} + serviceAccountName: {{ template "kube-prometheus-stack.operator.serviceAccountName" . }}-webhook + automountServiceAccountToken: {{ .Values.prometheusOperator.admissionWebhooks.deployment.automountServiceAccountToken }} +{{- if .Values.prometheusOperator.admissionWebhooks.deployment.hostNetwork }} + hostNetwork: true + dnsPolicy: ClusterFirstWithHostNet +{{- end }} + {{- with .Values.prometheusOperator.admissionWebhooks.deployment.nodeSelector }} + nodeSelector: +{{ toYaml . | indent 8 }} + {{- end }} + {{- with .Values.prometheusOperator.admissionWebhooks.deployment.affinity }} + affinity: +{{ toYaml . | indent 8 }} + {{- end }} + {{- with .Values.prometheusOperator.admissionWebhooks.deployment.tolerations }} + tolerations: +{{ toYaml . | indent 8 }} + {{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus-operator/admission-webhooks/deployment/pdb.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus-operator/admission-webhooks/deployment/pdb.yaml new file mode 100644 index 00000000000..52dd78f624a --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus-operator/admission-webhooks/deployment/pdb.yaml @@ -0,0 +1,15 @@ +{{- if .Values.prometheusOperator.admissionWebhooks.deployment.podDisruptionBudget -}} +apiVersion: policy/v1{{ ternary "" "beta1" ($.Capabilities.APIVersions.Has "policy/v1/PodDisruptionBudget") }} +kind: PodDisruptionBudget +metadata: + name: {{ template "kube-prometheus-stack.operator.fullname" . }}-webhook + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + {{- include "kube-prometheus-stack.prometheus-operator-webhook.labels" . | nindent 4 }} +spec: + selector: + matchLabels: + app: {{ template "kube-prometheus-stack.name" . }}-operator-webhook + release: {{ $.Release.Name | quote }} +{{ toYaml .Values.prometheusOperator.admissionWebhooks.deployment.podDisruptionBudget | indent 2 }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus-operator/admission-webhooks/deployment/service.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus-operator/admission-webhooks/deployment/service.yaml new file mode 100644 index 00000000000..b06c129123b --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus-operator/admission-webhooks/deployment/service.yaml @@ -0,0 +1,58 @@ +{{- if and .Values.prometheusOperator.enabled .Values.prometheusOperator.admissionWebhooks.deployment.enabled }} +apiVersion: v1 +kind: Service +metadata: + name: {{ template "kube-prometheus-stack.operator.fullname" . }}-webhook + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }}-operator-webhook + {{- include "kube-prometheus-stack.prometheus-operator-webhook.labels" . | nindent 4 }} +{{- if .Values.prometheusOperator.admissionWebhooks.deployment.service.labels }} +{{ toYaml .Values.prometheusOperator.admissionWebhooks.deployment.service.labels | indent 4 }} +{{- end }} +{{- if .Values.prometheusOperator.admissionWebhooks.deployment.service.annotations }} + annotations: +{{ toYaml .Values.prometheusOperator.admissionWebhooks.deployment.service.annotations | indent 4 }} +{{- end }} +spec: +{{- if .Values.prometheusOperator.admissionWebhooks.deployment.service.clusterIP }} + clusterIP: {{ .Values.prometheusOperator.admissionWebhooks.deployment.service.clusterIP }} +{{- end }} +{{- if .Values.prometheusOperator.admissionWebhooks.deployment.service.externalIPs }} + externalIPs: +{{ toYaml .Values.prometheusOperator.admissionWebhooks.deployment.service.externalIPs | indent 4 }} +{{- end }} +{{- if .Values.prometheusOperator.admissionWebhooks.deployment.service.loadBalancerIP }} + loadBalancerIP: {{ .Values.prometheusOperator.admissionWebhooks.deployment.service.loadBalancerIP }} +{{- end }} +{{- if .Values.prometheusOperator.admissionWebhooks.deployment.service.loadBalancerSourceRanges }} + loadBalancerSourceRanges: + {{- range $cidr := .Values.prometheusOperator.admissionWebhooks.deployment.service.loadBalancerSourceRanges }} + - {{ $cidr }} + {{- end }} +{{- end }} +{{- if ne .Values.prometheusOperator.admissionWebhooks.deployment.service.type "ClusterIP" }} + externalTrafficPolicy: {{ .Values.prometheusOperator.admissionWebhooks.deployment.service.externalTrafficPolicy }} +{{- end }} + ports: + {{- if not .Values.prometheusOperator.admissionWebhooks.deployment.tls.enabled }} + - name: http + {{- if eq .Values.prometheusOperator.admissionWebhooks.deployment.service.type "NodePort" }} + nodePort: {{ .Values.prometheusOperator.admissionWebhooks.deployment.service.nodePort }} + {{- end }} + port: 8080 + targetPort: http + {{- end }} + {{- if .Values.prometheusOperator.admissionWebhooks.deployment.tls.enabled }} + - name: https + {{- if eq .Values.prometheusOperator.admissionWebhooks.deployment.service.type "NodePort"}} + nodePort: {{ .Values.prometheusOperator.admissionWebhooks.deployment.service.nodePortTls }} + {{- end }} + port: 443 + targetPort: https + {{- end }} + selector: + app: {{ template "kube-prometheus-stack.name" . }}-operator-webhook + release: {{ $.Release.Name | quote }} + type: "{{ .Values.prometheusOperator.admissionWebhooks.deployment.service.type }}" +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus-operator/admission-webhooks/deployment/serviceaccount.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus-operator/admission-webhooks/deployment/serviceaccount.yaml new file mode 100644 index 00000000000..55511da36b7 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus-operator/admission-webhooks/deployment/serviceaccount.yaml @@ -0,0 +1,15 @@ +{{- if and .Values.prometheusOperator.enabled .Values.prometheusOperator.admissionWebhooks.deployment.enabled }} +apiVersion: v1 +kind: ServiceAccount +automountServiceAccountToken: {{ .Values.prometheusOperator.admissionWebhooks.deployment.serviceAccount.automountServiceAccountToken }} +metadata: + name: {{ template "kube-prometheus-stack.operator.admissionWebhooks.serviceAccountName" . }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }}-operator + {{- include "kube-prometheus-stack.prometheus-operator-webhook.labels" . | indent 4 }} +{{- if .Values.global.imagePullSecrets }} +imagePullSecrets: +{{ include "kube-prometheus-stack.imagePullSecrets" . | trim | indent 2 }} +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus-operator/admission-webhooks/job-patch/ciliumnetworkpolicy-createSecret.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus-operator/admission-webhooks/job-patch/ciliumnetworkpolicy-createSecret.yaml new file mode 100644 index 00000000000..f7543b0f1af --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus-operator/admission-webhooks/job-patch/ciliumnetworkpolicy-createSecret.yaml @@ -0,0 +1,36 @@ +{{- if and .Values.prometheusOperator.networkPolicy.enabled (eq .Values.prometheusOperator.networkPolicy.flavor "cilium") }} +{{- if and .Values.prometheusOperator.enabled .Values.prometheusOperator.admissionWebhooks.enabled .Values.prometheusOperator.admissionWebhooks.patch.enabled (not .Values.prometheusOperator.admissionWebhooks.certManager.enabled) }} +apiVersion: cilium.io/v2 +kind: CiliumNetworkPolicy +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-admission-create + namespace: {{ template "kube-prometheus-stack.namespace" . }} + annotations: + helm.sh/hook: pre-install,pre-upgrade + helm.sh/hook-delete-policy: before-hook-creation,hook-succeeded + ## Ensure this is run before the job + helm.sh/hook-weight: "-5" + {{- with .Values.prometheusOperator.admissionWebhooks.annotations }} + {{ toYaml . | nindent 4 }} + {{- end }} + labels: + app: {{ template "kube-prometheus-stack.name" $ }}-admission-create + {{- include "kube-prometheus-stack.prometheus-operator-webhook.labels" $ | nindent 4 }} +spec: + endpointSelector: + matchLabels: + app: {{ template "kube-prometheus-stack.name" $ }}-admission-create + {{- if .Values.prometheusOperator.networkPolicy.matchLabels }} + {{ toYaml .Values.prometheusOperator.networkPolicy.matchLabels | nindent 6 }} + {{- else }} + {{- include "kube-prometheus-stack.prometheus-operator-webhook.labels" $ | nindent 6 }} + {{- end }} + egress: + {{- if and .Values.prometheusOperator.networkPolicy.cilium .Values.prometheusOperator.networkPolicy.cilium.egress }} + {{ toYaml .Values.prometheusOperator.networkPolicy.cilium.egress | nindent 6 }} + {{- else }} + - toEntities: + - kube-apiserver + {{- end }} +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus-operator/admission-webhooks/job-patch/ciliumnetworkpolicy-patchWebhook.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus-operator/admission-webhooks/job-patch/ciliumnetworkpolicy-patchWebhook.yaml new file mode 100644 index 00000000000..4e3b0d92251 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus-operator/admission-webhooks/job-patch/ciliumnetworkpolicy-patchWebhook.yaml @@ -0,0 +1,36 @@ +{{- if and .Values.prometheusOperator.networkPolicy.enabled (eq .Values.prometheusOperator.networkPolicy.flavor "cilium") }} +{{- if and .Values.prometheusOperator.enabled .Values.prometheusOperator.admissionWebhooks.enabled .Values.prometheusOperator.admissionWebhooks.patch.enabled (not .Values.prometheusOperator.admissionWebhooks.certManager.enabled) }} +apiVersion: cilium.io/v2 +kind: CiliumNetworkPolicy +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-admission-patch + namespace: {{ template "kube-prometheus-stack.namespace" . }} + annotations: + helm.sh/hook: post-install,post-upgrade + helm.sh/hook-delete-policy: before-hook-creation,hook-succeeded + ## Ensure this is run before the job + helm.sh/hook-weight: "-5" + {{- with .Values.prometheusOperator.admissionWebhooks.patch.annotations }} + {{ toYaml . | nindent 4 }} + {{- end }} + labels: + app: {{ template "kube-prometheus-stack.name" $ }}-admission-patch + {{- include "kube-prometheus-stack.prometheus-operator-webhook.labels" $ | nindent 4 }} +spec: + endpointSelector: + matchLabels: + app: {{ template "kube-prometheus-stack.name" $ }}-admission-patch + {{- if .Values.prometheusOperator.networkPolicy.matchLabels }} + {{ toYaml .Values.prometheusOperator.networkPolicy.matchLabels | nindent 6 }} + {{- else }} + {{- include "kube-prometheus-stack.prometheus-operator-webhook.labels" $ | nindent 6 }} + {{- end }} + egress: + {{- if and .Values.prometheusOperator.networkPolicy.cilium .Values.prometheusOperator.networkPolicy.cilium.egress }} + {{ toYaml .Values.prometheusOperator.networkPolicy.cilium.egress | nindent 6 }} + {{- else }} + - toEntities: + - kube-apiserver + {{- end }} +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus-operator/admission-webhooks/job-patch/clusterrole.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus-operator/admission-webhooks/job-patch/clusterrole.yaml new file mode 100644 index 00000000000..b81257c1686 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus-operator/admission-webhooks/job-patch/clusterrole.yaml @@ -0,0 +1,33 @@ +{{- if and .Values.prometheusOperator.enabled .Values.prometheusOperator.admissionWebhooks.enabled .Values.prometheusOperator.admissionWebhooks.patch.enabled .Values.global.rbac.create (not .Values.prometheusOperator.admissionWebhooks.certManager.enabled) }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-admission + annotations: + "helm.sh/hook": pre-install,pre-upgrade,post-install,post-upgrade + "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded + labels: + app: {{ template "kube-prometheus-stack.name" $ }}-admission + {{- include "kube-prometheus-stack.prometheus-operator-webhook.labels" $ | nindent 4 }} +rules: + - apiGroups: + - admissionregistration.k8s.io + resources: + - validatingwebhookconfigurations + - mutatingwebhookconfigurations + verbs: + - get + - update +{{- if and (.Capabilities.APIVersions.Has "policy/v1beta1/PodSecurityPolicy") (or .Values.global.cattle.psp.enabled .Values.global.rbac.pspEnabled) }} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if semverCompare "> 1.15.0-0" $kubeTargetVersion }} + - apiGroups: ['policy'] +{{- else }} + - apiGroups: ['extensions'] +{{- end }} + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - {{ template "kube-prometheus-stack.fullname" . }}-admission +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus-operator/admission-webhooks/job-patch/clusterrolebinding.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus-operator/admission-webhooks/job-patch/clusterrolebinding.yaml new file mode 100644 index 00000000000..4cf1335b229 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus-operator/admission-webhooks/job-patch/clusterrolebinding.yaml @@ -0,0 +1,20 @@ +{{- if and .Values.prometheusOperator.enabled .Values.prometheusOperator.admissionWebhooks.enabled .Values.prometheusOperator.admissionWebhooks.patch.enabled .Values.global.rbac.create (not .Values.prometheusOperator.admissionWebhooks.certManager.enabled) }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-admission + annotations: + "helm.sh/hook": pre-install,pre-upgrade,post-install,post-upgrade + "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded + labels: + app: {{ template "kube-prometheus-stack.name" $ }}-admission + {{- include "kube-prometheus-stack.prometheus-operator-webhook.labels" $ | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "kube-prometheus-stack.fullname" . }}-admission +subjects: + - kind: ServiceAccount + name: {{ template "kube-prometheus-stack.fullname" . }}-admission + namespace: {{ template "kube-prometheus-stack.namespace" . }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus-operator/admission-webhooks/job-patch/job-createSecret.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus-operator/admission-webhooks/job-patch/job-createSecret.yaml new file mode 100644 index 00000000000..baed83db480 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus-operator/admission-webhooks/job-patch/job-createSecret.yaml @@ -0,0 +1,73 @@ +{{- if and .Values.prometheusOperator.enabled .Values.prometheusOperator.admissionWebhooks.enabled .Values.prometheusOperator.admissionWebhooks.patch.enabled (not .Values.prometheusOperator.admissionWebhooks.certManager.enabled) }} +apiVersion: batch/v1 +kind: Job +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-admission-create + namespace: {{ template "kube-prometheus-stack.namespace" . }} + annotations: + "helm.sh/hook": pre-install,pre-upgrade + "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded +{{- with .Values.prometheusOperator.admissionWebhooks.annotations }} +{{ toYaml . | indent 4 }} +{{- end }} + labels: + app: {{ template "kube-prometheus-stack.name" $ }}-admission-create + {{- include "kube-prometheus-stack.prometheus-operator-webhook.labels" $ | nindent 4 }} +spec: + {{- if .Capabilities.APIVersions.Has "batch/v1alpha1" }} + # Alpha feature since k8s 1.12 + ttlSecondsAfterFinished: 0 + {{- end }} + template: + metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-admission-create +{{- with .Values.prometheusOperator.admissionWebhooks.patch.podAnnotations }} + annotations: +{{ toYaml . | indent 8 }} +{{- end }} + labels: + app: {{ template "kube-prometheus-stack.name" $ }}-admission-create + {{- include "kube-prometheus-stack.prometheus-operator-webhook.labels" $ | nindent 8 }} + spec: + {{- if .Values.prometheusOperator.admissionWebhooks.patch.priorityClassName }} + priorityClassName: {{ .Values.prometheusOperator.admissionWebhooks.patch.priorityClassName }} + {{- end }} + containers: + - name: create + {{- $registry := include "monitoring_registry" . | default .Values.prometheusOperator.admissionWebhooks.patch.image.registry -}} + {{- if .Values.prometheusOperator.admissionWebhooks.patch.image.sha }} + image: {{ $registry }}/{{ .Values.prometheusOperator.admissionWebhooks.patch.image.repository }}:{{ .Values.prometheusOperator.admissionWebhooks.patch.image.tag }}@sha256:{{ .Values.prometheusOperator.admissionWebhooks.patch.image.sha }} + {{- else }} + image: {{ $registry }}/{{ .Values.prometheusOperator.admissionWebhooks.patch.image.repository }}:{{ .Values.prometheusOperator.admissionWebhooks.patch.image.tag }} + {{- end }} + imagePullPolicy: {{ .Values.prometheusOperator.admissionWebhooks.patch.image.pullPolicy }} + args: + - create + - --host={{- include "kube-prometheus-stack.operator.admission-webhook.dnsNames" . | replace "\n" "," }} + - --namespace={{ template "kube-prometheus-stack.namespace" . }} + - --secret-name={{ template "kube-prometheus-stack.fullname" . }}-admission + {{- with .Values.prometheusOperator.admissionWebhooks.createSecretJob }} + securityContext: + {{ toYaml .securityContext | nindent 12 }} + {{- end }} + resources: +{{ toYaml .Values.prometheusOperator.admissionWebhooks.patch.resources | indent 12 }} + restartPolicy: OnFailure + serviceAccountName: {{ template "kube-prometheus-stack.fullname" . }}-admission + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} +{{- with .Values.prometheusOperator.admissionWebhooks.patch.nodeSelector }} +{{ toYaml . | indent 8 }} +{{- end }} + {{- with .Values.prometheusOperator.admissionWebhooks.patch.affinity }} + affinity: +{{ toYaml . | indent 8 }} + {{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} +{{- with .Values.prometheusOperator.admissionWebhooks.patch.tolerations }} +{{ toYaml . | indent 8 }} + {{- end }} +{{- if .Values.prometheusOperator.admissionWebhooks.patch.securityContext }} + securityContext: +{{ toYaml .Values.prometheusOperator.admissionWebhooks.patch.securityContext | indent 8 }} +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus-operator/admission-webhooks/job-patch/job-patchWebhook.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus-operator/admission-webhooks/job-patch/job-patchWebhook.yaml new file mode 100644 index 00000000000..5639cc9e80c --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus-operator/admission-webhooks/job-patch/job-patchWebhook.yaml @@ -0,0 +1,74 @@ +{{- if and .Values.prometheusOperator.enabled .Values.prometheusOperator.admissionWebhooks.enabled .Values.prometheusOperator.admissionWebhooks.patch.enabled (not .Values.prometheusOperator.admissionWebhooks.certManager.enabled) }} +apiVersion: batch/v1 +kind: Job +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-admission-patch + namespace: {{ template "kube-prometheus-stack.namespace" . }} + annotations: + "helm.sh/hook": post-install,post-upgrade + "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded +{{- with .Values.prometheusOperator.admissionWebhooks.patch.annotations }} +{{ toYaml . | indent 4 }} +{{- end }} + labels: + app: {{ template "kube-prometheus-stack.name" $ }}-admission-patch + {{- include "kube-prometheus-stack.prometheus-operator-webhook.labels" $ | nindent 4 }} +spec: + {{- if .Capabilities.APIVersions.Has "batch/v1alpha1" }} + # Alpha feature since k8s 1.12 + ttlSecondsAfterFinished: 0 + {{- end }} + template: + metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-admission-patch +{{- with .Values.prometheusOperator.admissionWebhooks.patch.podAnnotations }} + annotations: +{{ toYaml . | indent 8 }} +{{- end }} + labels: + app: {{ template "kube-prometheus-stack.name" $ }}-admission-patch + {{- include "kube-prometheus-stack.prometheus-operator-webhook.labels" $ | nindent 8 }} + spec: + {{- if .Values.prometheusOperator.admissionWebhooks.patch.priorityClassName }} + priorityClassName: {{ .Values.prometheusOperator.admissionWebhooks.patch.priorityClassName }} + {{- end }} + containers: + - name: patch + {{- $registry := include "monitoring_registry" . | default .Values.prometheusOperator.admissionWebhooks.patch.image.registry -}} + {{- if .Values.prometheusOperator.admissionWebhooks.patch.image.sha }} + image: {{ $registry }}/{{ .Values.prometheusOperator.admissionWebhooks.patch.image.repository }}:{{ .Values.prometheusOperator.admissionWebhooks.patch.image.tag }}@sha256:{{ .Values.prometheusOperator.admissionWebhooks.patch.image.sha }} + {{- else }} + image: {{ $registry }}/{{ .Values.prometheusOperator.admissionWebhooks.patch.image.repository }}:{{ .Values.prometheusOperator.admissionWebhooks.patch.image.tag }} + {{- end }} + imagePullPolicy: {{ .Values.prometheusOperator.admissionWebhooks.patch.image.pullPolicy }} + args: + - patch + - --webhook-name={{ template "kube-prometheus-stack.fullname" . }}-admission + - --namespace={{ template "kube-prometheus-stack.namespace" . }} + - --secret-name={{ template "kube-prometheus-stack.fullname" . }}-admission + - --patch-failure-policy={{ .Values.prometheusOperator.admissionWebhooks.failurePolicy }} + {{- with .Values.prometheusOperator.admissionWebhooks.patchWebhookJob }} + securityContext: + {{ toYaml .securityContext | nindent 12 }} + {{- end }} + resources: +{{ toYaml .Values.prometheusOperator.admissionWebhooks.patch.resources | indent 12 }} + restartPolicy: OnFailure + serviceAccountName: {{ template "kube-prometheus-stack.fullname" . }}-admission + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} +{{- with .Values.prometheusOperator.admissionWebhooks.patch.nodeSelector }} +{{ toYaml . | indent 8 }} +{{- end }} + {{- with .Values.prometheusOperator.admissionWebhooks.patch.affinity }} + affinity: +{{ toYaml . | indent 8 }} + {{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} +{{- with .Values.prometheusOperator.admissionWebhooks.patch.tolerations }} +{{ toYaml . | indent 8 }} + {{- end }} +{{- if .Values.prometheusOperator.admissionWebhooks.patch.securityContext }} + securityContext: +{{ toYaml .Values.prometheusOperator.admissionWebhooks.patch.securityContext | indent 8 }} +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus-operator/admission-webhooks/job-patch/networkpolicy-createSecret.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus-operator/admission-webhooks/job-patch/networkpolicy-createSecret.yaml new file mode 100644 index 00000000000..864deb52a0a --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus-operator/admission-webhooks/job-patch/networkpolicy-createSecret.yaml @@ -0,0 +1,33 @@ +{{- if and .Values.prometheusOperator.networkPolicy.enabled (eq .Values.prometheusOperator.networkPolicy.flavor "kubernetes") }} +{{- if and .Values.prometheusOperator.enabled .Values.prometheusOperator.admissionWebhooks.enabled .Values.prometheusOperator.admissionWebhooks.patch.enabled (not .Values.prometheusOperator.admissionWebhooks.certManager.enabled) }} +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-admission-create + namespace: {{ template "kube-prometheus-stack.namespace" . }} + annotations: + "helm.sh/hook": pre-install,pre-upgrade + "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded + ## Ensure this is run before the job + "helm.sh/hook-weight": "-5" + {{- with .Values.prometheusOperator.admissionWebhooks.annotations }} + {{ toYaml . | nindent 4 }} + {{- end }} + labels: + app: {{ template "kube-prometheus-stack.name" $ }}-admission-create + {{- include "kube-prometheus-stack.prometheus-operator-webhook.labels" $ | nindent 4 }} +spec: + podSelector: + matchLabels: + app: {{ template "kube-prometheus-stack.name" $ }}-admission-create + {{- if .Values.prometheusOperator.networkPolicy.matchLabels }} + {{ toYaml .Values.prometheusOperator.networkPolicy.matchLabels | nindent 6 }} + {{- else }} + {{- include "kube-prometheus-stack.prometheus-operator-webhook.labels" $ | nindent 6 }} + {{- end }} + egress: + - {} + policyTypes: + - Egress +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus-operator/admission-webhooks/job-patch/networkpolicy-patchWebhook.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus-operator/admission-webhooks/job-patch/networkpolicy-patchWebhook.yaml new file mode 100644 index 00000000000..076c4670040 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus-operator/admission-webhooks/job-patch/networkpolicy-patchWebhook.yaml @@ -0,0 +1,33 @@ +{{- if and .Values.prometheusOperator.networkPolicy.enabled (eq .Values.prometheusOperator.networkPolicy.flavor "kubernetes") }} +{{- if and .Values.prometheusOperator.enabled .Values.prometheusOperator.admissionWebhooks.enabled .Values.prometheusOperator.admissionWebhooks.patch.enabled (not .Values.prometheusOperator.admissionWebhooks.certManager.enabled) }} +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-admission-patch + namespace: {{ template "kube-prometheus-stack.namespace" . }} + annotations: + "helm.sh/hook": post-install,post-upgrade + "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded + ## Ensure this is run before the job + "helm.sh/hook-weight": "-5" + {{- with .Values.prometheusOperator.admissionWebhooks.patch.annotations }} + {{ toYaml . | nindent 4 }} + {{- end }} + labels: + app: {{ template "kube-prometheus-stack.name" $ }}-admission-patch + {{- include "kube-prometheus-stack.prometheus-operator-webhook.labels" $ | nindent 4 }} +spec: + podSelector: + matchLabels: + app: {{ template "kube-prometheus-stack.name" $ }}-admission-patch + {{- if .Values.prometheusOperator.networkPolicy.matchLabels }} + {{ toYaml .Values.prometheusOperator.networkPolicy.matchLabels | nindent 6 }} + {{- else }} + {{- include "kube-prometheus-stack.prometheus-operator-webhook.labels" $ | nindent 6 }} + {{- end }} + egress: + - {} + policyTypes: + - Egress +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus-operator/admission-webhooks/job-patch/psp.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus-operator/admission-webhooks/job-patch/psp.yaml new file mode 100644 index 00000000000..0113b6a5d85 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus-operator/admission-webhooks/job-patch/psp.yaml @@ -0,0 +1,47 @@ +{{- if and (.Capabilities.APIVersions.Has "policy/v1beta1/PodSecurityPolicy") .Values.prometheusOperator.enabled .Values.prometheusOperator.admissionWebhooks.enabled .Values.prometheusOperator.admissionWebhooks.patch.enabled (or .Values.global.cattle.psp.enabled (and .Values.global.rbac.create .Values.global.rbac.pspEnabled)) (not .Values.prometheusOperator.admissionWebhooks.certManager.enabled) }} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-admission + annotations: + "helm.sh/hook": pre-install,pre-upgrade,post-install,post-upgrade + "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded +{{- if .Values.global.rbac.pspAnnotations }} +{{ toYaml .Values.global.rbac.pspAnnotations | indent 4 }} +{{- end }} + labels: + app: {{ template "kube-prometheus-stack.name" . }}-admission + {{- include "kube-prometheus-stack.prometheus-operator-webhook.labels" . | nindent 4 }} +spec: + privileged: false + # Allow core volume types. + volumes: + - 'configMap' + - 'emptyDir' + - 'projected' + - 'secret' + - 'downwardAPI' + - 'persistentVolumeClaim' + hostNetwork: false + hostIPC: false + hostPID: false + runAsUser: + # Permits the container to run with root privileges as well. + rule: 'RunAsAny' + seLinux: + # This policy assumes the nodes are using AppArmor rather than SELinux. + rule: 'RunAsAny' + supplementalGroups: + rule: 'MustRunAs' + ranges: + # Allow adding the root group. + - min: 0 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + # Allow adding the root group. + - min: 0 + max: 65535 + readOnlyRootFilesystem: false +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus-operator/admission-webhooks/job-patch/role.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus-operator/admission-webhooks/job-patch/role.yaml new file mode 100644 index 00000000000..f15abf4395e --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus-operator/admission-webhooks/job-patch/role.yaml @@ -0,0 +1,21 @@ +{{- if and .Values.prometheusOperator.enabled .Values.prometheusOperator.admissionWebhooks.enabled .Values.prometheusOperator.admissionWebhooks.patch.enabled .Values.global.rbac.create (not .Values.prometheusOperator.admissionWebhooks.certManager.enabled) }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-admission + namespace: {{ template "kube-prometheus-stack.namespace" . }} + annotations: + "helm.sh/hook": pre-install,pre-upgrade,post-install,post-upgrade + "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded + labels: + app: {{ template "kube-prometheus-stack.name" $ }}-admission + {{- include "kube-prometheus-stack.prometheus-operator-webhook.labels" $ | nindent 4 }} +rules: + - apiGroups: + - "" + resources: + - secrets + verbs: + - get + - create +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus-operator/admission-webhooks/job-patch/rolebinding.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus-operator/admission-webhooks/job-patch/rolebinding.yaml new file mode 100644 index 00000000000..30bde920b69 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus-operator/admission-webhooks/job-patch/rolebinding.yaml @@ -0,0 +1,21 @@ +{{- if and .Values.prometheusOperator.enabled .Values.prometheusOperator.admissionWebhooks.enabled .Values.prometheusOperator.admissionWebhooks.patch.enabled .Values.global.rbac.create (not .Values.prometheusOperator.admissionWebhooks.certManager.enabled) }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-admission + namespace: {{ template "kube-prometheus-stack.namespace" . }} + annotations: + "helm.sh/hook": pre-install,pre-upgrade,post-install,post-upgrade + "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded + labels: + app: {{ template "kube-prometheus-stack.name" $ }}-admission + {{- include "kube-prometheus-stack.prometheus-operator-webhook.labels" $ | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: {{ template "kube-prometheus-stack.fullname" . }}-admission +subjects: + - kind: ServiceAccount + name: {{ template "kube-prometheus-stack.fullname" . }}-admission + namespace: {{ template "kube-prometheus-stack.namespace" . }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus-operator/admission-webhooks/job-patch/serviceaccount.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus-operator/admission-webhooks/job-patch/serviceaccount.yaml new file mode 100644 index 00000000000..02594547d11 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus-operator/admission-webhooks/job-patch/serviceaccount.yaml @@ -0,0 +1,17 @@ +{{- if and .Values.prometheusOperator.enabled .Values.prometheusOperator.admissionWebhooks.enabled .Values.prometheusOperator.admissionWebhooks.patch.enabled .Values.global.rbac.create (not .Values.prometheusOperator.admissionWebhooks.certManager.enabled) }} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-admission + namespace: {{ template "kube-prometheus-stack.namespace" . }} + annotations: + "helm.sh/hook": pre-install,pre-upgrade,post-install,post-upgrade + "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded + labels: + app: {{ template "kube-prometheus-stack.name" $ }}-admission + {{- include "kube-prometheus-stack.prometheus-operator-webhook.labels" $ | nindent 4 }} +{{- if .Values.global.imagePullSecrets }} +imagePullSecrets: +{{ include "kube-prometheus-stack.imagePullSecrets" . | trim | indent 2 }} +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus-operator/admission-webhooks/mutatingWebhookConfiguration.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus-operator/admission-webhooks/mutatingWebhookConfiguration.yaml new file mode 100644 index 00000000000..da01f3b57ea --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus-operator/admission-webhooks/mutatingWebhookConfiguration.yaml @@ -0,0 +1,77 @@ +{{- if and .Values.prometheusOperator.enabled .Values.prometheusOperator.admissionWebhooks.enabled }} +apiVersion: admissionregistration.k8s.io/v1 +kind: MutatingWebhookConfiguration +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-admission +{{- if .Values.prometheusOperator.admissionWebhooks.certManager.enabled }} + annotations: + certmanager.k8s.io/inject-ca-from: {{ printf "%s/%s-admission" (include "kube-prometheus-stack.namespace" .) (include "kube-prometheus-stack.fullname" .) | quote }} + cert-manager.io/inject-ca-from: {{ printf "%s/%s-admission" (include "kube-prometheus-stack.namespace" .) (include "kube-prometheus-stack.fullname" .) | quote }} +{{- end }} + labels: + app: {{ template "kube-prometheus-stack.name" $ }}-admission + {{- include "kube-prometheus-stack.prometheus-operator-webhook.labels" $ | nindent 4 }} +webhooks: + - name: prometheusrulemutate.monitoring.coreos.com + {{- if eq .Values.prometheusOperator.admissionWebhooks.failurePolicy "IgnoreOnInstallOnly" }} + failurePolicy: {{ .Release.IsInstall | ternary "Ignore" "Fail" }} + {{- else if .Values.prometheusOperator.admissionWebhooks.failurePolicy }} + failurePolicy: {{ .Values.prometheusOperator.admissionWebhooks.failurePolicy }} + {{- else if .Values.prometheusOperator.admissionWebhooks.patch.enabled }} + failurePolicy: Ignore + {{- else }} + failurePolicy: Fail + {{- end }} + rules: + - apiGroups: + - monitoring.coreos.com + apiVersions: + - "*" + resources: + - prometheusrules + operations: + - CREATE + - UPDATE + clientConfig: + service: + namespace: {{ template "kube-prometheus-stack.namespace" . }} + name: {{ template "kube-prometheus-stack.operator.fullname" $ }}{{ if .Values.prometheusOperator.admissionWebhooks.deployment.enabled }}-webhook{{ end }} + path: /admission-prometheusrules/mutate + {{- if and .Values.prometheusOperator.admissionWebhooks.caBundle (not .Values.prometheusOperator.admissionWebhooks.patch.enabled) (not .Values.prometheusOperator.admissionWebhooks.certManager.enabled) }} + caBundle: {{ .Values.prometheusOperator.admissionWebhooks.caBundle }} + {{- end }} + timeoutSeconds: {{ .Values.prometheusOperator.admissionWebhooks.timeoutSeconds }} + admissionReviewVersions: ["v1", "v1beta1"] + sideEffects: None + {{- if or .Values.prometheusOperator.denyNamespaces .Values.prometheusOperator.namespaces .Values.prometheusOperator.admissionWebhooks.namespaceSelector }} + namespaceSelector: + {{- with (omit .Values.prometheusOperator.admissionWebhooks.namespaceSelector "matchExpressions") }} + {{- toYaml . | nindent 6 }} + {{- end }} + {{- if or .Values.prometheusOperator.denyNamespaces .Values.prometheusOperator.namespaces .Values.prometheusOperator.admissionWebhooks.namespaceSelector.matchExpressions }} + matchExpressions: + {{- with (.Values.prometheusOperator.admissionWebhooks.namespaceSelector.matchExpressions) }} + {{- toYaml . | nindent 6 }} + {{- end }} + {{- if .Values.prometheusOperator.denyNamespaces }} + - key: kubernetes.io/metadata.name + operator: NotIn + values: + {{- range $namespace := mustUniq .Values.prometheusOperator.denyNamespaces }} + - {{ $namespace }} + {{- end }} + {{- else if and .Values.prometheusOperator.namespaces .Values.prometheusOperator.namespaces.additional }} + - key: kubernetes.io/metadata.name + operator: In + values: + {{- if and .Values.prometheusOperator.namespaces.releaseNamespace (default .Values.prometheusOperator.namespaces.releaseNamespace true) }} + {{- $namespace := printf "%s" (include "kube-prometheus-stack.namespace" .) }} + - {{ $namespace }} + {{- end }} + {{- range $namespace := mustUniq .Values.prometheusOperator.namespaces.additional }} + - {{ $namespace }} + {{- end }} + {{- end }} + {{- end }} + {{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus-operator/admission-webhooks/validatingWebhookConfiguration.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus-operator/admission-webhooks/validatingWebhookConfiguration.yaml new file mode 100644 index 00000000000..4827871cca8 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus-operator/admission-webhooks/validatingWebhookConfiguration.yaml @@ -0,0 +1,77 @@ +{{- if and .Values.prometheusOperator.enabled .Values.prometheusOperator.admissionWebhooks.enabled }} +apiVersion: admissionregistration.k8s.io/v1 +kind: ValidatingWebhookConfiguration +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-admission +{{- if .Values.prometheusOperator.admissionWebhooks.certManager.enabled }} + annotations: + certmanager.k8s.io/inject-ca-from: {{ printf "%s/%s-admission" (include "kube-prometheus-stack.namespace" .) (include "kube-prometheus-stack.fullname" .) | quote }} + cert-manager.io/inject-ca-from: {{ printf "%s/%s-admission" (include "kube-prometheus-stack.namespace" .) (include "kube-prometheus-stack.fullname" .) | quote }} +{{- end }} + labels: + app: {{ template "kube-prometheus-stack.name" $ }}-admission + {{- include "kube-prometheus-stack.prometheus-operator-webhook.labels" $ | nindent 4 }} +webhooks: + - name: prometheusrulemutate.monitoring.coreos.com + {{- if eq .Values.prometheusOperator.admissionWebhooks.failurePolicy "IgnoreOnInstallOnly" }} + failurePolicy: {{ .Release.IsInstall | ternary "Ignore" "Fail" }} + {{- else if .Values.prometheusOperator.admissionWebhooks.failurePolicy }} + failurePolicy: {{ .Values.prometheusOperator.admissionWebhooks.failurePolicy }} + {{- else if .Values.prometheusOperator.admissionWebhooks.patch.enabled }} + failurePolicy: Ignore + {{- else }} + failurePolicy: Fail + {{- end }} + rules: + - apiGroups: + - monitoring.coreos.com + apiVersions: + - "*" + resources: + - prometheusrules + operations: + - CREATE + - UPDATE + clientConfig: + service: + namespace: {{ template "kube-prometheus-stack.namespace" . }} + name: {{ template "kube-prometheus-stack.operator.fullname" $ }}{{ if .Values.prometheusOperator.admissionWebhooks.deployment.enabled }}-webhook{{ end }} + path: /admission-prometheusrules/validate + {{- if and .Values.prometheusOperator.admissionWebhooks.caBundle (not .Values.prometheusOperator.admissionWebhooks.patch.enabled) (not .Values.prometheusOperator.admissionWebhooks.certManager.enabled) }} + caBundle: {{ .Values.prometheusOperator.admissionWebhooks.caBundle }} + {{- end }} + timeoutSeconds: {{ .Values.prometheusOperator.admissionWebhooks.timeoutSeconds }} + admissionReviewVersions: ["v1", "v1beta1"] + sideEffects: None + {{- if or .Values.prometheusOperator.denyNamespaces .Values.prometheusOperator.namespaces .Values.prometheusOperator.admissionWebhooks.namespaceSelector }} + namespaceSelector: + {{- with (omit .Values.prometheusOperator.admissionWebhooks.namespaceSelector "matchExpressions") }} + {{- toYaml . | nindent 6 }} + {{- end }} + {{- if or .Values.prometheusOperator.denyNamespaces .Values.prometheusOperator.namespaces .Values.prometheusOperator.admissionWebhooks.namespaceSelector.matchExpressions }} + matchExpressions: + {{- with (.Values.prometheusOperator.admissionWebhooks.namespaceSelector.matchExpressions) }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- if .Values.prometheusOperator.denyNamespaces }} + - key: kubernetes.io/metadata.name + operator: NotIn + values: + {{- range $namespace := mustUniq .Values.prometheusOperator.denyNamespaces }} + - {{ $namespace }} + {{- end }} + {{- else if and .Values.prometheusOperator.namespaces .Values.prometheusOperator.namespaces.additional }} + - key: kubernetes.io/metadata.name + operator: In + values: + {{- if and .Values.prometheusOperator.namespaces.releaseNamespace (default .Values.prometheusOperator.namespaces.releaseNamespace true) }} + {{- $namespace := printf "%s" (include "kube-prometheus-stack.namespace" .) }} + - {{ $namespace }} + {{- end }} + {{- range $namespace := mustUniq .Values.prometheusOperator.namespaces.additional }} + - {{ $namespace }} + {{- end }} + {{- end }} + {{- end }} + {{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus-operator/certmanager.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus-operator/certmanager.yaml new file mode 100644 index 00000000000..cb27e49f488 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus-operator/certmanager.yaml @@ -0,0 +1,55 @@ +{{- if .Values.prometheusOperator.admissionWebhooks.certManager.enabled -}} +{{- if not .Values.prometheusOperator.admissionWebhooks.certManager.issuerRef -}} +# Create a selfsigned Issuer, in order to create a root CA certificate for +# signing webhook serving certificates +apiVersion: cert-manager.io/v1 +kind: Issuer +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-self-signed-issuer + namespace: {{ template "kube-prometheus-stack.namespace" . }} +spec: + selfSigned: {} +--- +# Generate a CA Certificate used to sign certificates for the webhook +apiVersion: cert-manager.io/v1 +kind: Certificate +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-root-cert + namespace: {{ template "kube-prometheus-stack.namespace" . }} +spec: + secretName: {{ template "kube-prometheus-stack.fullname" . }}-root-cert + duration: {{ .Values.prometheusOperator.admissionWebhooks.certManager.rootCert.duration | default "43800h0m0s" | quote }} + issuerRef: + name: {{ template "kube-prometheus-stack.fullname" . }}-self-signed-issuer + commonName: "ca.webhook.kube-prometheus-stack" + isCA: true +--- +# Create an Issuer that uses the above generated CA certificate to issue certs +apiVersion: cert-manager.io/v1 +kind: Issuer +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-root-issuer + namespace: {{ template "kube-prometheus-stack.namespace" . }} +spec: + ca: + secretName: {{ template "kube-prometheus-stack.fullname" . }}-root-cert +{{- end }} +--- +# generate a server certificate for the apiservices to use +apiVersion: cert-manager.io/v1 +kind: Certificate +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-admission + namespace: {{ template "kube-prometheus-stack.namespace" . }} +spec: + secretName: {{ template "kube-prometheus-stack.fullname" . }}-admission + duration: {{ .Values.prometheusOperator.admissionWebhooks.certManager.admissionCert.duration | default "8760h0m0s" | quote }} + issuerRef: + {{- if .Values.prometheusOperator.admissionWebhooks.certManager.issuerRef }} + {{- toYaml .Values.prometheusOperator.admissionWebhooks.certManager.issuerRef | nindent 4 }} + {{- else }} + name: {{ template "kube-prometheus-stack.fullname" . }}-root-issuer + {{- end }} + dnsNames: + {{- include "kube-prometheus-stack.operator.admission-webhook.dnsNames" . | splitList "\n" | toYaml | nindent 4 }} +{{- end -}} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus-operator/ciliumnetworkpolicy.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus-operator/ciliumnetworkpolicy.yaml new file mode 100644 index 00000000000..07e2e999677 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus-operator/ciliumnetworkpolicy.yaml @@ -0,0 +1,40 @@ +{{- if and .Values.prometheusOperator.networkPolicy.enabled (eq .Values.prometheusOperator.networkPolicy.flavor "cilium") }} +apiVersion: cilium.io/v2 +kind: CiliumNetworkPolicy +metadata: + name: {{ template "kube-prometheus-stack.operator.fullname" . }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + {{- include "kube-prometheus-stack.prometheus-operator.labels" . | nindent 4 }} +spec: + endpointSelector: + matchLabels: + {{- if .Values.prometheusOperator.networkPolicy.matchLabels }} + app: {{ template "kube-prometheus-stack.name" . }}-operator + {{ toYaml .Values.prometheusOperator.networkPolicy.matchLabels | nindent 6 }} + {{- else }} + {{- include "kube-prometheus-stack.prometheus-operator.labels" $ | nindent 6 }} + {{- end }} + egress: + {{- if and .Values.prometheusOperator.networkPolicy.cilium .Values.prometheusOperator.networkPolicy.cilium.egress }} + {{ toYaml .Values.prometheusOperator.networkPolicy.cilium.egress | nindent 6 }} + {{- else }} + - toEntities: + - kube-apiserver + {{- end }} + ingress: + - toPorts: + - ports: + {{- if .Values.prometheusOperator.tls.enabled }} + - port: {{ .Values.prometheusOperator.tls.internalPort | quote }} + {{- else }} + - port: "8080" + {{- end }} + protocol: "TCP" + {{- if not .Values.prometheusOperator.tls.enabled }} + rules: + http: + - method: "GET" + path: "/metrics" + {{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus-operator/clusterrole.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus-operator/clusterrole.yaml new file mode 100644 index 00000000000..fd11b69eed4 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus-operator/clusterrole.yaml @@ -0,0 +1,109 @@ +{{- if and .Values.prometheusOperator.enabled .Values.global.rbac.create }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ template "kube-prometheus-stack.operator.fullname" . }} + labels: + {{- include "kube-prometheus-stack.prometheus-operator.labels" . | nindent 4 }} +rules: +- apiGroups: + - monitoring.coreos.com + resources: + - alertmanagers + - alertmanagers/finalizers + - alertmanagers/status + - alertmanagerconfigs + - prometheuses + - prometheuses/finalizers + - prometheuses/status + - prometheusagents + - prometheusagents/finalizers + - prometheusagents/status + - thanosrulers + - thanosrulers/finalizers + - thanosrulers/status + - scrapeconfigs + - servicemonitors + - podmonitors + - probes + - prometheusrules + verbs: + - '*' +- apiGroups: + - apps + resources: + - statefulsets + verbs: + - '*' +- apiGroups: + - "" + resources: + - configmaps + - secrets + verbs: + - '*' +- apiGroups: + - "" + resources: + - pods + verbs: + - list + - delete +- apiGroups: + - "" + resources: + - services + - services/finalizers + - endpoints + verbs: + - get + - create + - update + - delete +- apiGroups: + - "" + resources: + - nodes + verbs: + - list + - watch +- apiGroups: + - "" + resources: + - namespaces + verbs: + - get + - list + - watch +- apiGroups: + - "" + resources: + - events + verbs: + - patch + - create +- apiGroups: + - networking.k8s.io + resources: + - ingresses + verbs: + - get + - list + - watch +- apiGroups: + - storage.k8s.io + resources: + - storageclasses + verbs: + - get +{{- if .Capabilities.APIVersions.Has "discovery.k8s.io/v1/EndpointSlice" }} +- apiGroups: + - discovery.k8s.io + resources: + - endpointslices + verbs: + - get + - list + - watch +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus-operator/clusterrolebinding.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus-operator/clusterrolebinding.yaml new file mode 100644 index 00000000000..ad9e3ef6c53 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus-operator/clusterrolebinding.yaml @@ -0,0 +1,16 @@ +{{- if and .Values.prometheusOperator.enabled .Values.global.rbac.create }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ template "kube-prometheus-stack.operator.fullname" . }} + labels: + {{- include "kube-prometheus-stack.prometheus-operator.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "kube-prometheus-stack.operator.fullname" . }} +subjects: +- kind: ServiceAccount + name: {{ template "kube-prometheus-stack.operator.serviceAccountName" . }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus-operator/deployment.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus-operator/deployment.yaml new file mode 100644 index 00000000000..8a01b2912a2 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus-operator/deployment.yaml @@ -0,0 +1,204 @@ +{{- $namespace := printf "%s" (include "kube-prometheus-stack.namespace" .) }} +{{- $defaultKubeletSvcName := printf "%s-kubelet" (include "kube-prometheus-stack.fullname" .) }} +{{- if .Values.prometheusOperator.enabled }} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ template "kube-prometheus-stack.operator.fullname" . }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + {{- include "kube-prometheus-stack.prometheus-operator.labels" . | nindent 4 }} +{{- if .Values.prometheusOperator.labels }} +{{ toYaml .Values.prometheusOperator.labels | indent 4 }} +{{- end }} +{{- if .Values.prometheusOperator.annotations }} + annotations: +{{ toYaml .Values.prometheusOperator.annotations | indent 4 }} +{{- end }} +spec: + replicas: 1 + revisionHistoryLimit: {{ .Values.prometheusOperator.revisionHistoryLimit }} + selector: + matchLabels: + app: {{ template "kube-prometheus-stack.name" . }}-operator + release: {{ $.Release.Name | quote }} + {{- with .Values.prometheusOperator.strategy }} + strategy: + {{- toYaml . | nindent 4 }} + {{- end }} + template: + metadata: + labels: + {{- include "kube-prometheus-stack.prometheus-operator.labels" . | nindent 8 }} +{{- if .Values.prometheusOperator.podLabels }} +{{ toYaml .Values.prometheusOperator.podLabels | indent 8 }} +{{- end }} +{{- if .Values.prometheusOperator.podAnnotations }} + annotations: +{{ toYaml .Values.prometheusOperator.podAnnotations | indent 8 }} +{{- end }} + spec: + {{- if .Values.prometheusOperator.priorityClassName }} + priorityClassName: {{ .Values.prometheusOperator.priorityClassName }} + {{- end }} + {{- if .Values.global.imagePullSecrets }} + imagePullSecrets: + {{- include "kube-prometheus-stack.imagePullSecrets" . | indent 8 }} + {{- end }} + containers: + - name: {{ template "kube-prometheus-stack.name" . }} + {{- $base_registry := (include "monitoring_registry" .) }} + {{- $configReloaderRegistry := $base_registry | default .Values.prometheusOperator.prometheusConfigReloader.image.registry -}} + {{- $operatorRegistry := $base_registry | default .Values.prometheusOperator.image.registry -}} + {{- $thanosRegistry := $base_registry | default .Values.prometheusOperator.thanosImage.registry -}} + {{- if .Values.prometheusOperator.image.sha }} + image: "{{ $operatorRegistry }}/{{ .Values.prometheusOperator.image.repository }}:{{ .Values.prometheusOperator.image.tag | default .Chart.AppVersion }}@sha256:{{ .Values.prometheusOperator.image.sha }}" + {{- else }} + image: "{{ $operatorRegistry }}/{{ .Values.prometheusOperator.image.repository }}:{{ .Values.prometheusOperator.image.tag | default .Chart.AppVersion }}" + {{- end }} + imagePullPolicy: "{{ .Values.prometheusOperator.image.pullPolicy }}" + args: + {{- if .Values.prometheusOperator.kubeletService.enabled }} + - --kubelet-service={{ .Values.prometheusOperator.kubeletService.namespace }}/{{ default $defaultKubeletSvcName .Values.prometheusOperator.kubeletService.name }} + {{- end }} + {{- if .Values.prometheusOperator.logFormat }} + - --log-format={{ .Values.prometheusOperator.logFormat }} + {{- end }} + {{- if .Values.prometheusOperator.logLevel }} + - --log-level={{ .Values.prometheusOperator.logLevel }} + {{- end }} + {{- if .Values.prometheusOperator.denyNamespaces }} + - --deny-namespaces={{ tpl (.Values.prometheusOperator.denyNamespaces | join ",") $ }} + {{- end }} + {{- with $.Values.prometheusOperator.namespaces }} + {{- $namespaces := list }} + {{- if .releaseNamespace }} + {{- $namespaces = append $namespaces $namespace }} + {{- end }} + {{- if .additional }} + {{- range $ns := .additional }} + {{- $namespaces = append $namespaces (tpl $ns $) }} + {{- end }} + {{- end }} + - --namespaces={{ $namespaces | mustUniq | join "," }} + {{- end }} + - --localhost=127.0.0.1 + {{- if .Values.prometheusOperator.prometheusDefaultBaseImage }} + - --prometheus-default-base-image={{ $base_registry | default .Values.prometheusOperator.prometheusDefaultBaseImageRegistry }}/{{ .Values.prometheusOperator.prometheusDefaultBaseImage }} + {{- end }} + {{- if .Values.prometheusOperator.alertmanagerDefaultBaseImage }} + - --alertmanager-default-base-image={{ $base_registry | default .Values.prometheusOperator.alertmanagerDefaultBaseImageRegistry }}/{{ .Values.prometheusOperator.alertmanagerDefaultBaseImage }} + {{- end }} + {{- if .Values.prometheusOperator.prometheusConfigReloader.image.sha }} + - --prometheus-config-reloader={{ $configReloaderRegistry }}/{{ .Values.prometheusOperator.prometheusConfigReloader.image.repository }}:{{ .Values.prometheusOperator.prometheusConfigReloader.image.tag | default .Chart.AppVersion }}@sha256:{{ .Values.prometheusOperator.prometheusConfigReloader.image.sha }} + {{- else }} + - --prometheus-config-reloader={{ $configReloaderRegistry }}/{{ .Values.prometheusOperator.prometheusConfigReloader.image.repository }}:{{ .Values.prometheusOperator.prometheusConfigReloader.image.tag | default .Chart.AppVersion }} + {{- end }} + - --config-reloader-cpu-request={{ (((.Values.prometheusOperator.prometheusConfigReloader.resources).requests).cpu) | default 0 }} + - --config-reloader-cpu-limit={{ (((.Values.prometheusOperator.prometheusConfigReloader.resources).limits).cpu) | default 0 }} + - --config-reloader-memory-request={{ (((.Values.prometheusOperator.prometheusConfigReloader.resources).requests).memory) | default 0 }} + - --config-reloader-memory-limit={{ (((.Values.prometheusOperator.prometheusConfigReloader.resources).limits).memory) | default 0 }} + {{- if .Values.prometheusOperator.prometheusConfigReloader.enableProbe }} + - --enable-config-reloader-probes=true + {{- end }} + {{- if .Values.prometheusOperator.alertmanagerInstanceNamespaces }} + - --alertmanager-instance-namespaces={{ .Values.prometheusOperator.alertmanagerInstanceNamespaces | join "," }} + {{- end }} + {{- if .Values.prometheusOperator.alertmanagerInstanceSelector }} + - --alertmanager-instance-selector={{ .Values.prometheusOperator.alertmanagerInstanceSelector }} + {{- end }} + {{- if .Values.prometheusOperator.alertmanagerConfigNamespaces }} + - --alertmanager-config-namespaces={{ .Values.prometheusOperator.alertmanagerConfigNamespaces | join "," }} + {{- end }} + {{- if .Values.prometheusOperator.prometheusInstanceNamespaces }} + - --prometheus-instance-namespaces={{ .Values.prometheusOperator.prometheusInstanceNamespaces | join "," }} + {{- end }} + {{- if .Values.prometheusOperator.prometheusInstanceSelector }} + - --prometheus-instance-selector={{ .Values.prometheusOperator.prometheusInstanceSelector }} + {{- end }} + {{- if .Values.prometheusOperator.thanosImage.sha }} + - --thanos-default-base-image={{ $thanosRegistry }}/{{ .Values.prometheusOperator.thanosImage.repository }}:{{ .Values.prometheusOperator.thanosImage.tag }}@sha256:{{ .Values.prometheusOperator.thanosImage.sha }} + {{- else }} + - --thanos-default-base-image={{ $thanosRegistry }}/{{ .Values.prometheusOperator.thanosImage.repository }}:{{ .Values.prometheusOperator.thanosImage.tag }} + {{- end }} + {{- if .Values.prometheusOperator.thanosRulerInstanceNamespaces }} + - --thanos-ruler-instance-namespaces={{ .Values.prometheusOperator.thanosRulerInstanceNamespaces | join "," }} + {{- end }} + {{- if .Values.prometheusOperator.thanosRulerInstanceSelector }} + - --thanos-ruler-instance-selector={{ .Values.prometheusOperator.thanosRulerInstanceSelector }} + {{- end }} + {{- if .Values.prometheusOperator.secretFieldSelector }} + - --secret-field-selector={{ tpl (.Values.prometheusOperator.secretFieldSelector) $ }} + {{- end }} + {{- if .Values.prometheusOperator.clusterDomain }} + - --cluster-domain={{ .Values.prometheusOperator.clusterDomain }} + {{- end }} + {{- if .Values.prometheusOperator.tls.enabled }} + - --web.enable-tls=true + - --web.cert-file=/cert/{{ if .Values.prometheusOperator.admissionWebhooks.certManager.enabled }}tls.crt{{ else }}cert{{ end }} + - --web.key-file=/cert/{{ if .Values.prometheusOperator.admissionWebhooks.certManager.enabled }}tls.key{{ else }}key{{ end }} + - --web.listen-address=:{{ .Values.prometheusOperator.tls.internalPort }} + - --web.tls-min-version={{ .Values.prometheusOperator.tls.tlsMinVersion }} + ports: + - containerPort: {{ .Values.prometheusOperator.tls.internalPort }} + name: https + {{- else }} + ports: + - containerPort: 8080 + name: http + {{- end }} + env: + {{- range $key, $value := .Values.prometheusOperator.env }} + - name: {{ $key }} + value: {{ $value | quote }} + {{- end }} + resources: +{{ toYaml .Values.prometheusOperator.resources | indent 12 }} + securityContext: +{{ toYaml .Values.prometheusOperator.containerSecurityContext | indent 12 }} + volumeMounts: + {{- if .Values.prometheusOperator.tls.enabled }} + - name: tls-secret + mountPath: /cert + readOnly: true + {{- end }} + {{- with .Values.prometheusOperator.extraVolumeMounts }} + {{- toYaml . | nindent 12 }} + {{- end }} + volumes: + {{- if .Values.prometheusOperator.tls.enabled }} + - name: tls-secret + secret: + defaultMode: 420 + secretName: {{ template "kube-prometheus-stack.fullname" . }}-admission + {{- end }} + {{- with .Values.prometheusOperator.extraVolumes }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.prometheusOperator.dnsConfig }} + dnsConfig: +{{ toYaml . | indent 8 }} + {{- end }} +{{- if .Values.prometheusOperator.securityContext }} + securityContext: +{{ toYaml .Values.prometheusOperator.securityContext | indent 8 }} +{{- end }} + serviceAccountName: {{ template "kube-prometheus-stack.operator.serviceAccountName" . }} + automountServiceAccountToken: {{ .Values.prometheusOperator.automountServiceAccountToken }} +{{- if .Values.prometheusOperator.hostNetwork }} + hostNetwork: true + dnsPolicy: ClusterFirstWithHostNet +{{- end }} + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} +{{- with .Values.prometheusOperator.nodeSelector }} +{{ toYaml . | indent 8 }} +{{- end }} + {{- with .Values.prometheusOperator.affinity }} + affinity: +{{ toYaml . | indent 8 }} + {{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} +{{- with .Values.prometheusOperator.tolerations }} +{{ toYaml . | indent 8 }} +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus-operator/networkpolicy.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus-operator/networkpolicy.yaml new file mode 100644 index 00000000000..cfd5b0b8c75 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus-operator/networkpolicy.yaml @@ -0,0 +1,29 @@ +{{- if and .Values.prometheusOperator.networkPolicy.enabled (eq .Values.prometheusOperator.networkPolicy.flavor "kubernetes") }} +apiVersion: {{ template "kube-prometheus-stack.prometheus.networkPolicy.apiVersion" . }} +kind: NetworkPolicy +metadata: + name: {{ template "kube-prometheus-stack.operator.fullname" . }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + {{- include "kube-prometheus-stack.prometheus-operator.labels" . | nindent 4 }} +spec: + egress: + - {} + ingress: + - ports: + {{- if .Values.prometheusOperator.tls.enabled }} + - port: {{ .Values.prometheusOperator.tls.internalPort }} + {{- else }} + - port: 8080 + {{- end }} + policyTypes: + - Egress + - Ingress + podSelector: + matchLabels: + app: {{ template "kube-prometheus-stack.name" . }}-operator + release: {{ $.Release.Name | quote }} + {{- if .Values.prometheusOperator.networkPolicy.matchLabels }} + {{ toYaml .Values.prometheusOperator.networkPolicy.matchLabels | nindent 6 }} + {{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus-operator/psp-clusterrole.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus-operator/psp-clusterrole.yaml new file mode 100644 index 00000000000..61bc3d90401 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus-operator/psp-clusterrole.yaml @@ -0,0 +1,21 @@ +{{- if .Capabilities.APIVersions.Has "policy/v1beta1/PodSecurityPolicy" }} +{{- if and .Values.prometheusOperator.enabled (or .Values.global.cattle.psp.enabled (and .Values.global.rbac.create .Values.global.rbac.pspEnabled)) }} +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: {{ template "kube-prometheus-stack.operator.fullname" . }}-psp + labels: + {{- include "kube-prometheus-stack.prometheus-operator.labels" . | nindent 4 }} +rules: +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if semverCompare "> 1.15.0-0" $kubeTargetVersion }} +- apiGroups: ['policy'] +{{- else }} +- apiGroups: ['extensions'] +{{- end }} + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - {{ template "kube-prometheus-stack.operator.fullname" . }} +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus-operator/psp-clusterrolebinding.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus-operator/psp-clusterrolebinding.yaml new file mode 100644 index 00000000000..40e0fc5c153 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus-operator/psp-clusterrolebinding.yaml @@ -0,0 +1,18 @@ +{{- if and .Values.prometheusOperator.enabled (or .Values.global.cattle.psp.enabled (and .Values.global.rbac.create .Values.global.rbac.pspEnabled)) }} +{{- if .Capabilities.APIVersions.Has "policy/v1beta1/PodSecurityPolicy" }} +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: {{ template "kube-prometheus-stack.operator.fullname" . }}-psp + labels: + {{- include "kube-prometheus-stack.prometheus-operator.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "kube-prometheus-stack.operator.fullname" . }}-psp +subjects: + - kind: ServiceAccount + name: {{ template "kube-prometheus-stack.operator.serviceAccountName" . }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus-operator/psp.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus-operator/psp.yaml new file mode 100644 index 00000000000..28a9075d3e6 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus-operator/psp.yaml @@ -0,0 +1,46 @@ +{{- if and .Values.prometheusOperator.enabled (or .Values.global.cattle.psp.enabled (and .Values.global.rbac.create .Values.global.rbac.pspEnabled)) }} +{{- if .Capabilities.APIVersions.Has "policy/v1beta1/PodSecurityPolicy" }} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ template "kube-prometheus-stack.operator.fullname" . }} + labels: + {{- include "kube-prometheus-stack.prometheus-operator.labels" . | nindent 4 }} +{{- if .Values.global.rbac.pspAnnotations }} + annotations: +{{ toYaml .Values.global.rbac.pspAnnotations | indent 4 }} +{{- end }} +spec: + privileged: false + # Allow core volume types. + volumes: + - 'configMap' + - 'emptyDir' + - 'projected' + - 'secret' + - 'downwardAPI' + - 'persistentVolumeClaim' + hostNetwork: {{ .Values.prometheusOperator.hostNetwork }} + hostIPC: false + hostPID: false + runAsUser: + # Permits the container to run with root privileges as well. + rule: 'RunAsAny' + seLinux: + # This policy assumes the nodes are using AppArmor rather than SELinux. + rule: 'RunAsAny' + supplementalGroups: + rule: 'MustRunAs' + ranges: + # Allow adding the root group. + - min: 0 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + # Allow adding the root group. + - min: 0 + max: 65535 + readOnlyRootFilesystem: false +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus-operator/service.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus-operator/service.yaml new file mode 100644 index 00000000000..d45ab22d086 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus-operator/service.yaml @@ -0,0 +1,57 @@ +{{- if .Values.prometheusOperator.enabled }} +apiVersion: v1 +kind: Service +metadata: + name: {{ template "kube-prometheus-stack.operator.fullname" . }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + {{- include "kube-prometheus-stack.prometheus-operator.labels" . | nindent 4 }} +{{- if .Values.prometheusOperator.service.labels }} +{{ toYaml .Values.prometheusOperator.service.labels | indent 4 }} +{{- end }} +{{- if .Values.prometheusOperator.service.annotations }} + annotations: +{{ toYaml .Values.prometheusOperator.service.annotations | indent 4 }} +{{- end }} +spec: +{{- if .Values.prometheusOperator.service.clusterIP }} + clusterIP: {{ .Values.prometheusOperator.service.clusterIP }} +{{- end }} +{{- if .Values.prometheusOperator.service.externalIPs }} + externalIPs: +{{ toYaml .Values.prometheusOperator.service.externalIPs | indent 4 }} +{{- end }} +{{- if .Values.prometheusOperator.service.loadBalancerIP }} + loadBalancerIP: {{ .Values.prometheusOperator.service.loadBalancerIP }} +{{- end }} +{{- if .Values.prometheusOperator.service.loadBalancerSourceRanges }} + loadBalancerSourceRanges: + {{- range $cidr := .Values.prometheusOperator.service.loadBalancerSourceRanges }} + - {{ $cidr }} + {{- end }} +{{- end }} +{{- if ne .Values.prometheusOperator.service.type "ClusterIP" }} + externalTrafficPolicy: {{ .Values.prometheusOperator.service.externalTrafficPolicy }} +{{- end }} + ports: + {{- if not .Values.prometheusOperator.tls.enabled }} + - name: http + {{- if eq .Values.prometheusOperator.service.type "NodePort" }} + nodePort: {{ .Values.prometheusOperator.service.nodePort }} + {{- end }} + port: 8080 + targetPort: http + {{- end }} + {{- if .Values.prometheusOperator.tls.enabled }} + - name: https + {{- if eq .Values.prometheusOperator.service.type "NodePort"}} + nodePort: {{ .Values.prometheusOperator.service.nodePortTls }} + {{- end }} + port: 443 + targetPort: https + {{- end }} + selector: + app: {{ template "kube-prometheus-stack.name" . }}-operator + release: {{ $.Release.Name | quote }} + type: "{{ .Values.prometheusOperator.service.type }}" +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus-operator/serviceaccount.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus-operator/serviceaccount.yaml new file mode 100644 index 00000000000..4f84974f9b0 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus-operator/serviceaccount.yaml @@ -0,0 +1,14 @@ +{{- if and .Values.prometheusOperator.enabled .Values.prometheusOperator.serviceAccount.create }} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "kube-prometheus-stack.operator.serviceAccountName" . }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + {{- include "kube-prometheus-stack.prometheus-operator.labels" . | nindent 4 }} +automountServiceAccountToken: {{ .Values.prometheusOperator.serviceAccount.automountServiceAccountToken }} +{{- if .Values.global.imagePullSecrets }} +imagePullSecrets: +{{ include "kube-prometheus-stack.imagePullSecrets" . | trim | indent 2 }} +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus-operator/servicemonitor.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus-operator/servicemonitor.yaml new file mode 100644 index 00000000000..cbe79e1253f --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus-operator/servicemonitor.yaml @@ -0,0 +1,57 @@ +{{- if and .Values.prometheusOperator.enabled .Values.prometheusOperator.serviceMonitor.selfMonitor }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ template "kube-prometheus-stack.operator.fullname" . }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + {{- include "kube-prometheus-stack.prometheus-operator.labels" . | nindent 4 }} +{{- with .Values.prometheusOperator.serviceMonitor.additionalLabels }} +{{ toYaml . | indent 4 }} +{{- end }} +spec: + {{- include "servicemonitor.scrapeLimits" .Values.prometheusOperator.serviceMonitor | nindent 2 }} + endpoints: + {{- if .Values.prometheusOperator.tls.enabled }} + - port: https + scheme: https + tlsConfig: + serverName: {{ template "kube-prometheus-stack.operator.fullname" . }} + ca: + secret: + name: {{ template "kube-prometheus-stack.fullname" . }}-admission + key: {{ if .Values.prometheusOperator.admissionWebhooks.certManager.enabled }}ca.crt{{ else }}ca{{ end }} + optional: false + {{- else }} + - port: http + {{- end }} + honorLabels: true + {{- if .Values.prometheusOperator.serviceMonitor.interval }} + interval: {{ .Values.prometheusOperator.serviceMonitor.interval }} + {{- end }} + metricRelabelings: + {{- if .Values.prometheusOperator.serviceMonitor.metricRelabelings }} + {{ tpl (toYaml .Values.prometheusOperator.serviceMonitor.metricRelabelings | indent 6) . }} + {{- end }} + {{ if .Values.global.cattle.clusterId }} + - sourceLabels: [__address__] + targetLabel: cluster_id + replacement: {{ .Values.global.cattle.clusterId }} + {{- end }} + {{ if .Values.global.cattle.clusterName}} + - sourceLabels: [__address__] + targetLabel: cluster_name + replacement: {{ .Values.global.cattle.clusterName }} + {{- end }} +{{- if .Values.prometheusOperator.serviceMonitor.relabelings }} + relabelings: +{{ toYaml .Values.prometheusOperator.serviceMonitor.relabelings | indent 6 }} +{{- end }} + selector: + matchLabels: + app: {{ template "kube-prometheus-stack.name" . }}-operator + release: {{ $.Release.Name | quote }} + namespaceSelector: + matchNames: + - {{ printf "%s" (include "kube-prometheus-stack.namespace" .) | quote }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus-operator/verticalpodautoscaler.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus-operator/verticalpodautoscaler.yaml new file mode 100644 index 00000000000..f225d16dde3 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus-operator/verticalpodautoscaler.yaml @@ -0,0 +1,40 @@ +{{- if and (.Capabilities.APIVersions.Has "autoscaling.k8s.io/v1") (.Values.prometheusOperator.verticalPodAutoscaler.enabled) }} +apiVersion: autoscaling.k8s.io/v1 +kind: VerticalPodAutoscaler +metadata: + name: {{ template "kube-prometheus-stack.operator.fullname" . }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + {{- include "kube-prometheus-stack.prometheus-operator.labels" . | nindent 4 }} +spec: + {{- with .Values.prometheusOperator.verticalPodAutoscaler.recommenders }} + recommenders: + {{- toYaml . | nindent 4 }} + {{- end }} + resourcePolicy: + containerPolicies: + - containerName: {{ template "kube-prometheus-stack.name" . }} + {{- with .Values.prometheusOperator.verticalPodAutoscaler.controlledResources }} + controlledResources: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- if .Values.prometheusOperator.verticalPodAutoscaler.controlledValues }} + controlledValues: {{ .Values.prometheusOperator.verticalPodAutoscaler.controlledValues }} + {{- end }} + {{- if .Values.prometheusOperator.verticalPodAutoscaler.maxAllowed }} + maxAllowed: + {{- toYaml .Values.prometheusOperator.verticalPodAutoscaler.maxAllowed | nindent 8 }} + {{- end }} + {{- if .Values.prometheusOperator.verticalPodAutoscaler.minAllowed }} + minAllowed: + {{- toYaml .Values.prometheusOperator.verticalPodAutoscaler.minAllowed | nindent 8 }} + {{- end }} + targetRef: + apiVersion: apps/v1 + kind: Deployment + name: {{ template "kube-prometheus-stack.operator.fullname" . }} + {{- with .Values.prometheusOperator.verticalPodAutoscaler.updatePolicy }} + updatePolicy: + {{- toYaml . | nindent 4 }} + {{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/_rules.tpl b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/_rules.tpl new file mode 100644 index 00000000000..4a8213d0897 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/_rules.tpl @@ -0,0 +1,44 @@ +{{- /* +Generated file. Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- define "rules.names" }} +rules: + - "alertmanager.rules" + - "config-reloaders" + - "etcd" + - "general.rules" + - "k8s.rules.container-cpu-usage-seconds-total" + - "k8s.rules.container-memory-cache" + - "k8s.rules.container-memory-rss" + - "k8s.rules.container-memory-swap" + - "k8s.rules.container-memory-working-set-bytes" + - "k8s.rules.container-resource" + - "k8s.rules.pod-owner" + - "kube-apiserver-availability.rules" + - "kube-apiserver-burnrate.rules" + - "kube-apiserver-histogram.rules" + - "kube-apiserver-slos" + - "kube-prometheus-general.rules" + - "kube-prometheus-node-recording.rules" + - "kube-scheduler.rules" + - "kube-state-metrics" + - "kubelet.rules" + - "kubernetes-apps" + - "kubernetes-resources" + - "kubernetes-storage" + - "kubernetes-system" + - "kubernetes-system-kube-proxy" + - "kubernetes-system-apiserver" + - "kubernetes-system-kubelet" + - "kubernetes-system-controller-manager" + - "kubernetes-system-scheduler" + - "node-exporter.rules" + - "node-exporter" + - "node.rules" + - "node-network" + - "prometheus-operator" + - "prometheus" + - "windows.node.rules" + - "windows.pod.rules" +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/additionalAlertRelabelConfigs.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/additionalAlertRelabelConfigs.yaml new file mode 100644 index 00000000000..bff930981a6 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/additionalAlertRelabelConfigs.yaml @@ -0,0 +1,16 @@ +{{- if and .Values.prometheus.enabled .Values.prometheus.prometheusSpec.additionalAlertRelabelConfigs }} +apiVersion: v1 +kind: Secret +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-prometheus-am-relabel-confg + namespace: {{ template "kube-prometheus-stack.namespace" . }} +{{- if .Values.prometheus.prometheusSpec.additionalPrometheusSecretsAnnotations }} + annotations: +{{ toYaml .Values.prometheus.prometheusSpec.additionalPrometheusSecretsAnnotations | indent 4 }} +{{- end }} + labels: + app: {{ template "kube-prometheus-stack.name" . }}-prometheus-am-relabel-confg +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +data: + additional-alert-relabel-configs.yaml: {{ toYaml .Values.prometheus.prometheusSpec.additionalAlertRelabelConfigs | b64enc | quote }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/additionalAlertmanagerConfigs.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/additionalAlertmanagerConfigs.yaml new file mode 100644 index 00000000000..2fe8fdb8168 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/additionalAlertmanagerConfigs.yaml @@ -0,0 +1,16 @@ +{{- if and .Values.prometheus.enabled .Values.prometheus.prometheusSpec.additionalAlertManagerConfigs }} +apiVersion: v1 +kind: Secret +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-prometheus-am-confg + namespace: {{ template "kube-prometheus-stack.namespace" . }} +{{- if .Values.prometheus.prometheusSpec.additionalPrometheusSecretsAnnotations }} + annotations: +{{ toYaml .Values.prometheus.prometheusSpec.additionalPrometheusSecretsAnnotations | indent 4 }} +{{- end }} + labels: + app: {{ template "kube-prometheus-stack.name" . }}-prometheus-am-confg +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +data: + additional-alertmanager-configs.yaml: {{ tpl (toYaml .Values.prometheus.prometheusSpec.additionalAlertManagerConfigs) . | b64enc | quote }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/additionalPrometheusRules.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/additionalPrometheusRules.yaml new file mode 100644 index 00000000000..cb4aabaa7b5 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/additionalPrometheusRules.yaml @@ -0,0 +1,43 @@ +{{- if or .Values.additionalPrometheusRules .Values.additionalPrometheusRulesMap}} +apiVersion: v1 +kind: List +metadata: + name: {{ include "kube-prometheus-stack.fullname" $ }}-additional-prometheus-rules + namespace: {{ template "kube-prometheus-stack.namespace" . }} +items: +{{- if .Values.additionalPrometheusRulesMap }} +{{- range $prometheusRuleName, $prometheusRule := .Values.additionalPrometheusRulesMap }} + - apiVersion: monitoring.coreos.com/v1 + kind: PrometheusRule + metadata: + name: {{ template "kube-prometheus-stack.name" $ }}-{{ $prometheusRuleName }} + namespace: {{ template "kube-prometheus-stack.namespace" $ }} + labels: + app: {{ template "kube-prometheus-stack.name" $ }} +{{ include "kube-prometheus-stack.labels" $ | indent 8 }} + {{- if $prometheusRule.additionalLabels }} +{{ toYaml $prometheusRule.additionalLabels | indent 8 }} + {{- end }} + spec: + groups: +{{ toYaml $prometheusRule.groups| indent 8 }} +{{- end }} +{{- else }} +{{- range .Values.additionalPrometheusRules }} + - apiVersion: monitoring.coreos.com/v1 + kind: PrometheusRule + metadata: + name: {{ template "kube-prometheus-stack.name" $ }}-{{ .name }} + namespace: {{ template "kube-prometheus-stack.namespace" $ }} + labels: + app: {{ template "kube-prometheus-stack.name" $ }} +{{ include "kube-prometheus-stack.labels" $ | indent 8 }} + {{- if .additionalLabels }} +{{ toYaml .additionalLabels | indent 8 }} + {{- end }} + spec: + groups: +{{ toYaml .groups| indent 8 }} +{{- end }} +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/additionalScrapeConfigs.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/additionalScrapeConfigs.yaml new file mode 100644 index 00000000000..ebdf766fde0 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/additionalScrapeConfigs.yaml @@ -0,0 +1,20 @@ +{{- if and .Values.prometheus.enabled .Values.prometheus.prometheusSpec.additionalScrapeConfigs }} +apiVersion: v1 +kind: Secret +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-prometheus-scrape-confg + namespace: {{ template "kube-prometheus-stack.namespace" . }} +{{- if .Values.prometheus.prometheusSpec.additionalPrometheusSecretsAnnotations }} + annotations: +{{ toYaml .Values.prometheus.prometheusSpec.additionalPrometheusSecretsAnnotations | indent 4 }} +{{- end }} + labels: + app: {{ template "kube-prometheus-stack.name" . }}-prometheus-scrape-confg +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +data: +{{- if eq ( typeOf .Values.prometheus.prometheusSpec.additionalScrapeConfigs ) "string" }} + additional-scrape-configs.yaml: {{ tpl .Values.prometheus.prometheusSpec.additionalScrapeConfigs $ | b64enc | quote }} +{{- else }} + additional-scrape-configs.yaml: {{ tpl (toYaml .Values.prometheus.prometheusSpec.additionalScrapeConfigs) $ | b64enc | quote }} +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/ciliumnetworkpolicy.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/ciliumnetworkpolicy.yaml new file mode 100644 index 00000000000..74d61d7c130 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/ciliumnetworkpolicy.yaml @@ -0,0 +1,27 @@ +{{- if and .Values.prometheus.networkPolicy.enabled (eq .Values.prometheus.networkPolicy.flavor "cilium") }} +apiVersion: cilium.io/v2 +kind: CiliumNetworkPolicy +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-prometheus + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }}-prometheus + {{- include "kube-prometheus-stack.labels" . | nindent 4 }} +spec: + endpointSelector: + {{- if .Values.prometheus.networkPolicy.cilium.endpointSelector }} + {{- toYaml .Values.prometheus.networkPolicy.cilium.endpointSelector | nindent 4 }} + {{- else }} + matchExpressions: + - {key: app.kubernetes.io/name, operator: In, values: [prometheus]} + - {key: prometheus, operator: In, values: [{{ template "kube-prometheus-stack.prometheus.crname" . }}]} + {{- end }} + {{- if and .Values.prometheus.networkPolicy.cilium .Values.prometheus.networkPolicy.cilium.egress }} + egress: + {{ toYaml .Values.prometheus.networkPolicy.cilium.egress | nindent 4 }} + {{- end }} + {{- if and .Values.prometheus.networkPolicy.cilium .Values.prometheus.networkPolicy.cilium.ingress }} + ingress: + {{ toYaml .Values.prometheus.networkPolicy.cilium.ingress | nindent 4 }} + {{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/clusterrole.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/clusterrole.yaml new file mode 100644 index 00000000000..3585b5db115 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/clusterrole.yaml @@ -0,0 +1,30 @@ +{{- if and .Values.prometheus.enabled .Values.global.rbac.create }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-prometheus + labels: + app: {{ template "kube-prometheus-stack.name" . }}-prometheus +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +rules: +# This permission are not in the kube-prometheus repo +# they're grabbed from https://github.com/prometheus/prometheus/blob/master/documentation/examples/rbac-setup.yml +- apiGroups: [""] + resources: + - nodes + - nodes/metrics + - services + - endpoints + - pods + verbs: ["get", "list", "watch"] +- apiGroups: + - "networking.k8s.io" + resources: + - ingresses + verbs: ["get", "list", "watch"] +- nonResourceURLs: ["/metrics", "/metrics/cadvisor"] + verbs: ["get"] +{{- if .Values.prometheus.additionalRulesForClusterRole }} +{{ toYaml .Values.prometheus.additionalRulesForClusterRole | indent 0 }} +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/clusterrolebinding.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/clusterrolebinding.yaml new file mode 100644 index 00000000000..9fc4f65da42 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/clusterrolebinding.yaml @@ -0,0 +1,18 @@ +{{- if and .Values.prometheus.enabled .Values.global.rbac.create }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-prometheus + labels: + app: {{ template "kube-prometheus-stack.name" . }}-prometheus +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "kube-prometheus-stack.fullname" . }}-prometheus +subjects: + - kind: ServiceAccount + name: {{ template "kube-prometheus-stack.prometheus.serviceAccountName" . }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} +{{- end }} + diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/csi-secret.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/csi-secret.yaml new file mode 100644 index 00000000000..e05382f6333 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/csi-secret.yaml @@ -0,0 +1,12 @@ +{{- if and .Values.prometheus.prometheusSpec.thanos .Values.prometheus.prometheusSpec.thanos.secretProviderClass }} +--- +apiVersion: secrets-store.csi.x-k8s.io/v1alpha1 +kind: SecretProviderClass +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-prometheus + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }}-prometheus +spec: +{{ toYaml .Values.prometheus.prometheusSpec.thanos.secretProviderClass | indent 2 }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/extrasecret.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/extrasecret.yaml new file mode 100644 index 00000000000..17f3478a462 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/extrasecret.yaml @@ -0,0 +1,20 @@ +{{- if .Values.prometheus.extraSecret.data -}} +{{- $secretName := printf "prometheus-%s-extra" (include "kube-prometheus-stack.fullname" . ) -}} +apiVersion: v1 +kind: Secret +metadata: + name: {{ default $secretName .Values.prometheus.extraSecret.name }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} +{{- if .Values.prometheus.extraSecret.annotations }} + annotations: +{{ toYaml .Values.prometheus.extraSecret.annotations | indent 4 }} +{{- end }} + labels: + app: {{ template "kube-prometheus-stack.name" . }}-prometheus + app.kubernetes.io/component: prometheus +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +data: +{{- range $key, $val := .Values.prometheus.extraSecret.data }} + {{ $key }}: {{ $val | b64enc | quote }} +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/ingress.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/ingress.yaml new file mode 100644 index 00000000000..d2f6af5dd14 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/ingress.yaml @@ -0,0 +1,77 @@ +{{- if and .Values.prometheus.enabled .Values.prometheus.ingress.enabled -}} + {{- $pathType := .Values.prometheus.ingress.pathType | default "ImplementationSpecific" -}} + {{- $serviceName := printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "prometheus" -}} + {{- $servicePort := .Values.prometheus.ingress.servicePort | default .Values.prometheus.service.port -}} + {{- $routePrefix := list .Values.prometheus.prometheusSpec.routePrefix -}} + {{- $paths := .Values.prometheus.ingress.paths | default $routePrefix -}} + {{- $apiIsStable := eq (include "kube-prometheus-stack.ingress.isStable" .) "true" -}} + {{- $ingressSupportsPathType := eq (include "kube-prometheus-stack.ingress.supportsPathType" .) "true" -}} +apiVersion: {{ include "kube-prometheus-stack.ingress.apiVersion" . }} +kind: Ingress +metadata: +{{- if .Values.prometheus.ingress.annotations }} + annotations: + {{- tpl (toYaml .Values.prometheus.ingress.annotations) . | nindent 4 }} +{{- end }} + name: {{ $serviceName }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }}-prometheus +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.prometheus.ingress.labels }} +{{ toYaml .Values.prometheus.ingress.labels | indent 4 }} +{{- end }} +spec: + {{- if $apiIsStable }} + {{- if .Values.prometheus.ingress.ingressClassName }} + ingressClassName: {{ .Values.prometheus.ingress.ingressClassName }} + {{- end }} + {{- end }} + rules: + {{- if .Values.prometheus.ingress.hosts }} + {{- range $host := .Values.prometheus.ingress.hosts }} + - host: {{ tpl $host $ }} + http: + paths: + {{- range $p := $paths }} + - path: {{ tpl $p $ }} + {{- if and $pathType $ingressSupportsPathType }} + pathType: {{ $pathType }} + {{- end }} + backend: + {{- if $apiIsStable }} + service: + name: {{ $serviceName }} + port: + number: {{ $servicePort }} + {{- else }} + serviceName: {{ $serviceName }} + servicePort: {{ $servicePort }} + {{- end }} + {{- end -}} + {{- end -}} + {{- else }} + - http: + paths: + {{- range $p := $paths }} + - path: {{ tpl $p $ }} + {{- if and $pathType $ingressSupportsPathType }} + pathType: {{ $pathType }} + {{- end }} + backend: + {{- if $apiIsStable }} + service: + name: {{ $serviceName }} + port: + number: {{ $servicePort }} + {{- else }} + serviceName: {{ $serviceName }} + servicePort: {{ $servicePort }} + {{- end }} + {{- end -}} + {{- end -}} + {{- if .Values.prometheus.ingress.tls }} + tls: +{{ tpl (toYaml .Values.prometheus.ingress.tls | indent 4) . }} + {{- end -}} +{{- end -}} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/ingressThanosSidecar.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/ingressThanosSidecar.yaml new file mode 100644 index 00000000000..3f507cfa9f6 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/ingressThanosSidecar.yaml @@ -0,0 +1,77 @@ +{{- if and .Values.prometheus.enabled .Values.prometheus.thanosIngress.enabled }} +{{- $pathType := .Values.prometheus.thanosIngress.pathType | default "" }} +{{- $serviceName := printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "thanos-discovery" }} +{{- $thanosPort := .Values.prometheus.thanosIngress.servicePort -}} +{{- $routePrefix := list .Values.prometheus.prometheusSpec.routePrefix }} +{{- $paths := .Values.prometheus.thanosIngress.paths | default $routePrefix -}} +{{- $apiIsStable := eq (include "kube-prometheus-stack.ingress.isStable" .) "true" -}} +{{- $ingressSupportsPathType := eq (include "kube-prometheus-stack.ingress.supportsPathType" .) "true" -}} +apiVersion: {{ include "kube-prometheus-stack.ingress.apiVersion" . }} +kind: Ingress +metadata: +{{- if .Values.prometheus.thanosIngress.annotations }} + annotations: + {{- tpl (toYaml .Values.prometheus.thanosIngress.annotations) . | nindent 4 }} +{{- end }} + name: {{ template "kube-prometheus-stack.fullname" . }}-thanos-gateway + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }}-prometheus +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.prometheus.thanosIngress.labels }} +{{ toYaml .Values.prometheus.thanosIngress.labels | indent 4 }} +{{- end }} +spec: + {{- if $apiIsStable }} + {{- if .Values.prometheus.thanosIngress.ingressClassName }} + ingressClassName: {{ .Values.prometheus.thanosIngress.ingressClassName }} + {{- end }} + {{- end }} + rules: + {{- if .Values.prometheus.thanosIngress.hosts }} + {{- range $host := .Values.prometheus.thanosIngress.hosts }} + - host: {{ tpl $host $ }} + http: + paths: + {{- range $p := $paths }} + - path: {{ tpl $p $ }} + {{- if and $pathType $ingressSupportsPathType }} + pathType: {{ $pathType }} + {{- end }} + backend: + {{- if $apiIsStable }} + service: + name: {{ $serviceName }} + port: + number: {{ $thanosPort }} + {{- else }} + serviceName: {{ $serviceName }} + servicePort: {{ $thanosPort }} + {{- end }} + {{- end -}} + {{- end -}} + {{- else }} + - http: + paths: + {{- range $p := $paths }} + - path: {{ tpl $p $ }} + {{- if and $pathType $ingressSupportsPathType }} + pathType: {{ $pathType }} + {{- end }} + backend: + {{- if $apiIsStable }} + service: + name: {{ $serviceName }} + port: + number: {{ $thanosPort }} + {{- else }} + serviceName: {{ $serviceName }} + servicePort: {{ $thanosPort }} + {{- end }} + {{- end -}} + {{- end -}} + {{- if .Values.prometheus.thanosIngress.tls }} + tls: +{{ tpl (toYaml .Values.prometheus.thanosIngress.tls | indent 4) . }} + {{- end -}} +{{- end -}} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/ingressperreplica.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/ingressperreplica.yaml new file mode 100644 index 00000000000..1d76d135c8d --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/ingressperreplica.yaml @@ -0,0 +1,67 @@ +{{- if and .Values.prometheus.enabled .Values.prometheus.servicePerReplica.enabled .Values.prometheus.ingressPerReplica.enabled }} +{{- $pathType := .Values.prometheus.ingressPerReplica.pathType | default "" }} +{{- $count := .Values.prometheus.prometheusSpec.replicas | int -}} +{{- $servicePort := .Values.prometheus.servicePerReplica.port -}} +{{- $ingressValues := .Values.prometheus.ingressPerReplica -}} +{{- $apiIsStable := eq (include "kube-prometheus-stack.ingress.isStable" .) "true" -}} +{{- $ingressSupportsPathType := eq (include "kube-prometheus-stack.ingress.supportsPathType" .) "true" -}} +apiVersion: v1 +kind: List +metadata: + name: {{ include "kube-prometheus-stack.fullname" $ }}-prometheus-ingressperreplica + namespace: {{ template "kube-prometheus-stack.namespace" $ }} +items: +{{ range $i, $e := until $count }} + - kind: Ingress + apiVersion: {{ include "kube-prometheus-stack.ingress.apiVersion" $ }} + metadata: + name: {{ include "kube-prometheus-stack.fullname" $ }}-prometheus-{{ $i }} + namespace: {{ template "kube-prometheus-stack.namespace" $ }} + labels: + app: {{ include "kube-prometheus-stack.name" $ }}-prometheus + {{ include "kube-prometheus-stack.labels" $ | indent 8 }} + {{- if $ingressValues.labels }} +{{ toYaml $ingressValues.labels | indent 8 }} + {{- end }} + {{- if $ingressValues.annotations }} + annotations: + {{- tpl (toYaml $ingressValues.annotations) $ | nindent 8 }} + {{- end }} + spec: + {{- if $apiIsStable }} + {{- if $ingressValues.ingressClassName }} + ingressClassName: {{ $ingressValues.ingressClassName }} + {{- end }} + {{- end }} + rules: + - host: {{ $ingressValues.hostPrefix }}-{{ $i }}.{{ $ingressValues.hostDomain }} + http: + paths: + {{- range $p := $ingressValues.paths }} + - path: {{ tpl $p $ }} + {{- if and $pathType $ingressSupportsPathType }} + pathType: {{ $pathType }} + {{- end }} + backend: + {{- if $apiIsStable }} + service: + name: {{ include "kube-prometheus-stack.fullname" $ }}-prometheus-{{ $i }} + port: + number: {{ $servicePort }} + {{- else }} + serviceName: {{ include "kube-prometheus-stack.fullname" $ }}-prometheus-{{ $i }} + servicePort: {{ $servicePort }} + {{- end }} + {{- end -}} + {{- if or $ingressValues.tlsSecretName $ingressValues.tlsSecretPerReplica.enabled }} + tls: + - hosts: + - {{ $ingressValues.hostPrefix }}-{{ $i }}.{{ $ingressValues.hostDomain }} + {{- if $ingressValues.tlsSecretPerReplica.enabled }} + secretName: {{ $ingressValues.tlsSecretPerReplica.prefix }}-{{ $i }} + {{- else }} + secretName: {{ $ingressValues.tlsSecretName }} + {{- end }} + {{- end }} +{{- end -}} +{{- end -}} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/networkpolicy.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/networkpolicy.yaml new file mode 100644 index 00000000000..1296a790636 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/networkpolicy.yaml @@ -0,0 +1,34 @@ +{{- if and .Values.prometheus.networkPolicy.enabled (eq .Values.prometheus.networkPolicy.flavor "kubernetes") }} +apiVersion: {{ template "kube-prometheus-stack.prometheus.networkPolicy.apiVersion" . }} +kind: NetworkPolicy +metadata: + labels: + app: {{ template "kube-prometheus-stack.name" . }}-prometheus + {{- include "kube-prometheus-stack.labels" . | nindent 4 }} + name: {{ template "kube-prometheus-stack.fullname" . }}-prometheus + namespace: {{ template "kube-prometheus-stack.namespace" . }} +spec: + {{- if .Values.prometheus.networkPolicy.egress }} + egress: + {{- toYaml .Values.prometheus.networkPolicy.egress | nindent 4 }} + {{- end }} + {{- if .Values.prometheus.networkPolicy.ingress }} + ingress: + {{- toYaml .Values.prometheus.networkPolicy.ingress | nindent 4 }} + {{- end }} + policyTypes: + - Egress + - Ingress + podSelector: + {{- if .Values.prometheus.networkPolicy.podSelector }} + {{- toYaml .Values.prometheus.networkPolicy.podSelector | nindent 4 }} + {{- else }} + matchLabels: + {{- if .Values.prometheus.agentMode }} + app.kubernetes.io/name: prometheus-agent + {{- else }} + app.kubernetes.io/name: prometheus + {{- end }} + operator.prometheus.io/name: {{ template "kube-prometheus-stack.prometheus.crname" . }} + {{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/nginx-config.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/nginx-config.yaml new file mode 100644 index 00000000000..e4d91f9a9ec --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/nginx-config.yaml @@ -0,0 +1,68 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: prometheus-nginx-proxy-config + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }}-prometheus +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.prometheus.annotations }} + annotations: +{{ toYaml .Values.prometheus.annotations | indent 4 }} +{{- end }} +data: + nginx.conf: |- + worker_processes auto; + error_log /dev/stdout warn; + pid /var/cache/nginx/nginx.pid; + + events { + worker_connections 1024; + } + + http { + include /etc/nginx/mime.types; + log_format main '[$time_local - $status] $remote_addr - $remote_user $request ($http_referer)'; + + proxy_connect_timeout 10; + proxy_read_timeout 180; + proxy_send_timeout 5; + proxy_buffering off; + proxy_cache_path /var/cache/nginx/cache levels=1:2 keys_zone=my_zone:100m inactive=1d max_size=10g; + + server { + listen 8081; + access_log off; + + gzip on; + gzip_min_length 1k; + gzip_comp_level 2; + gzip_types text/plain application/javascript application/x-javascript text/css application/xml text/javascript image/jpeg image/gif image/png; + gzip_vary on; + gzip_disable "MSIE [1-6]\."; + + proxy_set_header Host $host; + + location / { + proxy_cache my_zone; + proxy_cache_valid 200 302 1d; + proxy_cache_valid 301 30d; + proxy_cache_valid any 5m; + proxy_cache_bypass $http_cache_control; + add_header X-Proxy-Cache $upstream_cache_status; + add_header Cache-Control "public"; + + proxy_pass http://localhost:9090/; + + sub_filter_once off; + sub_filter 'var PATH_PREFIX = "";' 'var PATH_PREFIX = ".";'; + + if ($request_filename ~ .*\.(?:js|css|jpg|jpeg|gif|png|ico|cur|gz|svg|svgz|mp4|ogg|ogv|webm)$) { + expires 90d; + } + + rewrite ^/k8s/clusters/.*/proxy(.*) /$1 break; + + } + } + } diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/podDisruptionBudget.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/podDisruptionBudget.yaml new file mode 100644 index 00000000000..48f3f1f5a62 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/podDisruptionBudget.yaml @@ -0,0 +1,25 @@ +{{- if and .Values.prometheus.enabled .Values.prometheus.podDisruptionBudget.enabled }} +apiVersion: {{ include "kube-prometheus-stack.pdb.apiVersion" . }} +kind: PodDisruptionBudget +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-prometheus + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }}-prometheus +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +spec: + {{- if .Values.prometheus.podDisruptionBudget.minAvailable }} + minAvailable: {{ .Values.prometheus.podDisruptionBudget.minAvailable }} + {{- end }} + {{- if .Values.prometheus.podDisruptionBudget.maxUnavailable }} + maxUnavailable: {{ .Values.prometheus.podDisruptionBudget.maxUnavailable }} + {{- end }} + selector: + matchLabels: + {{- if .Values.prometheus.agentMode }} + app.kubernetes.io/name: prometheus-agent + {{- else }} + app.kubernetes.io/name: prometheus + {{- end }} + operator.prometheus.io/name: {{ template "kube-prometheus-stack.prometheus.crname" . }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/podmonitors.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/podmonitors.yaml new file mode 100644 index 00000000000..4e748c23b5c --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/podmonitors.yaml @@ -0,0 +1,38 @@ +{{- if and .Values.prometheus.enabled .Values.prometheus.additionalPodMonitors }} +apiVersion: v1 +kind: List +items: +{{- range .Values.prometheus.additionalPodMonitors }} + - apiVersion: monitoring.coreos.com/v1 + kind: PodMonitor + metadata: + name: {{ .name }} + namespace: {{ template "kube-prometheus-stack.namespace" $ }} + labels: + app: {{ template "kube-prometheus-stack.name" $ }}-prometheus +{{ include "kube-prometheus-stack.labels" $ | indent 8 }} + {{- if .additionalLabels }} +{{ toYaml .additionalLabels | indent 8 }} + {{- end }} + spec: + {{- include "servicemonitor.scrapeLimits" . | nindent 6 }} + podMetricsEndpoints: +{{ toYaml .podMetricsEndpoints | indent 8 }} + {{- if .jobLabel }} + jobLabel: {{ .jobLabel }} + {{- end }} + {{- if .namespaceSelector }} + namespaceSelector: +{{ toYaml .namespaceSelector | indent 8 }} + {{- end }} + selector: +{{ toYaml .selector | indent 8 }} + {{- if .podTargetLabels }} + podTargetLabels: +{{ toYaml .podTargetLabels | indent 8 }} + {{- end }} + {{- if .sampleLimit }} + sampleLimit: {{ .sampleLimit }} + {{- end }} +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/prometheus.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/prometheus.yaml new file mode 100644 index 00000000000..5c3c8d4d1f3 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/prometheus.yaml @@ -0,0 +1,472 @@ +{{- if .Values.prometheus.enabled }} +{{- if .Values.prometheus.agentMode }} +apiVersion: monitoring.coreos.com/v1alpha1 +kind: PrometheusAgent +{{- else }} +apiVersion: monitoring.coreos.com/v1 +kind: Prometheus +{{- end }} +metadata: + name: {{ template "kube-prometheus-stack.prometheus.crname" . }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }}-prometheus +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.prometheus.annotations }} + annotations: +{{ toYaml .Values.prometheus.annotations | indent 4 }} +{{- end }} +spec: +{{- if and (not .Values.prometheus.agentMode) (or .Values.prometheus.prometheusSpec.alertingEndpoints .Values.alertmanager.enabled) }} + alerting: + alertmanagers: +{{- if .Values.prometheus.prometheusSpec.alertingEndpoints }} +{{ toYaml .Values.prometheus.prometheusSpec.alertingEndpoints | indent 6 }} +{{- else if .Values.alertmanager.enabled }} + - namespace: {{ template "kube-prometheus-stack.namespace" . }} + name: {{ template "kube-prometheus-stack.fullname" . }}-alertmanager + port: {{ .Values.alertmanager.alertmanagerSpec.portName }} + {{- if .Values.alertmanager.alertmanagerSpec.routePrefix }} + pathPrefix: "{{ .Values.alertmanager.alertmanagerSpec.routePrefix }}" + {{- end }} + {{- if .Values.alertmanager.alertmanagerSpec.scheme }} + scheme: {{ .Values.alertmanager.alertmanagerSpec.scheme }} + {{- end }} + {{- if .Values.alertmanager.alertmanagerSpec.tlsConfig }} + tlsConfig: +{{ toYaml .Values.alertmanager.alertmanagerSpec.tlsConfig | indent 10 }} + {{- end }} + apiVersion: {{ .Values.alertmanager.apiVersion }} +{{- end }} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.apiserverConfig }} + apiserverConfig: +{{ toYaml .Values.prometheus.prometheusSpec.apiserverConfig | indent 4}} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.image }} + {{- $registry := include "monitoring_registry" . | default .Values.prometheus.prometheusSpec.image.registry -}} + {{- if and .Values.prometheus.prometheusSpec.image.tag .Values.prometheus.prometheusSpec.image.sha }} + image: "{{ $registry }}/{{ .Values.prometheus.prometheusSpec.image.repository }}:{{ .Values.prometheus.prometheusSpec.image.tag }}@sha256:{{ .Values.prometheus.prometheusSpec.image.sha }}" + {{- else if .Values.prometheus.prometheusSpec.image.sha }} + image: "{{ $registry }}/{{ .Values.prometheus.prometheusSpec.image.repository }}@sha256:{{ .Values.prometheus.prometheusSpec.image.sha }}" + {{- else if .Values.prometheus.prometheusSpec.image.tag }} + image: "{{ $registry }}/{{ .Values.prometheus.prometheusSpec.image.repository }}:{{ .Values.prometheus.prometheusSpec.image.tag }}" + {{- else }} + image: "{{ $registry }}/{{ .Values.prometheus.prometheusSpec.image.repository }}" + {{- end }} + version: {{ default .Values.prometheus.prometheusSpec.image.tag .Values.prometheus.prometheusSpec.version }} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.additionalArgs }} + additionalArgs: +{{ toYaml .Values.prometheus.prometheusSpec.additionalArgs | indent 4}} +{{- end -}} +{{- if .Values.prometheus.prometheusSpec.externalLabels }} + externalLabels: +{{ tpl (toYaml .Values.prometheus.prometheusSpec.externalLabels | indent 4) . }} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.prometheusExternalLabelNameClear }} + prometheusExternalLabelName: "" +{{- else if .Values.prometheus.prometheusSpec.prometheusExternalLabelName }} + prometheusExternalLabelName: "{{ .Values.prometheus.prometheusSpec.prometheusExternalLabelName }}" +{{- end }} +{{- if .Values.prometheus.prometheusSpec.replicaExternalLabelNameClear }} + replicaExternalLabelName: "" +{{- else if .Values.prometheus.prometheusSpec.replicaExternalLabelName }} + replicaExternalLabelName: "{{ .Values.prometheus.prometheusSpec.replicaExternalLabelName }}" +{{- end }} +{{- if .Values.prometheus.prometheusSpec.enableRemoteWriteReceiver }} + enableRemoteWriteReceiver: {{ .Values.prometheus.prometheusSpec.enableRemoteWriteReceiver }} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.externalUrl }} + externalUrl: "{{ tpl .Values.prometheus.prometheusSpec.externalUrl . }}" +{{- else if and .Values.prometheus.ingress.enabled .Values.prometheus.ingress.hosts }} + externalUrl: "http://{{ tpl (index .Values.prometheus.ingress.hosts 0) . }}{{ .Values.prometheus.prometheusSpec.routePrefix }}" +{{- else if not (or (kindIs "invalid" .Values.global.cattle.url) (kindIs "invalid" .Values.global.cattle.clusterId)) }} + externalUrl: "{{ .Values.global.cattle.url }}/k8s/clusters/{{ .Values.global.cattle.clusterId }}/api/v1/namespaces/{{ template "kube-prometheus-stack.namespace" . }}/services/http:{{ template "kube-prometheus-stack.fullname" . }}-prometheus:{{ .Values.prometheus.service.port }}/proxy" +{{- else }} + externalUrl: http://{{ template "kube-prometheus-stack.fullname" . }}-prometheus.{{ template "kube-prometheus-stack.namespace" . }}:{{ .Values.prometheus.service.port }} +{{- end }} + nodeSelector: {{ include "linux-node-selector" . | nindent 4 }} +{{- if .Values.prometheus.prometheusSpec.nodeSelector }} +{{ toYaml .Values.prometheus.prometheusSpec.nodeSelector | indent 4 }} +{{- end }} + paused: {{ .Values.prometheus.prometheusSpec.paused }} + replicas: {{ .Values.prometheus.prometheusSpec.replicas }} + shards: {{ .Values.prometheus.prometheusSpec.shards }} + logLevel: {{ .Values.prometheus.prometheusSpec.logLevel }} + logFormat: {{ .Values.prometheus.prometheusSpec.logFormat }} + listenLocal: {{ .Values.prometheus.prometheusSpec.listenLocal }} +{{- if not .Values.prometheus.agentMode }} + enableAdminAPI: {{ .Values.prometheus.prometheusSpec.enableAdminAPI }} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.web }} + web: +{{ toYaml .Values.prometheus.prometheusSpec.web | indent 4 }} +{{- end }} +{{- if and (not .Values.prometheus.agentMode) .Values.prometheus.prometheusSpec.exemplars }} + exemplars: + {{ toYaml .Values.prometheus.prometheusSpec.exemplars | indent 4 }} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.enableFeatures }} + enableFeatures: +{{- range $enableFeatures := .Values.prometheus.prometheusSpec.enableFeatures }} + - {{ tpl $enableFeatures $ }} +{{- end }} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.scrapeInterval }} + scrapeInterval: {{ .Values.prometheus.prometheusSpec.scrapeInterval }} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.scrapeTimeout }} + scrapeTimeout: {{ .Values.prometheus.prometheusSpec.scrapeTimeout }} +{{- end }} +{{- if and (not .Values.prometheus.agentMode) .Values.prometheus.prometheusSpec.evaluationInterval }} + evaluationInterval: {{ .Values.prometheus.prometheusSpec.evaluationInterval }} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.resources }} + resources: +{{ toYaml .Values.prometheus.prometheusSpec.resources | indent 4 }} +{{- end }} +{{- if not .Values.prometheus.agentMode }} + retention: {{ .Values.prometheus.prometheusSpec.retention | quote }} +{{- if .Values.prometheus.prometheusSpec.retentionSize }} + retentionSize: {{ .Values.prometheus.prometheusSpec.retentionSize | quote }} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.tsdb }} + tsdb: + {{- if .Values.prometheus.prometheusSpec.tsdb.outOfOrderTimeWindow }} + outOfOrderTimeWindow: {{ .Values.prometheus.prometheusSpec.tsdb.outOfOrderTimeWindow }} + {{- end }} +{{- end }} +{{- end }} +{{- if eq .Values.prometheus.prometheusSpec.walCompression false }} + walCompression: false +{{ else }} + walCompression: true +{{- end }} +{{- if .Values.prometheus.prometheusSpec.routePrefix }} + routePrefix: {{ .Values.prometheus.prometheusSpec.routePrefix | quote }} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.secrets }} + secrets: +{{ toYaml .Values.prometheus.prometheusSpec.secrets | indent 4 }} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.configMaps }} + configMaps: +{{ toYaml .Values.prometheus.prometheusSpec.configMaps | indent 4 }} +{{- end }} + serviceAccountName: {{ template "kube-prometheus-stack.prometheus.serviceAccountName" . }} +{{- if .Values.prometheus.prometheusSpec.serviceMonitorSelector }} + serviceMonitorSelector: +{{ tpl (toYaml .Values.prometheus.prometheusSpec.serviceMonitorSelector | indent 4) . }} +{{ else if .Values.prometheus.prometheusSpec.serviceMonitorSelectorNilUsesHelmValues }} + serviceMonitorSelector: + matchLabels: + release: {{ $.Release.Name | quote }} +{{ else }} + serviceMonitorSelector: {} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.serviceMonitorNamespaceSelector }} + serviceMonitorNamespaceSelector: +{{ tpl (toYaml .Values.prometheus.prometheusSpec.serviceMonitorNamespaceSelector | indent 4) . }} +{{ else }} + serviceMonitorNamespaceSelector: {} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.podMonitorSelector }} + podMonitorSelector: +{{ tpl (toYaml .Values.prometheus.prometheusSpec.podMonitorSelector | indent 4) . }} +{{ else if .Values.prometheus.prometheusSpec.podMonitorSelectorNilUsesHelmValues }} + podMonitorSelector: + matchLabels: + release: {{ $.Release.Name | quote }} +{{ else }} + podMonitorSelector: {} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.podMonitorNamespaceSelector }} + podMonitorNamespaceSelector: +{{ tpl (toYaml .Values.prometheus.prometheusSpec.podMonitorNamespaceSelector | indent 4) . }} +{{ else }} + podMonitorNamespaceSelector: {} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.probeSelector }} + probeSelector: +{{ tpl (toYaml .Values.prometheus.prometheusSpec.probeSelector | indent 4) . }} +{{ else if .Values.prometheus.prometheusSpec.probeSelectorNilUsesHelmValues }} + probeSelector: + matchLabels: + release: {{ $.Release.Name | quote }} +{{ else }} + probeSelector: {} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.probeNamespaceSelector }} + probeNamespaceSelector: +{{ tpl (toYaml .Values.prometheus.prometheusSpec.probeNamespaceSelector | indent 4) . }} +{{ else }} + probeNamespaceSelector: {} +{{- end }} +{{- if and (not .Values.prometheus.agentMode) (or .Values.prometheus.prometheusSpec.remoteRead .Values.prometheus.prometheusSpec.additionalRemoteRead) }} + remoteRead: +{{- if .Values.prometheus.prometheusSpec.remoteRead }} +{{ tpl (toYaml .Values.prometheus.prometheusSpec.remoteRead | indent 4) . }} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.additionalRemoteRead }} +{{ toYaml .Values.prometheus.prometheusSpec.additionalRemoteRead | indent 4 }} +{{- end }} +{{- end }} +{{- if (or .Values.prometheus.prometheusSpec.remoteWrite .Values.prometheus.prometheusSpec.additionalRemoteWrite) }} + remoteWrite: +{{- if .Values.prometheus.prometheusSpec.remoteWrite }} +{{ tpl (toYaml .Values.prometheus.prometheusSpec.remoteWrite | indent 4) . }} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.additionalRemoteWrite }} +{{ toYaml .Values.prometheus.prometheusSpec.additionalRemoteWrite | indent 4 }} +{{- end }} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.securityContext }} + securityContext: +{{ toYaml .Values.prometheus.prometheusSpec.securityContext | indent 4 }} +{{- end }} +{{- if not .Values.prometheus.agentMode }} +{{- if .Values.prometheus.prometheusSpec.ruleNamespaceSelector }} + ruleNamespaceSelector: +{{ tpl (toYaml .Values.prometheus.prometheusSpec.ruleNamespaceSelector | indent 4) . }} +{{ else }} + ruleNamespaceSelector: {} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.ruleSelector }} + ruleSelector: +{{ tpl (toYaml .Values.prometheus.prometheusSpec.ruleSelector | indent 4) . }} +{{- else if .Values.prometheus.prometheusSpec.ruleSelectorNilUsesHelmValues }} + ruleSelector: + matchLabels: + release: {{ $.Release.Name | quote }} +{{ else }} + ruleSelector: {} +{{- end }} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.scrapeConfigSelector }} + scrapeConfigSelector: +{{ tpl (toYaml .Values.prometheus.prometheusSpec.scrapeConfigSelector | indent 4) . }} +{{ else if .Values.prometheus.prometheusSpec.scrapeConfigSelectorNilUsesHelmValues }} + scrapeConfigSelector: + matchLabels: + release: {{ $.Release.Name | quote }} +{{ else }} + scrapeConfigSelector: {} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.scrapeConfigNamespaceSelector }} + scrapeConfigNamespaceSelector: +{{ tpl (toYaml .Values.prometheus.prometheusSpec.scrapeConfigNamespaceSelector | indent 4) . }} +{{ else }} + scrapeConfigNamespaceSelector: {} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.storageSpec }} + storage: +{{ tpl (toYaml .Values.prometheus.prometheusSpec.storageSpec | indent 4) . }} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.podMetadata }} + podMetadata: +{{ tpl (toYaml .Values.prometheus.prometheusSpec.podMetadata | indent 4) . }} +{{- end }} +{{- if and (not .Values.prometheus.agentMode) .Values.prometheus.prometheusSpec.query }} + query: +{{ toYaml .Values.prometheus.prometheusSpec.query | indent 4}} +{{- end }} +{{- if or .Values.prometheus.prometheusSpec.podAntiAffinity .Values.prometheus.prometheusSpec.affinity }} + affinity: +{{- if .Values.prometheus.prometheusSpec.affinity }} +{{ toYaml .Values.prometheus.prometheusSpec.affinity | indent 4 }} +{{- end }} +{{- if eq .Values.prometheus.prometheusSpec.podAntiAffinity "hard" }} + podAntiAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - topologyKey: {{ .Values.prometheus.prometheusSpec.podAntiAffinityTopologyKey }} + labelSelector: + matchExpressions: + - {key: app.kubernetes.io/name, operator: In, values: [prometheus]} + - {key: prometheus, operator: In, values: [{{ template "kube-prometheus-stack.prometheus.crname" . }}]} +{{- else if eq .Values.prometheus.prometheusSpec.podAntiAffinity "soft" }} + podAntiAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 100 + podAffinityTerm: + topologyKey: {{ .Values.prometheus.prometheusSpec.podAntiAffinityTopologyKey }} + labelSelector: + matchExpressions: + - {key: app.kubernetes.io/name, operator: In, values: [prometheus]} + - {key: prometheus, operator: In, values: [{{ template "kube-prometheus-stack.prometheus.crname" . }}]} +{{- end }} +{{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 4 }} +{{- if .Values.prometheus.prometheusSpec.tolerations }} +{{ toYaml .Values.prometheus.prometheusSpec.tolerations | indent 4 }} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.topologySpreadConstraints }} + topologySpreadConstraints: +{{ toYaml .Values.prometheus.prometheusSpec.topologySpreadConstraints | indent 4 }} +{{- end }} +{{- if .Values.global.imagePullSecrets }} + imagePullSecrets: +{{ include "kube-prometheus-stack.imagePullSecrets" . | trim | indent 4 }} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.additionalScrapeConfigs }} + additionalScrapeConfigs: + name: {{ template "kube-prometheus-stack.fullname" . }}-prometheus-scrape-confg + key: additional-scrape-configs.yaml +{{- end }} +{{- if .Values.prometheus.prometheusSpec.additionalScrapeConfigsSecret.enabled }} + additionalScrapeConfigs: + name: {{ .Values.prometheus.prometheusSpec.additionalScrapeConfigsSecret.name }} + key: {{ .Values.prometheus.prometheusSpec.additionalScrapeConfigsSecret.key }} +{{- end }} +{{- if not .Values.prometheus.agentMode }} +{{- if or .Values.prometheus.prometheusSpec.additionalAlertManagerConfigs .Values.prometheus.prometheusSpec.additionalAlertManagerConfigsSecret }} + additionalAlertManagerConfigs: +{{- if .Values.prometheus.prometheusSpec.additionalAlertManagerConfigs }} + name: {{ template "kube-prometheus-stack.fullname" . }}-prometheus-am-confg + key: additional-alertmanager-configs.yaml +{{- end }} +{{- if .Values.prometheus.prometheusSpec.additionalAlertManagerConfigsSecret }} + name: {{ .Values.prometheus.prometheusSpec.additionalAlertManagerConfigsSecret.name }} + key: {{ .Values.prometheus.prometheusSpec.additionalAlertManagerConfigsSecret.key }} + {{- if hasKey .Values.prometheus.prometheusSpec.additionalAlertManagerConfigsSecret "optional" }} + optional: {{ .Values.prometheus.prometheusSpec.additionalAlertManagerConfigsSecret.optional }} + {{- end }} +{{- end }} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.additionalAlertRelabelConfigs }} + additionalAlertRelabelConfigs: + name: {{ template "kube-prometheus-stack.fullname" . }}-prometheus-am-relabel-confg + key: additional-alert-relabel-configs.yaml +{{- end }} +{{- if .Values.prometheus.prometheusSpec.additionalAlertRelabelConfigsSecret }} + additionalAlertRelabelConfigs: + name: {{ .Values.prometheus.prometheusSpec.additionalAlertRelabelConfigsSecret.name }} + key: {{ .Values.prometheus.prometheusSpec.additionalAlertRelabelConfigsSecret.key }} +{{- end }} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.containers }} + containers: +{{ tpl .Values.prometheus.prometheusSpec.containers $ | indent 4 }} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.initContainers }} + initContainers: +{{ toYaml .Values.prometheus.prometheusSpec.initContainers | indent 4 }} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.priorityClassName }} + priorityClassName: {{ .Values.prometheus.prometheusSpec.priorityClassName }} +{{- end }} +{{- if not .Values.prometheus.agentMode }} +{{- if .Values.prometheus.prometheusSpec.thanos }} + thanos: +{{- with (omit .Values.prometheus.prometheusSpec.thanos "objectStorageConfig")}} +{{ toYaml . | indent 4 }} +{{- end }} +{{- if ((.Values.prometheus.prometheusSpec.thanos.objectStorageConfig).existingSecret) }} + objectStorageConfig: + key: "{{.Values.prometheus.prometheusSpec.thanos.objectStorageConfig.existingSecret.key }}" + name: "{{.Values.prometheus.prometheusSpec.thanos.objectStorageConfig.existingSecret.name }}" +{{- else if ((.Values.prometheus.prometheusSpec.thanos.objectStorageConfig).secret) }} + objectStorageConfig: + key: object-storage-configs.yaml + name: {{ template "kube-prometheus-stack.fullname" . }}-prometheus +{{- end }} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.disableCompaction }} + disableCompaction: {{ .Values.prometheus.prometheusSpec.disableCompaction }} +{{- end }} +{{- end }} + portName: {{ .Values.prometheus.prometheusSpec.portName }} +{{- if .Values.prometheus.prometheusSpec.volumes }} + volumes: +{{ toYaml .Values.prometheus.prometheusSpec.volumes | indent 4 }} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.volumeMounts }} + volumeMounts: +{{ toYaml .Values.prometheus.prometheusSpec.volumeMounts | indent 4 }} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.arbitraryFSAccessThroughSMs }} + arbitraryFSAccessThroughSMs: +{{ toYaml .Values.prometheus.prometheusSpec.arbitraryFSAccessThroughSMs | indent 4 }} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.overrideHonorLabels }} + overrideHonorLabels: {{ .Values.prometheus.prometheusSpec.overrideHonorLabels }} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.overrideHonorTimestamps }} + overrideHonorTimestamps: {{ .Values.prometheus.prometheusSpec.overrideHonorTimestamps }} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.ignoreNamespaceSelectors }} + ignoreNamespaceSelectors: {{ .Values.prometheus.prometheusSpec.ignoreNamespaceSelectors }} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.enforcedNamespaceLabel }} + enforcedNamespaceLabel: {{ .Values.prometheus.prometheusSpec.enforcedNamespaceLabel }} +{{- $prometheusDefaultRulesExcludedFromEnforce := (include "rules.names" .) | fromYaml }} +{{- if not .Values.prometheus.agentMode }} + prometheusRulesExcludedFromEnforce: +{{- range $prometheusDefaultRulesExcludedFromEnforce.rules }} + - ruleNamespace: "{{ template "kube-prometheus-stack.namespace" $ }}" + ruleName: "{{ printf "%s-%s" (include "kube-prometheus-stack.fullname" $) . | trunc 63 | trimSuffix "-" }}" +{{- end }} +{{- if .Values.prometheus.prometheusSpec.prometheusRulesExcludedFromEnforce }} +{{ toYaml .Values.prometheus.prometheusSpec.prometheusRulesExcludedFromEnforce | indent 4 }} +{{- end }} +{{- end }} + excludedFromEnforcement: +{{- range $prometheusDefaultRulesExcludedFromEnforce.rules }} + - group: monitoring.coreos.com + resource: prometheusrules + namespace: "{{ template "kube-prometheus-stack.namespace" $ }}" + name: "{{ printf "%s-%s" (include "kube-prometheus-stack.fullname" $) . | trunc 63 | trimSuffix "-" }}" +{{- end }} +{{- if .Values.prometheus.prometheusSpec.excludedFromEnforcement }} +{{ tpl (toYaml .Values.prometheus.prometheusSpec.excludedFromEnforcement | indent 4) . }} +{{- end }} +{{- end }} +{{- if and (not .Values.prometheus.agentMode) .Values.prometheus.prometheusSpec.queryLogFile }} + queryLogFile: {{ .Values.prometheus.prometheusSpec.queryLogFile }} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.sampleLimit }} + sampleLimit: {{ .Values.prometheus.prometheusSpec.sampleLimit }} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.enforcedKeepDroppedTargets }} + enforcedKeepDroppedTargets: {{ .Values.prometheus.prometheusSpec.enforcedKeepDroppedTargets }} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.enforcedSampleLimit }} + enforcedSampleLimit: {{ .Values.prometheus.prometheusSpec.enforcedSampleLimit }} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.enforcedTargetLimit }} + enforcedTargetLimit: {{ .Values.prometheus.prometheusSpec.enforcedTargetLimit }} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.enforcedLabelLimit }} + enforcedLabelLimit: {{ .Values.prometheus.prometheusSpec.enforcedLabelLimit }} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.enforcedLabelNameLengthLimit }} + enforcedLabelNameLengthLimit: {{ .Values.prometheus.prometheusSpec.enforcedLabelNameLengthLimit }} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.enforcedLabelValueLengthLimit}} + enforcedLabelValueLengthLimit: {{ .Values.prometheus.prometheusSpec.enforcedLabelValueLengthLimit }} +{{- end }} +{{- if and (not .Values.prometheus.agentMode) .Values.prometheus.prometheusSpec.allowOverlappingBlocks }} + allowOverlappingBlocks: {{ .Values.prometheus.prometheusSpec.allowOverlappingBlocks }} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.minReadySeconds }} + minReadySeconds: {{ .Values.prometheus.prometheusSpec.minReadySeconds }} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.maximumStartupDurationSeconds }} + maximumStartupDurationSeconds: {{ .Values.prometheus.prometheusSpec.maximumStartupDurationSeconds }} +{{- end }} + hostNetwork: {{ .Values.prometheus.prometheusSpec.hostNetwork }} +{{- if .Values.prometheus.prometheusSpec.hostAliases }} + hostAliases: +{{ toYaml .Values.prometheus.prometheusSpec.hostAliases | indent 4 }} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.tracingConfig }} + tracingConfig: +{{ toYaml .Values.prometheus.prometheusSpec.tracingConfig | indent 4 }} +{{- end }} +{{- with .Values.prometheus.prometheusSpec.additionalConfig }} + {{- tpl (toYaml .) $ | nindent 2 }} +{{- end }} +{{- with .Values.prometheus.prometheusSpec.additionalConfigString }} + {{- tpl . $ | nindent 2 }} +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/psp-clusterrole.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/psp-clusterrole.yaml new file mode 100644 index 00000000000..71476cd18ba --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/psp-clusterrole.yaml @@ -0,0 +1,22 @@ +{{- if and .Values.prometheus.enabled (or .Values.global.cattle.psp.enabled (and .Values.global.rbac.create .Values.global.rbac.pspEnabled)) }} +{{- if .Capabilities.APIVersions.Has "policy/v1beta1/PodSecurityPolicy" }} +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-prometheus-psp + labels: + app: {{ template "kube-prometheus-stack.name" . }}-prometheus +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +rules: +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if semverCompare "> 1.15.0-0" $kubeTargetVersion }} +- apiGroups: ['policy'] +{{- else }} +- apiGroups: ['extensions'] +{{- end }} + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - {{ template "kube-prometheus-stack.fullname" . }}-prometheus +{{- end }} +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/psp-clusterrolebinding.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/psp-clusterrolebinding.yaml new file mode 100644 index 00000000000..a393928c78b --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/psp-clusterrolebinding.yaml @@ -0,0 +1,19 @@ +{{- if and .Values.prometheus.enabled (or .Values.global.cattle.psp.enabled (and .Values.global.rbac.create .Values.global.rbac.pspEnabled)) }} +{{- if .Capabilities.APIVersions.Has "policy/v1beta1/PodSecurityPolicy" }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-prometheus-psp + labels: + app: {{ template "kube-prometheus-stack.name" . }}-prometheus +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "kube-prometheus-stack.fullname" . }}-prometheus-psp +subjects: + - kind: ServiceAccount + name: {{ template "kube-prometheus-stack.prometheus.serviceAccountName" . }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/psp.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/psp.yaml new file mode 100644 index 00000000000..62d38541517 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/psp.yaml @@ -0,0 +1,58 @@ +{{- if and .Values.prometheus.enabled (or .Values.global.cattle.psp.enabled (and .Values.global.rbac.create .Values.global.rbac.pspEnabled)) }} +{{- if .Capabilities.APIVersions.Has "policy/v1beta1/PodSecurityPolicy" }} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-prometheus + labels: + app: {{ template "kube-prometheus-stack.name" . }}-prometheus +{{- if .Values.global.rbac.pspAnnotations }} + annotations: +{{ toYaml .Values.global.rbac.pspAnnotations | indent 4 }} +{{- end }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +spec: + privileged: false + # Allow core volume types. + volumes: + - 'configMap' + - 'emptyDir' + - 'projected' + - 'secret' + - 'downwardAPI' + - 'persistentVolumeClaim' +{{- if .Values.prometheus.podSecurityPolicy.volumes }} +{{ toYaml .Values.prometheus.podSecurityPolicy.volumes | indent 4 }} +{{- end }} + hostNetwork: false + hostIPC: false + hostPID: false + runAsUser: + # Permits the container to run with root privileges as well. + rule: 'RunAsAny' + seLinux: + # This policy assumes the nodes are using AppArmor rather than SELinux. + rule: 'RunAsAny' + supplementalGroups: + rule: 'MustRunAs' + ranges: + # Allow adding the root group. + - min: 0 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + # Allow adding the root group. + - min: 0 + max: 65535 + readOnlyRootFilesystem: false +{{- if .Values.prometheus.podSecurityPolicy.allowedCapabilities }} + allowedCapabilities: +{{ toYaml .Values.prometheus.podSecurityPolicy.allowedCapabilities | indent 4 }} +{{- end }} +{{- if .Values.prometheus.podSecurityPolicy.allowedHostPaths }} + allowedHostPaths: +{{ toYaml .Values.prometheus.podSecurityPolicy.allowedHostPaths | indent 4 }} +{{- end }} +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/alertmanager.rules.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/alertmanager.rules.yaml new file mode 100644 index 00000000000..b66f052ade3 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/alertmanager.rules.yaml @@ -0,0 +1,305 @@ +{{- /* +Generated from 'alertmanager.rules' group from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/a8ba97a150c75be42010c75d10b720c55e182f1a/manifests/alertmanager-prometheusRule.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.defaultRules.create .Values.defaultRules.rules.alertmanager }} +{{- $alertmanagerJob := printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "alertmanager" }} +{{- $namespace := printf "%s" (include "kube-prometheus-stack.namespace" .) }} +{{- if and .Values.alertmanager.enabled .Values.alertmanager.serviceMonitor.selfMonitor }} +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "alertmanager.rules" | trunc 63 | trimSuffix "-" }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.defaultRules.labels }} +{{ toYaml .Values.defaultRules.labels | indent 4 }} +{{- end }} +{{- if .Values.defaultRules.annotations }} + annotations: +{{ toYaml .Values.defaultRules.annotations | indent 4 }} +{{- end }} +spec: + groups: + - name: alertmanager.rules + rules: +{{- if not (.Values.defaultRules.disabled.AlertmanagerFailedReload | default false) }} + - alert: AlertmanagerFailedReload + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.alertmanager }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.alertmanager | indent 8 }} +{{- end }} + description: Configuration has failed to load for {{`{{`}} $labels.namespace {{`}}`}}/{{`{{`}} $labels.pod{{`}}`}}. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/alertmanager/alertmanagerfailedreload + summary: Reloading an Alertmanager configuration has failed. + expr: |- + # Without max_over_time, failed scrapes could create false negatives, see + # https://www.robustperception.io/alerting-on-gauges-in-prometheus-2-0 for details. + max_over_time(alertmanager_config_last_reload_successful{job="{{ $alertmanagerJob }}",namespace="{{ $namespace }}"}[5m]) == 0 + for: {{ dig "AlertmanagerFailedReload" "for" "10m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "AlertmanagerFailedReload" "severity" "critical" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.alertmanager }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.alertmanager }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.AlertmanagerMembersInconsistent | default false) }} + - alert: AlertmanagerMembersInconsistent + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.alertmanager }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.alertmanager | indent 8 }} +{{- end }} + description: Alertmanager {{`{{`}} $labels.namespace {{`}}`}}/{{`{{`}} $labels.pod{{`}}`}} has only found {{`{{`}} $value {{`}}`}} members of the {{`{{`}}$labels.job{{`}}`}} cluster. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/alertmanager/alertmanagermembersinconsistent + summary: A member of an Alertmanager cluster has not found all other cluster members. + expr: |- + # Without max_over_time, failed scrapes could create false negatives, see + # https://www.robustperception.io/alerting-on-gauges-in-prometheus-2-0 for details. + max_over_time(alertmanager_cluster_members{job="{{ $alertmanagerJob }}",namespace="{{ $namespace }}"}[5m]) + < on ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}namespace,service,cluster) group_left + count by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}namespace,service,cluster) (max_over_time(alertmanager_cluster_members{job="{{ $alertmanagerJob }}",namespace="{{ $namespace }}"}[5m])) + for: {{ dig "AlertmanagerMembersInconsistent" "for" "15m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "AlertmanagerMembersInconsistent" "severity" "critical" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.alertmanager }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.alertmanager }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.AlertmanagerFailedToSendAlerts | default false) }} + - alert: AlertmanagerFailedToSendAlerts + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.alertmanager }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.alertmanager | indent 8 }} +{{- end }} + description: Alertmanager {{`{{`}} $labels.namespace {{`}}`}}/{{`{{`}} $labels.pod{{`}}`}} failed to send {{`{{`}} $value | humanizePercentage {{`}}`}} of notifications to {{`{{`}} $labels.integration {{`}}`}}. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/alertmanager/alertmanagerfailedtosendalerts + summary: An Alertmanager instance failed to send notifications. + expr: |- + ( + rate(alertmanager_notifications_failed_total{job="{{ $alertmanagerJob }}",namespace="{{ $namespace }}"}[5m]) + / + ignoring (reason) group_left rate(alertmanager_notifications_total{job="{{ $alertmanagerJob }}",namespace="{{ $namespace }}"}[5m]) + ) + > 0.01 + for: {{ dig "AlertmanagerFailedToSendAlerts" "for" "5m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "AlertmanagerFailedToSendAlerts" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.alertmanager }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.alertmanager }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.AlertmanagerClusterFailedToSendAlerts | default false) }} + - alert: AlertmanagerClusterFailedToSendAlerts + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.alertmanager }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.alertmanager | indent 8 }} +{{- end }} + description: The minimum notification failure rate to {{`{{`}} $labels.integration {{`}}`}} sent from any instance in the {{`{{`}}$labels.job{{`}}`}} cluster is {{`{{`}} $value | humanizePercentage {{`}}`}}. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/alertmanager/alertmanagerclusterfailedtosendalerts + summary: All Alertmanager instances in a cluster failed to send notifications to a critical integration. + expr: |- + min by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}namespace,service, integration) ( + rate(alertmanager_notifications_failed_total{job="{{ $alertmanagerJob }}",namespace="{{ $namespace }}", integration=~`.*`}[5m]) + / + ignoring (reason) group_left rate(alertmanager_notifications_total{job="{{ $alertmanagerJob }}",namespace="{{ $namespace }}", integration=~`.*`}[5m]) + ) + > 0.01 + for: {{ dig "AlertmanagerClusterFailedToSendAlerts" "for" "5m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "AlertmanagerClusterFailedToSendAlerts" "severity" "critical" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.alertmanager }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.alertmanager }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.AlertmanagerClusterFailedToSendAlerts | default false) }} + - alert: AlertmanagerClusterFailedToSendAlerts + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.alertmanager }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.alertmanager | indent 8 }} +{{- end }} + description: The minimum notification failure rate to {{`{{`}} $labels.integration {{`}}`}} sent from any instance in the {{`{{`}}$labels.job{{`}}`}} cluster is {{`{{`}} $value | humanizePercentage {{`}}`}}. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/alertmanager/alertmanagerclusterfailedtosendalerts + summary: All Alertmanager instances in a cluster failed to send notifications to a non-critical integration. + expr: |- + min by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}namespace,service, integration) ( + rate(alertmanager_notifications_failed_total{job="{{ $alertmanagerJob }}",namespace="{{ $namespace }}", integration!~`.*`}[5m]) + / + ignoring (reason) group_left rate(alertmanager_notifications_total{job="{{ $alertmanagerJob }}",namespace="{{ $namespace }}", integration!~`.*`}[5m]) + ) + > 0.01 + for: {{ dig "AlertmanagerClusterFailedToSendAlerts" "for" "5m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "AlertmanagerClusterFailedToSendAlerts" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.alertmanager }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.alertmanager }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.AlertmanagerConfigInconsistent | default false) }} + - alert: AlertmanagerConfigInconsistent + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.alertmanager }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.alertmanager | indent 8 }} +{{- end }} + description: Alertmanager instances within the {{`{{`}}$labels.job{{`}}`}} cluster have different configurations. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/alertmanager/alertmanagerconfiginconsistent + summary: Alertmanager instances within the same cluster have different configurations. + expr: |- + count by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}namespace,service,cluster) ( + count_values by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}namespace,service,cluster) ("config_hash", alertmanager_config_hash{job="{{ $alertmanagerJob }}",namespace="{{ $namespace }}"}) + ) + != 1 + for: {{ dig "AlertmanagerConfigInconsistent" "for" "20m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "AlertmanagerConfigInconsistent" "severity" "critical" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.alertmanager }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.alertmanager }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.AlertmanagerClusterDown | default false) }} + - alert: AlertmanagerClusterDown + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.alertmanager }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.alertmanager | indent 8 }} +{{- end }} + description: '{{`{{`}} $value | humanizePercentage {{`}}`}} of Alertmanager instances within the {{`{{`}}$labels.job{{`}}`}} cluster have been up for less than half of the last 5m.' + runbook_url: {{ .Values.defaultRules.runbookUrl }}/alertmanager/alertmanagerclusterdown + summary: Half or more of the Alertmanager instances within the same cluster are down. + expr: |- + ( + count by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}namespace,service,cluster) ( + avg_over_time(up{job="{{ $alertmanagerJob }}",namespace="{{ $namespace }}"}[5m]) < 0.5 + ) + / + count by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}namespace,service,cluster) ( + up{job="{{ $alertmanagerJob }}",namespace="{{ $namespace }}"} + ) + ) + >= 0.5 + for: {{ dig "AlertmanagerClusterDown" "for" "5m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "AlertmanagerClusterDown" "severity" "critical" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.alertmanager }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.alertmanager }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.AlertmanagerClusterCrashlooping | default false) }} + - alert: AlertmanagerClusterCrashlooping + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.alertmanager }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.alertmanager | indent 8 }} +{{- end }} + description: '{{`{{`}} $value | humanizePercentage {{`}}`}} of Alertmanager instances within the {{`{{`}}$labels.job{{`}}`}} cluster have restarted at least 5 times in the last 10m.' + runbook_url: {{ .Values.defaultRules.runbookUrl }}/alertmanager/alertmanagerclustercrashlooping + summary: Half or more of the Alertmanager instances within the same cluster are crashlooping. + expr: |- + ( + count by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}namespace,service,cluster) ( + changes(process_start_time_seconds{job="{{ $alertmanagerJob }}",namespace="{{ $namespace }}"}[10m]) > 4 + ) + / + count by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}namespace,service,cluster) ( + up{job="{{ $alertmanagerJob }}",namespace="{{ $namespace }}"} + ) + ) + >= 0.5 + for: {{ dig "AlertmanagerClusterCrashlooping" "for" "5m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "AlertmanagerClusterCrashlooping" "severity" "critical" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.alertmanager }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.alertmanager }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/config-reloaders.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/config-reloaders.yaml new file mode 100644 index 00000000000..8416d6df400 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/config-reloaders.yaml @@ -0,0 +1,57 @@ +{{- /* +Generated from 'config-reloaders' group from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/a8ba97a150c75be42010c75d10b720c55e182f1a/manifests/prometheusOperator-prometheusRule.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.defaultRules.create .Values.defaultRules.rules.configReloaders }} +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "config-reloaders" | trunc 63 | trimSuffix "-" }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.defaultRules.labels }} +{{ toYaml .Values.defaultRules.labels | indent 4 }} +{{- end }} +{{- if .Values.defaultRules.annotations }} + annotations: +{{ toYaml .Values.defaultRules.annotations | indent 4 }} +{{- end }} +spec: + groups: + - name: config-reloaders + rules: +{{- if not (.Values.defaultRules.disabled.ConfigReloaderSidecarErrors | default false) }} + - alert: ConfigReloaderSidecarErrors + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.configReloaders }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.configReloaders | indent 8 }} +{{- end }} + description: 'Errors encountered while the {{`{{`}}$labels.pod{{`}}`}} config-reloader sidecar attempts to sync config in {{`{{`}}$labels.namespace{{`}}`}} namespace. + + As a result, configuration for service running in {{`{{`}}$labels.pod{{`}}`}} may be stale and cannot be updated anymore.' + runbook_url: {{ .Values.defaultRules.runbookUrl }}/prometheus-operator/configreloadersidecarerrors + summary: config-reloader sidecar has not had a successful reload for 10m + expr: max_over_time(reloader_last_reload_successful{namespace=~".+"}[5m]) == 0 + for: {{ dig "ConfigReloaderSidecarErrors" "for" "10m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "ConfigReloaderSidecarErrors" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.configReloaders }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.configReloaders }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/etcd.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/etcd.yaml new file mode 100644 index 00000000000..a1d7a508f8d --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/etcd.yaml @@ -0,0 +1,461 @@ +{{- /* +Generated from 'etcd' group from https://github.com/etcd-io/etcd.git +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.defaultRules.create .Values.defaultRules.rules.etcd }} +{{- if (include "exporter.kubeEtcd.enabled" .)}} +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "etcd" | trunc 63 | trimSuffix "-" }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.defaultRules.labels }} +{{ toYaml .Values.defaultRules.labels | indent 4 }} +{{- end }} +{{- if .Values.defaultRules.annotations }} + annotations: +{{ toYaml .Values.defaultRules.annotations | indent 4 }} +{{- end }} +spec: + groups: + - name: etcd + rules: +{{- if not (.Values.defaultRules.disabled.etcdMembersDown | default false) }} + - alert: etcdMembersDown + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.etcd }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.etcd | indent 8 }} +{{- end }} + description: 'etcd cluster "{{`{{`}} $labels.job {{`}}`}}": members are down ({{`{{`}} $value {{`}}`}}).' + summary: etcd cluster members are down. + expr: |- + max without (endpoint) ( + sum without (instance) (up{job=~".*etcd.*"} == bool 0) + or + count without (To) ( + sum without (instance) (rate(etcd_network_peer_sent_failures_total{job=~".*etcd.*"}[120s])) > 0.01 + ) + ) + > 0 + for: {{ dig "etcdMembersDown" "for" "10m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "etcdMembersDown" "severity" "critical" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.etcd }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.etcd }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.etcdInsufficientMembers | default false) }} + - alert: etcdInsufficientMembers + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.etcd }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.etcd | indent 8 }} +{{- end }} + description: 'etcd cluster "{{`{{`}} $labels.job {{`}}`}}": insufficient members ({{`{{`}} $value {{`}}`}}).' + summary: etcd cluster has insufficient number of members. + expr: sum(up{job=~".*etcd.*"} == bool 1) without (instance) < ((count(up{job=~".*etcd.*"}) without (instance) + 1) / 2) + for: {{ dig "etcdInsufficientMembers" "for" "3m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "etcdInsufficientMembers" "severity" "critical" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.etcd }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.etcd }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.etcdNoLeader | default false) }} + - alert: etcdNoLeader + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.etcd }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.etcd | indent 8 }} +{{- end }} + description: 'etcd cluster "{{`{{`}} $labels.job {{`}}`}}": member {{`{{`}} $labels.instance {{`}}`}} has no leader.' + summary: etcd cluster has no leader. + expr: etcd_server_has_leader{job=~".*etcd.*"} == 0 + for: {{ dig "etcdNoLeader" "for" "1m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "etcdNoLeader" "severity" "critical" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.etcd }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.etcd }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.etcdHighNumberOfLeaderChanges | default false) }} + - alert: etcdHighNumberOfLeaderChanges + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.etcd }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.etcd | indent 8 }} +{{- end }} + description: 'etcd cluster "{{`{{`}} $labels.job {{`}}`}}": {{`{{`}} $value {{`}}`}} leader changes within the last 15 minutes. Frequent elections may be a sign of insufficient resources, high network latency, or disruptions by other components and should be investigated.' + summary: etcd cluster has high number of leader changes. + expr: increase((max without (instance) (etcd_server_leader_changes_seen_total{job=~".*etcd.*"}) or 0*absent(etcd_server_leader_changes_seen_total{job=~".*etcd.*"}))[15m:1m]) >= 4 + for: {{ dig "etcdHighNumberOfLeaderChanges" "for" "5m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "etcdHighNumberOfLeaderChanges" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.etcd }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.etcd }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.etcdHighNumberOfFailedGRPCRequests | default false) }} + - alert: etcdHighNumberOfFailedGRPCRequests + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.etcd }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.etcd | indent 8 }} +{{- end }} + description: 'etcd cluster "{{`{{`}} $labels.job {{`}}`}}": {{`{{`}} $value {{`}}`}}% of requests for {{`{{`}} $labels.grpc_method {{`}}`}} failed on etcd instance {{`{{`}} $labels.instance {{`}}`}}.' + summary: etcd cluster has high number of failed grpc requests. + expr: |- + 100 * sum(rate(grpc_server_handled_total{job=~".*etcd.*", grpc_code=~"Unknown|FailedPrecondition|ResourceExhausted|Internal|Unavailable|DataLoss|DeadlineExceeded"}[5m])) without (grpc_type, grpc_code) + / + sum(rate(grpc_server_handled_total{job=~".*etcd.*"}[5m])) without (grpc_type, grpc_code) + > 1 + for: {{ dig "etcdHighNumberOfFailedGRPCRequests" "for" "10m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "etcdHighNumberOfFailedGRPCRequests" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.etcd }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.etcd }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.etcdHighNumberOfFailedGRPCRequests | default false) }} + - alert: etcdHighNumberOfFailedGRPCRequests + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.etcd }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.etcd | indent 8 }} +{{- end }} + description: 'etcd cluster "{{`{{`}} $labels.job {{`}}`}}": {{`{{`}} $value {{`}}`}}% of requests for {{`{{`}} $labels.grpc_method {{`}}`}} failed on etcd instance {{`{{`}} $labels.instance {{`}}`}}.' + summary: etcd cluster has high number of failed grpc requests. + expr: |- + 100 * sum(rate(grpc_server_handled_total{job=~".*etcd.*", grpc_code=~"Unknown|FailedPrecondition|ResourceExhausted|Internal|Unavailable|DataLoss|DeadlineExceeded"}[5m])) without (grpc_type, grpc_code) + / + sum(rate(grpc_server_handled_total{job=~".*etcd.*"}[5m])) without (grpc_type, grpc_code) + > 5 + for: {{ dig "etcdHighNumberOfFailedGRPCRequests" "for" "5m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "etcdHighNumberOfFailedGRPCRequests" "severity" "critical" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.etcd }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.etcd }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.etcdGRPCRequestsSlow | default false) }} + - alert: etcdGRPCRequestsSlow + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.etcd }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.etcd | indent 8 }} +{{- end }} + description: 'etcd cluster "{{`{{`}} $labels.job {{`}}`}}": 99th percentile of gRPC requests is {{`{{`}} $value {{`}}`}}s on etcd instance {{`{{`}} $labels.instance {{`}}`}} for {{`{{`}} $labels.grpc_method {{`}}`}} method.' + summary: etcd grpc requests are slow + expr: |- + histogram_quantile(0.99, sum(rate(grpc_server_handling_seconds_bucket{job=~".*etcd.*", grpc_method!="Defragment", grpc_type="unary"}[5m])) without(grpc_type)) + > 0.15 + for: {{ dig "etcdGRPCRequestsSlow" "for" "10m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "etcdGRPCRequestsSlow" "severity" "critical" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.etcd }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.etcd }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.etcdMemberCommunicationSlow | default false) }} + - alert: etcdMemberCommunicationSlow + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.etcd }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.etcd | indent 8 }} +{{- end }} + description: 'etcd cluster "{{`{{`}} $labels.job {{`}}`}}": member communication with {{`{{`}} $labels.To {{`}}`}} is taking {{`{{`}} $value {{`}}`}}s on etcd instance {{`{{`}} $labels.instance {{`}}`}}.' + summary: etcd cluster member communication is slow. + expr: |- + histogram_quantile(0.99, rate(etcd_network_peer_round_trip_time_seconds_bucket{job=~".*etcd.*"}[5m])) + > 0.15 + for: {{ dig "etcdMemberCommunicationSlow" "for" "10m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "etcdMemberCommunicationSlow" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.etcd }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.etcd }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.etcdHighNumberOfFailedProposals | default false) }} + - alert: etcdHighNumberOfFailedProposals + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.etcd }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.etcd | indent 8 }} +{{- end }} + description: 'etcd cluster "{{`{{`}} $labels.job {{`}}`}}": {{`{{`}} $value {{`}}`}} proposal failures within the last 30 minutes on etcd instance {{`{{`}} $labels.instance {{`}}`}}.' + summary: etcd cluster has high number of proposal failures. + expr: rate(etcd_server_proposals_failed_total{job=~".*etcd.*"}[15m]) > 5 + for: {{ dig "etcdHighNumberOfFailedProposals" "for" "15m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "etcdHighNumberOfFailedProposals" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.etcd }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.etcd }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.etcdHighFsyncDurations | default false) }} + - alert: etcdHighFsyncDurations + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.etcd }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.etcd | indent 8 }} +{{- end }} + description: 'etcd cluster "{{`{{`}} $labels.job {{`}}`}}": 99th percentile fsync durations are {{`{{`}} $value {{`}}`}}s on etcd instance {{`{{`}} $labels.instance {{`}}`}}.' + summary: etcd cluster 99th percentile fsync durations are too high. + expr: |- + histogram_quantile(0.99, rate(etcd_disk_wal_fsync_duration_seconds_bucket{job=~".*etcd.*"}[5m])) + > 0.5 + for: {{ dig "etcdHighFsyncDurations" "for" "10m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "etcdHighFsyncDurations" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.etcd }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.etcd }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.etcdHighFsyncDurations | default false) }} + - alert: etcdHighFsyncDurations + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.etcd }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.etcd | indent 8 }} +{{- end }} + description: 'etcd cluster "{{`{{`}} $labels.job {{`}}`}}": 99th percentile fsync durations are {{`{{`}} $value {{`}}`}}s on etcd instance {{`{{`}} $labels.instance {{`}}`}}.' + summary: etcd cluster 99th percentile fsync durations are too high. + expr: |- + histogram_quantile(0.99, rate(etcd_disk_wal_fsync_duration_seconds_bucket{job=~".*etcd.*"}[5m])) + > 1 + for: {{ dig "etcdHighFsyncDurations" "for" "10m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "etcdHighFsyncDurations" "severity" "critical" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.etcd }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.etcd }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.etcdHighCommitDurations | default false) }} + - alert: etcdHighCommitDurations + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.etcd }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.etcd | indent 8 }} +{{- end }} + description: 'etcd cluster "{{`{{`}} $labels.job {{`}}`}}": 99th percentile commit durations {{`{{`}} $value {{`}}`}}s on etcd instance {{`{{`}} $labels.instance {{`}}`}}.' + summary: etcd cluster 99th percentile commit durations are too high. + expr: |- + histogram_quantile(0.99, rate(etcd_disk_backend_commit_duration_seconds_bucket{job=~".*etcd.*"}[5m])) + > 0.25 + for: {{ dig "etcdHighCommitDurations" "for" "10m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "etcdHighCommitDurations" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.etcd }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.etcd }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.etcdDatabaseQuotaLowSpace | default false) }} + - alert: etcdDatabaseQuotaLowSpace + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.etcd }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.etcd | indent 8 }} +{{- end }} + description: 'etcd cluster "{{`{{`}} $labels.job {{`}}`}}": database size exceeds the defined quota on etcd instance {{`{{`}} $labels.instance {{`}}`}}, please defrag or increase the quota as the writes to etcd will be disabled when it is full.' + summary: etcd cluster database is running full. + expr: (last_over_time(etcd_mvcc_db_total_size_in_bytes{job=~".*etcd.*"}[5m]) / last_over_time(etcd_server_quota_backend_bytes{job=~".*etcd.*"}[5m]))*100 > 95 + for: {{ dig "etcdDatabaseQuotaLowSpace" "for" "10m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "etcdDatabaseQuotaLowSpace" "severity" "critical" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.etcd }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.etcd }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.etcdExcessiveDatabaseGrowth | default false) }} + - alert: etcdExcessiveDatabaseGrowth + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.etcd }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.etcd | indent 8 }} +{{- end }} + description: 'etcd cluster "{{`{{`}} $labels.job {{`}}`}}": Predicting running out of disk space in the next four hours, based on write observations within the past four hours on etcd instance {{`{{`}} $labels.instance {{`}}`}}, please check as it might be disruptive.' + summary: etcd cluster database growing very fast. + expr: predict_linear(etcd_mvcc_db_total_size_in_bytes{job=~".*etcd.*"}[4h], 4*60*60) > etcd_server_quota_backend_bytes{job=~".*etcd.*"} + for: {{ dig "etcdExcessiveDatabaseGrowth" "for" "10m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "etcdExcessiveDatabaseGrowth" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.etcd }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.etcd }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.etcdDatabaseHighFragmentationRatio | default false) }} + - alert: etcdDatabaseHighFragmentationRatio + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.etcd }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.etcd | indent 8 }} +{{- end }} + description: 'etcd cluster "{{`{{`}} $labels.job {{`}}`}}": database size in use on instance {{`{{`}} $labels.instance {{`}}`}} is {{`{{`}} $value | humanizePercentage {{`}}`}} of the actual allocated disk space, please run defragmentation (e.g. etcdctl defrag) to retrieve the unused fragmented disk space.' + runbook_url: https://etcd.io/docs/v3.5/op-guide/maintenance/#defragmentation + summary: etcd database size in use is less than 50% of the actual allocated storage. + expr: (last_over_time(etcd_mvcc_db_total_size_in_use_in_bytes{job=~".*etcd.*"}[5m]) / last_over_time(etcd_mvcc_db_total_size_in_bytes{job=~".*etcd.*"}[5m])) < 0.5 and etcd_mvcc_db_total_size_in_use_in_bytes{job=~".*etcd.*"} > 104857600 + for: {{ dig "etcdDatabaseHighFragmentationRatio" "for" "10m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "etcdDatabaseHighFragmentationRatio" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.etcd }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.etcd }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- end }} +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/general.rules.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/general.rules.yaml new file mode 100644 index 00000000000..8aca0b85f5e --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/general.rules.yaml @@ -0,0 +1,125 @@ +{{- /* +Generated from 'general.rules' group from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/a8ba97a150c75be42010c75d10b720c55e182f1a/manifests/kubePrometheus-prometheusRule.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.defaultRules.create .Values.defaultRules.rules.general }} +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "general.rules" | trunc 63 | trimSuffix "-" }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.defaultRules.labels }} +{{ toYaml .Values.defaultRules.labels | indent 4 }} +{{- end }} +{{- if .Values.defaultRules.annotations }} + annotations: +{{ toYaml .Values.defaultRules.annotations | indent 4 }} +{{- end }} +spec: + groups: + - name: general.rules + rules: +{{- if not (.Values.defaultRules.disabled.TargetDown | default false) }} + - alert: TargetDown + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.general }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.general | indent 8 }} +{{- end }} + description: '{{`{{`}} printf "%.4g" $value {{`}}`}}% of the {{`{{`}} $labels.job {{`}}`}}/{{`{{`}} $labels.service {{`}}`}} targets in {{`{{`}} $labels.namespace {{`}}`}} namespace are down.' + runbook_url: {{ .Values.defaultRules.runbookUrl }}/general/targetdown + summary: One or more targets are unreachable. + expr: 100 * (count(up == 0) BY (cluster, job, namespace, service) / count(up) BY (cluster, job, namespace, service)) > 10 + for: {{ dig "TargetDown" "for" "10m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "TargetDown" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.general }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.general }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.Watchdog | default false) }} + - alert: Watchdog + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.general }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.general | indent 8 }} +{{- end }} + description: 'This is an alert meant to ensure that the entire alerting pipeline is functional. + + This alert is always firing, therefore it should always be firing in Alertmanager + + and always fire against a receiver. There are integrations with various notification + + mechanisms that send a notification when this alert is not firing. For example the + + "DeadMansSnitch" integration in PagerDuty. + + ' + runbook_url: {{ .Values.defaultRules.runbookUrl }}/general/watchdog + summary: An alert that should always be firing to certify that Alertmanager is working properly. + expr: vector(1) + labels: + severity: {{ dig "Watchdog" "severity" "none" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.general }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.general }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.InfoInhibitor | default false) }} + - alert: InfoInhibitor + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.general }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.general | indent 8 }} +{{- end }} + description: 'This is an alert that is used to inhibit info alerts. + + By themselves, the info-level alerts are sometimes very noisy, but they are relevant when combined with + + other alerts. + + This alert fires whenever there''s a severity="info" alert, and stops firing when another alert with a + + severity of ''warning'' or ''critical'' starts firing on the same namespace. + + This alert should be routed to a null receiver and configured to inhibit alerts with severity="info". + + ' + runbook_url: {{ .Values.defaultRules.runbookUrl }}/general/infoinhibitor + summary: Info-level alert inhibition. + expr: ALERTS{severity = "info"} == 1 unless on ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}namespace) ALERTS{alertname != "InfoInhibitor", severity =~ "warning|critical", alertstate="firing"} == 1 + labels: + severity: {{ dig "InfoInhibitor" "severity" "none" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.general }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.general }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/k8s.rules.container_cpu_usage_seconds_total.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/k8s.rules.container_cpu_usage_seconds_total.yaml new file mode 100644 index 00000000000..9de5f5bc9c4 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/k8s.rules.container_cpu_usage_seconds_total.yaml @@ -0,0 +1,43 @@ +{{- /* +Generated from 'k8s.rules.container-cpu-usage-seconds-total' group from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/a8ba97a150c75be42010c75d10b720c55e182f1a/manifests/kubernetesControlPlane-prometheusRule.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.defaultRules.create .Values.defaultRules.rules.k8sContainerCpuUsageSecondsTotal }} +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "k8s.rules.container-cpu-usage-seconds-total" | trunc 63 | trimSuffix "-" }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.defaultRules.labels }} +{{ toYaml .Values.defaultRules.labels | indent 4 }} +{{- end }} +{{- if .Values.defaultRules.annotations }} + annotations: +{{ toYaml .Values.defaultRules.annotations | indent 4 }} +{{- end }} +spec: + groups: + - name: k8s.rules.container_cpu_usage_seconds_total + rules: + - expr: |- + sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, namespace, pod, container) ( + irate(container_cpu_usage_seconds_total{job="kubelet", metrics_path="/metrics/cadvisor", image!=""}[5m]) + ) * on ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, namespace, pod) group_left(node) topk by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, namespace, pod) ( + 1, max by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, namespace, pod, node) (kube_pod_info{node!=""}) + ) + record: node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.k8sContainerCpuUsageSecondsTotal }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.k8sContainerCpuUsageSecondsTotal }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/k8s.rules.container_memory_cache.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/k8s.rules.container_memory_cache.yaml new file mode 100644 index 00000000000..323f41f9cb5 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/k8s.rules.container_memory_cache.yaml @@ -0,0 +1,42 @@ +{{- /* +Generated from 'k8s.rules.container-memory-cache' group from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/a8ba97a150c75be42010c75d10b720c55e182f1a/manifests/kubernetesControlPlane-prometheusRule.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.defaultRules.create .Values.defaultRules.rules.k8sContainerMemoryCache }} +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "k8s.rules.container-memory-cache" | trunc 63 | trimSuffix "-" }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.defaultRules.labels }} +{{ toYaml .Values.defaultRules.labels | indent 4 }} +{{- end }} +{{- if .Values.defaultRules.annotations }} + annotations: +{{ toYaml .Values.defaultRules.annotations | indent 4 }} +{{- end }} +spec: + groups: + - name: k8s.rules.container_memory_cache + rules: + - expr: |- + container_memory_cache{job="kubelet", metrics_path="/metrics/cadvisor", image!=""} + * on ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, namespace, pod) group_left(node) topk by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, namespace, pod) (1, + max by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, namespace, pod, node) (kube_pod_info{node!=""}) + ) + record: node_namespace_pod_container:container_memory_cache + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.k8sContainerMemoryCache }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.k8sContainerMemoryCache }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/k8s.rules.container_memory_rss.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/k8s.rules.container_memory_rss.yaml new file mode 100644 index 00000000000..312d73c8892 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/k8s.rules.container_memory_rss.yaml @@ -0,0 +1,42 @@ +{{- /* +Generated from 'k8s.rules.container-memory-rss' group from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/a8ba97a150c75be42010c75d10b720c55e182f1a/manifests/kubernetesControlPlane-prometheusRule.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.defaultRules.create .Values.defaultRules.rules.k8sContainerMemoryRss }} +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "k8s.rules.container-memory-rss" | trunc 63 | trimSuffix "-" }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.defaultRules.labels }} +{{ toYaml .Values.defaultRules.labels | indent 4 }} +{{- end }} +{{- if .Values.defaultRules.annotations }} + annotations: +{{ toYaml .Values.defaultRules.annotations | indent 4 }} +{{- end }} +spec: + groups: + - name: k8s.rules.container_memory_rss + rules: + - expr: |- + container_memory_rss{job="kubelet", metrics_path="/metrics/cadvisor", image!=""} + * on ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, namespace, pod) group_left(node) topk by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, namespace, pod) (1, + max by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, namespace, pod, node) (kube_pod_info{node!=""}) + ) + record: node_namespace_pod_container:container_memory_rss + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.k8sContainerMemoryRss }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.k8sContainerMemoryRss }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/k8s.rules.container_memory_swap.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/k8s.rules.container_memory_swap.yaml new file mode 100644 index 00000000000..136595e8015 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/k8s.rules.container_memory_swap.yaml @@ -0,0 +1,42 @@ +{{- /* +Generated from 'k8s.rules.container-memory-swap' group from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/a8ba97a150c75be42010c75d10b720c55e182f1a/manifests/kubernetesControlPlane-prometheusRule.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.defaultRules.create .Values.defaultRules.rules.k8sContainerMemorySwap }} +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "k8s.rules.container-memory-swap" | trunc 63 | trimSuffix "-" }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.defaultRules.labels }} +{{ toYaml .Values.defaultRules.labels | indent 4 }} +{{- end }} +{{- if .Values.defaultRules.annotations }} + annotations: +{{ toYaml .Values.defaultRules.annotations | indent 4 }} +{{- end }} +spec: + groups: + - name: k8s.rules.container_memory_swap + rules: + - expr: |- + container_memory_swap{job="kubelet", metrics_path="/metrics/cadvisor", image!=""} + * on ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, namespace, pod) group_left(node) topk by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, namespace, pod) (1, + max by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, namespace, pod, node) (kube_pod_info{node!=""}) + ) + record: node_namespace_pod_container:container_memory_swap + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.k8sContainerMemorySwap }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.k8sContainerMemorySwap }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/k8s.rules.container_memory_working_set_bytes.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/k8s.rules.container_memory_working_set_bytes.yaml new file mode 100644 index 00000000000..d308b7473ab --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/k8s.rules.container_memory_working_set_bytes.yaml @@ -0,0 +1,42 @@ +{{- /* +Generated from 'k8s.rules.container-memory-working-set-bytes' group from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/a8ba97a150c75be42010c75d10b720c55e182f1a/manifests/kubernetesControlPlane-prometheusRule.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.defaultRules.create .Values.defaultRules.rules.k8sContainerMemoryWorkingSetBytes }} +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "k8s.rules.container-memory-working-set-bytes" | trunc 63 | trimSuffix "-" }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.defaultRules.labels }} +{{ toYaml .Values.defaultRules.labels | indent 4 }} +{{- end }} +{{- if .Values.defaultRules.annotations }} + annotations: +{{ toYaml .Values.defaultRules.annotations | indent 4 }} +{{- end }} +spec: + groups: + - name: k8s.rules.container_memory_working_set_bytes + rules: + - expr: |- + container_memory_working_set_bytes{job="kubelet", metrics_path="/metrics/cadvisor", image!=""} + * on ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, namespace, pod) group_left(node) topk by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, namespace, pod) (1, + max by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, namespace, pod, node) (kube_pod_info{node!=""}) + ) + record: node_namespace_pod_container:container_memory_working_set_bytes + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.k8sContainerMemoryWorkingSetBytes }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.k8sContainerMemoryWorkingSetBytes }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/k8s.rules.container_resource.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/k8s.rules.container_resource.yaml new file mode 100644 index 00000000000..2d896e59e48 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/k8s.rules.container_resource.yaml @@ -0,0 +1,168 @@ +{{- /* +Generated from 'k8s.rules.container-resource' group from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/a8ba97a150c75be42010c75d10b720c55e182f1a/manifests/kubernetesControlPlane-prometheusRule.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.defaultRules.create .Values.defaultRules.rules.k8sContainerResource }} +{{- $kubeStateMetricsJob := include "kube-prometheus-stack-kube-state-metrics.name" . }} +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "k8s.rules.container-resource" | trunc 63 | trimSuffix "-" }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.defaultRules.labels }} +{{ toYaml .Values.defaultRules.labels | indent 4 }} +{{- end }} +{{- if .Values.defaultRules.annotations }} + annotations: +{{ toYaml .Values.defaultRules.annotations | indent 4 }} +{{- end }} +spec: + groups: + - name: k8s.rules.container_resource + rules: + - expr: |- + kube_pod_container_resource_requests{resource="memory",job="{{ $kubeStateMetricsJob }}"} * on ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}namespace, pod, cluster) + group_left() max by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}namespace, pod, cluster) ( + (kube_pod_status_phase{phase=~"Pending|Running"} == 1) + ) + record: cluster:namespace:pod_memory:active:kube_pod_container_resource_requests + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.k8sContainerResource }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.k8sContainerResource }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: |- + sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}namespace, cluster) ( + sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}namespace, pod, cluster) ( + max by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}namespace, pod, container, cluster) ( + kube_pod_container_resource_requests{resource="memory",job="{{ $kubeStateMetricsJob }}"} + ) * on ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}namespace, pod, cluster) group_left() max by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}namespace, pod, cluster) ( + kube_pod_status_phase{phase=~"Pending|Running"} == 1 + ) + ) + ) + record: namespace_memory:kube_pod_container_resource_requests:sum + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.k8sContainerResource }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.k8sContainerResource }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: |- + kube_pod_container_resource_requests{resource="cpu",job="{{ $kubeStateMetricsJob }}"} * on ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}namespace, pod, cluster) + group_left() max by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}namespace, pod, cluster) ( + (kube_pod_status_phase{phase=~"Pending|Running"} == 1) + ) + record: cluster:namespace:pod_cpu:active:kube_pod_container_resource_requests + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.k8sContainerResource }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.k8sContainerResource }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: |- + sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}namespace, cluster) ( + sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}namespace, pod, cluster) ( + max by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}namespace, pod, container, cluster) ( + kube_pod_container_resource_requests{resource="cpu",job="{{ $kubeStateMetricsJob }}"} + ) * on ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}namespace, pod, cluster) group_left() max by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}namespace, pod, cluster) ( + kube_pod_status_phase{phase=~"Pending|Running"} == 1 + ) + ) + ) + record: namespace_cpu:kube_pod_container_resource_requests:sum + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.k8sContainerResource }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.k8sContainerResource }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: |- + kube_pod_container_resource_limits{resource="memory",job="{{ $kubeStateMetricsJob }}"} * on ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}namespace, pod, cluster) + group_left() max by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}namespace, pod, cluster) ( + (kube_pod_status_phase{phase=~"Pending|Running"} == 1) + ) + record: cluster:namespace:pod_memory:active:kube_pod_container_resource_limits + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.k8sContainerResource }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.k8sContainerResource }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: |- + sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}namespace, cluster) ( + sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}namespace, pod, cluster) ( + max by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}namespace, pod, container, cluster) ( + kube_pod_container_resource_limits{resource="memory",job="{{ $kubeStateMetricsJob }}"} + ) * on ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}namespace, pod, cluster) group_left() max by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}namespace, pod, cluster) ( + kube_pod_status_phase{phase=~"Pending|Running"} == 1 + ) + ) + ) + record: namespace_memory:kube_pod_container_resource_limits:sum + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.k8sContainerResource }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.k8sContainerResource }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: |- + kube_pod_container_resource_limits{resource="cpu",job="{{ $kubeStateMetricsJob }}"} * on ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}namespace, pod, cluster) + group_left() max by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}namespace, pod, cluster) ( + (kube_pod_status_phase{phase=~"Pending|Running"} == 1) + ) + record: cluster:namespace:pod_cpu:active:kube_pod_container_resource_limits + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.k8sContainerResource }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.k8sContainerResource }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: |- + sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}namespace, cluster) ( + sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}namespace, pod, cluster) ( + max by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}namespace, pod, container, cluster) ( + kube_pod_container_resource_limits{resource="cpu",job="{{ $kubeStateMetricsJob }}"} + ) * on ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}namespace, pod, cluster) group_left() max by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}namespace, pod, cluster) ( + kube_pod_status_phase{phase=~"Pending|Running"} == 1 + ) + ) + ) + record: namespace_cpu:kube_pod_container_resource_limits:sum + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.k8sContainerResource }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.k8sContainerResource }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/k8s.rules.pod_owner.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/k8s.rules.pod_owner.yaml new file mode 100644 index 00000000000..4915b25e732 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/k8s.rules.pod_owner.yaml @@ -0,0 +1,107 @@ +{{- /* +Generated from 'k8s.rules.pod-owner' group from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/a8ba97a150c75be42010c75d10b720c55e182f1a/manifests/kubernetesControlPlane-prometheusRule.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.defaultRules.create .Values.defaultRules.rules.k8sPodOwner }} +{{- $kubeStateMetricsJob := include "kube-prometheus-stack-kube-state-metrics.name" . }} +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "k8s.rules.pod-owner" | trunc 63 | trimSuffix "-" }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.defaultRules.labels }} +{{ toYaml .Values.defaultRules.labels | indent 4 }} +{{- end }} +{{- if .Values.defaultRules.annotations }} + annotations: +{{ toYaml .Values.defaultRules.annotations | indent 4 }} +{{- end }} +spec: + groups: + - name: k8s.rules.pod_owner + rules: + - expr: |- + max by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, namespace, workload, pod) ( + label_replace( + label_replace( + kube_pod_owner{job="{{ $kubeStateMetricsJob }}", owner_kind="ReplicaSet"}, + "replicaset", "$1", "owner_name", "(.*)" + ) * on ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}replicaset, namespace) group_left(owner_name) topk by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}replicaset, namespace) ( + 1, max by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}replicaset, namespace, owner_name) ( + kube_replicaset_owner{job="{{ $kubeStateMetricsJob }}"} + ) + ), + "workload", "$1", "owner_name", "(.*)" + ) + ) + labels: + workload_type: deployment + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.k8sPodOwner }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.k8sPodOwner }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + record: namespace_workload_pod:kube_pod_owner:relabel + - expr: |- + max by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, namespace, workload, pod) ( + label_replace( + kube_pod_owner{job="{{ $kubeStateMetricsJob }}", owner_kind="DaemonSet"}, + "workload", "$1", "owner_name", "(.*)" + ) + ) + labels: + workload_type: daemonset + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.k8sPodOwner }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.k8sPodOwner }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + record: namespace_workload_pod:kube_pod_owner:relabel + - expr: |- + max by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, namespace, workload, pod) ( + label_replace( + kube_pod_owner{job="{{ $kubeStateMetricsJob }}", owner_kind="StatefulSet"}, + "workload", "$1", "owner_name", "(.*)" + ) + ) + labels: + workload_type: statefulset + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.k8sPodOwner }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.k8sPodOwner }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + record: namespace_workload_pod:kube_pod_owner:relabel + - expr: |- + max by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, namespace, workload, pod) ( + label_replace( + kube_pod_owner{job="{{ $kubeStateMetricsJob }}", owner_kind="Job"}, + "workload", "$1", "owner_name", "(.*)" + ) + ) + labels: + workload_type: job + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.k8sPodOwner }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.k8sPodOwner }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + record: namespace_workload_pod:kube_pod_owner:relabel +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/k8s.rules.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/k8s.rules.yaml new file mode 100644 index 00000000000..c61bd222abf --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/k8s.rules.yaml @@ -0,0 +1,237 @@ +{{- /* +Generated from 'k8s.rules' group from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/main/manifests/kubernetesControlPlane-prometheusRule.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.defaultRules.create .Values.defaultRules.rules.k8s }} +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "k8s.rules" | trunc 63 | trimSuffix "-" }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.defaultRules.labels }} +{{ toYaml .Values.defaultRules.labels | indent 4 }} +{{- end }} +{{- if .Values.defaultRules.annotations }} + annotations: +{{ toYaml .Values.defaultRules.annotations | indent 4 }} +{{- end }} +spec: + groups: + - name: k8s.rules + rules: + - expr: |- + sum by (cluster, namespace, pod, container) ( + irate(container_cpu_usage_seconds_total{job="{{ include "exporter.kubelet.jobName" . }}", metrics_path="/metrics/cadvisor", image!=""}[5m]) + ) * on (cluster, namespace, pod) group_left(node) topk by (cluster, namespace, pod) ( + 1, max by(cluster, namespace, pod, node) (kube_pod_info{node!=""}) + ) + record: node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate + {{- if .Values.defaultRules.additionalRuleLabels }} + labels: + {{ toYaml .Values.defaultRules.additionalRuleLabels | nindent 8 }} + {{- end }} + - expr: |- + container_memory_working_set_bytes{job="kubelet", metrics_path="/metrics/cadvisor", image!=""} + * on (cluster, namespace, pod) group_left(node) topk by(cluster, namespace, pod) (1, + max by(cluster, namespace, pod, node) (kube_pod_info{node!=""}) + ) + record: node_namespace_pod_container:container_memory_working_set_bytes + {{- if .Values.defaultRules.additionalRuleLabels }} + labels: + {{ toYaml .Values.defaultRules.additionalRuleLabels | nindent 8 }} + {{- end }} + - expr: |- + container_memory_rss{job="kubelet", metrics_path="/metrics/cadvisor", image!=""} + * on (cluster, namespace, pod) group_left(node) topk by(cluster, namespace, pod) (1, + max by(cluster, namespace, pod, node) (kube_pod_info{node!=""}) + ) + record: node_namespace_pod_container:container_memory_rss + {{- if .Values.defaultRules.additionalRuleLabels }} + labels: + {{ toYaml .Values.defaultRules.additionalRuleLabels | nindent 8 }} + {{- end }} + - expr: |- + container_memory_cache{job="kubelet", metrics_path="/metrics/cadvisor", image!=""} + * on (cluster, namespace, pod) group_left(node) topk by(cluster, namespace, pod) (1, + max by(cluster, namespace, pod, node) (kube_pod_info{node!=""}) + ) + record: node_namespace_pod_container:container_memory_cache + {{- if .Values.defaultRules.additionalRuleLabels }} + labels: + {{ toYaml .Values.defaultRules.additionalRuleLabels | nindent 8 }} + {{- end }} + - expr: |- + container_memory_swap{job="kubelet", metrics_path="/metrics/cadvisor", image!=""} + * on (cluster, namespace, pod) group_left(node) topk by(cluster, namespace, pod) (1, + max by(cluster, namespace, pod, node) (kube_pod_info{node!=""}) + ) + record: node_namespace_pod_container:container_memory_swap + {{- if .Values.defaultRules.additionalRuleLabels }} + labels: + {{ toYaml .Values.defaultRules.additionalRuleLabels | nindent 8 }} + {{- end }} + - expr: |- + kube_pod_container_resource_requests{resource="memory",job="kube-state-metrics"} * on (namespace, pod, cluster) + group_left() max by (namespace, pod, cluster) ( + (kube_pod_status_phase{phase=~"Pending|Running"} == 1) + ) + record: cluster:namespace:pod_memory:active:kube_pod_container_resource_requests + {{- if .Values.defaultRules.additionalRuleLabels }} + labels: + {{ toYaml .Values.defaultRules.additionalRuleLabels | nindent 8 }} + {{- end }} + - expr: |- + sum by (namespace, cluster) ( + sum by (namespace, pod, cluster) ( + max by (namespace, pod, container, cluster) ( + kube_pod_container_resource_requests{resource="memory",job="kube-state-metrics"} + ) * on(namespace, pod, cluster) group_left() max by (namespace, pod, cluster) ( + kube_pod_status_phase{phase=~"Pending|Running"} == 1 + ) + ) + ) + record: namespace_memory:kube_pod_container_resource_requests:sum + {{- if .Values.defaultRules.additionalRuleLabels }} + labels: + {{ toYaml .Values.defaultRules.additionalRuleLabels | nindent 8 }} + {{- end }} + - expr: |- + kube_pod_container_resource_requests{resource="cpu",job="kube-state-metrics"} * on (namespace, pod, cluster) + group_left() max by (namespace, pod, cluster) ( + (kube_pod_status_phase{phase=~"Pending|Running"} == 1) + ) + record: cluster:namespace:pod_cpu:active:kube_pod_container_resource_requests + {{- if .Values.defaultRules.additionalRuleLabels }} + labels: + {{ toYaml .Values.defaultRules.additionalRuleLabels | nindent 8 }} + {{- end }} + - expr: |- + sum by (namespace, cluster) ( + sum by (namespace, pod, cluster) ( + max by (namespace, pod, container, cluster) ( + kube_pod_container_resource_requests{resource="cpu",job="kube-state-metrics"} + ) * on(namespace, pod, cluster) group_left() max by (namespace, pod, cluster) ( + kube_pod_status_phase{phase=~"Pending|Running"} == 1 + ) + ) + ) + record: namespace_cpu:kube_pod_container_resource_requests:sum + {{- if .Values.defaultRules.additionalRuleLabels }} + labels: + {{ toYaml .Values.defaultRules.additionalRuleLabels | nindent 8 }} + {{- end }} + - expr: |- + kube_pod_container_resource_limits{resource="memory",job="kube-state-metrics"} * on (namespace, pod, cluster) + group_left() max by (namespace, pod, cluster) ( + (kube_pod_status_phase{phase=~"Pending|Running"} == 1) + ) + record: cluster:namespace:pod_memory:active:kube_pod_container_resource_limits + {{- if .Values.defaultRules.additionalRuleLabels }} + labels: + {{ toYaml .Values.defaultRules.additionalRuleLabels | nindent 8 }} + {{- end }} + - expr: |- + sum by (namespace, cluster) ( + sum by (namespace, pod, cluster) ( + max by (namespace, pod, container, cluster) ( + kube_pod_container_resource_limits{resource="memory",job="kube-state-metrics"} + ) * on(namespace, pod, cluster) group_left() max by (namespace, pod, cluster) ( + kube_pod_status_phase{phase=~"Pending|Running"} == 1 + ) + ) + ) + record: namespace_memory:kube_pod_container_resource_limits:sum + {{- if .Values.defaultRules.additionalRuleLabels }} + labels: + {{ toYaml .Values.defaultRules.additionalRuleLabels | nindent 8 }} + {{- end }} + - expr: |- + kube_pod_container_resource_limits{resource="cpu",job="kube-state-metrics"} * on (namespace, pod, cluster) + group_left() max by (namespace, pod, cluster) ( + (kube_pod_status_phase{phase=~"Pending|Running"} == 1) + ) + record: cluster:namespace:pod_cpu:active:kube_pod_container_resource_limits + {{- if .Values.defaultRules.additionalRuleLabels }} + labels: + {{ toYaml .Values.defaultRules.additionalRuleLabels | nindent 8 }} + {{- end }} + - expr: |- + sum by (namespace, cluster) ( + sum by (namespace, pod, cluster) ( + max by (namespace, pod, container, cluster) ( + kube_pod_container_resource_limits{resource="cpu",job="kube-state-metrics"} + ) * on(namespace, pod, cluster) group_left() max by (namespace, pod, cluster) ( + kube_pod_status_phase{phase=~"Pending|Running"} == 1 + ) + ) + ) + record: namespace_cpu:kube_pod_container_resource_limits:sum + {{- if .Values.defaultRules.additionalRuleLabels }} + labels: + {{ toYaml .Values.defaultRules.additionalRuleLabels | nindent 8 }} + {{- end }} + - expr: |- + max by (cluster, namespace, workload, pod) ( + label_replace( + label_replace( + kube_pod_owner{job="kube-state-metrics", owner_kind="ReplicaSet"}, + "replicaset", "$1", "owner_name", "(.*)" + ) * on(replicaset, namespace) group_left(owner_name) topk by(replicaset, namespace) ( + 1, max by (replicaset, namespace, owner_name) ( + kube_replicaset_owner{job="kube-state-metrics"} + ) + ), + "workload", "$1", "owner_name", "(.*)" + ) + ) + labels: + workload_type: deployment + {{- if .Values.defaultRules.additionalRuleLabels }} + {{ toYaml .Values.defaultRules.additionalRuleLabels | nindent 8 }} + {{- end }} + record: namespace_workload_pod:kube_pod_owner:relabel + - expr: |- + max by (cluster, namespace, workload, pod) ( + label_replace( + kube_pod_owner{job="kube-state-metrics", owner_kind="DaemonSet"}, + "workload", "$1", "owner_name", "(.*)" + ) + ) + labels: + workload_type: daemonset + {{- if .Values.defaultRules.additionalRuleLabels }} + {{ toYaml .Values.defaultRules.additionalRuleLabels | nindent 8 }} + {{- end }} + record: namespace_workload_pod:kube_pod_owner:relabel + - expr: |- + max by (cluster, namespace, workload, pod) ( + label_replace( + kube_pod_owner{job="kube-state-metrics", owner_kind="StatefulSet"}, + "workload", "$1", "owner_name", "(.*)" + ) + ) + labels: + workload_type: statefulset + {{- if .Values.defaultRules.additionalRuleLabels }} + {{ toYaml .Values.defaultRules.additionalRuleLabels | nindent 8 }} + {{- end }} + record: namespace_workload_pod:kube_pod_owner:relabel + - expr: |- + max by (cluster, namespace, workload, pod) ( + label_replace( + kube_pod_owner{job="kube-state-metrics", owner_kind="Job"}, + "workload", "$1", "owner_name", "(.*)" + ) + ) + labels: + workload_type: job + {{- if .Values.defaultRules.additionalRuleLabels }} + {{ toYaml .Values.defaultRules.additionalRuleLabels | nindent 8 }} + {{- end }} + record: namespace_workload_pod:kube_pod_owner:relabel +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/kube-apiserver-availability.rules.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/kube-apiserver-availability.rules.yaml new file mode 100644 index 00000000000..6194e9c6142 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/kube-apiserver-availability.rules.yaml @@ -0,0 +1,273 @@ +{{- /* +Generated from 'kube-apiserver-availability.rules' group from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/a8ba97a150c75be42010c75d10b720c55e182f1a/manifests/kubernetesControlPlane-prometheusRule.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.defaultRules.create .Values.kubeApiServer.enabled .Values.defaultRules.rules.kubeApiserverAvailability }} +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "kube-apiserver-availability.rules" | trunc 63 | trimSuffix "-" }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.defaultRules.labels }} +{{ toYaml .Values.defaultRules.labels | indent 4 }} +{{- end }} +{{- if .Values.defaultRules.annotations }} + annotations: +{{ toYaml .Values.defaultRules.annotations | indent 4 }} +{{- end }} +spec: + groups: + - interval: 3m + name: kube-apiserver-availability.rules + rules: + - expr: avg_over_time(code_verb:apiserver_request_total:increase1h[30d]) * 24 * 30 + record: code_verb:apiserver_request_total:increase30d + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverAvailability }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverAvailability }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, code) (code_verb:apiserver_request_total:increase30d{verb=~"LIST|GET"}) + labels: + verb: read + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverAvailability }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverAvailability }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + record: code:apiserver_request_total:increase30d + - expr: sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, code) (code_verb:apiserver_request_total:increase30d{verb=~"POST|PUT|PATCH|DELETE"}) + labels: + verb: write + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverAvailability }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverAvailability }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + record: code:apiserver_request_total:increase30d + - expr: sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, verb, scope) (increase(apiserver_request_sli_duration_seconds_count{job="apiserver"}[1h])) + record: cluster_verb_scope:apiserver_request_sli_duration_seconds_count:increase1h + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverAvailability }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverAvailability }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, verb, scope) (avg_over_time(cluster_verb_scope:apiserver_request_sli_duration_seconds_count:increase1h[30d]) * 24 * 30) + record: cluster_verb_scope:apiserver_request_sli_duration_seconds_count:increase30d + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverAvailability }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverAvailability }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, verb, scope, le) (increase(apiserver_request_sli_duration_seconds_bucket[1h])) + record: cluster_verb_scope_le:apiserver_request_sli_duration_seconds_bucket:increase1h + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverAvailability }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverAvailability }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, verb, scope, le) (avg_over_time(cluster_verb_scope_le:apiserver_request_sli_duration_seconds_bucket:increase1h[30d]) * 24 * 30) + record: cluster_verb_scope_le:apiserver_request_sli_duration_seconds_bucket:increase30d + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverAvailability }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverAvailability }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: |- + 1 - ( + ( + # write too slow + sum by (cluster) (cluster_verb_scope:apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_count:increase30d{verb=~"POST|PUT|PATCH|DELETE"}) + - + sum by (cluster) (cluster_verb_scope_le:apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_bucket:increase30d{verb=~"POST|PUT|PATCH|DELETE",le="1"}) + ) + + ( + # read too slow + sum by (cluster) (cluster_verb_scope:apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_count:increase30d{verb=~"LIST|GET"}) + - + ( + ( + sum by (cluster) (cluster_verb_scope_le:apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_bucket:increase30d{verb=~"LIST|GET",scope=~"resource|",le="1"}) + or + vector(0) + ) + + + sum by (cluster) (cluster_verb_scope_le:apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_bucket:increase30d{verb=~"LIST|GET",scope="namespace",le="5"}) + + + sum by (cluster) (cluster_verb_scope_le:apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_bucket:increase30d{verb=~"LIST|GET",scope="cluster",le="30"}) + ) + ) + + # errors + sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) (code:apiserver_request_total:increase30d{code=~"5.."} or vector(0)) + ) + / + sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) (code:apiserver_request_total:increase30d) + labels: + verb: all + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverAvailability }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverAvailability }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + record: apiserver_request:availability30d + - expr: |- + 1 - ( + sum by (cluster) (cluster_verb_scope:apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_count:increase30d{verb=~"LIST|GET"}) + - + ( + # too slow + ( + sum by (cluster) (cluster_verb_scope_le:apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_bucket:increase30d{verb=~"LIST|GET",scope=~"resource|",le="1"}) + or + vector(0) + ) + + + sum by (cluster) (cluster_verb_scope_le:apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_bucket:increase30d{verb=~"LIST|GET",scope="namespace",le="5"}) + + + sum by (cluster) (cluster_verb_scope_le:apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_bucket:increase30d{verb=~"LIST|GET",scope="cluster",le="30"}) + ) + + + # errors + sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) (code:apiserver_request_total:increase30d{verb="read",code=~"5.."} or vector(0)) + ) + / + sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) (code:apiserver_request_total:increase30d{verb="read"}) + labels: + verb: read + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverAvailability }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverAvailability }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + record: apiserver_request:availability30d + - expr: |- + 1 - ( + ( + # too slow + sum by (cluster) (cluster_verb_scope:apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_count:increase30d{verb=~"POST|PUT|PATCH|DELETE"}) + - + sum by (cluster) (cluster_verb_scope_le:apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_bucket:increase30d{verb=~"POST|PUT|PATCH|DELETE",le="1"}) + ) + + + # errors + sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) (code:apiserver_request_total:increase30d{verb="write",code=~"5.."} or vector(0)) + ) + / + sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) (code:apiserver_request_total:increase30d{verb="write"}) + labels: + verb: write + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverAvailability }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverAvailability }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + record: apiserver_request:availability30d + - expr: sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster,code,resource) (rate(apiserver_request_total{job="apiserver",verb=~"LIST|GET"}[5m])) + labels: + verb: read + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverAvailability }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverAvailability }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + record: code_resource:apiserver_request_total:rate5m + - expr: sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster,code,resource) (rate(apiserver_request_total{job="apiserver",verb=~"POST|PUT|PATCH|DELETE"}[5m])) + labels: + verb: write + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverAvailability }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverAvailability }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + record: code_resource:apiserver_request_total:rate5m + - expr: sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, code, verb) (increase(apiserver_request_total{job="apiserver",verb=~"LIST|GET|POST|PUT|PATCH|DELETE",code=~"2.."}[1h])) + record: code_verb:apiserver_request_total:increase1h + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverAvailability }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverAvailability }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, code, verb) (increase(apiserver_request_total{job="apiserver",verb=~"LIST|GET|POST|PUT|PATCH|DELETE",code=~"3.."}[1h])) + record: code_verb:apiserver_request_total:increase1h + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverAvailability }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverAvailability }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, code, verb) (increase(apiserver_request_total{job="apiserver",verb=~"LIST|GET|POST|PUT|PATCH|DELETE",code=~"4.."}[1h])) + record: code_verb:apiserver_request_total:increase1h + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverAvailability }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverAvailability }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, code, verb) (increase(apiserver_request_total{job="apiserver",verb=~"LIST|GET|POST|PUT|PATCH|DELETE",code=~"5.."}[1h])) + record: code_verb:apiserver_request_total:increase1h + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverAvailability }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverAvailability }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/kube-apiserver-burnrate.rules.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/kube-apiserver-burnrate.rules.yaml new file mode 100644 index 00000000000..e6666a6f41b --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/kube-apiserver-burnrate.rules.yaml @@ -0,0 +1,440 @@ +{{- /* +Generated from 'kube-apiserver-burnrate.rules' group from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/a8ba97a150c75be42010c75d10b720c55e182f1a/manifests/kubernetesControlPlane-prometheusRule.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.defaultRules.create .Values.kubeApiServer.enabled .Values.defaultRules.rules.kubeApiserverBurnrate }} +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "kube-apiserver-burnrate.rules" | trunc 63 | trimSuffix "-" }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.defaultRules.labels }} +{{ toYaml .Values.defaultRules.labels | indent 4 }} +{{- end }} +{{- if .Values.defaultRules.annotations }} + annotations: +{{ toYaml .Values.defaultRules.annotations | indent 4 }} +{{- end }} +spec: + groups: + - name: kube-apiserver-burnrate.rules + rules: + - expr: |- + ( + ( + # too slow + sum by (cluster) (rate(apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_count{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward"}[1d])) + - + ( + ( + sum by (cluster) (rate(apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward",scope=~"resource|",le="1"}[1d])) + or + vector(0) + ) + + + sum by (cluster) (rate(apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward",scope="namespace",le="5"}[1d])) + + + sum by (cluster) (rate(apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward",scope="cluster",le="30"}[1d])) + ) + ) + + + # errors + sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) (rate(apiserver_request_total{job="apiserver",verb=~"LIST|GET",code=~"5.."}[1d])) + ) + / + sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) (rate(apiserver_request_total{job="apiserver",verb=~"LIST|GET"}[1d])) + labels: + verb: read + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverBurnrate }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverBurnrate }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + record: apiserver_request:burnrate1d + - expr: |- + ( + ( + # too slow + sum by (cluster) (rate(apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_count{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward"}[1h])) + - + ( + ( + sum by (cluster) (rate(apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward",scope=~"resource|",le="1"}[1h])) + or + vector(0) + ) + + + sum by (cluster) (rate(apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward",scope="namespace",le="5"}[1h])) + + + sum by (cluster) (rate(apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward",scope="cluster",le="30"}[1h])) + ) + ) + + + # errors + sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) (rate(apiserver_request_total{job="apiserver",verb=~"LIST|GET",code=~"5.."}[1h])) + ) + / + sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) (rate(apiserver_request_total{job="apiserver",verb=~"LIST|GET"}[1h])) + labels: + verb: read + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverBurnrate }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverBurnrate }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + record: apiserver_request:burnrate1h + - expr: |- + ( + ( + # too slow + sum by (cluster) (rate(apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_count{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward"}[2h])) + - + ( + ( + sum by (cluster) (rate(apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward",scope=~"resource|",le="1"}[2h])) + or + vector(0) + ) + + + sum by (cluster) (rate(apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward",scope="namespace",le="5"}[2h])) + + + sum by (cluster) (rate(apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward",scope="cluster",le="30"}[2h])) + ) + ) + + + # errors + sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) (rate(apiserver_request_total{job="apiserver",verb=~"LIST|GET",code=~"5.."}[2h])) + ) + / + sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) (rate(apiserver_request_total{job="apiserver",verb=~"LIST|GET"}[2h])) + labels: + verb: read + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverBurnrate }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverBurnrate }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + record: apiserver_request:burnrate2h + - expr: |- + ( + ( + # too slow + sum by (cluster) (rate(apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_count{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward"}[30m])) + - + ( + ( + sum by (cluster) (rate(apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward",scope=~"resource|",le="1"}[30m])) + or + vector(0) + ) + + + sum by (cluster) (rate(apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward",scope="namespace",le="5"}[30m])) + + + sum by (cluster) (rate(apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward",scope="cluster",le="30"}[30m])) + ) + ) + + + # errors + sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) (rate(apiserver_request_total{job="apiserver",verb=~"LIST|GET",code=~"5.."}[30m])) + ) + / + sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) (rate(apiserver_request_total{job="apiserver",verb=~"LIST|GET"}[30m])) + labels: + verb: read + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverBurnrate }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverBurnrate }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + record: apiserver_request:burnrate30m + - expr: |- + ( + ( + # too slow + sum by (cluster) (rate(apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_count{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward"}[3d])) + - + ( + ( + sum by (cluster) (rate(apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward",scope=~"resource|",le="1"}[3d])) + or + vector(0) + ) + + + sum by (cluster) (rate(apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward",scope="namespace",le="5"}[3d])) + + + sum by (cluster) (rate(apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward",scope="cluster",le="30"}[3d])) + ) + ) + + + # errors + sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) (rate(apiserver_request_total{job="apiserver",verb=~"LIST|GET",code=~"5.."}[3d])) + ) + / + sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) (rate(apiserver_request_total{job="apiserver",verb=~"LIST|GET"}[3d])) + labels: + verb: read + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverBurnrate }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverBurnrate }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + record: apiserver_request:burnrate3d + - expr: |- + ( + ( + # too slow + sum by (cluster) (rate(apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_count{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward"}[5m])) + - + ( + ( + sum by (cluster) (rate(apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward",scope=~"resource|",le="1"}[5m])) + or + vector(0) + ) + + + sum by (cluster) (rate(apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward",scope="namespace",le="5"}[5m])) + + + sum by (cluster) (rate(apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward",scope="cluster",le="30"}[5m])) + ) + ) + + + # errors + sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) (rate(apiserver_request_total{job="apiserver",verb=~"LIST|GET",code=~"5.."}[5m])) + ) + / + sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) (rate(apiserver_request_total{job="apiserver",verb=~"LIST|GET"}[5m])) + labels: + verb: read + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverBurnrate }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverBurnrate }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + record: apiserver_request:burnrate5m + - expr: |- + ( + ( + # too slow + sum by (cluster) (rate(apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_count{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward"}[6h])) + - + ( + ( + sum by (cluster) (rate(apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward",scope=~"resource|",le="1"}[6h])) + or + vector(0) + ) + + + sum by (cluster) (rate(apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward",scope="namespace",le="5"}[6h])) + + + sum by (cluster) (rate(apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward",scope="cluster",le="30"}[6h])) + ) + ) + + + # errors + sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) (rate(apiserver_request_total{job="apiserver",verb=~"LIST|GET",code=~"5.."}[6h])) + ) + / + sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) (rate(apiserver_request_total{job="apiserver",verb=~"LIST|GET"}[6h])) + labels: + verb: read + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverBurnrate }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverBurnrate }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + record: apiserver_request:burnrate6h + - expr: |- + ( + ( + # too slow + sum by (cluster) (rate(apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_count{job="apiserver",verb=~"POST|PUT|PATCH|DELETE",subresource!~"proxy|attach|log|exec|portforward"}[1d])) + - + sum by (cluster) (rate(apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_bucket{job="apiserver",verb=~"POST|PUT|PATCH|DELETE",subresource!~"proxy|attach|log|exec|portforward",le="1"}[1d])) + ) + + + sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) (rate(apiserver_request_total{job="apiserver",verb=~"POST|PUT|PATCH|DELETE",code=~"5.."}[1d])) + ) + / + sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) (rate(apiserver_request_total{job="apiserver",verb=~"POST|PUT|PATCH|DELETE"}[1d])) + labels: + verb: write + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverBurnrate }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverBurnrate }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + record: apiserver_request:burnrate1d + - expr: |- + ( + ( + # too slow + sum by (cluster) (rate(apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_count{job="apiserver",verb=~"POST|PUT|PATCH|DELETE",subresource!~"proxy|attach|log|exec|portforward"}[1h])) + - + sum by (cluster) (rate(apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_bucket{job="apiserver",verb=~"POST|PUT|PATCH|DELETE",subresource!~"proxy|attach|log|exec|portforward",le="1"}[1h])) + ) + + + sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) (rate(apiserver_request_total{job="apiserver",verb=~"POST|PUT|PATCH|DELETE",code=~"5.."}[1h])) + ) + / + sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) (rate(apiserver_request_total{job="apiserver",verb=~"POST|PUT|PATCH|DELETE"}[1h])) + labels: + verb: write + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverBurnrate }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverBurnrate }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + record: apiserver_request:burnrate1h + - expr: |- + ( + ( + # too slow + sum by (cluster) (rate(apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_count{job="apiserver",verb=~"POST|PUT|PATCH|DELETE",subresource!~"proxy|attach|log|exec|portforward"}[2h])) + - + sum by (cluster) (rate(apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_bucket{job="apiserver",verb=~"POST|PUT|PATCH|DELETE",subresource!~"proxy|attach|log|exec|portforward",le="1"}[2h])) + ) + + + sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) (rate(apiserver_request_total{job="apiserver",verb=~"POST|PUT|PATCH|DELETE",code=~"5.."}[2h])) + ) + / + sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) (rate(apiserver_request_total{job="apiserver",verb=~"POST|PUT|PATCH|DELETE"}[2h])) + labels: + verb: write + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverBurnrate }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverBurnrate }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + record: apiserver_request:burnrate2h + - expr: |- + ( + ( + # too slow + sum by (cluster) (rate(apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_count{job="apiserver",verb=~"POST|PUT|PATCH|DELETE",subresource!~"proxy|attach|log|exec|portforward"}[30m])) + - + sum by (cluster) (rate(apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_bucket{job="apiserver",verb=~"POST|PUT|PATCH|DELETE",subresource!~"proxy|attach|log|exec|portforward",le="1"}[30m])) + ) + + + sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) (rate(apiserver_request_total{job="apiserver",verb=~"POST|PUT|PATCH|DELETE",code=~"5.."}[30m])) + ) + / + sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) (rate(apiserver_request_total{job="apiserver",verb=~"POST|PUT|PATCH|DELETE"}[30m])) + labels: + verb: write + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverBurnrate }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverBurnrate }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + record: apiserver_request:burnrate30m + - expr: |- + ( + ( + # too slow + sum by (cluster) (rate(apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_count{job="apiserver",verb=~"POST|PUT|PATCH|DELETE",subresource!~"proxy|attach|log|exec|portforward"}[3d])) + - + sum by (cluster) (rate(apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_bucket{job="apiserver",verb=~"POST|PUT|PATCH|DELETE",subresource!~"proxy|attach|log|exec|portforward",le="1"}[3d])) + ) + + + sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) (rate(apiserver_request_total{job="apiserver",verb=~"POST|PUT|PATCH|DELETE",code=~"5.."}[3d])) + ) + / + sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) (rate(apiserver_request_total{job="apiserver",verb=~"POST|PUT|PATCH|DELETE"}[3d])) + labels: + verb: write + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverBurnrate }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverBurnrate }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + record: apiserver_request:burnrate3d + - expr: |- + ( + ( + # too slow + sum by (cluster) (rate(apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_count{job="apiserver",verb=~"POST|PUT|PATCH|DELETE",subresource!~"proxy|attach|log|exec|portforward"}[5m])) + - + sum by (cluster) (rate(apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_bucket{job="apiserver",verb=~"POST|PUT|PATCH|DELETE",subresource!~"proxy|attach|log|exec|portforward",le="1"}[5m])) + ) + + + sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) (rate(apiserver_request_total{job="apiserver",verb=~"POST|PUT|PATCH|DELETE",code=~"5.."}[5m])) + ) + / + sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) (rate(apiserver_request_total{job="apiserver",verb=~"POST|PUT|PATCH|DELETE"}[5m])) + labels: + verb: write + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverBurnrate }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverBurnrate }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + record: apiserver_request:burnrate5m + - expr: |- + ( + ( + # too slow + sum by (cluster) (rate(apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_count{job="apiserver",verb=~"POST|PUT|PATCH|DELETE",subresource!~"proxy|attach|log|exec|portforward"}[6h])) + - + sum by (cluster) (rate(apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_bucket{job="apiserver",verb=~"POST|PUT|PATCH|DELETE",subresource!~"proxy|attach|log|exec|portforward",le="1"}[6h])) + ) + + + sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) (rate(apiserver_request_total{job="apiserver",verb=~"POST|PUT|PATCH|DELETE",code=~"5.."}[6h])) + ) + / + sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) (rate(apiserver_request_total{job="apiserver",verb=~"POST|PUT|PATCH|DELETE"}[6h])) + labels: + verb: write + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverBurnrate }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverBurnrate }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + record: apiserver_request:burnrate6h +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/kube-apiserver-histogram.rules.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/kube-apiserver-histogram.rules.yaml new file mode 100644 index 00000000000..d1453419520 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/kube-apiserver-histogram.rules.yaml @@ -0,0 +1,53 @@ +{{- /* +Generated from 'kube-apiserver-histogram.rules' group from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/a8ba97a150c75be42010c75d10b720c55e182f1a/manifests/kubernetesControlPlane-prometheusRule.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.defaultRules.create .Values.kubeApiServer.enabled .Values.defaultRules.rules.kubeApiserverHistogram }} +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "kube-apiserver-histogram.rules" | trunc 63 | trimSuffix "-" }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.defaultRules.labels }} +{{ toYaml .Values.defaultRules.labels | indent 4 }} +{{- end }} +{{- if .Values.defaultRules.annotations }} + annotations: +{{ toYaml .Values.defaultRules.annotations | indent 4 }} +{{- end }} +spec: + groups: + - name: kube-apiserver-histogram.rules + rules: + - expr: histogram_quantile(0.99, sum by (cluster, le, resource) (rate(apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward"}[5m]))) > 0 + labels: + quantile: '0.99' + verb: read + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverHistogram }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverHistogram }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + record: cluster_quantile:apiserver_request_sli_duration_seconds:histogram_quantile + - expr: histogram_quantile(0.99, sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, le, resource) (rate(apiserver_request_sli_duration_seconds_bucket{job="apiserver",verb=~"POST|PUT|PATCH|DELETE",subresource!~"proxy|attach|log|exec|portforward"}[5m]))) > 0 + labels: + quantile: '0.99' + verb: write + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverHistogram }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverHistogram }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + record: cluster_quantile:apiserver_request_sli_duration_seconds:histogram_quantile +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/kube-apiserver-slos.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/kube-apiserver-slos.yaml new file mode 100644 index 00000000000..30ef9a42933 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/kube-apiserver-slos.yaml @@ -0,0 +1,159 @@ +{{- /* +Generated from 'kube-apiserver-slos' group from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/a8ba97a150c75be42010c75d10b720c55e182f1a/manifests/kubernetesControlPlane-prometheusRule.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.defaultRules.create .Values.kubeApiServer.enabled .Values.defaultRules.rules.kubeApiserverSlos }} +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "kube-apiserver-slos" | trunc 63 | trimSuffix "-" }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.defaultRules.labels }} +{{ toYaml .Values.defaultRules.labels | indent 4 }} +{{- end }} +{{- if .Values.defaultRules.annotations }} + annotations: +{{ toYaml .Values.defaultRules.annotations | indent 4 }} +{{- end }} +spec: + groups: + - name: kube-apiserver-slos + rules: +{{- if not (.Values.defaultRules.disabled.KubeAPIErrorBudgetBurn | default false) }} + - alert: KubeAPIErrorBudgetBurn + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubeApiserverSlos }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubeApiserverSlos | indent 8 }} +{{- end }} + description: The API server is burning too much error budget. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubeapierrorbudgetburn + summary: The API server is burning too much error budget. + expr: |- + sum(apiserver_request:burnrate1h) > (14.40 * 0.01000) + and + sum(apiserver_request:burnrate5m) > (14.40 * 0.01000) + for: {{ dig "KubeAPIErrorBudgetBurn" "for" "2m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + long: 1h + severity: {{ dig "KubeAPIErrorBudgetBurn" "severity" "critical" .Values.customRules }} + short: 5m + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverSlos }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverSlos }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.KubeAPIErrorBudgetBurn | default false) }} + - alert: KubeAPIErrorBudgetBurn + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubeApiserverSlos }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubeApiserverSlos | indent 8 }} +{{- end }} + description: The API server is burning too much error budget. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubeapierrorbudgetburn + summary: The API server is burning too much error budget. + expr: |- + sum(apiserver_request:burnrate6h) > (6.00 * 0.01000) + and + sum(apiserver_request:burnrate30m) > (6.00 * 0.01000) + for: {{ dig "KubeAPIErrorBudgetBurn" "for" "15m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + long: 6h + severity: {{ dig "KubeAPIErrorBudgetBurn" "severity" "critical" .Values.customRules }} + short: 30m + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverSlos }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverSlos }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.KubeAPIErrorBudgetBurn | default false) }} + - alert: KubeAPIErrorBudgetBurn + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubeApiserverSlos }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubeApiserverSlos | indent 8 }} +{{- end }} + description: The API server is burning too much error budget. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubeapierrorbudgetburn + summary: The API server is burning too much error budget. + expr: |- + sum(apiserver_request:burnrate1d) > (3.00 * 0.01000) + and + sum(apiserver_request:burnrate2h) > (3.00 * 0.01000) + for: {{ dig "KubeAPIErrorBudgetBurn" "for" "1h" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + long: 1d + severity: {{ dig "KubeAPIErrorBudgetBurn" "severity" "warning" .Values.customRules }} + short: 2h + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverSlos }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverSlos }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.KubeAPIErrorBudgetBurn | default false) }} + - alert: KubeAPIErrorBudgetBurn + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubeApiserverSlos }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubeApiserverSlos | indent 8 }} +{{- end }} + description: The API server is burning too much error budget. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubeapierrorbudgetburn + summary: The API server is burning too much error budget. + expr: |- + sum(apiserver_request:burnrate3d) > (1.00 * 0.01000) + and + sum(apiserver_request:burnrate6h) > (1.00 * 0.01000) + for: {{ dig "KubeAPIErrorBudgetBurn" "for" "3h" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + long: 3d + severity: {{ dig "KubeAPIErrorBudgetBurn" "severity" "warning" .Values.customRules }} + short: 6h + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverSlos }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverSlos }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/kube-prometheus-general.rules.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/kube-prometheus-general.rules.yaml new file mode 100644 index 00000000000..fcf35f389b4 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/kube-prometheus-general.rules.yaml @@ -0,0 +1,49 @@ +{{- /* +Generated from 'kube-prometheus-general.rules' group from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/a8ba97a150c75be42010c75d10b720c55e182f1a/manifests/kubePrometheus-prometheusRule.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.defaultRules.create .Values.defaultRules.rules.kubePrometheusGeneral }} +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "kube-prometheus-general.rules" | trunc 63 | trimSuffix "-" }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.defaultRules.labels }} +{{ toYaml .Values.defaultRules.labels | indent 4 }} +{{- end }} +{{- if .Values.defaultRules.annotations }} + annotations: +{{ toYaml .Values.defaultRules.annotations | indent 4 }} +{{- end }} +spec: + groups: + - name: kube-prometheus-general.rules + rules: + - expr: count without(instance, pod, node) (up == 1) + record: count:up1 + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubePrometheusGeneral }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubePrometheusGeneral }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: count without(instance, pod, node) (up == 0) + record: count:up0 + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubePrometheusGeneral }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubePrometheusGeneral }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/kube-prometheus-node-recording.rules.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/kube-prometheus-node-recording.rules.yaml new file mode 100644 index 00000000000..7a0d2023243 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/kube-prometheus-node-recording.rules.yaml @@ -0,0 +1,93 @@ +{{- /* +Generated from 'kube-prometheus-node-recording.rules' group from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/a8ba97a150c75be42010c75d10b720c55e182f1a/manifests/kubePrometheus-prometheusRule.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.defaultRules.create .Values.defaultRules.rules.kubePrometheusNodeRecording }} +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "kube-prometheus-node-recording.rules" | trunc 63 | trimSuffix "-" }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.defaultRules.labels }} +{{ toYaml .Values.defaultRules.labels | indent 4 }} +{{- end }} +{{- if .Values.defaultRules.annotations }} + annotations: +{{ toYaml .Values.defaultRules.annotations | indent 4 }} +{{- end }} +spec: + groups: + - name: kube-prometheus-node-recording.rules + rules: + - expr: sum(rate(node_cpu_seconds_total{mode!="idle",mode!="iowait",mode!="steal"}[3m])) BY (instance) + record: instance:node_cpu:rate:sum + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubePrometheusNodeRecording }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubePrometheusNodeRecording }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: sum(rate(node_network_receive_bytes_total[3m])) BY (instance) + record: instance:node_network_receive_bytes:rate:sum + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubePrometheusNodeRecording }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubePrometheusNodeRecording }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: sum(rate(node_network_transmit_bytes_total[3m])) BY (instance) + record: instance:node_network_transmit_bytes:rate:sum + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubePrometheusNodeRecording }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubePrometheusNodeRecording }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: sum(rate(node_cpu_seconds_total{mode!="idle",mode!="iowait",mode!="steal"}[5m])) WITHOUT (cpu, mode) / ON(instance) GROUP_LEFT() count(sum(node_cpu_seconds_total) BY (instance, cpu)) BY (instance) + record: instance:node_cpu:ratio + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubePrometheusNodeRecording }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubePrometheusNodeRecording }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: sum(rate(node_cpu_seconds_total{mode!="idle",mode!="iowait",mode!="steal"}[5m])) + record: cluster:node_cpu:sum_rate5m + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubePrometheusNodeRecording }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubePrometheusNodeRecording }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: cluster:node_cpu:sum_rate5m / count(sum(node_cpu_seconds_total) BY (instance, cpu)) + record: cluster:node_cpu:ratio + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubePrometheusNodeRecording }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubePrometheusNodeRecording }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/kube-scheduler.rules.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/kube-scheduler.rules.yaml new file mode 100644 index 00000000000..c9d61ce37b2 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/kube-scheduler.rules.yaml @@ -0,0 +1,135 @@ +{{- /* +Generated from 'kube-scheduler.rules' group from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/a8ba97a150c75be42010c75d10b720c55e182f1a/manifests/kubernetesControlPlane-prometheusRule.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.defaultRules.create .Values.kubeScheduler.enabled .Values.defaultRules.rules.kubeSchedulerRecording }} +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "kube-scheduler.rules" | trunc 63 | trimSuffix "-" }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.defaultRules.labels }} +{{ toYaml .Values.defaultRules.labels | indent 4 }} +{{- end }} +{{- if .Values.defaultRules.annotations }} + annotations: +{{ toYaml .Values.defaultRules.annotations | indent 4 }} +{{- end }} +spec: + groups: + - name: kube-scheduler.rules + rules: + - expr: histogram_quantile(0.99, sum(rate(scheduler_e2e_scheduling_duration_seconds_bucket{job="{{ include "exporter.kubeScheduler.jobName" . }}"}[5m])) without(instance, pod)) + labels: + quantile: '0.99' + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubeSchedulerRecording }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubeSchedulerRecording }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + record: cluster_quantile:scheduler_e2e_scheduling_duration_seconds:histogram_quantile + - expr: histogram_quantile(0.99, sum(rate(scheduler_scheduling_algorithm_duration_seconds_bucket{job="{{ include "exporter.kubeScheduler.jobName" . }}"}[5m])) without(instance, pod)) + labels: + quantile: '0.99' + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubeSchedulerRecording }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubeSchedulerRecording }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + record: cluster_quantile:scheduler_scheduling_algorithm_duration_seconds:histogram_quantile + - expr: histogram_quantile(0.99, sum(rate(scheduler_binding_duration_seconds_bucket{job="{{ include "exporter.kubeScheduler.jobName" . }}"}[5m])) without(instance, pod)) + labels: + quantile: '0.99' + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubeSchedulerRecording }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubeSchedulerRecording }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + record: cluster_quantile:scheduler_binding_duration_seconds:histogram_quantile + - expr: histogram_quantile(0.9, sum(rate(scheduler_e2e_scheduling_duration_seconds_bucket{job="{{ include "exporter.kubeScheduler.jobName" . }}"}[5m])) without(instance, pod)) + labels: + quantile: '0.9' + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubeSchedulerRecording }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubeSchedulerRecording }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + record: cluster_quantile:scheduler_e2e_scheduling_duration_seconds:histogram_quantile + - expr: histogram_quantile(0.9, sum(rate(scheduler_scheduling_algorithm_duration_seconds_bucket{job="{{ include "exporter.kubeScheduler.jobName" . }}"}[5m])) without(instance, pod)) + labels: + quantile: '0.9' + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubeSchedulerRecording }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubeSchedulerRecording }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + record: cluster_quantile:scheduler_scheduling_algorithm_duration_seconds:histogram_quantile + - expr: histogram_quantile(0.9, sum(rate(scheduler_binding_duration_seconds_bucket{job="{{ include "exporter.kubeScheduler.jobName" . }}"}[5m])) without(instance, pod)) + labels: + quantile: '0.9' + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubeSchedulerRecording }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubeSchedulerRecording }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + record: cluster_quantile:scheduler_binding_duration_seconds:histogram_quantile + - expr: histogram_quantile(0.5, sum(rate(scheduler_e2e_scheduling_duration_seconds_bucket{job="{{ include "exporter.kubeScheduler.jobName" . }}"}[5m])) without(instance, pod)) + labels: + quantile: '0.5' + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubeSchedulerRecording }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubeSchedulerRecording }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + record: cluster_quantile:scheduler_e2e_scheduling_duration_seconds:histogram_quantile + - expr: histogram_quantile(0.5, sum(rate(scheduler_scheduling_algorithm_duration_seconds_bucket{job="{{ include "exporter.kubeScheduler.jobName" . }}"}[5m])) without(instance, pod)) + labels: + quantile: '0.5' + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubeSchedulerRecording }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubeSchedulerRecording }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + record: cluster_quantile:scheduler_scheduling_algorithm_duration_seconds:histogram_quantile + - expr: histogram_quantile(0.5, sum(rate(scheduler_binding_duration_seconds_bucket{job="{{ include "exporter.kubeScheduler.jobName" . }}"}[5m])) without(instance, pod)) + labels: + quantile: '0.5' + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubeSchedulerRecording }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubeSchedulerRecording }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + record: cluster_quantile:scheduler_binding_duration_seconds:histogram_quantile +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/kube-state-metrics.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/kube-state-metrics.yaml new file mode 100644 index 00000000000..d1ad3cae5e9 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/kube-state-metrics.yaml @@ -0,0 +1,152 @@ +{{- /* +Generated from 'kube-state-metrics' group from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/a8ba97a150c75be42010c75d10b720c55e182f1a/manifests/kubeStateMetrics-prometheusRule.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.defaultRules.create .Values.defaultRules.rules.kubeStateMetrics }} +{{- $kubeStateMetricsJob := include "kube-prometheus-stack-kube-state-metrics.name" . }} +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "kube-state-metrics" | trunc 63 | trimSuffix "-" }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.defaultRules.labels }} +{{ toYaml .Values.defaultRules.labels | indent 4 }} +{{- end }} +{{- if .Values.defaultRules.annotations }} + annotations: +{{ toYaml .Values.defaultRules.annotations | indent 4 }} +{{- end }} +spec: + groups: + - name: kube-state-metrics + rules: +{{- if not (.Values.defaultRules.disabled.KubeStateMetricsListErrors | default false) }} + - alert: KubeStateMetricsListErrors + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubeStateMetrics }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubeStateMetrics | indent 8 }} +{{- end }} + description: kube-state-metrics is experiencing errors at an elevated rate in list operations. This is likely causing it to not be able to expose metrics about Kubernetes objects correctly or at all. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kube-state-metrics/kubestatemetricslisterrors + summary: kube-state-metrics is experiencing errors in list operations. + expr: |- + (sum(rate(kube_state_metrics_list_total{job="{{ $kubeStateMetricsJob }}",result="error"}[5m])) by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) + / + sum(rate(kube_state_metrics_list_total{job="{{ $kubeStateMetricsJob }}"}[5m])) by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster)) + > 0.01 + for: {{ dig "KubeStateMetricsListErrors" "for" "15m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "KubeStateMetricsListErrors" "severity" "critical" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubeStateMetrics }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubeStateMetrics }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.KubeStateMetricsWatchErrors | default false) }} + - alert: KubeStateMetricsWatchErrors + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubeStateMetrics }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubeStateMetrics | indent 8 }} +{{- end }} + description: kube-state-metrics is experiencing errors at an elevated rate in watch operations. This is likely causing it to not be able to expose metrics about Kubernetes objects correctly or at all. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kube-state-metrics/kubestatemetricswatcherrors + summary: kube-state-metrics is experiencing errors in watch operations. + expr: |- + (sum(rate(kube_state_metrics_watch_total{job="{{ $kubeStateMetricsJob }}",result="error"}[5m])) by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) + / + sum(rate(kube_state_metrics_watch_total{job="{{ $kubeStateMetricsJob }}"}[5m])) by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster)) + > 0.01 + for: {{ dig "KubeStateMetricsWatchErrors" "for" "15m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "KubeStateMetricsWatchErrors" "severity" "critical" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubeStateMetrics }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubeStateMetrics }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.KubeStateMetricsShardingMismatch | default false) }} + - alert: KubeStateMetricsShardingMismatch + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubeStateMetrics }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubeStateMetrics | indent 8 }} +{{- end }} + description: kube-state-metrics pods are running with different --total-shards configuration, some Kubernetes objects may be exposed multiple times or not exposed at all. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kube-state-metrics/kubestatemetricsshardingmismatch + summary: kube-state-metrics sharding is misconfigured. + expr: stdvar (kube_state_metrics_total_shards{job="{{ $kubeStateMetricsJob }}"}) by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) != 0 + for: {{ dig "KubeStateMetricsShardingMismatch" "for" "15m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "KubeStateMetricsShardingMismatch" "severity" "critical" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubeStateMetrics }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubeStateMetrics }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.KubeStateMetricsShardsMissing | default false) }} + - alert: KubeStateMetricsShardsMissing + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubeStateMetrics }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubeStateMetrics | indent 8 }} +{{- end }} + description: kube-state-metrics shards are missing, some Kubernetes objects are not being exposed. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kube-state-metrics/kubestatemetricsshardsmissing + summary: kube-state-metrics shards are missing. + expr: |- + 2^max(kube_state_metrics_total_shards{job="{{ $kubeStateMetricsJob }}"}) by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) - 1 + - + sum( 2 ^ max by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, shard_ordinal) (kube_state_metrics_shard_ordinal{job="{{ $kubeStateMetricsJob }}"}) ) by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) + != 0 + for: {{ dig "KubeStateMetricsShardsMissing" "for" "15m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "KubeStateMetricsShardsMissing" "severity" "critical" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubeStateMetrics }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubeStateMetrics }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/kubelet.rules.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/kubelet.rules.yaml new file mode 100644 index 00000000000..39fdddf3fe1 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/kubelet.rules.yaml @@ -0,0 +1,65 @@ +{{- /* +Generated from 'kubelet.rules' group from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/a8ba97a150c75be42010c75d10b720c55e182f1a/manifests/kubernetesControlPlane-prometheusRule.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.defaultRules.create .Values.defaultRules.rules.kubelet }} +{{- if (include "exporter.kubelet.enabled" .)}} +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "kubelet.rules" | trunc 63 | trimSuffix "-" }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.defaultRules.labels }} +{{ toYaml .Values.defaultRules.labels | indent 4 }} +{{- end }} +{{- if .Values.defaultRules.annotations }} + annotations: +{{ toYaml .Values.defaultRules.annotations | indent 4 }} +{{- end }} +spec: + groups: + - name: kubelet.rules + rules: + - expr: histogram_quantile(0.99, sum(rate(kubelet_pleg_relist_duration_seconds_bucket{job="kubelet", metrics_path="/metrics"}[5m])) by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, instance, le) * on ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, instance) group_left(node) kubelet_node_name{job="kubelet", metrics_path="/metrics"}) + labels: + quantile: '0.99' + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubelet }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubelet }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + record: node_quantile:kubelet_pleg_relist_duration_seconds:histogram_quantile + - expr: histogram_quantile(0.9, sum(rate(kubelet_pleg_relist_duration_seconds_bucket{job="kubelet", metrics_path="/metrics"}[5m])) by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, instance, le) * on ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, instance) group_left(node) kubelet_node_name{job="kubelet", metrics_path="/metrics"}) + labels: + quantile: '0.9' + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubelet }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubelet }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + record: node_quantile:kubelet_pleg_relist_duration_seconds:histogram_quantile + - expr: histogram_quantile(0.5, sum(rate(kubelet_pleg_relist_duration_seconds_bucket{job="kubelet", metrics_path="/metrics"}[5m])) by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, instance, le) * on ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, instance) group_left(node) kubelet_node_name{job="kubelet", metrics_path="/metrics"}) + labels: + quantile: '0.5' + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubelet }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubelet }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + record: node_quantile:kubelet_pleg_relist_duration_seconds:histogram_quantile +{{- end }} +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/kubernetes-apps.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/kubernetes-apps.yaml new file mode 100644 index 00000000000..2a861a522cb --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/kubernetes-apps.yaml @@ -0,0 +1,568 @@ +{{- /* +Generated from 'kubernetes-apps' group from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/a8ba97a150c75be42010c75d10b720c55e182f1a/manifests/kubernetesControlPlane-prometheusRule.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.defaultRules.create .Values.defaultRules.rules.kubernetesApps }} +{{- $kubeStateMetricsJob := include "kube-prometheus-stack-kube-state-metrics.name" . }} +{{- $targetNamespace := .Values.defaultRules.appNamespacesTarget }} +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "kubernetes-apps" | trunc 63 | trimSuffix "-" }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.defaultRules.labels }} +{{ toYaml .Values.defaultRules.labels | indent 4 }} +{{- end }} +{{- if .Values.defaultRules.annotations }} + annotations: +{{ toYaml .Values.defaultRules.annotations | indent 4 }} +{{- end }} +spec: + groups: + - name: kubernetes-apps + rules: +{{- if not (.Values.defaultRules.disabled.KubePodCrashLooping | default false) }} + - alert: KubePodCrashLooping + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesApps }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesApps | indent 8 }} +{{- end }} + description: 'Pod {{`{{`}} $labels.namespace {{`}}`}}/{{`{{`}} $labels.pod {{`}}`}} ({{`{{`}} $labels.container {{`}}`}}) is in waiting state (reason: "CrashLoopBackOff").' + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubepodcrashlooping + summary: Pod is crash looping. + expr: max_over_time(kube_pod_container_status_waiting_reason{reason="CrashLoopBackOff", job="{{ $kubeStateMetricsJob }}", namespace=~"{{ $targetNamespace }}"}[5m]) >= 1 + for: {{ dig "KubePodCrashLooping" "for" "15m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "KubePodCrashLooping" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubernetesApps }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubernetesApps }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.KubePodNotReady | default false) }} + - alert: KubePodNotReady + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesApps }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesApps | indent 8 }} +{{- end }} + description: Pod {{`{{`}} $labels.namespace {{`}}`}}/{{`{{`}} $labels.pod {{`}}`}} has been in a non-ready state for longer than 15 minutes. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubepodnotready + summary: Pod has been in a non-ready state for more than 15 minutes. + expr: |- + sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}namespace, pod, cluster) ( + max by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}namespace, pod, cluster) ( + kube_pod_status_phase{job="{{ $kubeStateMetricsJob }}", namespace=~"{{ $targetNamespace }}", phase=~"Pending|Unknown|Failed"} + ) * on ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}namespace, pod, cluster) group_left(owner_kind) topk by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}namespace, pod, cluster) ( + 1, max by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}namespace, pod, owner_kind, cluster) (kube_pod_owner{owner_kind!="Job"}) + ) + ) > 0 + for: {{ dig "KubePodNotReady" "for" "15m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "KubePodNotReady" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubernetesApps }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubernetesApps }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.KubeDeploymentGenerationMismatch | default false) }} + - alert: KubeDeploymentGenerationMismatch + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesApps }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesApps | indent 8 }} +{{- end }} + description: Deployment generation for {{`{{`}} $labels.namespace {{`}}`}}/{{`{{`}} $labels.deployment {{`}}`}} does not match, this indicates that the Deployment has failed but has not been rolled back. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubedeploymentgenerationmismatch + summary: Deployment generation mismatch due to possible roll-back + expr: |- + kube_deployment_status_observed_generation{job="{{ $kubeStateMetricsJob }}", namespace=~"{{ $targetNamespace }}"} + != + kube_deployment_metadata_generation{job="{{ $kubeStateMetricsJob }}", namespace=~"{{ $targetNamespace }}"} + for: {{ dig "KubeDeploymentGenerationMismatch" "for" "15m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "KubeDeploymentGenerationMismatch" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubernetesApps }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubernetesApps }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.KubeDeploymentReplicasMismatch | default false) }} + - alert: KubeDeploymentReplicasMismatch + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesApps }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesApps | indent 8 }} +{{- end }} + description: Deployment {{`{{`}} $labels.namespace {{`}}`}}/{{`{{`}} $labels.deployment {{`}}`}} has not matched the expected number of replicas for longer than 15 minutes. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubedeploymentreplicasmismatch + summary: Deployment has not matched the expected number of replicas. + expr: |- + ( + kube_deployment_spec_replicas{job="{{ $kubeStateMetricsJob }}", namespace=~"{{ $targetNamespace }}"} + > + kube_deployment_status_replicas_available{job="{{ $kubeStateMetricsJob }}", namespace=~"{{ $targetNamespace }}"} + ) and ( + changes(kube_deployment_status_replicas_updated{job="{{ $kubeStateMetricsJob }}", namespace=~"{{ $targetNamespace }}"}[10m]) + == + 0 + ) + for: {{ dig "KubeDeploymentReplicasMismatch" "for" "15m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "KubeDeploymentReplicasMismatch" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubernetesApps }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubernetesApps }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.KubeDeploymentRolloutStuck | default false) }} + - alert: KubeDeploymentRolloutStuck + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesApps }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesApps | indent 8 }} +{{- end }} + description: Rollout of deployment {{`{{`}} $labels.namespace {{`}}`}}/{{`{{`}} $labels.deployment {{`}}`}} is not progressing for longer than 15 minutes. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubedeploymentrolloutstuck + summary: Deployment rollout is not progressing. + expr: |- + kube_deployment_status_condition{condition="Progressing", status="false",job="{{ $kubeStateMetricsJob }}", namespace=~"{{ $targetNamespace }}"} + != 0 + for: {{ dig "KubeDeploymentRolloutStuck" "for" "15m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "KubeDeploymentRolloutStuck" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubernetesApps }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubernetesApps }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.KubeStatefulSetReplicasMismatch | default false) }} + - alert: KubeStatefulSetReplicasMismatch + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesApps }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesApps | indent 8 }} +{{- end }} + description: StatefulSet {{`{{`}} $labels.namespace {{`}}`}}/{{`{{`}} $labels.statefulset {{`}}`}} has not matched the expected number of replicas for longer than 15 minutes. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubestatefulsetreplicasmismatch + summary: StatefulSet has not matched the expected number of replicas. + expr: |- + ( + kube_statefulset_status_replicas_ready{job="{{ $kubeStateMetricsJob }}", namespace=~"{{ $targetNamespace }}"} + != + kube_statefulset_status_replicas{job="{{ $kubeStateMetricsJob }}", namespace=~"{{ $targetNamespace }}"} + ) and ( + changes(kube_statefulset_status_replicas_updated{job="{{ $kubeStateMetricsJob }}", namespace=~"{{ $targetNamespace }}"}[10m]) + == + 0 + ) + for: {{ dig "KubeStatefulSetReplicasMismatch" "for" "15m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "KubeStatefulSetReplicasMismatch" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubernetesApps }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubernetesApps }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.KubeStatefulSetGenerationMismatch | default false) }} + - alert: KubeStatefulSetGenerationMismatch + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesApps }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesApps | indent 8 }} +{{- end }} + description: StatefulSet generation for {{`{{`}} $labels.namespace {{`}}`}}/{{`{{`}} $labels.statefulset {{`}}`}} does not match, this indicates that the StatefulSet has failed but has not been rolled back. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubestatefulsetgenerationmismatch + summary: StatefulSet generation mismatch due to possible roll-back + expr: |- + kube_statefulset_status_observed_generation{job="{{ $kubeStateMetricsJob }}", namespace=~"{{ $targetNamespace }}"} + != + kube_statefulset_metadata_generation{job="{{ $kubeStateMetricsJob }}", namespace=~"{{ $targetNamespace }}"} + for: {{ dig "KubeStatefulSetGenerationMismatch" "for" "15m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "KubeStatefulSetGenerationMismatch" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubernetesApps }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubernetesApps }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.KubeStatefulSetUpdateNotRolledOut | default false) }} + - alert: KubeStatefulSetUpdateNotRolledOut + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesApps }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesApps | indent 8 }} +{{- end }} + description: StatefulSet {{`{{`}} $labels.namespace {{`}}`}}/{{`{{`}} $labels.statefulset {{`}}`}} update has not been rolled out. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubestatefulsetupdatenotrolledout + summary: StatefulSet update has not been rolled out. + expr: |- + ( + max without (revision) ( + kube_statefulset_status_current_revision{job="{{ $kubeStateMetricsJob }}", namespace=~"{{ $targetNamespace }}"} + unless + kube_statefulset_status_update_revision{job="{{ $kubeStateMetricsJob }}", namespace=~"{{ $targetNamespace }}"} + ) + * + ( + kube_statefulset_replicas{job="{{ $kubeStateMetricsJob }}", namespace=~"{{ $targetNamespace }}"} + != + kube_statefulset_status_replicas_updated{job="{{ $kubeStateMetricsJob }}", namespace=~"{{ $targetNamespace }}"} + ) + ) and ( + changes(kube_statefulset_status_replicas_updated{job="{{ $kubeStateMetricsJob }}", namespace=~"{{ $targetNamespace }}"}[5m]) + == + 0 + ) + for: {{ dig "KubeStatefulSetUpdateNotRolledOut" "for" "15m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "KubeStatefulSetUpdateNotRolledOut" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubernetesApps }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubernetesApps }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.KubeDaemonSetRolloutStuck | default false) }} + - alert: KubeDaemonSetRolloutStuck + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesApps }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesApps | indent 8 }} +{{- end }} + description: DaemonSet {{`{{`}} $labels.namespace {{`}}`}}/{{`{{`}} $labels.daemonset {{`}}`}} has not finished or progressed for at least 15 minutes. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubedaemonsetrolloutstuck + summary: DaemonSet rollout is stuck. + expr: |- + ( + ( + kube_daemonset_status_current_number_scheduled{job="{{ $kubeStateMetricsJob }}", namespace=~"{{ $targetNamespace }}"} + != + kube_daemonset_status_desired_number_scheduled{job="{{ $kubeStateMetricsJob }}", namespace=~"{{ $targetNamespace }}"} + ) or ( + kube_daemonset_status_number_misscheduled{job="{{ $kubeStateMetricsJob }}", namespace=~"{{ $targetNamespace }}"} + != + 0 + ) or ( + kube_daemonset_status_updated_number_scheduled{job="{{ $kubeStateMetricsJob }}", namespace=~"{{ $targetNamespace }}"} + != + kube_daemonset_status_desired_number_scheduled{job="{{ $kubeStateMetricsJob }}", namespace=~"{{ $targetNamespace }}"} + ) or ( + kube_daemonset_status_number_available{job="{{ $kubeStateMetricsJob }}", namespace=~"{{ $targetNamespace }}"} + != + kube_daemonset_status_desired_number_scheduled{job="{{ $kubeStateMetricsJob }}", namespace=~"{{ $targetNamespace }}"} + ) + ) and ( + changes(kube_daemonset_status_updated_number_scheduled{job="{{ $kubeStateMetricsJob }}", namespace=~"{{ $targetNamespace }}"}[5m]) + == + 0 + ) + for: {{ dig "KubeDaemonSetRolloutStuck" "for" "15m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "KubeDaemonSetRolloutStuck" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubernetesApps }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubernetesApps }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.KubeContainerWaiting | default false) }} + - alert: KubeContainerWaiting + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesApps }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesApps | indent 8 }} +{{- end }} + description: pod/{{`{{`}} $labels.pod {{`}}`}} in namespace {{`{{`}} $labels.namespace {{`}}`}} on container {{`{{`}} $labels.container{{`}}`}} has been in waiting state for longer than 1 hour. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubecontainerwaiting + summary: Pod container waiting longer than 1 hour + expr: sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}namespace, pod, container, cluster) (kube_pod_container_status_waiting_reason{job="{{ $kubeStateMetricsJob }}", namespace=~"{{ $targetNamespace }}"}) > 0 + for: {{ dig "KubeContainerWaiting" "for" "1h" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "KubeContainerWaiting" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubernetesApps }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubernetesApps }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.KubeDaemonSetNotScheduled | default false) }} + - alert: KubeDaemonSetNotScheduled + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesApps }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesApps | indent 8 }} +{{- end }} + description: '{{`{{`}} $value {{`}}`}} Pods of DaemonSet {{`{{`}} $labels.namespace {{`}}`}}/{{`{{`}} $labels.daemonset {{`}}`}} are not scheduled.' + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubedaemonsetnotscheduled + summary: DaemonSet pods are not scheduled. + expr: |- + kube_daemonset_status_desired_number_scheduled{job="{{ $kubeStateMetricsJob }}", namespace=~"{{ $targetNamespace }}"} + - + kube_daemonset_status_current_number_scheduled{job="{{ $kubeStateMetricsJob }}", namespace=~"{{ $targetNamespace }}"} > 0 + for: {{ dig "KubeDaemonSetNotScheduled" "for" "10m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "KubeDaemonSetNotScheduled" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubernetesApps }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubernetesApps }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.KubeDaemonSetMisScheduled | default false) }} + - alert: KubeDaemonSetMisScheduled + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesApps }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesApps | indent 8 }} +{{- end }} + description: '{{`{{`}} $value {{`}}`}} Pods of DaemonSet {{`{{`}} $labels.namespace {{`}}`}}/{{`{{`}} $labels.daemonset {{`}}`}} are running where they are not supposed to run.' + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubedaemonsetmisscheduled + summary: DaemonSet pods are misscheduled. + expr: kube_daemonset_status_number_misscheduled{job="{{ $kubeStateMetricsJob }}", namespace=~"{{ $targetNamespace }}"} > 0 + for: {{ dig "KubeDaemonSetMisScheduled" "for" "15m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "KubeDaemonSetMisScheduled" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubernetesApps }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubernetesApps }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.KubeJobNotCompleted | default false) }} + - alert: KubeJobNotCompleted + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesApps }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesApps | indent 8 }} +{{- end }} + description: Job {{`{{`}} $labels.namespace {{`}}`}}/{{`{{`}} $labels.job_name {{`}}`}} is taking more than {{`{{`}} "43200" | humanizeDuration {{`}}`}} to complete. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubejobnotcompleted + summary: Job did not complete in time + expr: |- + time() - max by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}namespace, job_name, cluster) (kube_job_status_start_time{job="{{ $kubeStateMetricsJob }}", namespace=~"{{ $targetNamespace }}"} + and + kube_job_status_active{job="{{ $kubeStateMetricsJob }}", namespace=~"{{ $targetNamespace }}"} > 0) > 43200 + labels: + severity: {{ dig "KubeJobNotCompleted" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubernetesApps }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubernetesApps }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.KubeJobFailed | default false) }} + - alert: KubeJobFailed + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesApps }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesApps | indent 8 }} +{{- end }} + description: Job {{`{{`}} $labels.namespace {{`}}`}}/{{`{{`}} $labels.job_name {{`}}`}} failed to complete. Removing failed job after investigation should clear this alert. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubejobfailed + summary: Job failed to complete. + expr: kube_job_failed{job="{{ $kubeStateMetricsJob }}", namespace=~"{{ $targetNamespace }}"} > 0 + for: {{ dig "KubeJobFailed" "for" "15m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "KubeJobFailed" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubernetesApps }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubernetesApps }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.KubeHpaReplicasMismatch | default false) }} + - alert: KubeHpaReplicasMismatch + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesApps }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesApps | indent 8 }} +{{- end }} + description: HPA {{`{{`}} $labels.namespace {{`}}`}}/{{`{{`}} $labels.horizontalpodautoscaler {{`}}`}} has not matched the desired number of replicas for longer than 15 minutes. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubehpareplicasmismatch + summary: HPA has not matched desired number of replicas. + expr: |- + (kube_horizontalpodautoscaler_status_desired_replicas{job="{{ $kubeStateMetricsJob }}", namespace=~"{{ $targetNamespace }}"} + != + kube_horizontalpodautoscaler_status_current_replicas{job="{{ $kubeStateMetricsJob }}", namespace=~"{{ $targetNamespace }}"}) + and + (kube_horizontalpodautoscaler_status_current_replicas{job="{{ $kubeStateMetricsJob }}", namespace=~"{{ $targetNamespace }}"} + > + kube_horizontalpodautoscaler_spec_min_replicas{job="{{ $kubeStateMetricsJob }}", namespace=~"{{ $targetNamespace }}"}) + and + (kube_horizontalpodautoscaler_status_current_replicas{job="{{ $kubeStateMetricsJob }}", namespace=~"{{ $targetNamespace }}"} + < + kube_horizontalpodautoscaler_spec_max_replicas{job="{{ $kubeStateMetricsJob }}", namespace=~"{{ $targetNamespace }}"}) + and + changes(kube_horizontalpodautoscaler_status_current_replicas{job="{{ $kubeStateMetricsJob }}", namespace=~"{{ $targetNamespace }}"}[15m]) == 0 + for: {{ dig "KubeHpaReplicasMismatch" "for" "15m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "KubeHpaReplicasMismatch" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubernetesApps }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubernetesApps }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.KubeHpaMaxedOut | default false) }} + - alert: KubeHpaMaxedOut + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesApps }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesApps | indent 8 }} +{{- end }} + description: HPA {{`{{`}} $labels.namespace {{`}}`}}/{{`{{`}} $labels.horizontalpodautoscaler {{`}}`}} has been running at max replicas for longer than 15 minutes. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubehpamaxedout + summary: HPA is running at max replicas + expr: |- + kube_horizontalpodautoscaler_status_current_replicas{job="{{ $kubeStateMetricsJob }}", namespace=~"{{ $targetNamespace }}"} + == + kube_horizontalpodautoscaler_spec_max_replicas{job="{{ $kubeStateMetricsJob }}", namespace=~"{{ $targetNamespace }}"} + for: {{ dig "KubeHpaMaxedOut" "for" "15m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "KubeHpaMaxedOut" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubernetesApps }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubernetesApps }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/kubernetes-resources.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/kubernetes-resources.yaml new file mode 100644 index 00000000000..1d32f9bbadc --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/kubernetes-resources.yaml @@ -0,0 +1,282 @@ +{{- /* +Generated from 'kubernetes-resources' group from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/a8ba97a150c75be42010c75d10b720c55e182f1a/manifests/kubernetesControlPlane-prometheusRule.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.defaultRules.create .Values.defaultRules.rules.kubernetesResources }} +{{- $kubeStateMetricsJob := include "kube-prometheus-stack-kube-state-metrics.name" . }} +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "kubernetes-resources" | trunc 63 | trimSuffix "-" }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.defaultRules.labels }} +{{ toYaml .Values.defaultRules.labels | indent 4 }} +{{- end }} +{{- if .Values.defaultRules.annotations }} + annotations: +{{ toYaml .Values.defaultRules.annotations | indent 4 }} +{{- end }} +spec: + groups: + - name: kubernetes-resources + rules: +{{- if not (.Values.defaultRules.disabled.KubeCPUOvercommit | default false) }} + - alert: KubeCPUOvercommit + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesResources }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesResources | indent 8 }} +{{- end }} + description: Cluster {{`{{`}} $labels.cluster {{`}}`}} has overcommitted CPU resource requests for Pods by {{`{{`}} $value {{`}}`}} CPU shares and cannot tolerate node failure. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubecpuovercommit + summary: Cluster has overcommitted CPU resource requests. + expr: |- + sum(namespace_cpu:kube_pod_container_resource_requests:sum{job="{{ $kubeStateMetricsJob }}",}) by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) - (sum(kube_node_status_allocatable{job="{{ $kubeStateMetricsJob }}",resource="cpu"}) by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) - max(kube_node_status_allocatable{job="{{ $kubeStateMetricsJob }}",resource="cpu"}) by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster)) > 0 + and + (sum(kube_node_status_allocatable{job="{{ $kubeStateMetricsJob }}",resource="cpu"}) by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) - max(kube_node_status_allocatable{job="{{ $kubeStateMetricsJob }}",resource="cpu"}) by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster)) > 0 + for: {{ dig "KubeCPUOvercommit" "for" "10m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "KubeCPUOvercommit" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubernetesResources }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubernetesResources }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.KubeMemoryOvercommit | default false) }} + - alert: KubeMemoryOvercommit + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesResources }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesResources | indent 8 }} +{{- end }} + description: Cluster {{`{{`}} $labels.cluster {{`}}`}} has overcommitted memory resource requests for Pods by {{`{{`}} $value | humanize {{`}}`}} bytes and cannot tolerate node failure. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubememoryovercommit + summary: Cluster has overcommitted memory resource requests. + expr: |- + sum(namespace_memory:kube_pod_container_resource_requests:sum{}) by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) - (sum(kube_node_status_allocatable{resource="memory", job="{{ $kubeStateMetricsJob }}"}) by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) - max(kube_node_status_allocatable{resource="memory", job="{{ $kubeStateMetricsJob }}"}) by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster)) > 0 + and + (sum(kube_node_status_allocatable{resource="memory", job="{{ $kubeStateMetricsJob }}"}) by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) - max(kube_node_status_allocatable{resource="memory", job="{{ $kubeStateMetricsJob }}"}) by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster)) > 0 + for: {{ dig "KubeMemoryOvercommit" "for" "10m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "KubeMemoryOvercommit" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubernetesResources }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubernetesResources }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.KubeCPUQuotaOvercommit | default false) }} + - alert: KubeCPUQuotaOvercommit + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesResources }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesResources | indent 8 }} +{{- end }} + description: Cluster {{`{{`}} $labels.cluster {{`}}`}} has overcommitted CPU resource requests for Namespaces. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubecpuquotaovercommit + summary: Cluster has overcommitted CPU resource requests. + expr: |- + sum(min without(resource) (kube_resourcequota{job="{{ $kubeStateMetricsJob }}", type="hard", resource=~"(cpu|requests.cpu)"})) by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) + / + sum(kube_node_status_allocatable{resource="cpu", job="{{ $kubeStateMetricsJob }}"}) by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) + > 1.5 + for: {{ dig "KubeCPUQuotaOvercommit" "for" "5m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "KubeCPUQuotaOvercommit" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubernetesResources }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubernetesResources }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.KubeMemoryQuotaOvercommit | default false) }} + - alert: KubeMemoryQuotaOvercommit + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesResources }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesResources | indent 8 }} +{{- end }} + description: Cluster {{`{{`}} $labels.cluster {{`}}`}} has overcommitted memory resource requests for Namespaces. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubememoryquotaovercommit + summary: Cluster has overcommitted memory resource requests. + expr: |- + sum(min without(resource) (kube_resourcequota{job="{{ $kubeStateMetricsJob }}", type="hard", resource=~"(memory|requests.memory)"})) by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) + / + sum(kube_node_status_allocatable{resource="memory", job="{{ $kubeStateMetricsJob }}"}) by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) + > 1.5 + for: {{ dig "KubeMemoryQuotaOvercommit" "for" "5m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "KubeMemoryQuotaOvercommit" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubernetesResources }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubernetesResources }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.KubeQuotaAlmostFull | default false) }} + - alert: KubeQuotaAlmostFull + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesResources }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesResources | indent 8 }} +{{- end }} + description: Namespace {{`{{`}} $labels.namespace {{`}}`}} is using {{`{{`}} $value | humanizePercentage {{`}}`}} of its {{`{{`}} $labels.resource {{`}}`}} quota. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubequotaalmostfull + summary: Namespace quota is going to be full. + expr: |- + kube_resourcequota{job="{{ $kubeStateMetricsJob }}", type="used"} + / ignoring(instance, job, type) + (kube_resourcequota{job="{{ $kubeStateMetricsJob }}", type="hard"} > 0) + > 0.9 < 1 + for: {{ dig "KubeQuotaAlmostFull" "for" "15m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "KubeQuotaAlmostFull" "severity" "info" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubernetesResources }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubernetesResources }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.KubeQuotaFullyUsed | default false) }} + - alert: KubeQuotaFullyUsed + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesResources }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesResources | indent 8 }} +{{- end }} + description: Namespace {{`{{`}} $labels.namespace {{`}}`}} is using {{`{{`}} $value | humanizePercentage {{`}}`}} of its {{`{{`}} $labels.resource {{`}}`}} quota. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubequotafullyused + summary: Namespace quota is fully used. + expr: |- + kube_resourcequota{job="{{ $kubeStateMetricsJob }}", type="used"} + / ignoring(instance, job, type) + (kube_resourcequota{job="{{ $kubeStateMetricsJob }}", type="hard"} > 0) + == 1 + for: {{ dig "KubeQuotaFullyUsed" "for" "15m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "KubeQuotaFullyUsed" "severity" "info" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubernetesResources }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubernetesResources }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.KubeQuotaExceeded | default false) }} + - alert: KubeQuotaExceeded + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesResources }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesResources | indent 8 }} +{{- end }} + description: Namespace {{`{{`}} $labels.namespace {{`}}`}} is using {{`{{`}} $value | humanizePercentage {{`}}`}} of its {{`{{`}} $labels.resource {{`}}`}} quota. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubequotaexceeded + summary: Namespace quota has exceeded the limits. + expr: |- + kube_resourcequota{job="{{ $kubeStateMetricsJob }}", type="used"} + / ignoring(instance, job, type) + (kube_resourcequota{job="{{ $kubeStateMetricsJob }}", type="hard"} > 0) + > 1 + for: {{ dig "KubeQuotaExceeded" "for" "15m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "KubeQuotaExceeded" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubernetesResources }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubernetesResources }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.CPUThrottlingHigh | default false) }} + - alert: CPUThrottlingHigh + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesResources }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesResources | indent 8 }} +{{- end }} + description: '{{`{{`}} $value | humanizePercentage {{`}}`}} throttling of CPU in namespace {{`{{`}} $labels.namespace {{`}}`}} for container {{`{{`}} $labels.container {{`}}`}} in pod {{`{{`}} $labels.pod {{`}}`}}.' + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/cputhrottlinghigh + summary: Processes experience elevated CPU throttling. + expr: |- + sum(increase(container_cpu_cfs_throttled_periods_total{container!="", }[5m])) by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, container, pod, namespace) + / + sum(increase(container_cpu_cfs_periods_total{}[5m])) by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, container, pod, namespace) + > ( 25 / 100 ) + for: {{ dig "CPUThrottlingHigh" "for" "15m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "CPUThrottlingHigh" "severity" "info" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubernetesResources }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubernetesResources }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/kubernetes-storage.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/kubernetes-storage.yaml new file mode 100644 index 00000000000..b9884456532 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/kubernetes-storage.yaml @@ -0,0 +1,216 @@ +{{- /* +Generated from 'kubernetes-storage' group from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/a8ba97a150c75be42010c75d10b720c55e182f1a/manifests/kubernetesControlPlane-prometheusRule.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.defaultRules.create .Values.defaultRules.rules.kubernetesStorage }} +{{- $kubeStateMetricsJob := include "kube-prometheus-stack-kube-state-metrics.name" . }} +{{- $targetNamespace := .Values.defaultRules.appNamespacesTarget }} +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "kubernetes-storage" | trunc 63 | trimSuffix "-" }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.defaultRules.labels }} +{{ toYaml .Values.defaultRules.labels | indent 4 }} +{{- end }} +{{- if .Values.defaultRules.annotations }} + annotations: +{{ toYaml .Values.defaultRules.annotations | indent 4 }} +{{- end }} +spec: + groups: + - name: kubernetes-storage + rules: +{{- if not (.Values.defaultRules.disabled.KubePersistentVolumeFillingUp | default false) }} + - alert: KubePersistentVolumeFillingUp + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesStorage }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesStorage | indent 8 }} +{{- end }} + description: The PersistentVolume claimed by {{`{{`}} $labels.persistentvolumeclaim {{`}}`}} in Namespace {{`{{`}} $labels.namespace {{`}}`}} {{`{{`}} with $labels.cluster -{{`}}`}} on Cluster {{`{{`}} . {{`}}`}} {{`{{`}}- end {{`}}`}} is only {{`{{`}} $value | humanizePercentage {{`}}`}} free. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubepersistentvolumefillingup + summary: PersistentVolume is filling up. + expr: |- + kubelet_volume_stats_available_bytes{job="{{ include "exporter.kubelet.jobName" . }}", namespace=~"{{ $targetNamespace }}", metrics_path="/metrics"} + / + kubelet_volume_stats_capacity_bytes{job="{{ include "exporter.kubelet.jobName" . }}", namespace=~"{{ $targetNamespace }}", metrics_path="/metrics"} + < 0.03 + and + kubelet_volume_stats_used_bytes{job="{{ include "exporter.kubelet.jobName" . }}", namespace=~"{{ $targetNamespace }}", metrics_path="/metrics"} > 0 + unless on(namespace, persistentvolumeclaim) + kube_persistentvolumeclaim_access_mode{ access_mode="ReadOnlyMany"} == 1 + unless on ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, namespace, persistentvolumeclaim) + kube_persistentvolumeclaim_labels{label_excluded_from_alerts="true"} == 1 + for: {{ dig "KubePersistentVolumeFillingUp" "for" "1m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "KubePersistentVolumeFillingUp" "severity" "critical" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubernetesStorage }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubernetesStorage }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.KubePersistentVolumeFillingUp | default false) }} + - alert: KubePersistentVolumeFillingUp + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesStorage }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesStorage | indent 8 }} +{{- end }} + description: Based on recent sampling, the PersistentVolume claimed by {{`{{`}} $labels.persistentvolumeclaim {{`}}`}} in Namespace {{`{{`}} $labels.namespace {{`}}`}} {{`{{`}} with $labels.cluster -{{`}}`}} on Cluster {{`{{`}} . {{`}}`}} {{`{{`}}- end {{`}}`}} is expected to fill up within four days. Currently {{`{{`}} $value | humanizePercentage {{`}}`}} is available. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubepersistentvolumefillingup + summary: PersistentVolume is filling up. + expr: |- + ( + kubelet_volume_stats_available_bytes{job="{{ include "exporter.kubelet.jobName" . }}", namespace=~"{{ $targetNamespace }}", metrics_path="/metrics"} + / + kubelet_volume_stats_capacity_bytes{job="{{ include "exporter.kubelet.jobName" . }}", namespace=~"{{ $targetNamespace }}", metrics_path="/metrics"} + ) < 0.15 + and + kubelet_volume_stats_used_bytes{job="{{ include "exporter.kubelet.jobName" . }}", namespace=~"{{ $targetNamespace }}", metrics_path="/metrics"} > 0 + and + predict_linear(kubelet_volume_stats_available_bytes{job="{{ include "exporter.kubelet.jobName" . }}", namespace=~"{{ $targetNamespace }}", metrics_path="/metrics"}[6h], 4 * 24 * 3600) < 0 + unless on(namespace, persistentvolumeclaim) + kube_persistentvolumeclaim_access_mode{ access_mode="ReadOnlyMany"} == 1 + unless on ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, namespace, persistentvolumeclaim) + kube_persistentvolumeclaim_labels{label_excluded_from_alerts="true"} == 1 + for: {{ dig "KubePersistentVolumeFillingUp" "for" "1h" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "KubePersistentVolumeFillingUp" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubernetesStorage }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubernetesStorage }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.KubePersistentVolumeInodesFillingUp | default false) }} + - alert: KubePersistentVolumeInodesFillingUp + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesStorage }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesStorage | indent 8 }} +{{- end }} + description: The PersistentVolume claimed by {{`{{`}} $labels.persistentvolumeclaim {{`}}`}} in Namespace {{`{{`}} $labels.namespace {{`}}`}} {{`{{`}} with $labels.cluster -{{`}}`}} on Cluster {{`{{`}} . {{`}}`}} {{`{{`}}- end {{`}}`}} only has {{`{{`}} $value | humanizePercentage {{`}}`}} free inodes. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubepersistentvolumeinodesfillingup + summary: PersistentVolumeInodes are filling up. + expr: |- + ( + kubelet_volume_stats_inodes_free{job="{{ include "exporter.kubelet.jobName" . }}", namespace=~"{{ $targetNamespace }}", metrics_path="/metrics"} + / + kubelet_volume_stats_inodes{job="{{ include "exporter.kubelet.jobName" . }}", namespace=~"{{ $targetNamespace }}", metrics_path="/metrics"} + ) < 0.03 + and + kubelet_volume_stats_inodes_used{job="{{ include "exporter.kubelet.jobName" . }}", namespace=~"{{ $targetNamespace }}", metrics_path="/metrics"} > 0 + unless on(namespace, persistentvolumeclaim) + kube_persistentvolumeclaim_access_mode{ access_mode="ReadOnlyMany"} == 1 + unless on ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, namespace, persistentvolumeclaim) + kube_persistentvolumeclaim_labels{label_excluded_from_alerts="true"} == 1 + for: {{ dig "KubePersistentVolumeInodesFillingUp" "for" "1m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "KubePersistentVolumeInodesFillingUp" "severity" "critical" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubernetesStorage }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubernetesStorage }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.KubePersistentVolumeInodesFillingUp | default false) }} + - alert: KubePersistentVolumeInodesFillingUp + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesStorage }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesStorage | indent 8 }} +{{- end }} + description: Based on recent sampling, the PersistentVolume claimed by {{`{{`}} $labels.persistentvolumeclaim {{`}}`}} in Namespace {{`{{`}} $labels.namespace {{`}}`}} {{`{{`}} with $labels.cluster -{{`}}`}} on Cluster {{`{{`}} . {{`}}`}} {{`{{`}}- end {{`}}`}} is expected to run out of inodes within four days. Currently {{`{{`}} $value | humanizePercentage {{`}}`}} of its inodes are free. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubepersistentvolumeinodesfillingup + summary: PersistentVolumeInodes are filling up. + expr: |- + ( + kubelet_volume_stats_inodes_free{job="{{ include "exporter.kubelet.jobName" . }}", namespace=~"{{ $targetNamespace }}", metrics_path="/metrics"} + / + kubelet_volume_stats_inodes{job="{{ include "exporter.kubelet.jobName" . }}", namespace=~"{{ $targetNamespace }}", metrics_path="/metrics"} + ) < 0.15 + and + kubelet_volume_stats_inodes_used{job="{{ include "exporter.kubelet.jobName" . }}", namespace=~"{{ $targetNamespace }}", metrics_path="/metrics"} > 0 + and + predict_linear(kubelet_volume_stats_inodes_free{job="{{ include "exporter.kubelet.jobName" . }}", namespace=~"{{ $targetNamespace }}", metrics_path="/metrics"}[6h], 4 * 24 * 3600) < 0 + unless on(namespace, persistentvolumeclaim) + kube_persistentvolumeclaim_access_mode{ access_mode="ReadOnlyMany"} == 1 + unless on ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, namespace, persistentvolumeclaim) + kube_persistentvolumeclaim_labels{label_excluded_from_alerts="true"} == 1 + for: {{ dig "KubePersistentVolumeInodesFillingUp" "for" "1h" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "KubePersistentVolumeInodesFillingUp" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubernetesStorage }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubernetesStorage }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.KubePersistentVolumeErrors | default false) }} + - alert: KubePersistentVolumeErrors + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesStorage }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesStorage | indent 8 }} +{{- end }} + description: The persistent volume {{`{{`}} $labels.persistentvolume {{`}}`}} {{`{{`}} with $labels.cluster -{{`}}`}} on Cluster {{`{{`}} . {{`}}`}} {{`{{`}}- end {{`}}`}} has status {{`{{`}} $labels.phase {{`}}`}}. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubepersistentvolumeerrors + summary: PersistentVolume is having issues with provisioning. + expr: kube_persistentvolume_status_phase{phase=~"Failed|Pending",job="{{ $kubeStateMetricsJob }}"} > 0 + for: {{ dig "KubePersistentVolumeErrors" "for" "5m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "KubePersistentVolumeErrors" "severity" "critical" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubernetesStorage }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubernetesStorage }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/kubernetes-system-apiserver.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/kubernetes-system-apiserver.yaml new file mode 100644 index 00000000000..af34a23f88c --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/kubernetes-system-apiserver.yaml @@ -0,0 +1,193 @@ +{{- /* +Generated from 'kubernetes-system-apiserver' group from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/a8ba97a150c75be42010c75d10b720c55e182f1a/manifests/kubernetesControlPlane-prometheusRule.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.defaultRules.create .Values.defaultRules.rules.kubernetesSystem }} +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "kubernetes-system-apiserver" | trunc 63 | trimSuffix "-" }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.defaultRules.labels }} +{{ toYaml .Values.defaultRules.labels | indent 4 }} +{{- end }} +{{- if .Values.defaultRules.annotations }} + annotations: +{{ toYaml .Values.defaultRules.annotations | indent 4 }} +{{- end }} +spec: + groups: + - name: kubernetes-system-apiserver + rules: +{{- if not (.Values.defaultRules.disabled.KubeClientCertificateExpiration | default false) }} + - alert: KubeClientCertificateExpiration + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesSystem }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesSystem | indent 8 }} +{{- end }} + description: A client certificate used to authenticate to kubernetes apiserver is expiring in less than 7.0 days. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubeclientcertificateexpiration + summary: Client certificate is about to expire. + expr: apiserver_client_certificate_expiration_seconds_count{job="apiserver"} > 0 and on ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}job) histogram_quantile(0.01, sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}job, le) (rate(apiserver_client_certificate_expiration_seconds_bucket{job="apiserver"}[5m]))) < 604800 + for: {{ dig "KubeClientCertificateExpiration" "for" "5m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "KubeClientCertificateExpiration" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubernetesSystem }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubernetesSystem }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.KubeClientCertificateExpiration | default false) }} + - alert: KubeClientCertificateExpiration + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesSystem }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesSystem | indent 8 }} +{{- end }} + description: A client certificate used to authenticate to kubernetes apiserver is expiring in less than 24.0 hours. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubeclientcertificateexpiration + summary: Client certificate is about to expire. + expr: apiserver_client_certificate_expiration_seconds_count{job="apiserver"} > 0 and on ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}job) histogram_quantile(0.01, sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}job, le) (rate(apiserver_client_certificate_expiration_seconds_bucket{job="apiserver"}[5m]))) < 86400 + for: {{ dig "KubeClientCertificateExpiration" "for" "5m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "KubeClientCertificateExpiration" "severity" "critical" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubernetesSystem }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubernetesSystem }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.KubeAggregatedAPIErrors | default false) }} + - alert: KubeAggregatedAPIErrors + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesSystem }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesSystem | indent 8 }} +{{- end }} + description: Kubernetes aggregated API {{`{{`}} $labels.name {{`}}`}}/{{`{{`}} $labels.namespace {{`}}`}} has reported errors. It has appeared unavailable {{`{{`}} $value | humanize {{`}}`}} times averaged over the past 10m. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubeaggregatedapierrors + summary: Kubernetes aggregated API has reported errors. + expr: sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}name, namespace, cluster)(increase(aggregator_unavailable_apiservice_total{job="apiserver"}[10m])) > 4 + labels: + severity: {{ dig "KubeAggregatedAPIErrors" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubernetesSystem }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubernetesSystem }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.KubeAggregatedAPIDown | default false) }} + - alert: KubeAggregatedAPIDown + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesSystem }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesSystem | indent 8 }} +{{- end }} + description: Kubernetes aggregated API {{`{{`}} $labels.name {{`}}`}}/{{`{{`}} $labels.namespace {{`}}`}} has been only {{`{{`}} $value | humanize {{`}}`}}% available over the last 10m. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubeaggregatedapidown + summary: Kubernetes aggregated API is down. + expr: (1 - max by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}name, namespace, cluster)(avg_over_time(aggregator_unavailable_apiservice{job="apiserver"}[10m]))) * 100 < 85 + for: {{ dig "KubeAggregatedAPIDown" "for" "5m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "KubeAggregatedAPIDown" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubernetesSystem }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubernetesSystem }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if .Values.kubeApiServer.enabled }} +{{- if not (.Values.defaultRules.disabled.KubeAPIDown | default false) }} + - alert: KubeAPIDown + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesSystem }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesSystem | indent 8 }} +{{- end }} + description: KubeAPI has disappeared from Prometheus target discovery. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubeapidown + summary: Target disappeared from Prometheus target discovery. + expr: absent(up{job="apiserver"} == 1) + for: {{ dig "KubeAPIDown" "for" "15m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "KubeAPIDown" "severity" "critical" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubernetesSystem }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubernetesSystem }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.KubeAPITerminatedRequests | default false) }} + - alert: KubeAPITerminatedRequests + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesSystem }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesSystem | indent 8 }} +{{- end }} + description: The kubernetes apiserver has terminated {{`{{`}} $value | humanizePercentage {{`}}`}} of its incoming requests. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubeapiterminatedrequests + summary: The kubernetes apiserver has terminated {{`{{`}} $value | humanizePercentage {{`}}`}} of its incoming requests. + expr: sum(rate(apiserver_request_terminations_total{job="apiserver"}[10m])) / ( sum(rate(apiserver_request_total{job="apiserver"}[10m])) + sum(rate(apiserver_request_terminations_total{job="apiserver"}[10m])) ) > 0.20 + for: {{ dig "KubeAPITerminatedRequests" "for" "5m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "KubeAPITerminatedRequests" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubernetesSystem }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubernetesSystem }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/kubernetes-system-controller-manager.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/kubernetes-system-controller-manager.yaml new file mode 100644 index 00000000000..205bd598001 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/kubernetes-system-controller-manager.yaml @@ -0,0 +1,55 @@ +{{- /* +Generated from 'kubernetes-system-controller-manager' group from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/a8ba97a150c75be42010c75d10b720c55e182f1a/manifests/kubernetesControlPlane-prometheusRule.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.defaultRules.create .Values.defaultRules.rules.kubeControllerManager }} +{{- if (include "exporter.kubeControllerManager.enabled" .)}} +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "kubernetes-system-controller-manager" | trunc 63 | trimSuffix "-" }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.defaultRules.labels }} +{{ toYaml .Values.defaultRules.labels | indent 4 }} +{{- end }} +{{- if .Values.defaultRules.annotations }} + annotations: +{{ toYaml .Values.defaultRules.annotations | indent 4 }} +{{- end }} +spec: + groups: + - name: kubernetes-system-controller-manager + rules: +{{- if not (.Values.defaultRules.disabled.KubeControllerManagerDown | default false) }} + - alert: KubeControllerManagerDown + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubeControllerManager }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubeControllerManager | indent 8 }} +{{- end }} + description: KubeControllerManager has disappeared from Prometheus target discovery. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubecontrollermanagerdown + summary: Target disappeared from Prometheus target discovery. + expr: absent(up{job="{{ include "exporter.kubeControllerManager.jobName" . }}"} == 1) + for: 15m + labels: + severity: {{ dig "KubeControllerManagerDown" "severity" "critical" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubeControllerManager }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubeControllerManager }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- end }} +{{- end }} + diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/kubernetes-system-kube-proxy.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/kubernetes-system-kube-proxy.yaml new file mode 100644 index 00000000000..66b1d62001e --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/kubernetes-system-kube-proxy.yaml @@ -0,0 +1,56 @@ +{{- /* +Generated from 'kubernetes-system-kube-proxy' group from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/main/manifests/kubernetesControlPlane-prometheusRule.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.defaultRules.create .Values.defaultRules.rules.kubeProxy }} +{{- if (include "exporter.kubeProxy.enabled" .)}} +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "kubernetes-system-kube-proxy" | trunc 63 | trimSuffix "-" }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.defaultRules.labels }} +{{ toYaml .Values.defaultRules.labels | indent 4 }} +{{- end }} +{{- if .Values.defaultRules.annotations }} + annotations: +{{ toYaml .Values.defaultRules.annotations | indent 4 }} +{{- end }} +spec: + groups: + - name: kubernetes-system-kube-proxy + rules: +{{- if not (.Values.defaultRules.disabled.KubeProxyDown | default false) }} + - alert: KubeProxyDown + annotations: + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupAnnotations.kubeProxy }} + {{- with .Values.defaultRules.additionalRuleAnnotations }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupAnnotations.kubeProxy }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + description: KubeProxy has disappeared from Prometheus target discovery. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubeproxydown + summary: Target disappeared from Prometheus target discovery. + expr: absent(up{job="{{ include "exporter.kubeProxy.jobName" . }}"} == 1) + for: 15m + labels: + severity: {{ dig "KubeProxyDown" "labelsSeverity" "critical" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubeProxy }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubeProxy }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/kubernetes-system-kubelet.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/kubernetes-system-kubelet.yaml new file mode 100644 index 00000000000..2a55676735a --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/kubernetes-system-kubelet.yaml @@ -0,0 +1,379 @@ +{{- /* +Generated from 'kubernetes-system-kubelet' group from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/a8ba97a150c75be42010c75d10b720c55e182f1a/manifests/kubernetesControlPlane-prometheusRule.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.defaultRules.create .Values.defaultRules.rules.kubernetesSystem }} +{{- $kubeStateMetricsJob := include "kube-prometheus-stack-kube-state-metrics.name" . }} +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "kubernetes-system-kubelet" | trunc 63 | trimSuffix "-" }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.defaultRules.labels }} +{{ toYaml .Values.defaultRules.labels | indent 4 }} +{{- end }} +{{- if .Values.defaultRules.annotations }} + annotations: +{{ toYaml .Values.defaultRules.annotations | indent 4 }} +{{- end }} +spec: + groups: + - name: kubernetes-system-kubelet + rules: +{{- if not (.Values.defaultRules.disabled.KubeNodeNotReady | default false) }} + - alert: KubeNodeNotReady + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesSystem }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesSystem | indent 8 }} +{{- end }} + description: '{{`{{`}} $labels.node {{`}}`}} has been unready for more than 15 minutes.' + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubenodenotready + summary: Node is not ready. + expr: kube_node_status_condition{job="{{ $kubeStateMetricsJob }}",condition="Ready",status="true"} == 0 + for: {{ dig "KubeNodeNotReady" "for" "15m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "KubeNodeNotReady" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubernetesSystem }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubernetesSystem }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.KubeNodeUnreachable | default false) }} + - alert: KubeNodeUnreachable + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesSystem }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesSystem | indent 8 }} +{{- end }} + description: '{{`{{`}} $labels.node {{`}}`}} is unreachable and some workloads may be rescheduled.' + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubenodeunreachable + summary: Node is unreachable. + expr: (kube_node_spec_taint{job="{{ $kubeStateMetricsJob }}",key="node.kubernetes.io/unreachable",effect="NoSchedule"} unless ignoring(key,value) kube_node_spec_taint{job="{{ $kubeStateMetricsJob }}",key=~"ToBeDeletedByClusterAutoscaler|cloud.google.com/impending-node-termination|aws-node-termination-handler/spot-itn"}) == 1 + for: {{ dig "KubeNodeUnreachable" "for" "15m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "KubeNodeUnreachable" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubernetesSystem }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubernetesSystem }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.KubeletTooManyPods | default false) }} + - alert: KubeletTooManyPods + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesSystem }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesSystem | indent 8 }} +{{- end }} + description: Kubelet '{{`{{`}} $labels.node {{`}}`}}' is running at {{`{{`}} $value | humanizePercentage {{`}}`}} of its Pod capacity. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubelettoomanypods + summary: Kubelet is running at capacity. + expr: |- + count by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, node) ( + (kube_pod_status_phase{job="{{ $kubeStateMetricsJob }}",phase="Running"} == 1) * on ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}instance,pod,namespace,cluster) group_left(node) topk by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}instance,pod,namespace,cluster) (1, kube_pod_info{job="{{ $kubeStateMetricsJob }}"}) + ) + / + max by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, node) ( + kube_node_status_capacity{job="{{ $kubeStateMetricsJob }}",resource="pods"} != 1 + ) > 0.95 + for: {{ dig "KubeletTooManyPods" "for" "15m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "KubeletTooManyPods" "severity" "info" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubernetesSystem }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubernetesSystem }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.KubeNodeReadinessFlapping | default false) }} + - alert: KubeNodeReadinessFlapping + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesSystem }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesSystem | indent 8 }} +{{- end }} + description: The readiness status of node {{`{{`}} $labels.node {{`}}`}} has changed {{`{{`}} $value {{`}}`}} times in the last 15 minutes. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubenodereadinessflapping + summary: Node readiness status is flapping. + expr: sum(changes(kube_node_status_condition{job="{{ $kubeStateMetricsJob }}",status="true",condition="Ready"}[15m])) by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, node) > 2 + for: {{ dig "KubeNodeReadinessFlapping" "for" "15m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "KubeNodeReadinessFlapping" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubernetesSystem }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubernetesSystem }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.KubeletPlegDurationHigh | default false) }} + - alert: KubeletPlegDurationHigh + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesSystem }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesSystem | indent 8 }} +{{- end }} + description: The Kubelet Pod Lifecycle Event Generator has a 99th percentile duration of {{`{{`}} $value {{`}}`}} seconds on node {{`{{`}} $labels.node {{`}}`}}. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubeletplegdurationhigh + summary: Kubelet Pod Lifecycle Event Generator is taking too long to relist. + expr: node_quantile:kubelet_pleg_relist_duration_seconds:histogram_quantile{quantile="0.99"} >= 10 + for: {{ dig "KubeletPlegDurationHigh" "for" "5m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "KubeletPlegDurationHigh" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubernetesSystem }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubernetesSystem }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.KubeletPodStartUpLatencyHigh | default false) }} + - alert: KubeletPodStartUpLatencyHigh + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesSystem }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesSystem | indent 8 }} +{{- end }} + description: Kubelet Pod startup 99th percentile latency is {{`{{`}} $value {{`}}`}} seconds on node {{`{{`}} $labels.node {{`}}`}}. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubeletpodstartuplatencyhigh + summary: Kubelet Pod startup latency is too high. + expr: histogram_quantile(0.99, sum(rate(kubelet_pod_worker_duration_seconds_bucket{job="{{ include "exporter.kubelet.jobName" . }}", metrics_path="/metrics"}[5m])) by (cluster, instance, le)) * on(cluster, instance) group_left(node) kubelet_node_name{job="{{ include "exporter.kubelet.jobName" . }}", metrics_path="/metrics"} > 60 + for: 15m + labels: + severity: {{ dig "KubeletPodStartUpLatencyHigh" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubernetesSystem }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubernetesSystem }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.KubeletClientCertificateExpiration | default false) }} + - alert: KubeletClientCertificateExpiration + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesSystem }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesSystem | indent 8 }} +{{- end }} + description: Client certificate for Kubelet on node {{`{{`}} $labels.node {{`}}`}} expires in {{`{{`}} $value | humanizeDuration {{`}}`}}. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubeletclientcertificateexpiration + summary: Kubelet client certificate is about to expire. + expr: kubelet_certificate_manager_client_ttl_seconds < 604800 + labels: + severity: {{ dig "KubeletClientCertificateExpiration" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubernetesSystem }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubernetesSystem }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.KubeletClientCertificateExpiration | default false) }} + - alert: KubeletClientCertificateExpiration + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesSystem }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesSystem | indent 8 }} +{{- end }} + description: Client certificate for Kubelet on node {{`{{`}} $labels.node {{`}}`}} expires in {{`{{`}} $value | humanizeDuration {{`}}`}}. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubeletclientcertificateexpiration + summary: Kubelet client certificate is about to expire. + expr: kubelet_certificate_manager_client_ttl_seconds < 86400 + labels: + severity: {{ dig "KubeletClientCertificateExpiration" "severity" "critical" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubernetesSystem }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubernetesSystem }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.KubeletServerCertificateExpiration | default false) }} + - alert: KubeletServerCertificateExpiration + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesSystem }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesSystem | indent 8 }} +{{- end }} + description: Server certificate for Kubelet on node {{`{{`}} $labels.node {{`}}`}} expires in {{`{{`}} $value | humanizeDuration {{`}}`}}. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubeletservercertificateexpiration + summary: Kubelet server certificate is about to expire. + expr: kubelet_certificate_manager_server_ttl_seconds < 604800 + labels: + severity: {{ dig "KubeletServerCertificateExpiration" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubernetesSystem }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubernetesSystem }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.KubeletServerCertificateExpiration | default false) }} + - alert: KubeletServerCertificateExpiration + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesSystem }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesSystem | indent 8 }} +{{- end }} + description: Server certificate for Kubelet on node {{`{{`}} $labels.node {{`}}`}} expires in {{`{{`}} $value | humanizeDuration {{`}}`}}. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubeletservercertificateexpiration + summary: Kubelet server certificate is about to expire. + expr: kubelet_certificate_manager_server_ttl_seconds < 86400 + labels: + severity: {{ dig "KubeletServerCertificateExpiration" "severity" "critical" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubernetesSystem }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubernetesSystem }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.KubeletClientCertificateRenewalErrors | default false) }} + - alert: KubeletClientCertificateRenewalErrors + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesSystem }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesSystem | indent 8 }} +{{- end }} + description: Kubelet on node {{`{{`}} $labels.node {{`}}`}} has failed to renew its client certificate ({{`{{`}} $value | humanize {{`}}`}} errors in the last 5 minutes). + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubeletclientcertificaterenewalerrors + summary: Kubelet has failed to renew its client certificate. + expr: increase(kubelet_certificate_manager_client_expiration_renew_errors[5m]) > 0 + for: {{ dig "KubeletClientCertificateRenewalErrors" "for" "15m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "KubeletClientCertificateRenewalErrors" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubernetesSystem }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubernetesSystem }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.KubeletServerCertificateRenewalErrors | default false) }} + - alert: KubeletServerCertificateRenewalErrors + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesSystem }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesSystem | indent 8 }} +{{- end }} + description: Kubelet on node {{`{{`}} $labels.node {{`}}`}} has failed to renew its server certificate ({{`{{`}} $value | humanize {{`}}`}} errors in the last 5 minutes). + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubeletservercertificaterenewalerrors + summary: Kubelet has failed to renew its server certificate. + expr: increase(kubelet_server_expiration_renew_errors[5m]) > 0 + for: {{ dig "KubeletServerCertificateRenewalErrors" "for" "15m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "KubeletServerCertificateRenewalErrors" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubernetesSystem }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubernetesSystem }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if (include "exporter.kubelet.enabled" .)}} +{{- if not (.Values.defaultRules.disabled.KubeletDown | default false) }} + - alert: KubeletDown + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesSystem }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesSystem | indent 8 }} +{{- end }} + description: Kubelet has disappeared from Prometheus target discovery. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubeletdown + summary: Target disappeared from Prometheus target discovery. + expr: absent(up{job="{{ include "exporter.kubelet.jobName" . }}", metrics_path="/metrics"} == 1) + for: 15m + labels: + severity: {{ dig "KubeletDown" "severity" "critical" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubernetesSystem }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubernetesSystem }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- end }} +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/kubernetes-system-scheduler.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/kubernetes-system-scheduler.yaml new file mode 100644 index 00000000000..9890b1c9598 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/kubernetes-system-scheduler.yaml @@ -0,0 +1,54 @@ +{{- /* +Generated from 'kubernetes-system-scheduler' group from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/a8ba97a150c75be42010c75d10b720c55e182f1a/manifests/kubernetesControlPlane-prometheusRule.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.defaultRules.create .Values.kubeScheduler.enabled .Values.defaultRules.rules.kubeSchedulerAlerting }} +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "kubernetes-system-scheduler" | trunc 63 | trimSuffix "-" }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.defaultRules.labels }} +{{ toYaml .Values.defaultRules.labels | indent 4 }} +{{- end }} +{{- if .Values.defaultRules.annotations }} + annotations: +{{ toYaml .Values.defaultRules.annotations | indent 4 }} +{{- end }} +spec: + groups: + - name: kubernetes-system-scheduler + rules: +{{- if .Values.kubeScheduler.enabled }} +{{- if not (.Values.defaultRules.disabled.KubeSchedulerDown | default false) }} + - alert: KubeSchedulerDown + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubeSchedulerAlerting }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubeSchedulerAlerting | indent 8 }} +{{- end }} + description: KubeScheduler has disappeared from Prometheus target discovery. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubeschedulerdown + summary: Target disappeared from Prometheus target discovery. + expr: absent(up{job="{{ include "exporter.kubeScheduler.jobName" . }}"} == 1) + for: 15m + labels: + severity: {{ dig "KubeSchedulerDown" "severity" "critical" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubeSchedulerAlerting }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubeSchedulerAlerting }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- end }} +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/kubernetes-system.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/kubernetes-system.yaml new file mode 100644 index 00000000000..621326d0add --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/kubernetes-system.yaml @@ -0,0 +1,87 @@ +{{- /* +Generated from 'kubernetes-system' group from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/a8ba97a150c75be42010c75d10b720c55e182f1a/manifests/kubernetesControlPlane-prometheusRule.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.defaultRules.create .Values.defaultRules.rules.kubernetesSystem }} +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "kubernetes-system" | trunc 63 | trimSuffix "-" }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.defaultRules.labels }} +{{ toYaml .Values.defaultRules.labels | indent 4 }} +{{- end }} +{{- if .Values.defaultRules.annotations }} + annotations: +{{ toYaml .Values.defaultRules.annotations | indent 4 }} +{{- end }} +spec: + groups: + - name: kubernetes-system + rules: +{{- if not (.Values.defaultRules.disabled.KubeVersionMismatch | default false) }} + - alert: KubeVersionMismatch + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesSystem }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesSystem | indent 8 }} +{{- end }} + description: There are {{`{{`}} $value {{`}}`}} different semantic versions of Kubernetes components running. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubeversionmismatch + summary: Different semantic versions of Kubernetes components running. + expr: count by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) (count by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}git_version, cluster) (label_replace(kubernetes_build_info{job!~"kube-dns|coredns"},"git_version","$1","git_version","(v[0-9]*.[0-9]*).*"))) > 1 + for: {{ dig "KubeVersionMismatch" "for" "15m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "KubeVersionMismatch" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubernetesSystem }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubernetesSystem }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.KubeClientErrors | default false) }} + - alert: KubeClientErrors + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesSystem }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesSystem | indent 8 }} +{{- end }} + description: Kubernetes API server client '{{`{{`}} $labels.job {{`}}`}}/{{`{{`}} $labels.instance {{`}}`}}' is experiencing {{`{{`}} $value | humanizePercentage {{`}}`}} errors.' + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubeclienterrors + summary: Kubernetes API server client is experiencing errors. + expr: |- + (sum(rate(rest_client_requests_total{job="apiserver",code=~"5.."}[5m])) by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, instance, job, namespace) + / + sum(rate(rest_client_requests_total{job="apiserver"}[5m])) by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, instance, job, namespace)) + > 0.01 + for: {{ dig "KubeClientErrors" "for" "15m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "KubeClientErrors" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubernetesSystem }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubernetesSystem }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/node-exporter.rules.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/node-exporter.rules.yaml new file mode 100644 index 00000000000..5d4711ae088 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/node-exporter.rules.yaml @@ -0,0 +1,188 @@ +{{- /* +Generated from 'node-exporter.rules' group from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/a8ba97a150c75be42010c75d10b720c55e182f1a/manifests/nodeExporter-prometheusRule.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.defaultRules.create .Values.defaultRules.rules.nodeExporterRecording }} +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "node-exporter.rules" | trunc 63 | trimSuffix "-" }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.defaultRules.labels }} +{{ toYaml .Values.defaultRules.labels | indent 4 }} +{{- end }} +{{- if .Values.defaultRules.annotations }} + annotations: +{{ toYaml .Values.defaultRules.annotations | indent 4 }} +{{- end }} +spec: + groups: + - name: node-exporter.rules + rules: + - expr: |- + count without (cpu, mode) ( + node_cpu_seconds_total{job="node-exporter",mode="idle"} + ) + record: instance:node_num_cpu:sum + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.nodeExporterRecording }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.nodeExporterRecording }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: |- + 1 - avg without (cpu) ( + sum without (mode) (rate(node_cpu_seconds_total{job="node-exporter", mode=~"idle|iowait|steal"}[5m])) + ) + record: instance:node_cpu_utilisation:rate5m + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.nodeExporterRecording }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.nodeExporterRecording }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: |- + ( + node_load1{job="node-exporter"} + / + instance:node_num_cpu:sum{job="node-exporter"} + ) + record: instance:node_load1_per_cpu:ratio + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.nodeExporterRecording }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.nodeExporterRecording }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: |- + 1 - ( + ( + node_memory_MemAvailable_bytes{job="node-exporter"} + or + ( + node_memory_Buffers_bytes{job="node-exporter"} + + + node_memory_Cached_bytes{job="node-exporter"} + + + node_memory_MemFree_bytes{job="node-exporter"} + + + node_memory_Slab_bytes{job="node-exporter"} + ) + ) + / + node_memory_MemTotal_bytes{job="node-exporter"} + ) + record: instance:node_memory_utilisation:ratio + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.nodeExporterRecording }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.nodeExporterRecording }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: rate(node_vmstat_pgmajfault{job="node-exporter"}[5m]) + record: instance:node_vmstat_pgmajfault:rate5m + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.nodeExporterRecording }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.nodeExporterRecording }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: rate(node_disk_io_time_seconds_total{job="node-exporter", device=~"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)"}[5m]) + record: instance_device:node_disk_io_time_seconds:rate5m + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.nodeExporterRecording }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.nodeExporterRecording }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: rate(node_disk_io_time_weighted_seconds_total{job="node-exporter", device=~"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)"}[5m]) + record: instance_device:node_disk_io_time_weighted_seconds:rate5m + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.nodeExporterRecording }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.nodeExporterRecording }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: |- + sum without (device) ( + rate(node_network_receive_bytes_total{job="node-exporter", device!="lo"}[5m]) + ) + record: instance:node_network_receive_bytes_excluding_lo:rate5m + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.nodeExporterRecording }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.nodeExporterRecording }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: |- + sum without (device) ( + rate(node_network_transmit_bytes_total{job="node-exporter", device!="lo"}[5m]) + ) + record: instance:node_network_transmit_bytes_excluding_lo:rate5m + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.nodeExporterRecording }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.nodeExporterRecording }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: |- + sum without (device) ( + rate(node_network_receive_drop_total{job="node-exporter", device!="lo"}[5m]) + ) + record: instance:node_network_receive_drop_excluding_lo:rate5m + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.nodeExporterRecording }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.nodeExporterRecording }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: |- + sum without (device) ( + rate(node_network_transmit_drop_total{job="node-exporter", device!="lo"}[5m]) + ) + record: instance:node_network_transmit_drop_excluding_lo:rate5m + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.nodeExporterRecording }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.nodeExporterRecording }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/node-exporter.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/node-exporter.yaml new file mode 100644 index 00000000000..14738cedfae --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/node-exporter.yaml @@ -0,0 +1,801 @@ +{{- /* +Generated from 'node-exporter' group from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/a8ba97a150c75be42010c75d10b720c55e182f1a/manifests/nodeExporter-prometheusRule.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.defaultRules.create .Values.defaultRules.rules.nodeExporterAlerting }} +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "node-exporter" | trunc 63 | trimSuffix "-" }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.defaultRules.labels }} +{{ toYaml .Values.defaultRules.labels | indent 4 }} +{{- end }} +{{- if .Values.defaultRules.annotations }} + annotations: +{{ toYaml .Values.defaultRules.annotations | indent 4 }} +{{- end }} +spec: + groups: + - name: node-exporter + rules: +{{- if not (.Values.defaultRules.disabled.NodeFilesystemSpaceFillingUp | default false) }} + - alert: NodeFilesystemSpaceFillingUp + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting | indent 8 }} +{{- end }} + description: Filesystem on {{`{{`}} $labels.device {{`}}`}}, mounted on {{`{{`}} $labels.mountpoint {{`}}`}}, at {{`{{`}} $labels.instance {{`}}`}} has only {{`{{`}} printf "%.2f" $value {{`}}`}}% available space left and is filling up. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/node/nodefilesystemspacefillingup + summary: Filesystem is predicted to run out of space within the next 24 hours. + expr: |- + ( + node_filesystem_avail_bytes{job="node-exporter",fstype!="",mountpoint!=""} / node_filesystem_size_bytes{job="node-exporter",fstype!="",mountpoint!=""} * 100 < 15 + and + predict_linear(node_filesystem_avail_bytes{job="node-exporter",fstype!="",mountpoint!=""}[6h], 24*60*60) < 0 + and + node_filesystem_readonly{job="node-exporter",fstype!="",mountpoint!=""} == 0 + ) + for: {{ dig "NodeFilesystemSpaceFillingUp" "for" "1h" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "NodeFilesystemSpaceFillingUp" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.NodeFilesystemSpaceFillingUp | default false) }} + - alert: NodeFilesystemSpaceFillingUp + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting | indent 8 }} +{{- end }} + description: Filesystem on {{`{{`}} $labels.device {{`}}`}}, mounted on {{`{{`}} $labels.mountpoint {{`}}`}}, at {{`{{`}} $labels.instance {{`}}`}} has only {{`{{`}} printf "%.2f" $value {{`}}`}}% available space left and is filling up fast. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/node/nodefilesystemspacefillingup + summary: Filesystem is predicted to run out of space within the next 4 hours. + expr: |- + ( + node_filesystem_avail_bytes{job="node-exporter",fstype!="",mountpoint!=""} / node_filesystem_size_bytes{job="node-exporter",fstype!="",mountpoint!=""} * 100 < 10 + and + predict_linear(node_filesystem_avail_bytes{job="node-exporter",fstype!="",mountpoint!=""}[6h], 4*60*60) < 0 + and + node_filesystem_readonly{job="node-exporter",fstype!="",mountpoint!=""} == 0 + ) + for: {{ dig "NodeFilesystemSpaceFillingUp" "for" "1h" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "NodeFilesystemSpaceFillingUp" "severity" "critical" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.NodeFilesystemAlmostOutOfSpace | default false) }} + - alert: NodeFilesystemAlmostOutOfSpace + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting | indent 8 }} +{{- end }} + description: Filesystem on {{`{{`}} $labels.device {{`}}`}}, mounted on {{`{{`}} $labels.mountpoint {{`}}`}}, at {{`{{`}} $labels.instance {{`}}`}} has only {{`{{`}} printf "%.2f" $value {{`}}`}}% available space left. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/node/nodefilesystemalmostoutofspace + summary: Filesystem has less than 5% space left. + expr: |- + ( + node_filesystem_avail_bytes{job="node-exporter",fstype!="",mountpoint!=""} / node_filesystem_size_bytes{job="node-exporter",fstype!="",mountpoint!=""} * 100 < 5 + and + node_filesystem_readonly{job="node-exporter",fstype!="",mountpoint!=""} == 0 + ) + for: {{ dig "NodeFilesystemAlmostOutOfSpace" "for" "30m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "NodeFilesystemAlmostOutOfSpace" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.NodeFilesystemAlmostOutOfSpace | default false) }} + - alert: NodeFilesystemAlmostOutOfSpace + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting | indent 8 }} +{{- end }} + description: Filesystem on {{`{{`}} $labels.device {{`}}`}}, mounted on {{`{{`}} $labels.mountpoint {{`}}`}}, at {{`{{`}} $labels.instance {{`}}`}} has only {{`{{`}} printf "%.2f" $value {{`}}`}}% available space left. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/node/nodefilesystemalmostoutofspace + summary: Filesystem has less than 3% space left. + expr: |- + ( + node_filesystem_avail_bytes{job="node-exporter",fstype!="",mountpoint!=""} / node_filesystem_size_bytes{job="node-exporter",fstype!="",mountpoint!=""} * 100 < 3 + and + node_filesystem_readonly{job="node-exporter",fstype!="",mountpoint!=""} == 0 + ) + for: {{ dig "NodeFilesystemAlmostOutOfSpace" "for" "30m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "NodeFilesystemAlmostOutOfSpace" "severity" "critical" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.NodeFilesystemFilesFillingUp | default false) }} + - alert: NodeFilesystemFilesFillingUp + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting | indent 8 }} +{{- end }} + description: Filesystem on {{`{{`}} $labels.device {{`}}`}}, mounted on {{`{{`}} $labels.mountpoint {{`}}`}}, at {{`{{`}} $labels.instance {{`}}`}} has only {{`{{`}} printf "%.2f" $value {{`}}`}}% available inodes left and is filling up. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/node/nodefilesystemfilesfillingup + summary: Filesystem is predicted to run out of inodes within the next 24 hours. + expr: |- + ( + node_filesystem_files_free{job="node-exporter",fstype!="",mountpoint!=""} / node_filesystem_files{job="node-exporter",fstype!="",mountpoint!=""} * 100 < 40 + and + predict_linear(node_filesystem_files_free{job="node-exporter",fstype!="",mountpoint!=""}[6h], 24*60*60) < 0 + and + node_filesystem_readonly{job="node-exporter",fstype!="",mountpoint!=""} == 0 + ) + for: {{ dig "NodeFilesystemFilesFillingUp" "for" "1h" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "NodeFilesystemFilesFillingUp" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.NodeFilesystemFilesFillingUp | default false) }} + - alert: NodeFilesystemFilesFillingUp + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting | indent 8 }} +{{- end }} + description: Filesystem on {{`{{`}} $labels.device {{`}}`}}, mounted on {{`{{`}} $labels.mountpoint {{`}}`}}, at {{`{{`}} $labels.instance {{`}}`}} has only {{`{{`}} printf "%.2f" $value {{`}}`}}% available inodes left and is filling up fast. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/node/nodefilesystemfilesfillingup + summary: Filesystem is predicted to run out of inodes within the next 4 hours. + expr: |- + ( + node_filesystem_files_free{job="node-exporter",fstype!="",mountpoint!=""} / node_filesystem_files{job="node-exporter",fstype!="",mountpoint!=""} * 100 < 20 + and + predict_linear(node_filesystem_files_free{job="node-exporter",fstype!="",mountpoint!=""}[6h], 4*60*60) < 0 + and + node_filesystem_readonly{job="node-exporter",fstype!="",mountpoint!=""} == 0 + ) + for: {{ dig "NodeFilesystemFilesFillingUp" "for" "1h" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "NodeFilesystemFilesFillingUp" "severity" "critical" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.NodeFilesystemAlmostOutOfFiles | default false) }} + - alert: NodeFilesystemAlmostOutOfFiles + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting | indent 8 }} +{{- end }} + description: Filesystem on {{`{{`}} $labels.device {{`}}`}}, mounted on {{`{{`}} $labels.mountpoint {{`}}`}}, at {{`{{`}} $labels.instance {{`}}`}} has only {{`{{`}} printf "%.2f" $value {{`}}`}}% available inodes left. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/node/nodefilesystemalmostoutoffiles + summary: Filesystem has less than 5% inodes left. + expr: |- + ( + node_filesystem_files_free{job="node-exporter",fstype!="",mountpoint!=""} / node_filesystem_files{job="node-exporter",fstype!="",mountpoint!=""} * 100 < 5 + and + node_filesystem_readonly{job="node-exporter",fstype!="",mountpoint!=""} == 0 + ) + for: {{ dig "NodeFilesystemAlmostOutOfFiles" "for" "1h" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "NodeFilesystemAlmostOutOfFiles" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.NodeFilesystemAlmostOutOfFiles | default false) }} + - alert: NodeFilesystemAlmostOutOfFiles + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting | indent 8 }} +{{- end }} + description: Filesystem on {{`{{`}} $labels.device {{`}}`}}, mounted on {{`{{`}} $labels.mountpoint {{`}}`}}, at {{`{{`}} $labels.instance {{`}}`}} has only {{`{{`}} printf "%.2f" $value {{`}}`}}% available inodes left. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/node/nodefilesystemalmostoutoffiles + summary: Filesystem has less than 3% inodes left. + expr: |- + ( + node_filesystem_files_free{job="node-exporter",fstype!="",mountpoint!=""} / node_filesystem_files{job="node-exporter",fstype!="",mountpoint!=""} * 100 < 3 + and + node_filesystem_readonly{job="node-exporter",fstype!="",mountpoint!=""} == 0 + ) + for: {{ dig "NodeFilesystemAlmostOutOfFiles" "for" "1h" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "NodeFilesystemAlmostOutOfFiles" "severity" "critical" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.NodeNetworkReceiveErrs | default false) }} + - alert: NodeNetworkReceiveErrs + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting | indent 8 }} +{{- end }} + description: '{{`{{`}} $labels.instance {{`}}`}} interface {{`{{`}} $labels.device {{`}}`}} has encountered {{`{{`}} printf "%.0f" $value {{`}}`}} receive errors in the last two minutes.' + runbook_url: {{ .Values.defaultRules.runbookUrl }}/node/nodenetworkreceiveerrs + summary: Network interface is reporting many receive errors. + expr: rate(node_network_receive_errs_total{job="node-exporter"}[2m]) / rate(node_network_receive_packets_total{job="node-exporter"}[2m]) > 0.01 + for: {{ dig "NodeNetworkReceiveErrs" "for" "1h" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "NodeNetworkReceiveErrs" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.NodeNetworkTransmitErrs | default false) }} + - alert: NodeNetworkTransmitErrs + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting | indent 8 }} +{{- end }} + description: '{{`{{`}} $labels.instance {{`}}`}} interface {{`{{`}} $labels.device {{`}}`}} has encountered {{`{{`}} printf "%.0f" $value {{`}}`}} transmit errors in the last two minutes.' + runbook_url: {{ .Values.defaultRules.runbookUrl }}/node/nodenetworktransmiterrs + summary: Network interface is reporting many transmit errors. + expr: rate(node_network_transmit_errs_total{job="node-exporter"}[2m]) / rate(node_network_transmit_packets_total{job="node-exporter"}[2m]) > 0.01 + for: {{ dig "NodeNetworkTransmitErrs" "for" "1h" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "NodeNetworkTransmitErrs" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.NodeHighNumberConntrackEntriesUsed | default false) }} + - alert: NodeHighNumberConntrackEntriesUsed + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting | indent 8 }} +{{- end }} + description: '{{`{{`}} $value | humanizePercentage {{`}}`}} of conntrack entries are used.' + runbook_url: {{ .Values.defaultRules.runbookUrl }}/node/nodehighnumberconntrackentriesused + summary: Number of conntrack are getting close to the limit. + expr: (node_nf_conntrack_entries{job="node-exporter"} / node_nf_conntrack_entries_limit) > 0.75 + labels: + severity: {{ dig "NodeHighNumberConntrackEntriesUsed" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.NodeTextFileCollectorScrapeError | default false) }} + - alert: NodeTextFileCollectorScrapeError + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting | indent 8 }} +{{- end }} + description: Node Exporter text file collector on {{`{{`}} $labels.instance {{`}}`}} failed to scrape. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/node/nodetextfilecollectorscrapeerror + summary: Node Exporter text file collector failed to scrape. + expr: node_textfile_scrape_error{job="node-exporter"} == 1 + labels: + severity: {{ dig "NodeTextFileCollectorScrapeError" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.NodeClockSkewDetected | default false) }} + - alert: NodeClockSkewDetected + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting | indent 8 }} +{{- end }} + description: Clock at {{`{{`}} $labels.instance {{`}}`}} is out of sync by more than 0.05s. Ensure NTP is configured correctly on this host. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/node/nodeclockskewdetected + summary: Clock skew detected. + expr: |- + ( + node_timex_offset_seconds{job="node-exporter"} > 0.05 + and + deriv(node_timex_offset_seconds{job="node-exporter"}[5m]) >= 0 + ) + or + ( + node_timex_offset_seconds{job="node-exporter"} < -0.05 + and + deriv(node_timex_offset_seconds{job="node-exporter"}[5m]) <= 0 + ) + for: {{ dig "NodeClockSkewDetected" "for" "10m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "NodeClockSkewDetected" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.NodeClockNotSynchronising | default false) }} + - alert: NodeClockNotSynchronising + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting | indent 8 }} +{{- end }} + description: Clock at {{`{{`}} $labels.instance {{`}}`}} is not synchronising. Ensure NTP is configured on this host. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/node/nodeclocknotsynchronising + summary: Clock not synchronising. + expr: |- + min_over_time(node_timex_sync_status{job="node-exporter"}[5m]) == 0 + and + node_timex_maxerror_seconds{job="node-exporter"} >= 16 + for: {{ dig "NodeClockNotSynchronising" "for" "10m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "NodeClockNotSynchronising" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.NodeRAIDDegraded | default false) }} + - alert: NodeRAIDDegraded + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting | indent 8 }} +{{- end }} + description: RAID array '{{`{{`}} $labels.device {{`}}`}}' at {{`{{`}} $labels.instance {{`}}`}} is in degraded state due to one or more disks failures. Number of spare drives is insufficient to fix issue automatically. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/node/noderaiddegraded + summary: RAID Array is degraded. + expr: node_md_disks_required{job="node-exporter",device=~"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)"} - ignoring (state) (node_md_disks{state="active",job="node-exporter",device=~"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)"}) > 0 + for: {{ dig "NodeRAIDDegraded" "for" "15m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "NodeRAIDDegraded" "severity" "critical" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.NodeRAIDDiskFailure | default false) }} + - alert: NodeRAIDDiskFailure + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting | indent 8 }} +{{- end }} + description: At least one device in RAID array at {{`{{`}} $labels.instance {{`}}`}} failed. Array '{{`{{`}} $labels.device {{`}}`}}' needs attention and possibly a disk swap. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/node/noderaiddiskfailure + summary: Failed device in RAID array. + expr: node_md_disks{state="failed",job="node-exporter",device=~"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)"} > 0 + labels: + severity: {{ dig "NodeRAIDDiskFailure" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.NodeFileDescriptorLimit | default false) }} + - alert: NodeFileDescriptorLimit + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting | indent 8 }} +{{- end }} + description: File descriptors limit at {{`{{`}} $labels.instance {{`}}`}} is currently at {{`{{`}} printf "%.2f" $value {{`}}`}}%. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/node/nodefiledescriptorlimit + summary: Kernel is predicted to exhaust file descriptors limit soon. + expr: |- + ( + node_filefd_allocated{job="node-exporter"} * 100 / node_filefd_maximum{job="node-exporter"} > 70 + ) + for: {{ dig "NodeFileDescriptorLimit" "for" "15m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "NodeFileDescriptorLimit" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.NodeFileDescriptorLimit | default false) }} + - alert: NodeFileDescriptorLimit + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting | indent 8 }} +{{- end }} + description: File descriptors limit at {{`{{`}} $labels.instance {{`}}`}} is currently at {{`{{`}} printf "%.2f" $value {{`}}`}}%. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/node/nodefiledescriptorlimit + summary: Kernel is predicted to exhaust file descriptors limit soon. + expr: |- + ( + node_filefd_allocated{job="node-exporter"} * 100 / node_filefd_maximum{job="node-exporter"} > 90 + ) + for: {{ dig "NodeFileDescriptorLimit" "for" "15m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "NodeFileDescriptorLimit" "severity" "critical" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.NodeCPUHighUsage | default false) }} + - alert: NodeCPUHighUsage + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting | indent 8 }} +{{- end }} + description: 'CPU usage at {{`{{`}} $labels.instance {{`}}`}} has been above 90% for the last 15 minutes, is currently at {{`{{`}} printf "%.2f" $value {{`}}`}}%. + + ' + runbook_url: {{ .Values.defaultRules.runbookUrl }}/node/nodecpuhighusage + summary: High CPU usage. + expr: sum without(mode) (avg without (cpu) (rate(node_cpu_seconds_total{job="node-exporter", mode!="idle"}[2m]))) * 100 > 90 + for: {{ dig "NodeCPUHighUsage" "for" "15m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "NodeCPUHighUsage" "severity" "info" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.NodeSystemSaturation | default false) }} + - alert: NodeSystemSaturation + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting | indent 8 }} +{{- end }} + description: 'System load per core at {{`{{`}} $labels.instance {{`}}`}} has been above 2 for the last 15 minutes, is currently at {{`{{`}} printf "%.2f" $value {{`}}`}}. + + This might indicate this instance resources saturation and can cause it becoming unresponsive. + + ' + runbook_url: {{ .Values.defaultRules.runbookUrl }}/node/nodesystemsaturation + summary: System saturated, load per core is very high. + expr: |- + node_load1{job="node-exporter"} + / count without (cpu, mode) (node_cpu_seconds_total{job="node-exporter", mode="idle"}) > 2 + for: {{ dig "NodeSystemSaturation" "for" "15m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "NodeSystemSaturation" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.NodeMemoryMajorPagesFaults | default false) }} + - alert: NodeMemoryMajorPagesFaults + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting | indent 8 }} +{{- end }} + description: 'Memory major pages are occurring at very high rate at {{`{{`}} $labels.instance {{`}}`}}, 500 major page faults per second for the last 15 minutes, is currently at {{`{{`}} printf "%.2f" $value {{`}}`}}. + + Please check that there is enough memory available at this instance. + + ' + runbook_url: {{ .Values.defaultRules.runbookUrl }}/node/nodememorymajorpagesfaults + summary: Memory major page faults are occurring at very high rate. + expr: rate(node_vmstat_pgmajfault{job="node-exporter"}[5m]) > 500 + for: {{ dig "NodeMemoryMajorPagesFaults" "for" "15m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "NodeMemoryMajorPagesFaults" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.NodeMemoryHighUtilization | default false) }} + - alert: NodeMemoryHighUtilization + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting | indent 8 }} +{{- end }} + description: 'Memory is filling up at {{`{{`}} $labels.instance {{`}}`}}, has been above 90% for the last 15 minutes, is currently at {{`{{`}} printf "%.2f" $value {{`}}`}}%. + + ' + runbook_url: {{ .Values.defaultRules.runbookUrl }}/node/nodememoryhighutilization + summary: Host is running out of memory. + expr: 100 - (node_memory_MemAvailable_bytes{job="node-exporter"} / node_memory_MemTotal_bytes{job="node-exporter"} * 100) > 90 + for: {{ dig "NodeMemoryHighUtilization" "for" "15m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "NodeMemoryHighUtilization" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.NodeDiskIOSaturation | default false) }} + - alert: NodeDiskIOSaturation + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting | indent 8 }} +{{- end }} + description: 'Disk IO queue (aqu-sq) is high on {{`{{`}} $labels.device {{`}}`}} at {{`{{`}} $labels.instance {{`}}`}}, has been above 10 for the last 30 minutes, is currently at {{`{{`}} printf "%.2f" $value {{`}}`}}. + + This symptom might indicate disk saturation. + + ' + runbook_url: {{ .Values.defaultRules.runbookUrl }}/node/nodediskiosaturation + summary: Disk IO queue is high. + expr: rate(node_disk_io_time_weighted_seconds_total{job="node-exporter", device=~"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)"}[5m]) > 10 + for: {{ dig "NodeDiskIOSaturation" "for" "30m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "NodeDiskIOSaturation" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.NodeSystemdServiceFailed | default false) }} + - alert: NodeSystemdServiceFailed + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting | indent 8 }} +{{- end }} + description: Systemd service {{`{{`}} $labels.name {{`}}`}} has entered failed state at {{`{{`}} $labels.instance {{`}}`}} + runbook_url: {{ .Values.defaultRules.runbookUrl }}/node/nodesystemdservicefailed + summary: Systemd service has entered failed state. + expr: node_systemd_unit_state{job="node-exporter", state="failed"} == 1 + for: {{ dig "NodeSystemdServiceFailed" "for" "5m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "NodeSystemdServiceFailed" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.NodeBondingDegraded | default false) }} + - alert: NodeBondingDegraded + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting | indent 8 }} +{{- end }} + description: Bonding interface {{`{{`}} $labels.master {{`}}`}} on {{`{{`}} $labels.instance {{`}}`}} is in degraded state due to one or more slave failures. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/node/nodebondingdegraded + summary: Bonding interface is degraded + expr: (node_bonding_slaves - node_bonding_active) != 0 + for: {{ dig "NodeBondingDegraded" "for" "5m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "NodeBondingDegraded" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/node-network.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/node-network.yaml new file mode 100644 index 00000000000..8dc60ef66be --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/node-network.yaml @@ -0,0 +1,55 @@ +{{- /* +Generated from 'node-network' group from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/a8ba97a150c75be42010c75d10b720c55e182f1a/manifests/kubePrometheus-prometheusRule.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.defaultRules.create .Values.defaultRules.rules.network }} +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "node-network" | trunc 63 | trimSuffix "-" }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.defaultRules.labels }} +{{ toYaml .Values.defaultRules.labels | indent 4 }} +{{- end }} +{{- if .Values.defaultRules.annotations }} + annotations: +{{ toYaml .Values.defaultRules.annotations | indent 4 }} +{{- end }} +spec: + groups: + - name: node-network + rules: +{{- if not (.Values.defaultRules.disabled.NodeNetworkInterfaceFlapping | default false) }} + - alert: NodeNetworkInterfaceFlapping + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.network }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.network | indent 8 }} +{{- end }} + description: Network interface "{{`{{`}} $labels.device {{`}}`}}" changing its up status often on node-exporter {{`{{`}} $labels.namespace {{`}}`}}/{{`{{`}} $labels.pod {{`}}`}} + runbook_url: {{ .Values.defaultRules.runbookUrl }}/general/nodenetworkinterfaceflapping + summary: Network interface is often changing its status + expr: changes(node_network_up{job="node-exporter",device!~"veth.+"}[2m]) > 2 + for: {{ dig "NodeNetworkInterfaceFlapping" "for" "2m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "NodeNetworkInterfaceFlapping" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.network }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.network }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/node.rules.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/node.rules.yaml new file mode 100644 index 00000000000..e2911b905e3 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/node.rules.yaml @@ -0,0 +1,109 @@ +{{- /* +Generated from 'node.rules' group from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/a8ba97a150c75be42010c75d10b720c55e182f1a/manifests/kubernetesControlPlane-prometheusRule.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.defaultRules.create .Values.defaultRules.rules.node }} +{{- $kubeStateMetricsJob := include "kube-prometheus-stack-kube-state-metrics.name" . }} +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "node.rules" | trunc 63 | trimSuffix "-" }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.defaultRules.labels }} +{{ toYaml .Values.defaultRules.labels | indent 4 }} +{{- end }} +{{- if .Values.defaultRules.annotations }} + annotations: +{{ toYaml .Values.defaultRules.annotations | indent 4 }} +{{- end }} +spec: + groups: + - name: node.rules + rules: + - expr: |- + topk by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, namespace, pod) (1, + max by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, node, namespace, pod) ( + label_replace(kube_pod_info{job="{{ $kubeStateMetricsJob }}",node!=""}, "pod", "$1", "pod", "(.*)") + )) + record: 'node_namespace_pod:kube_pod_info:' + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.node }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.node }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: |- + count by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, node) ( + node_cpu_seconds_total{mode="idle",job="node-exporter"} + * on ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, namespace, pod) group_left(node) + topk by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, namespace, pod) (1, node_namespace_pod:kube_pod_info:) + ) + record: node:node_num_cpu:sum + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.node }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.node }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: |- + sum( + node_memory_MemAvailable_bytes{job="node-exporter"} or + ( + node_memory_Buffers_bytes{job="node-exporter"} + + node_memory_Cached_bytes{job="node-exporter"} + + node_memory_MemFree_bytes{job="node-exporter"} + + node_memory_Slab_bytes{job="node-exporter"} + ) + ) by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) + record: :node_memory_MemAvailable_bytes:sum + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.node }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.node }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: |- + avg by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, node) ( + sum without (mode) ( + rate(node_cpu_seconds_total{mode!="idle",mode!="iowait",mode!="steal",job="node-exporter"}[5m]) + ) + ) + record: node:node_cpu_utilization:ratio_rate5m + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.node }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.node }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: |- + avg by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) ( + node:node_cpu_utilization:ratio_rate5m + ) + record: cluster:node_cpu:ratio_rate5m + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.node }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.node }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/prometheus-operator.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/prometheus-operator.yaml new file mode 100644 index 00000000000..1f288dffe36 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/prometheus-operator.yaml @@ -0,0 +1,253 @@ +{{- /* +Generated from 'prometheus-operator' group from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/a8ba97a150c75be42010c75d10b720c55e182f1a/manifests/prometheusOperator-prometheusRule.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.defaultRules.create .Values.defaultRules.rules.prometheusOperator }} +{{- $operatorJob := printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "operator" }} +{{- $namespace := printf "%s" (include "kube-prometheus-stack.namespace" .) }} +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "prometheus-operator" | trunc 63 | trimSuffix "-" }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.defaultRules.labels }} +{{ toYaml .Values.defaultRules.labels | indent 4 }} +{{- end }} +{{- if .Values.defaultRules.annotations }} + annotations: +{{ toYaml .Values.defaultRules.annotations | indent 4 }} +{{- end }} +spec: + groups: + - name: prometheus-operator + rules: +{{- if not (.Values.defaultRules.disabled.PrometheusOperatorListErrors | default false) }} + - alert: PrometheusOperatorListErrors + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.prometheusOperator }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.prometheusOperator | indent 8 }} +{{- end }} + description: Errors while performing List operations in controller {{`{{`}}$labels.controller{{`}}`}} in {{`{{`}}$labels.namespace{{`}}`}} namespace. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/prometheus-operator/prometheusoperatorlisterrors + summary: Errors while performing list operations in controller. + expr: (sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster,controller,namespace) (rate(prometheus_operator_list_operations_failed_total{job="{{ $operatorJob }}",namespace="{{ $namespace }}"}[10m])) / sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster,controller,namespace) (rate(prometheus_operator_list_operations_total{job="{{ $operatorJob }}",namespace="{{ $namespace }}"}[10m]))) > 0.4 + for: {{ dig "PrometheusOperatorListErrors" "for" "15m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "PrometheusOperatorListErrors" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.prometheusOperator }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.prometheusOperator }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.PrometheusOperatorWatchErrors | default false) }} + - alert: PrometheusOperatorWatchErrors + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.prometheusOperator }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.prometheusOperator | indent 8 }} +{{- end }} + description: Errors while performing watch operations in controller {{`{{`}}$labels.controller{{`}}`}} in {{`{{`}}$labels.namespace{{`}}`}} namespace. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/prometheus-operator/prometheusoperatorwatcherrors + summary: Errors while performing watch operations in controller. + expr: (sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster,controller,namespace) (rate(prometheus_operator_watch_operations_failed_total{job="{{ $operatorJob }}",namespace="{{ $namespace }}"}[5m])) / sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster,controller,namespace) (rate(prometheus_operator_watch_operations_total{job="{{ $operatorJob }}",namespace="{{ $namespace }}"}[5m]))) > 0.4 + for: {{ dig "PrometheusOperatorWatchErrors" "for" "15m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "PrometheusOperatorWatchErrors" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.prometheusOperator }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.prometheusOperator }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.PrometheusOperatorSyncFailed | default false) }} + - alert: PrometheusOperatorSyncFailed + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.prometheusOperator }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.prometheusOperator | indent 8 }} +{{- end }} + description: Controller {{`{{`}} $labels.controller {{`}}`}} in {{`{{`}} $labels.namespace {{`}}`}} namespace fails to reconcile {{`{{`}} $value {{`}}`}} objects. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/prometheus-operator/prometheusoperatorsyncfailed + summary: Last controller reconciliation failed + expr: min_over_time(prometheus_operator_syncs{status="failed",job="{{ $operatorJob }}",namespace="{{ $namespace }}"}[5m]) > 0 + for: {{ dig "PrometheusOperatorSyncFailed" "for" "10m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "PrometheusOperatorSyncFailed" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.prometheusOperator }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.prometheusOperator }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.PrometheusOperatorReconcileErrors | default false) }} + - alert: PrometheusOperatorReconcileErrors + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.prometheusOperator }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.prometheusOperator | indent 8 }} +{{- end }} + description: '{{`{{`}} $value | humanizePercentage {{`}}`}} of reconciling operations failed for {{`{{`}} $labels.controller {{`}}`}} controller in {{`{{`}} $labels.namespace {{`}}`}} namespace.' + runbook_url: {{ .Values.defaultRules.runbookUrl }}/prometheus-operator/prometheusoperatorreconcileerrors + summary: Errors while reconciling objects. + expr: (sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster,controller,namespace) (rate(prometheus_operator_reconcile_errors_total{job="{{ $operatorJob }}",namespace="{{ $namespace }}"}[5m]))) / (sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster,controller,namespace) (rate(prometheus_operator_reconcile_operations_total{job="{{ $operatorJob }}",namespace="{{ $namespace }}"}[5m]))) > 0.1 + for: {{ dig "PrometheusOperatorReconcileErrors" "for" "10m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "PrometheusOperatorReconcileErrors" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.prometheusOperator }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.prometheusOperator }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.PrometheusOperatorStatusUpdateErrors | default false) }} + - alert: PrometheusOperatorStatusUpdateErrors + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.prometheusOperator }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.prometheusOperator | indent 8 }} +{{- end }} + description: '{{`{{`}} $value | humanizePercentage {{`}}`}} of status update operations failed for {{`{{`}} $labels.controller {{`}}`}} controller in {{`{{`}} $labels.namespace {{`}}`}} namespace.' + runbook_url: {{ .Values.defaultRules.runbookUrl }}/prometheus-operator/prometheusoperatorstatusupdateerrors + summary: Errors while updating objects status. + expr: (sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster,controller,namespace) (rate(prometheus_operator_status_update_errors_total{job="{{ $operatorJob }}",namespace="{{ $namespace }}"}[5m]))) / (sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster,controller,namespace) (rate(prometheus_operator_status_update_operations_total{job="{{ $operatorJob }}",namespace="{{ $namespace }}"}[5m]))) > 0.1 + for: {{ dig "PrometheusOperatorStatusUpdateErrors" "for" "10m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "PrometheusOperatorStatusUpdateErrors" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.prometheusOperator }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.prometheusOperator }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.PrometheusOperatorNodeLookupErrors | default false) }} + - alert: PrometheusOperatorNodeLookupErrors + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.prometheusOperator }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.prometheusOperator | indent 8 }} +{{- end }} + description: Errors while reconciling Prometheus in {{`{{`}} $labels.namespace {{`}}`}} Namespace. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/prometheus-operator/prometheusoperatornodelookuperrors + summary: Errors while reconciling Prometheus. + expr: rate(prometheus_operator_node_address_lookup_errors_total{job="{{ $operatorJob }}",namespace="{{ $namespace }}"}[5m]) > 0.1 + for: {{ dig "PrometheusOperatorNodeLookupErrors" "for" "10m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "PrometheusOperatorNodeLookupErrors" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.prometheusOperator }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.prometheusOperator }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.PrometheusOperatorNotReady | default false) }} + - alert: PrometheusOperatorNotReady + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.prometheusOperator }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.prometheusOperator | indent 8 }} +{{- end }} + description: Prometheus operator in {{`{{`}} $labels.namespace {{`}}`}} namespace isn't ready to reconcile {{`{{`}} $labels.controller {{`}}`}} resources. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/prometheus-operator/prometheusoperatornotready + summary: Prometheus operator not ready + expr: min by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster,controller,namespace) (max_over_time(prometheus_operator_ready{job="{{ $operatorJob }}",namespace="{{ $namespace }}"}[5m]) == 0) + for: {{ dig "PrometheusOperatorNotReady" "for" "5m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "PrometheusOperatorNotReady" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.prometheusOperator }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.prometheusOperator }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.PrometheusOperatorRejectedResources | default false) }} + - alert: PrometheusOperatorRejectedResources + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.prometheusOperator }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.prometheusOperator | indent 8 }} +{{- end }} + description: Prometheus operator in {{`{{`}} $labels.namespace {{`}}`}} namespace rejected {{`{{`}} printf "%0.0f" $value {{`}}`}} {{`{{`}} $labels.controller {{`}}`}}/{{`{{`}} $labels.resource {{`}}`}} resources. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/prometheus-operator/prometheusoperatorrejectedresources + summary: Resources rejected by Prometheus operator + expr: min_over_time(prometheus_operator_managed_resources{state="rejected",job="{{ $operatorJob }}",namespace="{{ $namespace }}"}[5m]) > 0 + for: {{ dig "PrometheusOperatorRejectedResources" "for" "5m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "PrometheusOperatorRejectedResources" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.prometheusOperator }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.prometheusOperator }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/prometheus.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/prometheus.yaml new file mode 100644 index 00000000000..9dfeb1f9dbe --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/prometheus.yaml @@ -0,0 +1,707 @@ +{{- /* +Generated from 'prometheus' group from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/a8ba97a150c75be42010c75d10b720c55e182f1a/manifests/prometheus-prometheusRule.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.defaultRules.create .Values.defaultRules.rules.prometheus }} +{{- $prometheusJob := printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "prometheus" }} +{{- $namespace := printf "%s" (include "kube-prometheus-stack.namespace" .) }} +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "prometheus" | trunc 63 | trimSuffix "-" }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.defaultRules.labels }} +{{ toYaml .Values.defaultRules.labels | indent 4 }} +{{- end }} +{{- if .Values.defaultRules.annotations }} + annotations: +{{ toYaml .Values.defaultRules.annotations | indent 4 }} +{{- end }} +spec: + groups: + - name: prometheus + rules: +{{- if not (.Values.defaultRules.disabled.PrometheusBadConfig | default false) }} + - alert: PrometheusBadConfig + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.prometheus }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.prometheus | indent 8 }} +{{- end }} + description: Prometheus {{`{{`}}$labels.namespace{{`}}`}}/{{`{{`}}$labels.pod{{`}}`}} has failed to reload its configuration. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/prometheus/prometheusbadconfig + summary: Failed Prometheus configuration reload. + expr: |- + # Without max_over_time, failed scrapes could create false negatives, see + # https://www.robustperception.io/alerting-on-gauges-in-prometheus-2-0 for details. + max_over_time(prometheus_config_last_reload_successful{job="{{ $prometheusJob }}",namespace="{{ $namespace }}"}[5m]) == 0 + for: {{ dig "PrometheusBadConfig" "for" "10m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "PrometheusBadConfig" "severity" "critical" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.prometheus }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.prometheus }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.PrometheusSDRefreshFailure | default false) }} + - alert: PrometheusSDRefreshFailure + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.prometheus }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.prometheus | indent 8 }} +{{- end }} + description: Prometheus {{`{{`}}$labels.namespace{{`}}`}}/{{`{{`}}$labels.pod{{`}}`}} has failed to refresh SD with mechanism {{`{{`}}$labels.mechanism{{`}}`}}. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/prometheus/prometheussdrefreshfailure + summary: Failed Prometheus SD refresh. + expr: increase(prometheus_sd_refresh_failures_total{job="{{ $prometheusJob }}",namespace="{{ $namespace }}"}[10m]) > 0 + for: {{ dig "PrometheusSDRefreshFailure" "for" "20m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "PrometheusSDRefreshFailure" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.prometheus }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.prometheus }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.PrometheusNotificationQueueRunningFull | default false) }} + - alert: PrometheusNotificationQueueRunningFull + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.prometheus }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.prometheus | indent 8 }} +{{- end }} + description: Alert notification queue of Prometheus {{`{{`}}$labels.namespace{{`}}`}}/{{`{{`}}$labels.pod{{`}}`}} is running full. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/prometheus/prometheusnotificationqueuerunningfull + summary: Prometheus alert notification queue predicted to run full in less than 30m. + expr: |- + # Without min_over_time, failed scrapes could create false negatives, see + # https://www.robustperception.io/alerting-on-gauges-in-prometheus-2-0 for details. + ( + predict_linear(prometheus_notifications_queue_length{job="{{ $prometheusJob }}",namespace="{{ $namespace }}"}[5m], 60 * 30) + > + min_over_time(prometheus_notifications_queue_capacity{job="{{ $prometheusJob }}",namespace="{{ $namespace }}"}[5m]) + ) + for: {{ dig "PrometheusNotificationQueueRunningFull" "for" "15m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "PrometheusNotificationQueueRunningFull" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.prometheus }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.prometheus }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.PrometheusErrorSendingAlertsToSomeAlertmanagers | default false) }} + - alert: PrometheusErrorSendingAlertsToSomeAlertmanagers + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.prometheus }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.prometheus | indent 8 }} +{{- end }} + description: '{{`{{`}} printf "%.1f" $value {{`}}`}}% errors while sending alerts from Prometheus {{`{{`}}$labels.namespace{{`}}`}}/{{`{{`}}$labels.pod{{`}}`}} to Alertmanager {{`{{`}}$labels.alertmanager{{`}}`}}.' + runbook_url: {{ .Values.defaultRules.runbookUrl }}/prometheus/prometheuserrorsendingalertstosomealertmanagers + summary: Prometheus has encountered more than 1% errors sending alerts to a specific Alertmanager. + expr: |- + ( + rate(prometheus_notifications_errors_total{job="{{ $prometheusJob }}",namespace="{{ $namespace }}"}[5m]) + / + rate(prometheus_notifications_sent_total{job="{{ $prometheusJob }}",namespace="{{ $namespace }}"}[5m]) + ) + * 100 + > 1 + for: {{ dig "PrometheusErrorSendingAlertsToSomeAlertmanagers" "for" "15m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "PrometheusErrorSendingAlertsToSomeAlertmanagers" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.prometheus }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.prometheus }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.PrometheusNotConnectedToAlertmanagers | default false) }} + - alert: PrometheusNotConnectedToAlertmanagers + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.prometheus }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.prometheus | indent 8 }} +{{- end }} + description: Prometheus {{`{{`}}$labels.namespace{{`}}`}}/{{`{{`}}$labels.pod{{`}}`}} is not connected to any Alertmanagers. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/prometheus/prometheusnotconnectedtoalertmanagers + summary: Prometheus is not connected to any Alertmanagers. + expr: |- + # Without max_over_time, failed scrapes could create false negatives, see + # https://www.robustperception.io/alerting-on-gauges-in-prometheus-2-0 for details. + max_over_time(prometheus_notifications_alertmanagers_discovered{job="{{ $prometheusJob }}",namespace="{{ $namespace }}"}[5m]) < 1 + for: {{ dig "PrometheusNotConnectedToAlertmanagers" "for" "10m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "PrometheusNotConnectedToAlertmanagers" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.prometheus }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.prometheus }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.PrometheusTSDBReloadsFailing | default false) }} + - alert: PrometheusTSDBReloadsFailing + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.prometheus }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.prometheus | indent 8 }} +{{- end }} + description: Prometheus {{`{{`}}$labels.namespace{{`}}`}}/{{`{{`}}$labels.pod{{`}}`}} has detected {{`{{`}}$value | humanize{{`}}`}} reload failures over the last 3h. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/prometheus/prometheustsdbreloadsfailing + summary: Prometheus has issues reloading blocks from disk. + expr: increase(prometheus_tsdb_reloads_failures_total{job="{{ $prometheusJob }}",namespace="{{ $namespace }}"}[3h]) > 0 + for: {{ dig "PrometheusTSDBReloadsFailing" "for" "4h" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "PrometheusTSDBReloadsFailing" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.prometheus }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.prometheus }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.PrometheusTSDBCompactionsFailing | default false) }} + - alert: PrometheusTSDBCompactionsFailing + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.prometheus }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.prometheus | indent 8 }} +{{- end }} + description: Prometheus {{`{{`}}$labels.namespace{{`}}`}}/{{`{{`}}$labels.pod{{`}}`}} has detected {{`{{`}}$value | humanize{{`}}`}} compaction failures over the last 3h. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/prometheus/prometheustsdbcompactionsfailing + summary: Prometheus has issues compacting blocks. + expr: increase(prometheus_tsdb_compactions_failed_total{job="{{ $prometheusJob }}",namespace="{{ $namespace }}"}[3h]) > 0 + for: {{ dig "PrometheusTSDBCompactionsFailing" "for" "4h" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "PrometheusTSDBCompactionsFailing" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.prometheus }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.prometheus }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.PrometheusNotIngestingSamples | default false) }} + - alert: PrometheusNotIngestingSamples + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.prometheus }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.prometheus | indent 8 }} +{{- end }} + description: Prometheus {{`{{`}}$labels.namespace{{`}}`}}/{{`{{`}}$labels.pod{{`}}`}} is not ingesting samples. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/prometheus/prometheusnotingestingsamples + summary: Prometheus is not ingesting samples. + expr: |- + ( + sum without(type) (rate(prometheus_tsdb_head_samples_appended_total{job="{{ $prometheusJob }}",namespace="{{ $namespace }}"}[5m])) <= 0 + and + ( + sum without(scrape_job) (prometheus_target_metadata_cache_entries{job="{{ $prometheusJob }}",namespace="{{ $namespace }}"}) > 0 + or + sum without(rule_group) (prometheus_rule_group_rules{job="{{ $prometheusJob }}",namespace="{{ $namespace }}"}) > 0 + ) + ) + for: {{ dig "PrometheusNotIngestingSamples" "for" "10m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "PrometheusNotIngestingSamples" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.prometheus }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.prometheus }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.PrometheusDuplicateTimestamps | default false) }} + - alert: PrometheusDuplicateTimestamps + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.prometheus }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.prometheus | indent 8 }} +{{- end }} + description: Prometheus {{`{{`}}$labels.namespace{{`}}`}}/{{`{{`}}$labels.pod{{`}}`}} is dropping {{`{{`}} printf "%.4g" $value {{`}}`}} samples/s with different values but duplicated timestamp. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/prometheus/prometheusduplicatetimestamps + summary: Prometheus is dropping samples with duplicate timestamps. + expr: rate(prometheus_target_scrapes_sample_duplicate_timestamp_total{job="{{ $prometheusJob }}",namespace="{{ $namespace }}"}[5m]) > 0 + for: {{ dig "PrometheusDuplicateTimestamps" "for" "10m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "PrometheusDuplicateTimestamps" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.prometheus }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.prometheus }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.PrometheusOutOfOrderTimestamps | default false) }} + - alert: PrometheusOutOfOrderTimestamps + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.prometheus }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.prometheus | indent 8 }} +{{- end }} + description: Prometheus {{`{{`}}$labels.namespace{{`}}`}}/{{`{{`}}$labels.pod{{`}}`}} is dropping {{`{{`}} printf "%.4g" $value {{`}}`}} samples/s with timestamps arriving out of order. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/prometheus/prometheusoutofordertimestamps + summary: Prometheus drops samples with out-of-order timestamps. + expr: rate(prometheus_target_scrapes_sample_out_of_order_total{job="{{ $prometheusJob }}",namespace="{{ $namespace }}"}[5m]) > 0 + for: {{ dig "PrometheusOutOfOrderTimestamps" "for" "10m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "PrometheusOutOfOrderTimestamps" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.prometheus }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.prometheus }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.PrometheusRemoteStorageFailures | default false) }} + - alert: PrometheusRemoteStorageFailures + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.prometheus }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.prometheus | indent 8 }} +{{- end }} + description: Prometheus {{`{{`}}$labels.namespace{{`}}`}}/{{`{{`}}$labels.pod{{`}}`}} failed to send {{`{{`}} printf "%.1f" $value {{`}}`}}% of the samples to {{`{{`}} $labels.remote_name{{`}}`}}:{{`{{`}} $labels.url {{`}}`}} + runbook_url: {{ .Values.defaultRules.runbookUrl }}/prometheus/prometheusremotestoragefailures + summary: Prometheus fails to send samples to remote storage. + expr: |- + ( + (rate(prometheus_remote_storage_failed_samples_total{job="{{ $prometheusJob }}",namespace="{{ $namespace }}"}[5m]) or rate(prometheus_remote_storage_samples_failed_total{job="{{ $prometheusJob }}",namespace="{{ $namespace }}"}[5m])) + / + ( + (rate(prometheus_remote_storage_failed_samples_total{job="{{ $prometheusJob }}",namespace="{{ $namespace }}"}[5m]) or rate(prometheus_remote_storage_samples_failed_total{job="{{ $prometheusJob }}",namespace="{{ $namespace }}"}[5m])) + + + (rate(prometheus_remote_storage_succeeded_samples_total{job="{{ $prometheusJob }}",namespace="{{ $namespace }}"}[5m]) or rate(prometheus_remote_storage_samples_total{job="{{ $prometheusJob }}",namespace="{{ $namespace }}"}[5m])) + ) + ) + * 100 + > 1 + for: {{ dig "PrometheusRemoteStorageFailures" "for" "15m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "PrometheusRemoteStorageFailures" "severity" "critical" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.prometheus }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.prometheus }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.PrometheusRemoteWriteBehind | default false) }} + - alert: PrometheusRemoteWriteBehind + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.prometheus }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.prometheus | indent 8 }} +{{- end }} + description: Prometheus {{`{{`}}$labels.namespace{{`}}`}}/{{`{{`}}$labels.pod{{`}}`}} remote write is {{`{{`}} printf "%.1f" $value {{`}}`}}s behind for {{`{{`}} $labels.remote_name{{`}}`}}:{{`{{`}} $labels.url {{`}}`}}. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/prometheus/prometheusremotewritebehind + summary: Prometheus remote write is behind. + expr: |- + # Without max_over_time, failed scrapes could create false negatives, see + # https://www.robustperception.io/alerting-on-gauges-in-prometheus-2-0 for details. + ( + max_over_time(prometheus_remote_storage_highest_timestamp_in_seconds{job="{{ $prometheusJob }}",namespace="{{ $namespace }}"}[5m]) + - ignoring(remote_name, url) group_right + max_over_time(prometheus_remote_storage_queue_highest_sent_timestamp_seconds{job="{{ $prometheusJob }}",namespace="{{ $namespace }}"}[5m]) + ) + > 120 + for: {{ dig "PrometheusRemoteWriteBehind" "for" "15m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "PrometheusRemoteWriteBehind" "severity" "critical" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.prometheus }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.prometheus }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.PrometheusRemoteWriteDesiredShards | default false) }} + - alert: PrometheusRemoteWriteDesiredShards + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.prometheus }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.prometheus | indent 8 }} +{{- end }} + description: Prometheus {{`{{`}}$labels.namespace{{`}}`}}/{{`{{`}}$labels.pod{{`}}`}} remote write desired shards calculation wants to run {{`{{`}} $value {{`}}`}} shards for queue {{`{{`}} $labels.remote_name{{`}}`}}:{{`{{`}} $labels.url {{`}}`}}, which is more than the max of {{`{{`}} printf `prometheus_remote_storage_shards_max{instance="%s",job="{{ $prometheusJob }}",namespace="{{ $namespace }}"}` $labels.instance | query | first | value {{`}}`}}. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/prometheus/prometheusremotewritedesiredshards + summary: Prometheus remote write desired shards calculation wants to run more than configured max shards. + expr: |- + # Without max_over_time, failed scrapes could create false negatives, see + # https://www.robustperception.io/alerting-on-gauges-in-prometheus-2-0 for details. + ( + max_over_time(prometheus_remote_storage_shards_desired{job="{{ $prometheusJob }}",namespace="{{ $namespace }}"}[5m]) + > + max_over_time(prometheus_remote_storage_shards_max{job="{{ $prometheusJob }}",namespace="{{ $namespace }}"}[5m]) + ) + for: {{ dig "PrometheusRemoteWriteDesiredShards" "for" "15m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "PrometheusRemoteWriteDesiredShards" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.prometheus }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.prometheus }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.PrometheusRuleFailures | default false) }} + - alert: PrometheusRuleFailures + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.prometheus }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.prometheus | indent 8 }} +{{- end }} + description: Prometheus {{`{{`}}$labels.namespace{{`}}`}}/{{`{{`}}$labels.pod{{`}}`}} has failed to evaluate {{`{{`}} printf "%.0f" $value {{`}}`}} rules in the last 5m. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/prometheus/prometheusrulefailures + summary: Prometheus is failing rule evaluations. + expr: increase(prometheus_rule_evaluation_failures_total{job="{{ $prometheusJob }}",namespace="{{ $namespace }}"}[5m]) > 0 + for: {{ dig "PrometheusRuleFailures" "for" "15m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "PrometheusRuleFailures" "severity" "critical" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.prometheus }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.prometheus }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.PrometheusMissingRuleEvaluations | default false) }} + - alert: PrometheusMissingRuleEvaluations + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.prometheus }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.prometheus | indent 8 }} +{{- end }} + description: Prometheus {{`{{`}}$labels.namespace{{`}}`}}/{{`{{`}}$labels.pod{{`}}`}} has missed {{`{{`}} printf "%.0f" $value {{`}}`}} rule group evaluations in the last 5m. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/prometheus/prometheusmissingruleevaluations + summary: Prometheus is missing rule evaluations due to slow rule group evaluation. + expr: increase(prometheus_rule_group_iterations_missed_total{job="{{ $prometheusJob }}",namespace="{{ $namespace }}"}[5m]) > 0 + for: {{ dig "PrometheusMissingRuleEvaluations" "for" "15m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "PrometheusMissingRuleEvaluations" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.prometheus }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.prometheus }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.PrometheusTargetLimitHit | default false) }} + - alert: PrometheusTargetLimitHit + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.prometheus }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.prometheus | indent 8 }} +{{- end }} + description: Prometheus {{`{{`}}$labels.namespace{{`}}`}}/{{`{{`}}$labels.pod{{`}}`}} has dropped {{`{{`}} printf "%.0f" $value {{`}}`}} targets because the number of targets exceeded the configured target_limit. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/prometheus/prometheustargetlimithit + summary: Prometheus has dropped targets because some scrape configs have exceeded the targets limit. + expr: increase(prometheus_target_scrape_pool_exceeded_target_limit_total{job="{{ $prometheusJob }}",namespace="{{ $namespace }}"}[5m]) > 0 + for: {{ dig "PrometheusTargetLimitHit" "for" "15m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "PrometheusTargetLimitHit" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.prometheus }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.prometheus }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.PrometheusLabelLimitHit | default false) }} + - alert: PrometheusLabelLimitHit + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.prometheus }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.prometheus | indent 8 }} +{{- end }} + description: Prometheus {{`{{`}}$labels.namespace{{`}}`}}/{{`{{`}}$labels.pod{{`}}`}} has dropped {{`{{`}} printf "%.0f" $value {{`}}`}} targets because some samples exceeded the configured label_limit, label_name_length_limit or label_value_length_limit. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/prometheus/prometheuslabellimithit + summary: Prometheus has dropped targets because some scrape configs have exceeded the labels limit. + expr: increase(prometheus_target_scrape_pool_exceeded_label_limits_total{job="{{ $prometheusJob }}",namespace="{{ $namespace }}"}[5m]) > 0 + for: {{ dig "PrometheusLabelLimitHit" "for" "15m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "PrometheusLabelLimitHit" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.prometheus }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.prometheus }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.PrometheusScrapeBodySizeLimitHit | default false) }} + - alert: PrometheusScrapeBodySizeLimitHit + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.prometheus }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.prometheus | indent 8 }} +{{- end }} + description: Prometheus {{`{{`}}$labels.namespace{{`}}`}}/{{`{{`}}$labels.pod{{`}}`}} has failed {{`{{`}} printf "%.0f" $value {{`}}`}} scrapes in the last 5m because some targets exceeded the configured body_size_limit. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/prometheus/prometheusscrapebodysizelimithit + summary: Prometheus has dropped some targets that exceeded body size limit. + expr: increase(prometheus_target_scrapes_exceeded_body_size_limit_total{job="{{ $prometheusJob }}",namespace="{{ $namespace }}"}[5m]) > 0 + for: {{ dig "PrometheusScrapeBodySizeLimitHit" "for" "15m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "PrometheusScrapeBodySizeLimitHit" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.prometheus }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.prometheus }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.PrometheusScrapeSampleLimitHit | default false) }} + - alert: PrometheusScrapeSampleLimitHit + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.prometheus }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.prometheus | indent 8 }} +{{- end }} + description: Prometheus {{`{{`}}$labels.namespace{{`}}`}}/{{`{{`}}$labels.pod{{`}}`}} has failed {{`{{`}} printf "%.0f" $value {{`}}`}} scrapes in the last 5m because some targets exceeded the configured sample_limit. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/prometheus/prometheusscrapesamplelimithit + summary: Prometheus has failed scrapes that have exceeded the configured sample limit. + expr: increase(prometheus_target_scrapes_exceeded_sample_limit_total{job="{{ $prometheusJob }}",namespace="{{ $namespace }}"}[5m]) > 0 + for: {{ dig "PrometheusScrapeSampleLimitHit" "for" "15m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "PrometheusScrapeSampleLimitHit" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.prometheus }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.prometheus }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.PrometheusTargetSyncFailure | default false) }} + - alert: PrometheusTargetSyncFailure + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.prometheus }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.prometheus | indent 8 }} +{{- end }} + description: '{{`{{`}} printf "%.0f" $value {{`}}`}} targets in Prometheus {{`{{`}}$labels.namespace{{`}}`}}/{{`{{`}}$labels.pod{{`}}`}} have failed to sync because invalid configuration was supplied.' + runbook_url: {{ .Values.defaultRules.runbookUrl }}/prometheus/prometheustargetsyncfailure + summary: Prometheus has failed to sync targets. + expr: increase(prometheus_target_sync_failed_total{job="{{ $prometheusJob }}",namespace="{{ $namespace }}"}[30m]) > 0 + for: {{ dig "PrometheusTargetSyncFailure" "for" "5m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "PrometheusTargetSyncFailure" "severity" "critical" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.prometheus }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.prometheus }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.PrometheusHighQueryLoad | default false) }} + - alert: PrometheusHighQueryLoad + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.prometheus }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.prometheus | indent 8 }} +{{- end }} + description: Prometheus {{`{{`}}$labels.namespace{{`}}`}}/{{`{{`}}$labels.pod{{`}}`}} query API has less than 20% available capacity in its query engine for the last 15 minutes. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/prometheus/prometheushighqueryload + summary: Prometheus is reaching its maximum capacity serving concurrent requests. + expr: avg_over_time(prometheus_engine_queries{job="{{ $prometheusJob }}",namespace="{{ $namespace }}"}[5m]) / max_over_time(prometheus_engine_queries_concurrent_max{job="{{ $prometheusJob }}",namespace="{{ $namespace }}"}[5m]) > 0.8 + for: {{ dig "PrometheusHighQueryLoad" "for" "15m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "PrometheusHighQueryLoad" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.prometheus }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.prometheus }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.PrometheusErrorSendingAlertsToAnyAlertmanager | default false) }} + - alert: PrometheusErrorSendingAlertsToAnyAlertmanager + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.prometheus }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.prometheus | indent 8 }} +{{- end }} + description: '{{`{{`}} printf "%.1f" $value {{`}}`}}% minimum errors while sending alerts from Prometheus {{`{{`}}$labels.namespace{{`}}`}}/{{`{{`}}$labels.pod{{`}}`}} to any Alertmanager.' + runbook_url: {{ .Values.defaultRules.runbookUrl }}/prometheus/prometheuserrorsendingalertstoanyalertmanager + summary: Prometheus encounters more than 3% errors sending alerts to any Alertmanager. + expr: |- + min without (alertmanager) ( + rate(prometheus_notifications_errors_total{job="{{ $prometheusJob }}",namespace="{{ $namespace }}",alertmanager!~``}[5m]) + / + rate(prometheus_notifications_sent_total{job="{{ $prometheusJob }}",namespace="{{ $namespace }}",alertmanager!~``}[5m]) + ) + * 100 + > 3 + for: {{ dig "PrometheusErrorSendingAlertsToAnyAlertmanager" "for" "15m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "PrometheusErrorSendingAlertsToAnyAlertmanager" "severity" "critical" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.prometheus }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.prometheus }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/windows.node.rules.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/windows.node.rules.yaml new file mode 100644 index 00000000000..7c255538617 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/windows.node.rules.yaml @@ -0,0 +1,301 @@ +{{- /* +Generated from 'windows.node.rules' group from https://github.com/kubernetes-monitoring/kubernetes-mixin.git +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.defaultRules.create .Values.windowsMonitoring.enabled .Values.defaultRules.rules.windows }} +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "windows.node.rules" | trunc 63 | trimSuffix "-" }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.defaultRules.labels }} +{{ toYaml .Values.defaultRules.labels | indent 4 }} +{{- end }} +{{- if .Values.defaultRules.annotations }} + annotations: +{{ toYaml .Values.defaultRules.annotations | indent 4 }} +{{- end }} +spec: + groups: + - name: windows.node.rules + rules: + - expr: |- + count by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) ( + windows_system_system_up_time{job="windows-exporter"} + ) + record: node:windows_node:sum + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.windows }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.windows }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: |- + count by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, instance) (sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, instance, core) ( + windows_cpu_time_total{job="windows-exporter"} + )) + record: node:windows_node_num_cpu:sum + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.windows }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.windows }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: 1 - avg by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) (rate(windows_cpu_time_total{job="windows-exporter",mode="idle"}[1m])) + record: :windows_node_cpu_utilisation:avg1m + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.windows }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.windows }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: |- + 1 - avg by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, instance) ( + rate(windows_cpu_time_total{job="windows-exporter",mode="idle"}[1m]) + ) + record: node:windows_node_cpu_utilisation:avg1m + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.windows }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.windows }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: |- + 1 - + sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) (windows_memory_available_bytes{job="windows-exporter"}) + / + sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) (windows_os_visible_memory_bytes{job="windows-exporter"}) + record: ':windows_node_memory_utilisation:' + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.windows }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.windows }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) (windows_memory_available_bytes{job="windows-exporter"} + windows_memory_cache_bytes{job="windows-exporter"}) + record: :windows_node_memory_MemFreeCached_bytes:sum + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.windows }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.windows }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: (windows_memory_cache_bytes{job="windows-exporter"} + windows_memory_modified_page_list_bytes{job="windows-exporter"} + windows_memory_standby_cache_core_bytes{job="windows-exporter"} + windows_memory_standby_cache_normal_priority_bytes{job="windows-exporter"} + windows_memory_standby_cache_reserve_bytes{job="windows-exporter"}) + record: node:windows_node_memory_totalCached_bytes:sum + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.windows }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.windows }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) (windows_os_visible_memory_bytes{job="windows-exporter"}) + record: :windows_node_memory_MemTotal_bytes:sum + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.windows }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.windows }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: |- + sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, instance) ( + (windows_memory_available_bytes{job="windows-exporter"}) + ) + record: node:windows_node_memory_bytes_available:sum + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.windows }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.windows }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: |- + sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, instance) ( + windows_os_visible_memory_bytes{job="windows-exporter"} + ) + record: node:windows_node_memory_bytes_total:sum + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.windows }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.windows }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: |- + (node:windows_node_memory_bytes_total:sum - node:windows_node_memory_bytes_available:sum) + / + scalar(sum(node:windows_node_memory_bytes_total:sum)) + record: node:windows_node_memory_utilisation:ratio + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.windows }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.windows }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: 1 - (node:windows_node_memory_bytes_available:sum / node:windows_node_memory_bytes_total:sum) + record: 'node:windows_node_memory_utilisation:' + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.windows }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.windows }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: irate(windows_memory_swap_page_operations_total{job="windows-exporter"}[5m]) + record: node:windows_node_memory_swap_io_pages:irate + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.windows }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.windows }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: |- + avg by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) (irate(windows_logical_disk_read_seconds_total{job="windows-exporter"}[1m]) + + irate(windows_logical_disk_write_seconds_total{job="windows-exporter"}[1m]) + ) + record: :windows_node_disk_utilisation:avg_irate + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.windows }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.windows }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: |- + avg by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, instance) ( + (irate(windows_logical_disk_read_seconds_total{job="windows-exporter"}[1m]) + + irate(windows_logical_disk_write_seconds_total{job="windows-exporter"}[1m])) + ) + record: node:windows_node_disk_utilisation:avg_irate + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.windows }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.windows }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: |- + max by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster,instance,volume)( + (windows_logical_disk_size_bytes{job="windows-exporter"} + - windows_logical_disk_free_bytes{job="windows-exporter"}) + / windows_logical_disk_size_bytes{job="windows-exporter"} + ) + record: 'node:windows_node_filesystem_usage:' + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.windows }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.windows }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: max by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, instance, volume) (windows_logical_disk_free_bytes{job="windows-exporter"} / windows_logical_disk_size_bytes{job="windows-exporter"}) + record: 'node:windows_node_filesystem_avail:' + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.windows }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.windows }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) (irate(windows_net_bytes_total{job="windows-exporter"}[1m])) + record: :windows_node_net_utilisation:sum_irate + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.windows }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.windows }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: |- + sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, instance) ( + (irate(windows_net_bytes_total{job="windows-exporter"}[1m])) + ) + record: node:windows_node_net_utilisation:sum_irate + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.windows }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.windows }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: |- + sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) (irate(windows_net_packets_received_discarded_total{job="windows-exporter"}[1m])) + + sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) (irate(windows_net_packets_outbound_discarded_total{job="windows-exporter"}[1m])) + record: :windows_node_net_saturation:sum_irate + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.windows }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.windows }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: |- + sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, instance) ( + (irate(windows_net_packets_received_discarded_total{job="windows-exporter"}[1m]) + + irate(windows_net_packets_outbound_discarded_total{job="windows-exporter"}[1m])) + ) + record: node:windows_node_net_saturation:sum_irate + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.windows }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.windows }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/windows.pod.rules.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/windows.pod.rules.yaml new file mode 100644 index 00000000000..86340b5c057 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/rules-1.14/windows.pod.rules.yaml @@ -0,0 +1,158 @@ +{{- /* +Generated from 'windows.pod.rules' group from https://github.com/kubernetes-monitoring/kubernetes-mixin.git +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.defaultRules.create .Values.windowsMonitoring.enabled .Values.defaultRules.rules.windows }} +{{- $kubeStateMetricsJob := include "kube-prometheus-stack-kube-state-metrics.name" . }} +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "windows.pod.rules" | trunc 63 | trimSuffix "-" }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.defaultRules.labels }} +{{ toYaml .Values.defaultRules.labels | indent 4 }} +{{- end }} +{{- if .Values.defaultRules.annotations }} + annotations: +{{ toYaml .Values.defaultRules.annotations | indent 4 }} +{{- end }} +spec: + groups: + - name: windows.pod.rules + rules: + - expr: windows_container_available{job="windows-exporter", container_id != ""} * on ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}container_id, cluster) group_left(container, pod, namespace) max(kube_pod_container_info{job="{{ $kubeStateMetricsJob }}", container_id != ""}) by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}container, container_id, pod, namespace, cluster) + record: windows_pod_container_available + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.windows }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.windows }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: windows_container_cpu_usage_seconds_total{job="windows-exporter", container_id != ""} * on ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}container_id, cluster) group_left(container, pod, namespace) max(kube_pod_container_info{job="{{ $kubeStateMetricsJob }}", container_id != ""}) by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}container, container_id, pod, namespace, cluster) + record: windows_container_total_runtime + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.windows }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.windows }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: windows_container_memory_usage_commit_bytes{job="windows-exporter", container_id != ""} * on ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}container_id, cluster) group_left(container, pod, namespace) max(kube_pod_container_info{job="{{ $kubeStateMetricsJob }}", container_id != ""}) by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}container, container_id, pod, namespace, cluster) + record: windows_container_memory_usage + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.windows }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.windows }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: windows_container_memory_usage_private_working_set_bytes{job="windows-exporter", container_id != ""} * on ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}container_id, cluster) group_left(container, pod, namespace) max(kube_pod_container_info{job="{{ $kubeStateMetricsJob }}", container_id != ""}) by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}container, container_id, pod, namespace, cluster) + record: windows_container_private_working_set_usage + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.windows }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.windows }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: windows_container_network_receive_bytes_total{job="windows-exporter", container_id != ""} * on ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}container_id, cluster) group_left(container, pod, namespace) max(kube_pod_container_info{job="{{ $kubeStateMetricsJob }}", container_id != ""}) by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}container, container_id, pod, namespace, cluster) + record: windows_container_network_received_bytes_total + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.windows }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.windows }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: windows_container_network_transmit_bytes_total{job="windows-exporter", container_id != ""} * on ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}container_id, cluster) group_left(container, pod, namespace) max(kube_pod_container_info{job="{{ $kubeStateMetricsJob }}", container_id != ""}) by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}container, container_id, pod, namespace, cluster) + record: windows_container_network_transmitted_bytes_total + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.windows }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.windows }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: |- + max by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, namespace, pod, container) ( + kube_pod_container_resource_requests{resource="memory",job="{{ $kubeStateMetricsJob }}"} + ) * on ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}container,pod,namespace,cluster) (windows_pod_container_available) + record: kube_pod_windows_container_resource_memory_request + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.windows }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.windows }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: kube_pod_container_resource_limits{resource="memory",job="{{ $kubeStateMetricsJob }}"} * on ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}container,pod,namespace,cluster) (windows_pod_container_available) + record: kube_pod_windows_container_resource_memory_limit + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.windows }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.windows }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: |- + max by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, namespace, pod, container) ( + kube_pod_container_resource_requests{resource="cpu",job="{{ $kubeStateMetricsJob }}"} + ) * on ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}container,pod,namespace,cluster) (windows_pod_container_available) + record: kube_pod_windows_container_resource_cpu_cores_request + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.windows }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.windows }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: kube_pod_container_resource_limits{resource="cpu",job="{{ $kubeStateMetricsJob }}"} * on ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}container,pod,namespace,cluster) (windows_pod_container_available) + record: kube_pod_windows_container_resource_cpu_cores_limit + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.windows }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.windows }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: |- + sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, namespace, pod, container) ( + rate(windows_container_total_runtime{}[5m]) + ) + record: namespace_pod_container:windows_container_cpu_usage_seconds_total:sum_rate + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.windows }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.windows }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/secret.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/secret.yaml new file mode 100644 index 00000000000..e4a1e73c7b8 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/secret.yaml @@ -0,0 +1,15 @@ +{{- if and .Values.prometheus.enabled .Values.prometheus.prometheusSpec.thanos .Values.prometheus.prometheusSpec.thanos.objectStorageConfig}} +{{- if and .Values.prometheus.prometheusSpec.thanos.objectStorageConfig.secret (not .Values.prometheus.prometheusSpec.thanos.objectStorageConfig.existingSecret) }} +apiVersion: v1 +kind: Secret +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-prometheus + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }}-prometheus + app.kubernetes.io/component: prometheus +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +data: + object-storage-configs.yaml: {{ toYaml .Values.prometheus.prometheusSpec.thanos.objectStorageConfig.secret | b64enc | quote }} +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/service.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/service.yaml new file mode 100644 index 00000000000..d61b9d6ef39 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/service.yaml @@ -0,0 +1,80 @@ +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if .Values.prometheus.enabled }} +apiVersion: v1 +kind: Service +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-prometheus + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }}-prometheus + self-monitor: {{ .Values.prometheus.serviceMonitor.selfMonitor | quote }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.prometheus.service.labels }} +{{ toYaml .Values.prometheus.service.labels | indent 4 }} +{{- end }} +{{- if .Values.prometheus.service.annotations }} + annotations: +{{ toYaml .Values.prometheus.service.annotations | indent 4 }} +{{- end }} +spec: +{{- if .Values.prometheus.service.clusterIP }} + clusterIP: {{ .Values.prometheus.service.clusterIP }} +{{- end }} +{{- if .Values.prometheus.service.externalIPs }} + externalIPs: +{{ toYaml .Values.prometheus.service.externalIPs | indent 4 }} +{{- end }} +{{- if .Values.prometheus.service.loadBalancerIP }} + loadBalancerIP: {{ .Values.prometheus.service.loadBalancerIP }} +{{- end }} +{{- if .Values.prometheus.service.loadBalancerSourceRanges }} + loadBalancerSourceRanges: + {{- range $cidr := .Values.prometheus.service.loadBalancerSourceRanges }} + - {{ $cidr }} + {{- end }} +{{- end }} +{{- if ne .Values.prometheus.service.type "ClusterIP" }} + externalTrafficPolicy: {{ .Values.prometheus.service.externalTrafficPolicy }} +{{- end }} + ports: + - name: {{ .Values.prometheus.prometheusSpec.portName }} + {{- if eq .Values.prometheus.service.type "NodePort" }} + nodePort: {{ .Values.prometheus.service.nodePort }} + {{- end }} + port: {{ .Values.prometheus.service.port }} + targetPort: {{ .Values.prometheus.service.targetPort }} + - name: reloader-web + {{- if semverCompare "> 1.20.0-0" $kubeTargetVersion }} + appProtocol: http + {{- end }} + port: {{ .Values.prometheus.service.reloaderWebPort }} + targetPort: reloader-web + {{- if .Values.prometheus.thanosIngress.enabled }} + - name: grpc + {{- if eq .Values.prometheus.service.type "NodePort" }} + nodePort: {{ .Values.prometheus.thanosIngress.nodePort }} + {{- end }} + port: {{ .Values.prometheus.thanosIngress.servicePort }} + targetPort: {{ .Values.prometheus.thanosIngress.servicePort }} + {{- end }} +{{- if .Values.prometheus.service.additionalPorts }} +{{ toYaml .Values.prometheus.service.additionalPorts | indent 2 }} +{{- end }} + publishNotReadyAddresses: {{ .Values.prometheus.service.publishNotReadyAddresses }} + selector: + {{- if .Values.prometheus.agentMode }} + app.kubernetes.io/name: prometheus-agent + {{- else }} + app.kubernetes.io/name: prometheus + {{- end }} + operator.prometheus.io/name: {{ template "kube-prometheus-stack.prometheus.crname" . }} +{{- if .Values.prometheus.service.sessionAffinity }} + sessionAffinity: {{ .Values.prometheus.service.sessionAffinity }} +{{- end }} +{{- if eq .Values.prometheus.service.sessionAffinity "ClientIP" }} + sessionAffinityConfig: + clientIP: + timeoutSeconds: {{ .Values.prometheus.service.sessionAffinityConfig.clientIP.timeoutSeconds }} +{{- end }} + type: "{{ .Values.prometheus.service.type }}" +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/serviceThanosSidecar.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/serviceThanosSidecar.yaml new file mode 100644 index 00000000000..15b89c8c238 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/serviceThanosSidecar.yaml @@ -0,0 +1,39 @@ +{{- if and .Values.prometheus.enabled .Values.prometheus.thanosService.enabled }} +apiVersion: v1 +kind: Service +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-thanos-discovery + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }}-thanos-discovery +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.prometheus.thanosService.labels }} +{{ toYaml .Values.prometheus.thanosService.labels | indent 4 }} +{{- end }} +{{- if .Values.prometheus.thanosService.annotations }} + annotations: +{{ toYaml .Values.prometheus.thanosService.annotations | indent 4 }} +{{- end }} +spec: + type: {{ .Values.prometheus.thanosService.type }} + clusterIP: {{ .Values.prometheus.thanosService.clusterIP }} +{{- if ne .Values.prometheus.thanosService.type "ClusterIP" }} + externalTrafficPolicy: {{ .Values.prometheus.thanosService.externalTrafficPolicy }} +{{- end }} + ports: + - name: {{ .Values.prometheus.thanosService.portName }} + port: {{ .Values.prometheus.thanosService.port }} + targetPort: {{ .Values.prometheus.thanosService.targetPort }} + {{- if eq .Values.prometheus.thanosService.type "NodePort" }} + nodePort: {{ .Values.prometheus.thanosService.nodePort }} + {{- end }} + - name: {{ .Values.prometheus.thanosService.httpPortName }} + port: {{ .Values.prometheus.thanosService.httpPort }} + targetPort: {{ .Values.prometheus.thanosService.targetHttpPort }} + {{- if eq .Values.prometheus.thanosService.type "NodePort" }} + nodePort: {{ .Values.prometheus.thanosService.httpNodePort }} + {{- end }} + selector: + app.kubernetes.io/name: prometheus + operator.prometheus.io/name: {{ template "kube-prometheus-stack.prometheus.crname" . }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/serviceThanosSidecarExternal.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/serviceThanosSidecarExternal.yaml new file mode 100644 index 00000000000..453eed7f1bb --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/serviceThanosSidecarExternal.yaml @@ -0,0 +1,46 @@ +{{- if and .Values.prometheus.enabled .Values.prometheus.thanosServiceExternal.enabled }} +apiVersion: v1 +kind: Service +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-thanos-external + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.prometheus.thanosServiceExternal.labels }} +{{ toYaml .Values.prometheus.thanosServiceExternal.labels | indent 4 }} +{{- end }} +{{- if .Values.prometheus.thanosServiceExternal.annotations }} + annotations: +{{ toYaml .Values.prometheus.thanosServiceExternal.annotations | indent 4 }} +{{- end }} +spec: + type: {{ .Values.prometheus.thanosServiceExternal.type }} +{{- if .Values.prometheus.thanosServiceExternal.loadBalancerIP }} + loadBalancerIP: {{ .Values.prometheus.thanosServiceExternal.loadBalancerIP }} +{{- end }} +{{- if .Values.prometheus.thanosServiceExternal.loadBalancerSourceRanges }} + loadBalancerSourceRanges: + {{- range $cidr := .Values.prometheus.thanosServiceExternal.loadBalancerSourceRanges }} + - {{ $cidr }} + {{- end }} +{{- end }} +{{- if ne .Values.prometheus.thanosServiceExternal.type "ClusterIP" }} + externalTrafficPolicy: {{ .Values.prometheus.thanosServiceExternal.externalTrafficPolicy }} +{{- end }} + ports: + - name: {{ .Values.prometheus.thanosServiceExternal.portName }} + port: {{ .Values.prometheus.thanosServiceExternal.port }} + targetPort: {{ .Values.prometheus.thanosServiceExternal.targetPort }} + {{- if eq .Values.prometheus.thanosServiceExternal.type "NodePort" }} + nodePort: {{ .Values.prometheus.thanosServiceExternal.nodePort }} + {{- end }} + - name: {{ .Values.prometheus.thanosServiceExternal.httpPortName }} + port: {{ .Values.prometheus.thanosServiceExternal.httpPort }} + targetPort: {{ .Values.prometheus.thanosServiceExternal.targetHttpPort }} + {{- if eq .Values.prometheus.thanosServiceExternal.type "NodePort" }} + nodePort: {{ .Values.prometheus.thanosServiceExternal.httpNodePort }} + {{- end }} + selector: + app.kubernetes.io/name: prometheus + operator.prometheus.io/name: {{ template "kube-prometheus-stack.prometheus.crname" . }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/serviceaccount.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/serviceaccount.yaml new file mode 100644 index 00000000000..e97b989bbdf --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/serviceaccount.yaml @@ -0,0 +1,21 @@ +{{- if and .Values.prometheus.enabled .Values.prometheus.serviceAccount.create }} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "kube-prometheus-stack.prometheus.serviceAccountName" . }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }}-prometheus + app.kubernetes.io/name: {{ template "kube-prometheus-stack.name" . }}-prometheus + app.kubernetes.io/component: prometheus +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.prometheus.serviceAccount.annotations }} + annotations: +{{ toYaml .Values.prometheus.serviceAccount.annotations | indent 4 }} +{{- end }} +automountServiceAccountToken: {{ .Values.prometheus.serviceAccount.automountServiceAccountToken }} +{{- if .Values.global.imagePullSecrets }} +imagePullSecrets: +{{ include "kube-prometheus-stack.imagePullSecrets" . | trim | indent 2 }} +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/servicemonitor.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/servicemonitor.yaml new file mode 100644 index 00000000000..a36f3e33ca4 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/servicemonitor.yaml @@ -0,0 +1,97 @@ +{{- if and .Values.prometheus.enabled .Values.prometheus.serviceMonitor.selfMonitor }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-prometheus + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }}-prometheus +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- with .Values.prometheus.serviceMonitor.additionalLabels }} +{{- toYaml . | nindent 4 }} +{{- end }} +spec: + {{- include "servicemonitor.scrapeLimits" .Values.prometheus.serviceMonitor | nindent 2 }} + selector: + matchLabels: + app: {{ template "kube-prometheus-stack.name" . }}-prometheus + release: {{ $.Release.Name | quote }} + self-monitor: "true" + namespaceSelector: + matchNames: + - {{ printf "%s" (include "kube-prometheus-stack.namespace" .) | quote }} + endpoints: + - port: {{ .Values.prometheus.prometheusSpec.portName }} + {{- if .Values.prometheus.serviceMonitor.interval }} + interval: {{ .Values.prometheus.serviceMonitor.interval }} + {{- end }} + {{- if .Values.prometheus.serviceMonitor.scheme }} + scheme: {{ .Values.prometheus.serviceMonitor.scheme }} + {{- end }} + {{- if .Values.prometheus.serviceMonitor.tlsConfig }} + tlsConfig: {{- toYaml .Values.prometheus.serviceMonitor.tlsConfig | nindent 6 }} + {{- end }} + {{- if .Values.prometheus.serviceMonitor.bearerTokenFile }} + bearerTokenFile: {{ .Values.prometheus.serviceMonitor.bearerTokenFile }} + {{- end }} + path: "{{ trimSuffix "/" .Values.prometheus.prometheusSpec.routePrefix }}/metrics" + metricRelabelings: + {{- if .Values.prometheus.serviceMonitor.metricRelabelings }} + {{- tpl (toYaml .Values.prometheus.serviceMonitor.metricRelabelings | nindent 6) . }} + {{- end }} + {{ if .Values.global.cattle.clusterId }} + - sourceLabels: [__address__] + targetLabel: cluster_id + replacement: {{ .Values.global.cattle.clusterId }} + {{- end }} + {{ if .Values.global.cattle.clusterName}} + - sourceLabels: [__address__] + targetLabel: cluster_name + replacement: {{ .Values.global.cattle.clusterName }} + {{- end }} + {{- if .Values.prometheus.serviceMonitor.relabelings }} + relabelings: {{- toYaml .Values.prometheus.serviceMonitor.relabelings | nindent 6 }} + {{- end }} + - port: reloader-web + {{- if .Values.prometheus.serviceMonitor.interval }} + interval: {{ .Values.prometheus.serviceMonitor.interval }} + {{- end }} + {{- if .Values.prometheus.serviceMonitor.scheme }} + scheme: {{ .Values.prometheus.serviceMonitor.scheme }} + {{- end }} + {{- if .Values.prometheus.serviceMonitor.tlsConfig }} + tlsConfig: {{- toYaml .Values.prometheus.serviceMonitor.tlsConfig | nindent 6 }} + {{- end }} + path: "/metrics" + {{- if .Values.prometheus.serviceMonitor.metricRelabelings }} + metricRelabelings: {{- tpl (toYaml .Values.prometheus.serviceMonitor.metricRelabelings | nindent 6) . }} + {{- end }} + {{- if .Values.prometheus.serviceMonitor.relabelings }} + relabelings: {{- toYaml .Values.prometheus.serviceMonitor.relabelings | nindent 6 }} + {{- end }} + {{- range .Values.prometheus.serviceMonitor.additionalEndpoints }} + - port: {{ .port }} + {{- if or $.Values.prometheus.serviceMonitor.interval .interval }} + interval: {{ default $.Values.prometheus.serviceMonitor.interval .interval }} + {{- end }} + {{- if or $.Values.prometheus.serviceMonitor.proxyUrl .proxyUrl }} + proxyUrl: {{ default $.Values.prometheus.serviceMonitor.proxyUrl .proxyUrl }} + {{- end }} + {{- if or $.Values.prometheus.serviceMonitor.scheme .scheme }} + scheme: {{ default $.Values.prometheus.serviceMonitor.scheme .scheme }} + {{- end }} + {{- if or $.Values.prometheus.serviceMonitor.bearerTokenFile .bearerTokenFile }} + bearerTokenFile: {{ default $.Values.prometheus.serviceMonitor.bearerTokenFile .bearerTokenFile }} + {{- end }} + {{- if or $.Values.prometheus.serviceMonitor.tlsConfig .tlsConfig }} + tlsConfig: {{- default $.Values.prometheus.serviceMonitor.tlsConfig .tlsConfig | toYaml | nindent 6 }} + {{- end }} + path: {{ .path }} + {{- if or $.Values.prometheus.serviceMonitor.metricRelabelings .metricRelabelings }} + metricRelabelings: {{- tpl (default $.Values.prometheus.serviceMonitor.metricRelabelings .metricRelabelings | toYaml | nindent 6) . }} + {{- end }} + {{- if or $.Values.prometheus.serviceMonitor.relabelings .relabelings }} + relabelings: {{- default $.Values.prometheus.serviceMonitor.relabelings .relabelings | toYaml | nindent 6 }} + {{- end }} + {{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/servicemonitorThanosSidecar.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/servicemonitorThanosSidecar.yaml new file mode 100644 index 00000000000..0f70aabb58a --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/servicemonitorThanosSidecar.yaml @@ -0,0 +1,55 @@ +{{- if and .Values.prometheus.thanosService.enabled .Values.prometheus.thanosServiceMonitor.enabled }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-thanos-sidecar + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }}-thanos-sidecar +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- with .Values.prometheus.thanosServiceMonitor.additionalLabels }} +{{- toYaml . | nindent 4 }} +{{- end }} +spec: + {{- include "servicemonitor.scrapeLimits" .Values.prometheus.thanosServiceMonitor | nindent 2 }} + selector: + matchLabels: + app: {{ template "kube-prometheus-stack.name" . }}-thanos-discovery + release: {{ $.Release.Name | quote }} + namespaceSelector: + matchNames: + - {{ printf "%s" (include "kube-prometheus-stack.namespace" .) | quote }} + endpoints: + - port: {{ .Values.prometheus.thanosService.httpPortName }} + {{- if .Values.prometheus.thanosServiceMonitor.interval }} + interval: {{ .Values.prometheus.thanosServiceMonitor.interval }} + {{- end }} + {{- if .Values.prometheus.thanosServiceMonitor.scheme }} + scheme: {{ .Values.prometheus.thanosServiceMonitor.scheme }} + {{- end }} + {{- if .Values.prometheus.thanosServiceMonitor.tlsConfig }} + tlsConfig: {{ toYaml .Values.prometheus.thanosServiceMonitor.tlsConfig | nindent 6 }} + {{- end }} + {{- if .Values.prometheus.thanosServiceMonitor.bearerTokenFile }} + bearerTokenFile: {{ .Values.prometheus.thanosServiceMonitor.bearerTokenFile }} + {{- end }} + path: "/metrics" + metricRelabelings: + {{- if .Values.prometheus.thanosServiceMonitor.metricRelabelings}} + {{ tpl (toYaml .Values.prometheus.thanosServiceMonitor.metricRelabelings | indent 6) . }} + {{- end }} + {{ if .Values.global.cattle.clusterId }} + - sourceLabels: [__address__] + targetLabel: cluster_id + replacement: {{ .Values.global.cattle.clusterId }} + {{- end }} + {{ if .Values.global.cattle.clusterName}} + - sourceLabels: [__address__] + targetLabel: cluster_name + replacement: {{ .Values.global.cattle.clusterName }} + {{- end }} +{{- if .Values.prometheus.thanosServiceMonitor.relabelings }} + relabelings: +{{ toYaml .Values.prometheus.thanosServiceMonitor.relabelings | indent 6 }} +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/servicemonitors.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/servicemonitors.yaml new file mode 100644 index 00000000000..a7a301babc4 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/servicemonitors.yaml @@ -0,0 +1,47 @@ +{{- if and .Values.prometheus.enabled .Values.prometheus.additionalServiceMonitors }} +apiVersion: v1 +kind: List +items: +{{- range .Values.prometheus.additionalServiceMonitors }} + - apiVersion: monitoring.coreos.com/v1 + kind: ServiceMonitor + metadata: + name: {{ .name }} + namespace: {{ template "kube-prometheus-stack.namespace" $ }} + labels: + app: {{ template "kube-prometheus-stack.name" $ }}-prometheus +{{ include "kube-prometheus-stack.labels" $ | indent 8 }} + {{- if .additionalLabels }} +{{ toYaml .additionalLabels | indent 8 }} + {{- end }} + spec: + {{- include "servicemonitor.scrapeLimits" . | nindent 6 }} + endpoints: +{{ toYaml .endpoints | indent 8 }} + {{- if .jobLabel }} + jobLabel: {{ .jobLabel }} + {{- end }} + {{- if .namespaceSelector }} + namespaceSelector: +{{ toYaml .namespaceSelector | indent 8 }} + {{- end }} + selector: +{{ toYaml .selector | indent 8 }} + {{- if .targetLabels }} + targetLabels: +{{ toYaml .targetLabels | indent 8 }} + {{- end }} + {{- if .podTargetLabels }} + podTargetLabels: +{{ toYaml .podTargetLabels | indent 8 }} + {{- end }} + {{- if .metricRelabelings }} + metricRelabelings: +{{ toYaml .metricRelabelings | indent 8 }} + {{- end }} + {{- if .relabelings }} + relabelings: +{{ toYaml .relabelings | indent 8 }} + {{- end }} +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/serviceperreplica.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/serviceperreplica.yaml new file mode 100644 index 00000000000..4bc7f7b8695 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/prometheus/serviceperreplica.yaml @@ -0,0 +1,54 @@ +{{- if and .Values.prometheus.enabled .Values.prometheus.servicePerReplica.enabled }} +{{- $count := .Values.prometheus.prometheusSpec.replicas | int -}} +{{- $serviceValues := .Values.prometheus.servicePerReplica -}} +apiVersion: v1 +kind: List +metadata: + name: {{ include "kube-prometheus-stack.fullname" $ }}-prometheus-serviceperreplica + namespace: {{ template "kube-prometheus-stack.namespace" . }} +items: +{{- range $i, $e := until $count }} + - apiVersion: v1 + kind: Service + metadata: + name: {{ include "kube-prometheus-stack.fullname" $ }}-prometheus-{{ $i }} + namespace: {{ template "kube-prometheus-stack.namespace" $ }} + labels: + app: {{ include "kube-prometheus-stack.name" $ }}-prometheus +{{ include "kube-prometheus-stack.labels" $ | indent 8 }} + {{- if $serviceValues.annotations }} + annotations: +{{ toYaml $serviceValues.annotations | indent 8 }} + {{- end }} + spec: + {{- if $serviceValues.clusterIP }} + clusterIP: {{ $serviceValues.clusterIP }} + {{- end }} + {{- if $serviceValues.loadBalancerSourceRanges }} + loadBalancerSourceRanges: + {{- range $cidr := $serviceValues.loadBalancerSourceRanges }} + - {{ $cidr }} + {{- end }} + {{- end }} + {{- if ne $serviceValues.type "ClusterIP" }} + externalTrafficPolicy: {{ $serviceValues.externalTrafficPolicy }} + {{- end }} + ports: + - name: {{ $.Values.prometheus.prometheusSpec.portName }} + {{- if eq $serviceValues.type "NodePort" }} + nodePort: {{ $serviceValues.nodePort }} + {{- end }} + port: {{ $serviceValues.port }} + targetPort: {{ $serviceValues.targetPort }} + selector: + {{- if $.Values.prometheus.agentMode }} + app.kubernetes.io/name: prometheus-agent + statefulset.kubernetes.io/pod-name: prom-agent-{{ include "kube-prometheus-stack.prometheus.crname" $ }}-{{ $i }} + {{- else }} + app.kubernetes.io/name: prometheus + statefulset.kubernetes.io/pod-name: prometheus-{{ include "kube-prometheus-stack.prometheus.crname" $ }}-{{ $i }} + {{- end }} + operator.prometheus.io/name: {{ template "kube-prometheus-stack.prometheus.crname" $ }} + type: "{{ $serviceValues.type }}" +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/rancher-monitoring/clusterrole.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/rancher-monitoring/clusterrole.yaml new file mode 100644 index 00000000000..56ca9f5eaeb --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/rancher-monitoring/clusterrole.yaml @@ -0,0 +1,135 @@ +{{- if and .Values.global.rbac.create .Values.global.rbac.userRoles.create }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: monitoring-admin + labels: {{ include "kube-prometheus-stack.labels" . | nindent 4 }} + {{- if .Values.global.rbac.userRoles.aggregateToDefaultRoles }} + rbac.authorization.k8s.io/aggregate-to-admin: "true" + {{- end }} +rules: +- apiGroups: + - monitoring.coreos.com + resources: + - alertmanagers + - prometheuses + - prometheuses/finalizers + - alertmanagers/finalizers + verbs: + - 'get' + - 'list' + - 'watch' +- apiGroups: + - monitoring.coreos.com + resources: + - thanosrulers + - thanosrulers/finalizers + - servicemonitors + - podmonitors + - prometheusrules + - podmonitors + - probes + - probes/finalizers + - alertmanagerconfigs + verbs: + - '*' +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: monitoring-edit + labels: {{ include "kube-prometheus-stack.labels" . | nindent 4 }} + {{- if .Values.global.rbac.userRoles.aggregateToDefaultRoles }} + rbac.authorization.k8s.io/aggregate-to-edit: "true" + {{- end }} +rules: +- apiGroups: + - monitoring.coreos.com + resources: + - alertmanagers + - prometheuses + - prometheuses/finalizers + - alertmanagers/finalizers + verbs: + - 'get' + - 'list' + - 'watch' +- apiGroups: + - monitoring.coreos.com + resources: + - thanosrulers + - thanosrulers/finalizers + - servicemonitors + - podmonitors + - prometheusrules + - podmonitors + - probes + - alertmanagerconfigs + verbs: + - '*' +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: monitoring-view + labels: {{ include "kube-prometheus-stack.labels" . | nindent 4 }} + {{- if .Values.global.rbac.userRoles.aggregateToDefaultRoles }} + rbac.authorization.k8s.io/aggregate-to-view: "true" + {{- end }} +rules: +- apiGroups: + - monitoring.coreos.com + resources: + - alertmanagers + - prometheuses + - prometheuses/finalizers + - alertmanagers/finalizers + - thanosrulers + - thanosrulers/finalizers + - servicemonitors + - podmonitors + - prometheusrules + - podmonitors + - probes + - probes/finalizers + - alertmanagerconfigs + verbs: + - 'get' + - 'list' + - 'watch' +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: monitoring-ui-view + labels: {{ include "kube-prometheus-stack.labels" . | nindent 4 }} +rules: +- apiGroups: + - "" + resources: + - services/proxy + resourceNames: + - "http:{{ template "kube-prometheus-stack.fullname" . }}-prometheus:{{ .Values.prometheus.service.port }}" + - "https:{{ template "kube-prometheus-stack.fullname" . }}-prometheus:{{ .Values.prometheus.service.port }}" + - "http:{{ template "kube-prometheus-stack.fullname" . }}-alertmanager:{{ .Values.alertmanager.service.port }}" + - "https:{{ template "kube-prometheus-stack.fullname" . }}-alertmanager:{{ .Values.alertmanager.service.port }}" +{{- if .Values.grafana.enabled }} + - "http:{{ include "call-nested" (list . "grafana" "grafana.fullname") }}:{{ .Values.grafana.service.port }}" + - "https:{{ include "call-nested" (list . "grafana" "grafana.fullname") }}:{{ .Values.grafana.service.port }}" +{{- end }} + verbs: + - 'get' + - 'create' +- apiGroups: + - "" + resourceNames: + - {{ template "kube-prometheus-stack.fullname" . }}-prometheus + - {{ template "kube-prometheus-stack.fullname" . }}-alertmanager +{{- if .Values.grafana.enabled }} + - {{ include "call-nested" (list . "grafana" "grafana.fullname") }} +{{- end }} + resources: + - endpoints + verbs: + - list +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/rancher-monitoring/config-role.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/rancher-monitoring/config-role.yaml new file mode 100644 index 00000000000..f48ffc827e1 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/rancher-monitoring/config-role.yaml @@ -0,0 +1,48 @@ +{{- if and .Values.global.rbac.create .Values.global.rbac.userRoles.create }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: monitoring-config-admin + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: {{ include "kube-prometheus-stack.labels" . | nindent 4 }} +rules: +- apiGroups: + - "" + resources: + - configmaps + - secrets + verbs: + - '*' +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: monitoring-config-edit + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: {{ include "kube-prometheus-stack.labels" . | nindent 4 }} +rules: +- apiGroups: + - "" + resources: + - configmaps + - secrets + verbs: + - '*' +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: monitoring-config-view + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: {{ include "kube-prometheus-stack.labels" . | nindent 4 }} +rules: +- apiGroups: + - "" + resources: + - configmaps + - secrets + verbs: + - 'get' + - 'list' + - 'watch' +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/rancher-monitoring/dashboard-role.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/rancher-monitoring/dashboard-role.yaml new file mode 100644 index 00000000000..d2f81976a2b --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/rancher-monitoring/dashboard-role.yaml @@ -0,0 +1,47 @@ +{{- if and .Values.global.rbac.create .Values.global.rbac.userRoles.create .Values.grafana.enabled }} +{{- if .Values.grafana.defaultDashboardsEnabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: monitoring-dashboard-admin + namespace: {{ .Values.grafana.defaultDashboards.namespace }} + labels: {{ include "kube-prometheus-stack.labels" . | nindent 4 }} +rules: +- apiGroups: + - "" + resources: + - configmaps + verbs: + - '*' +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: monitoring-dashboard-edit + namespace: {{ .Values.grafana.defaultDashboards.namespace }} + labels: {{ include "kube-prometheus-stack.labels" . | nindent 4 }} +rules: +- apiGroups: + - "" + resources: + - configmaps + verbs: + - '*' +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: monitoring-dashboard-view + namespace: {{ .Values.grafana.defaultDashboards.namespace }} + labels: {{ include "kube-prometheus-stack.labels" . | nindent 4 }} +rules: +- apiGroups: + - "" + resources: + - configmaps + verbs: + - 'get' + - 'list' + - 'watch' +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/rancher-monitoring/dashboards/addons/ingress-nginx-dashboard.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/rancher-monitoring/dashboards/addons/ingress-nginx-dashboard.yaml new file mode 100644 index 00000000000..7b51a0bf7ab --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/rancher-monitoring/dashboards/addons/ingress-nginx-dashboard.yaml @@ -0,0 +1,18 @@ +{{- if and .Values.grafana.enabled .Values.grafana.defaultDashboardsEnabled .Values.ingressNginx.enabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: {{ .Values.grafana.defaultDashboards.namespace }} + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" $) "ingress-nginx" | trunc 63 | trimSuffix "-" }} + {{- if .Values.grafana.sidecar.dashboards.annotations }} + annotations: {{ toYaml .Values.grafana.sidecar.dashboards.annotations | nindent 4 }} + {{- end }} + labels: + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: "1" + {{- end }} + app: {{ template "kube-prometheus-stack.name" $ }}-grafana +{{ include "kube-prometheus-stack.labels" $ | indent 4 }} +data: +{{ (.Files.Glob "files/ingress-nginx/*").AsConfig | indent 2 }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/rancher-monitoring/dashboards/rancher/cluster-dashboards.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/rancher-monitoring/dashboards/rancher/cluster-dashboards.yaml new file mode 100644 index 00000000000..d73b257451c --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/rancher-monitoring/dashboards/rancher/cluster-dashboards.yaml @@ -0,0 +1,17 @@ +{{- if and .Values.grafana.enabled .Values.grafana.defaultDashboardsEnabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: {{ .Values.grafana.defaultDashboards.namespace }} + name: rancher-default-dashboards-cluster + annotations: +{{ toYaml .Values.grafana.sidecar.dashboards.annotations | indent 4 }} + labels: + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: "1" + {{- end }} + app: {{ template "kube-prometheus-stack.name" $ }}-grafana +{{ include "kube-prometheus-stack.labels" $ | indent 4 }} +data: +{{ (.Files.Glob "files/rancher/cluster/*").AsConfig | indent 2 }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/rancher-monitoring/dashboards/rancher/default-dashboard.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/rancher-monitoring/dashboards/rancher/default-dashboard.yaml new file mode 100644 index 00000000000..8865efa9323 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/rancher-monitoring/dashboards/rancher/default-dashboard.yaml @@ -0,0 +1,17 @@ +{{- if and .Values.grafana.enabled .Values.grafana.defaultDashboardsEnabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: {{ .Values.grafana.defaultDashboards.namespace }} + name: rancher-default-dashboards-home + annotations: +{{ toYaml .Values.grafana.sidecar.dashboards.annotations | indent 4 }} + labels: + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: "1" + {{- end }} + app: {{ template "kube-prometheus-stack.name" $ }}-grafana +{{ include "kube-prometheus-stack.labels" $ | indent 4 }} +data: +{{ (.Files.Glob "files/rancher/home/*").AsConfig | indent 2 }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/rancher-monitoring/dashboards/rancher/fleet-dashboards.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/rancher-monitoring/dashboards/rancher/fleet-dashboards.yaml new file mode 100644 index 00000000000..9b05cea2e8c --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/rancher-monitoring/dashboards/rancher/fleet-dashboards.yaml @@ -0,0 +1,17 @@ +{{- if and .Values.grafana.enabled .Values.grafana.defaultDashboardsEnabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: {{ .Values.grafana.defaultDashboards.namespace }} + name: rancher-fleet-dashboards + annotations: +{{ toYaml .Values.grafana.sidecar.dashboards.annotations | indent 4 }} + labels: + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: "1" + {{- end }} + app: {{ template "kube-prometheus-stack.name" $ }}-grafana +{{ include "kube-prometheus-stack.labels" $ | indent 4 }} +data: +{{ (.Files.Glob "files/rancher/fleet/*").AsConfig | indent 2 }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/rancher-monitoring/dashboards/rancher/k8s-dashboards.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/rancher-monitoring/dashboards/rancher/k8s-dashboards.yaml new file mode 100644 index 00000000000..2afae10ef72 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/rancher-monitoring/dashboards/rancher/k8s-dashboards.yaml @@ -0,0 +1,31 @@ +{{- $files := (.Files.Glob "files/rancher/k8s/*").AsConfig }} +{{- $filesDict := (fromYaml $files) }} +{{- if not (include "exporter.kubeEtcd.enabled" .) }} +{{- $filesDict = (unset $filesDict "rancher-etcd-nodes.json") -}} +{{- $filesDict = (unset $filesDict "rancher-etcd.json") -}} +{{- end }} +{{- if not (include "exporter.kubeControllerManager.enabled" .) }} +{{- $filesDict = (unset $filesDict "rancher-k8s-components-nodes.json") -}} +{{- $filesDict = (unset $filesDict "rancher-k8s-components.json") -}} +{{- else }} +{{- $_ := (set $filesDict "rancher-k8s-components-nodes.json" (get $filesDict "rancher-k8s-components-nodes.json" | replace "kube-controller-manager" (include "exporter.kubeControllerManager.jobName" .))) -}} +{{- $_ := (set $filesDict "rancher-k8s-components.json" (get $filesDict "rancher-k8s-components.json" | replace "kube-controller-manager" (include "exporter.kubeControllerManager.jobName" .))) -}} +{{- end }} +{{ $files = (toYaml $filesDict) }} +{{- if and .Values.grafana.enabled .Values.grafana.defaultDashboardsEnabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: {{ .Values.grafana.defaultDashboards.namespace }} + name: rancher-default-dashboards-k8s + annotations: +{{ toYaml .Values.grafana.sidecar.dashboards.annotations | indent 4 }} + labels: + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: "1" + {{- end }} + app: {{ template "kube-prometheus-stack.name" $ }}-grafana +{{ include "kube-prometheus-stack.labels" $ | indent 4 }} +data: +{{ $files | indent 2 }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/rancher-monitoring/dashboards/rancher/nodes-dashboards.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/rancher-monitoring/dashboards/rancher/nodes-dashboards.yaml new file mode 100644 index 00000000000..172c36e9d18 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/rancher-monitoring/dashboards/rancher/nodes-dashboards.yaml @@ -0,0 +1,17 @@ +{{- if and .Values.grafana.enabled .Values.grafana.defaultDashboardsEnabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: {{ .Values.grafana.defaultDashboards.namespace }} + name: rancher-default-dashboards-nodes + annotations: +{{ toYaml .Values.grafana.sidecar.dashboards.annotations | indent 4 }} + labels: + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: "1" + {{- end }} + app: {{ template "kube-prometheus-stack.name" $ }}-grafana +{{ include "kube-prometheus-stack.labels" $ | indent 4 }} +data: +{{ (.Files.Glob "files/rancher/nodes/*").AsConfig | indent 2 }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/rancher-monitoring/dashboards/rancher/performance-dashboards.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/rancher-monitoring/dashboards/rancher/performance-dashboards.yaml new file mode 100644 index 00000000000..19836ec4e43 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/rancher-monitoring/dashboards/rancher/performance-dashboards.yaml @@ -0,0 +1,18 @@ +{{- $selector := (include "rancher.serviceMonitor.selector" .) -}} +{{- if and .Values.grafana.enabled .Values.grafana.defaultDashboardsEnabled .Values.rancherMonitoring.enabled $selector }} +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: {{ .Values.grafana.defaultDashboards.namespace }} + name: rancher-default-dashboards-performance-debugging + annotations: +{{ toYaml .Values.grafana.sidecar.dashboards.annotations | indent 4 }} + labels: + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: "1" + {{- end }} + app: {{ template "kube-prometheus-stack.name" $ }}-grafana +{{ include "kube-prometheus-stack.labels" $ | indent 4 }} +data: +{{ (.Files.Glob "files/rancher/performance/*").AsConfig | indent 2 }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/rancher-monitoring/dashboards/rancher/pods-dashboards.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/rancher-monitoring/dashboards/rancher/pods-dashboards.yaml new file mode 100644 index 00000000000..940f18869b7 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/rancher-monitoring/dashboards/rancher/pods-dashboards.yaml @@ -0,0 +1,17 @@ +{{- if and .Values.grafana.enabled .Values.grafana.defaultDashboardsEnabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: {{ .Values.grafana.defaultDashboards.namespace }} + name: rancher-default-dashboards-pods + annotations: +{{ toYaml .Values.grafana.sidecar.dashboards.annotations | indent 4 }} + labels: + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: "1" + {{- end }} + app: {{ template "kube-prometheus-stack.name" $ }}-grafana +{{ include "kube-prometheus-stack.labels" $ | indent 4 }} +data: +{{ (.Files.Glob "files/rancher/pods/*").AsConfig | indent 2 }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/rancher-monitoring/dashboards/rancher/workload-dashboards.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/rancher-monitoring/dashboards/rancher/workload-dashboards.yaml new file mode 100644 index 00000000000..d146dacdd0f --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/rancher-monitoring/dashboards/rancher/workload-dashboards.yaml @@ -0,0 +1,17 @@ +{{- if and .Values.grafana.enabled .Values.grafana.defaultDashboardsEnabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: {{ .Values.grafana.defaultDashboards.namespace }} + name: rancher-default-dashboards-workloads + annotations: +{{ toYaml .Values.grafana.sidecar.dashboards.annotations | indent 4 }} + labels: + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: "1" + {{- end }} + app: {{ template "kube-prometheus-stack.name" $ }}-grafana +{{ include "kube-prometheus-stack.labels" $ | indent 4 }} +data: +{{ (.Files.Glob "files/rancher/workloads/*").AsConfig | indent 2 }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/rancher-monitoring/exporters/fleet/servicemonitor.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/rancher-monitoring/exporters/fleet/servicemonitor.yaml new file mode 100644 index 00000000000..90d24c20615 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/rancher-monitoring/exporters/fleet/servicemonitor.yaml @@ -0,0 +1,53 @@ +{{- if .Values.rancherMonitoring.enabled }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + labels: {{ include "kube-prometheus-stack.labels" . | nindent 4 }} + name: monitoring-fleet-controller + namespace: cattle-fleet-system +spec: + endpoints: + - port: metrics + metricRelabelings: + {{ if .Values.global.cattle.clusterId }} + - sourceLabels: [__address__] + targetLabel: cluster_id + replacement: {{ .Values.global.cattle.clusterId }} + {{- end }} + {{ if .Values.global.cattle.clusterName}} + - sourceLabels: [__address__] + targetLabel: cluster_name + replacement: {{ .Values.global.cattle.clusterName }} + {{- end }} + jobLabel: fleet + selector: + matchLabels: + app: fleet-controller +{{- end }} +--- +{{- if .Values.rancherMonitoring.enabled }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + labels: {{ include "kube-prometheus-stack.labels" . | nindent 4 }} + name: monitoring-gitops-controller + namespace: cattle-fleet-system +spec: + endpoints: + - port: metrics + metricRelabelings: + {{ if .Values.global.cattle.clusterId }} + - sourceLabels: [__address__] + targetLabel: cluster_id + replacement: {{ .Values.global.cattle.clusterId }} + {{- end }} + {{ if .Values.global.cattle.clusterName}} + - sourceLabels: [__address__] + targetLabel: cluster_name + replacement: {{ .Values.global.cattle.clusterName }} + {{- end }} + jobLabel: gitops + selector: + matchLabels: + app: gitjob +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/rancher-monitoring/exporters/ingress-nginx/network-policy.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/rancher-monitoring/exporters/ingress-nginx/network-policy.yaml new file mode 100644 index 00000000000..9d3e515f397 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/rancher-monitoring/exporters/ingress-nginx/network-policy.yaml @@ -0,0 +1,19 @@ +{{- if .Values.rke2IngressNginx.networkPolicy.enabled }} +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + annotations: + np.rke2.io/ingress: resolved + name: rke2-ingress-network-policy + namespace: {{ include "rke2-ingress-nginx.namespace" . }} +spec: + ingress: + - ports: + - port: {{ .Values.rke2IngressNginx.metricsPort }} + protocol: TCP + podSelector: + matchLabels: + app.kubernetes.io/name: rke2-ingress-nginx + policyTypes: + - Ingress +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/rancher-monitoring/exporters/ingress-nginx/service.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/rancher-monitoring/exporters/ingress-nginx/service.yaml new file mode 100644 index 00000000000..53a9ad68978 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/rancher-monitoring/exporters/ingress-nginx/service.yaml @@ -0,0 +1,27 @@ +{{- if and (not .Values.ingressNginx.enabled) (.Values.rkeIngressNginx.enabled) }} +{{- fail "Cannot set .Values.rkeIngressNginx.enabled=true when .Values.ingressNginx.enabled=false" }} +{{- end }} +{{- if and .Values.ingressNginx.enabled (not .Values.rkeIngressNginx.enabled) }} +apiVersion: v1 +kind: Service +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-ingress-nginx + labels: + app: {{ template "kube-prometheus-stack.name" . }}-ingress-nginx + jobLabel: ingress-nginx +{{ include "kube-prometheus-stack.labels" . | indent 4 }} + namespace: {{ .Values.ingressNginx.namespace }} +spec: + clusterIP: None + ports: + - name: http-metrics + port: {{ .Values.ingressNginx.service.port }} + protocol: TCP + targetPort: {{ .Values.ingressNginx.service.targetPort }} + selector: + {{- if .Values.ingressNginx.service.selector }} +{{ toYaml .Values.ingressNginx.service.selector | indent 4 }} + {{- else }} + app: ingress-nginx + {{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/rancher-monitoring/exporters/ingress-nginx/servicemonitor.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/rancher-monitoring/exporters/ingress-nginx/servicemonitor.yaml new file mode 100644 index 00000000000..b0f92e63b59 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/rancher-monitoring/exporters/ingress-nginx/servicemonitor.yaml @@ -0,0 +1,49 @@ +{{- if and (not .Values.ingressNginx.enabled) (.Values.rkeIngressNginx.enabled) }} +{{- fail "Cannot set .Values.rkeIngressNginx.enabled=true when .Values.ingressNginx.enabled=false" }} +{{- end }} +{{- if and .Values.ingressNginx.enabled (not .Values.rkeIngressNginx.enabled) }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-ingress-nginx + namespace: {{ .Values.ingressNginx.namespace }} + labels: + app: {{ template "kube-prometheus-stack.name" . }}-ingress-nginx +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +spec: + jobLabel: jobLabel + selector: + matchLabels: + app: {{ template "kube-prometheus-stack.name" . }}-ingress-nginx + release: {{ $.Release.Name | quote }} + namespaceSelector: + matchNames: + - {{ .Values.ingressNginx.namespace }} + endpoints: + - port: http-metrics + {{- if .Values.ingressNginx.serviceMonitor.interval}} + interval: {{ .Values.ingressNginx.serviceMonitor.interval }} + {{- end }} + {{- if .Values.ingressNginx.serviceMonitor.proxyUrl }} + proxyUrl: {{ .Values.ingressNginx.serviceMonitor.proxyUrl}} + {{- end }} + bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token + metricRelabelings: + {{- if .Values.ingressNginx.serviceMonitor.metricRelabelings }} + {{ tpl (toYaml .Values.ingressNginx.serviceMonitor.metricRelabelings | indent 4) . }} + {{- end }} + {{ if .Values.global.cattle.clusterId }} + - sourceLabels: [__address__] + targetLabel: cluster_id + replacement: {{ .Values.global.cattle.clusterId }} + {{- end }} + {{ if .Values.global.cattle.clusterName}} + - sourceLabels: [__address__] + targetLabel: cluster_name + replacement: {{ .Values.global.cattle.clusterName }} + {{- end }} +{{- if .Values.ingressNginx.serviceMonitor.relabelings }} + relabelings: +{{ toYaml .Values.ingressNginx.serviceMonitor.relabelings | indent 4 }} +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/rancher-monitoring/exporters/rancher/servicemonitor.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/rancher-monitoring/exporters/rancher/servicemonitor.yaml new file mode 100644 index 00000000000..1fba8f23f7e --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/rancher-monitoring/exporters/rancher/servicemonitor.yaml @@ -0,0 +1,58 @@ +{{- $selector := (include "rancher.serviceMonitor.selector" .) -}} +{{- if and .Values.rancherMonitoring.enabled $selector }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + labels: {{ include "kube-prometheus-stack.labels" . | nindent 4 }} + name: rancher + namespace: cattle-system +spec: + endpoints: + - bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token + port: http + tlsConfig: + caFile: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt + insecureSkipVerify: true + serverName: rancher + metricRelabelings: + {{ if .Values.global.cattle.clusterId }} + - sourceLabels: [__address__] + targetLabel: cluster_id + replacement: {{ .Values.global.cattle.clusterId }} + {{- end }} + {{ if .Values.global.cattle.clusterName}} + - sourceLabels: [__address__] + targetLabel: cluster_name + replacement: {{ .Values.global.cattle.clusterName }} + {{- end }} + jobLabel: rancher +{{- if .Values.rancherMonitoring.namespaceSelector }} + namespaceSelector: {{ .Values.rancherMonitoring.namespaceSelector | toYaml | nindent 4 }} +{{- end }} + selector: {{ include "rancher.serviceMonitor.selector" . | nindent 4 }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-rancher-metrics +rules: +- apiGroups: + - management.cattle.io + resources: + - ranchermetrics + verbs: + - get +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-rancher-metrics +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "kube-prometheus-stack.fullname" . }}-rancher-metrics +subjects: + - kind: ServiceAccount + name: {{ template "kube-prometheus-stack.fullname" . }}-prometheus + namespace: {{ template "kube-prometheus-stack.namespace" . }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/rancher-monitoring/hardened.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/rancher-monitoring/hardened.yaml new file mode 100644 index 00000000000..f9a66151eec --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/rancher-monitoring/hardened.yaml @@ -0,0 +1,147 @@ +{{- $namespaces := dict "_0" .Release.Namespace -}} +{{- if and .Values.grafana.enabled .Values.grafana.defaultDashboardsEnabled (not .Values.grafana.defaultDashboards.useExistingNamespace) -}} +{{- $_ := set $namespaces "_1" .Values.grafana.defaultDashboards.namespace -}} +{{- end -}} +apiVersion: batch/v1 +kind: Job +metadata: + name: {{ .Chart.Name }}-patch-sa + namespace: {{ .Release.Namespace }} + labels: + app: {{ .Chart.Name }}-patch-sa + annotations: + "helm.sh/hook": post-install, post-upgrade + "helm.sh/hook-delete-policy": hook-succeeded, before-hook-creation +spec: + template: + metadata: + name: {{ .Chart.Name }}-patch-sa + labels: + app: {{ .Chart.Name }}-patch-sa + spec: + serviceAccountName: {{ .Chart.Name }}-patch-sa + securityContext: + runAsNonRoot: true + runAsUser: 1000 + restartPolicy: Never + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} + containers: + {{- range $_, $ns := $namespaces }} + - name: patch-sa-{{ $ns }} + image: {{ template "system_default_registry" $ }}{{ $.Values.global.kubectl.repository }}:{{ $.Values.global.kubectl.tag }} + imagePullPolicy: {{ $.Values.global.kubectl.pullPolicy }} + command: ["kubectl", "patch", "serviceaccount", "default", "-p", "{\"automountServiceAccountToken\": false}"] + args: ["-n", "{{ $ns }}"] + {{- end }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ .Chart.Name }}-patch-sa + labels: + app: {{ .Chart.Name }}-patch-sa +rules: +- apiGroups: + - "" + resources: + - serviceaccounts + verbs: ['get', 'patch'] +{{- if .Values.global.cattle.psp.enabled }} +- apiGroups: ['policy'] + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - {{ .Chart.Name }}-patch-sa +{{- end }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ .Chart.Name }}-patch-sa + labels: + app: {{ .Chart.Name }}-patch-sa +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ .Chart.Name }}-patch-sa +subjects: +- kind: ServiceAccount + name: {{ .Chart.Name }}-patch-sa + namespace: {{ .Release.Namespace }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ .Chart.Name }}-patch-sa + namespace: {{ .Release.Namespace }} + labels: + app: {{ .Chart.Name }}-patch-sa +--- +{{- if .Values.global.cattle.psp.enabled }} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ .Chart.Name }}-patch-sa + namespace: {{ .Release.Namespace }} + labels: + app: {{ .Chart.Name }}-patch-sa +spec: + privileged: false + hostNetwork: false + hostIPC: false + hostPID: false + runAsUser: + rule: 'MustRunAsNonRoot' + seLinux: + rule: 'RunAsAny' + supplementalGroups: + rule: 'MustRunAs' + ranges: + - min: 1 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + - min: 1 + max: 65535 + readOnlyRootFilesystem: false + volumes: + - 'secret' +{{- range $_, $ns := $namespaces }} +--- +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: default-allow-all + namespace: {{ $ns }} +spec: + podSelector: {} + ingress: + - {} + egress: + - {} + policyTypes: + - Ingress + - Egress +{{- end }} +{{- end }} +--- +{{- if .Values.hardened.k3s.networkPolicy.enabled }} +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: rancher-monitoring-coredns-allow-all + namespace: kube-system +spec: + ingress: + - {} + egress: + - {} + policyTypes: + - Ingress + - Egress + podSelector: + matchLabels: + k8s-app: kube-dns +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/rancher-monitoring/upgrade/configmap.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/rancher-monitoring/upgrade/configmap.yaml new file mode 100644 index 00000000000..53cb898214f --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/rancher-monitoring/upgrade/configmap.yaml @@ -0,0 +1,13 @@ +{{- if .Values.upgrade.enabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-upgrade + namespace: {{ template "kube-prometheus-stack.namespace" . }} + annotations: + "helm.sh/hook": pre-upgrade, pre-rollback + "helm.sh/hook-delete-policy": before-hook-creation, hook-succeeded, hook-failed + "helm.sh/hook-weight": "0" +data: +{{ (.Files.Glob "files/upgrade/scripts/*").AsConfig | indent 2 }} +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/rancher-monitoring/upgrade/job.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/rancher-monitoring/upgrade/job.yaml new file mode 100644 index 00000000000..8f2771740ce --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/rancher-monitoring/upgrade/job.yaml @@ -0,0 +1,46 @@ +{{- if .Values.upgrade.enabled }} +apiVersion: batch/v1 +kind: Job +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-upgrade + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.fullname" . }}-upgrade + annotations: + "helm.sh/hook": pre-upgrade, pre-rollback + "helm.sh/hook-delete-policy": before-hook-creation, hook-succeeded, hook-failed + "helm.sh/hook-weight": "2" +spec: + template: + metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-upgrade + labels: + app: {{ template "kube-prometheus-stack.fullname" . }}-upgrade + spec: + serviceAccountName: {{ template "kube-prometheus-stack.fullname" . }}-upgrade + securityContext: + runAsNonRoot: false + runAsUser: 0 + restartPolicy: Never + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} + containers: + - name: run-scripts + image: {{ template "system_default_registry" . }}{{ .Values.upgrade.image.repository }}:{{ .Values.upgrade.image.tag }} + imagePullPolicy: {{ $.Values.global.kubectl.pullPolicy }} + command: + - /bin/sh + - -c + - > + for s in $(find /etc/scripts -type f); do + echo "Running $s..."; + cat $s | bash + done; + volumeMounts: + - name: upgrade + mountPath: /etc/scripts + volumes: + - name: upgrade + configMap: + name: {{ template "kube-prometheus-stack.fullname" . }}-upgrade +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/rancher-monitoring/upgrade/rbac.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/rancher-monitoring/upgrade/rbac.yaml new file mode 100644 index 00000000000..e929a19925b --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/rancher-monitoring/upgrade/rbac.yaml @@ -0,0 +1,131 @@ +{{- if .Values.upgrade.enabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-upgrade + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.fullname" . }}-upgrade + annotations: + "helm.sh/hook": pre-upgrade, pre-rollback + "helm.sh/hook-delete-policy": before-hook-creation, hook-succeeded + "helm.sh/hook-weight": "1" +rules: +- apiGroups: + - apps + resources: + - deployments + - daemonsets + - statefulsets + verbs: + - 'list' + - 'delete' +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-upgrade + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.fullname" . }}-upgrade + annotations: + "helm.sh/hook": pre-upgrade, pre-rollback + "helm.sh/hook-delete-policy": before-hook-creation, hook-succeeded, hook-failed + "helm.sh/hook-weight": "1" +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: {{ template "kube-prometheus-stack.fullname" . }}-upgrade +subjects: +- kind: ServiceAccount + name: {{ template "kube-prometheus-stack.fullname" . }}-upgrade + namespace: {{ template "kube-prometheus-stack.namespace" . }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-upgrade + labels: + app: {{ template "kube-prometheus-stack.fullname" . }}-upgrade + annotations: + "helm.sh/hook": pre-upgrade, pre-rollback + "helm.sh/hook-delete-policy": before-hook-creation, hook-succeeded, hook-failed + "helm.sh/hook-weight": "1" +rules: +{{- if .Values.global.cattle.psp.enabled }} +- apiGroups: ['policy'] + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - {{ template "kube-prometheus-stack.fullname" . }}-upgrade +{{- end }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-upgrade + labels: + app: {{ template "kube-prometheus-stack.fullname" . }}-upgrade + annotations: + "helm.sh/hook": pre-upgrade, pre-rollback + "helm.sh/hook-delete-policy": before-hook-creation, hook-succeeded, hook-failed + "helm.sh/hook-weight": "1" +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "kube-prometheus-stack.fullname" . }}-upgrade +subjects: +- kind: ServiceAccount + name: {{ template "kube-prometheus-stack.fullname" . }}-upgrade + namespace: {{ template "kube-prometheus-stack.namespace" . }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-upgrade + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.fullname" . }}-upgrade + annotations: + "helm.sh/hook": pre-upgrade, pre-rollback + "helm.sh/hook-delete-policy": before-hook-creation, hook-succeeded, hook-failed + "helm.sh/hook-weight": "1" +--- +{{- if .Values.global.cattle.psp.enabled }} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-upgrade + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.fullname" . }}-upgrade + annotations: + "helm.sh/hook": pre-upgrade, pre-rollback + "helm.sh/hook-delete-policy": before-hook-creation, hook-succeeded, hook-failed + "helm.sh/hook-weight": "1" +spec: + privileged: false + allowPrivilegeEscalation: false + hostNetwork: false + hostIPC: false + hostPID: false + runAsUser: + rule: 'RunAsAny' + seLinux: + rule: 'RunAsAny' + supplementalGroups: + rule: 'MustRunAs' + ranges: + - min: 1 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + - min: 1 + max: 65535 + readOnlyRootFilesystem: false + volumes: + - 'configMap' + - 'secret' +{{- end }} +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/thanos-ruler/extrasecret.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/thanos-ruler/extrasecret.yaml new file mode 100644 index 00000000000..587fca2dca7 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/thanos-ruler/extrasecret.yaml @@ -0,0 +1,20 @@ +{{- if .Values.thanosRuler.extraSecret.data -}} +{{- $secretName := printf "%s-extra" (include "kube-prometheus-stack.thanosRuler.name" . ) -}} +apiVersion: v1 +kind: Secret +metadata: + name: {{ default $secretName .Values.thanosRuler.extraSecret.name }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} +{{- if .Values.thanosRuler.extraSecret.annotations }} + annotations: +{{ toYaml .Values.thanosRuler.extraSecret.annotations | indent 4 }} +{{- end }} + labels: + app: {{ template "kube-prometheus-stack.thanosRuler.name" . }} + app.kubernetes.io/component: thanos-ruler +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +data: +{{- range $key, $val := .Values.thanosRuler.extraSecret.data }} + {{ $key }}: {{ $val | b64enc | quote }} +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/thanos-ruler/ingress.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/thanos-ruler/ingress.yaml new file mode 100644 index 00000000000..e245ad448e8 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/thanos-ruler/ingress.yaml @@ -0,0 +1,77 @@ +{{- if and .Values.thanosRuler.enabled .Values.thanosRuler.ingress.enabled }} +{{- $pathType := .Values.thanosRuler.ingress.pathType | default "ImplementationSpecific" }} +{{- $serviceName := include "kube-prometheus-stack.thanosRuler.name" . }} +{{- $servicePort := .Values.thanosRuler.service.port -}} +{{- $routePrefix := list .Values.thanosRuler.thanosRulerSpec.routePrefix }} +{{- $paths := .Values.thanosRuler.ingress.paths | default $routePrefix -}} +{{- $apiIsStable := eq (include "kube-prometheus-stack.ingress.isStable" .) "true" -}} +{{- $ingressSupportsPathType := eq (include "kube-prometheus-stack.ingress.supportsPathType" .) "true" -}} +apiVersion: {{ include "kube-prometheus-stack.ingress.apiVersion" . }} +kind: Ingress +metadata: + name: {{ $serviceName }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} +{{- if .Values.thanosRuler.ingress.annotations }} + annotations: + {{- tpl (toYaml .Values.thanosRuler.ingress.annotations) . | nindent 4 }} +{{- end }} + labels: + app: {{ template "kube-prometheus-stack.thanosRuler.name" . }} +{{- if .Values.thanosRuler.ingress.labels }} +{{ toYaml .Values.thanosRuler.ingress.labels | indent 4 }} +{{- end }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +spec: + {{- if $apiIsStable }} + {{- if .Values.thanosRuler.ingress.ingressClassName }} + ingressClassName: {{ .Values.thanosRuler.ingress.ingressClassName }} + {{- end }} + {{- end }} + rules: + {{- if .Values.thanosRuler.ingress.hosts }} + {{- range $host := .Values.thanosRuler.ingress.hosts }} + - host: {{ tpl $host $ }} + http: + paths: + {{- range $p := $paths }} + - path: {{ tpl $p $ }} + {{- if and $pathType $ingressSupportsPathType }} + pathType: {{ $pathType }} + {{- end }} + backend: + {{- if $apiIsStable }} + service: + name: {{ $serviceName }} + port: + number: {{ $servicePort }} + {{- else }} + serviceName: {{ $serviceName }} + servicePort: {{ $servicePort }} + {{- end }} + {{- end -}} + {{- end -}} + {{- else }} + - http: + paths: + {{- range $p := $paths }} + - path: {{ tpl $p $ }} + {{- if and $pathType $ingressSupportsPathType }} + pathType: {{ $pathType }} + {{- end }} + backend: + {{- if $apiIsStable }} + service: + name: {{ $serviceName }} + port: + number: {{ $servicePort }} + {{- else }} + serviceName: {{ $serviceName }} + servicePort: {{ $servicePort }} + {{- end }} + {{- end -}} + {{- end -}} + {{- if .Values.thanosRuler.ingress.tls }} + tls: +{{ tpl (toYaml .Values.thanosRuler.ingress.tls | indent 4) . }} + {{- end -}} +{{- end -}} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/thanos-ruler/podDisruptionBudget.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/thanos-ruler/podDisruptionBudget.yaml new file mode 100644 index 00000000000..83e54edf9b5 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/thanos-ruler/podDisruptionBudget.yaml @@ -0,0 +1,21 @@ +{{- if and .Values.thanosRuler.enabled .Values.thanosRuler.podDisruptionBudget.enabled }} +apiVersion: {{ include "kube-prometheus-stack.pdb.apiVersion" . }} +kind: PodDisruptionBudget +metadata: + name: {{ template "kube-prometheus-stack.thanosRuler.name" . }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.thanosRuler.name" . }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +spec: + {{- if .Values.thanosRuler.podDisruptionBudget.minAvailable }} + minAvailable: {{ .Values.thanosRuler.podDisruptionBudget.minAvailable }} + {{- end }} + {{- if .Values.thanosRuler.podDisruptionBudget.maxUnavailable }} + maxUnavailable: {{ .Values.thanosRuler.podDisruptionBudget.maxUnavailable }} + {{- end }} + selector: + matchLabels: + app.kubernetes.io/name: thanos-ruler + thanos-ruler: {{ template "kube-prometheus-stack.thanosRuler.name" . }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/thanos-ruler/ruler.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/thanos-ruler/ruler.yaml new file mode 100644 index 00000000000..b2812215635 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/thanos-ruler/ruler.yaml @@ -0,0 +1,189 @@ +{{- if .Values.thanosRuler.enabled }} +apiVersion: monitoring.coreos.com/v1 +kind: ThanosRuler +metadata: + name: {{ template "kube-prometheus-stack.thanosRuler.name" . }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ include "kube-prometheus-stack.thanosRuler.name" . }} +{{- include "kube-prometheus-stack.labels" . | indent 4 -}} +{{- if .Values.thanosRuler.annotations }} + annotations: +{{ toYaml .Values.thanosRuler.annotations | indent 4 }} +{{- end }} +spec: +{{- if .Values.thanosRuler.thanosRulerSpec.image }} + {{- $registry := include "monitoring_registry" . | default .Values.thanosRuler.thanosRulerSpec.image.registry -}} + {{- if and .Values.thanosRuler.thanosRulerSpec.image.tag .Values.thanosRuler.thanosRulerSpec.image.sha }} + image: "{{ $registry }}/{{ .Values.thanosRuler.thanosRulerSpec.image.repository }}:{{ .Values.thanosRuler.thanosRulerSpec.image.tag }}@sha256:{{ .Values.thanosRuler.thanosRulerSpec.image.sha }}" + {{- else if .Values.thanosRuler.thanosRulerSpec.image.sha }} + image: "{{ $registry }}/{{ .Values.thanosRuler.thanosRulerSpec.image.repository }}@sha256:{{ .Values.thanosRuler.thanosRulerSpec.image.sha }}" + {{- else if .Values.thanosRuler.thanosRulerSpec.image.tag }} + image: "{{ $registry }}/{{ .Values.thanosRuler.thanosRulerSpec.image.repository }}:{{ .Values.thanosRuler.thanosRulerSpec.image.tag }}" + {{- else }} + image: "{{ $registry }}/{{ .Values.thanosRuler.thanosRulerSpec.image.repository }}" + {{- end }} + {{- if .Values.thanosRuler.thanosRulerSpec.image.sha }} + sha: {{ .Values.thanosRuler.thanosRulerSpec.image.sha }} + {{- end }} +{{- end }} + replicas: {{ .Values.thanosRuler.thanosRulerSpec.replicas }} + listenLocal: {{ .Values.thanosRuler.thanosRulerSpec.listenLocal }} + serviceAccountName: {{ template "kube-prometheus-stack.thanosRuler.serviceAccountName" . }} +{{- if .Values.thanosRuler.thanosRulerSpec.externalPrefix }} + externalPrefix: "{{ tpl .Values.thanosRuler.thanosRulerSpec.externalPrefix . }}" +{{- else if and .Values.thanosRuler.ingress.enabled .Values.thanosRuler.ingress.hosts }} + externalPrefix: "http://{{ tpl (index .Values.thanosRuler.ingress.hosts 0) . }}{{ .Values.thanosRuler.thanosRulerSpec.routePrefix }}" +{{- else }} + externalPrefix: http://{{ template "kube-prometheus-stack.thanosRuler.name" . }}.{{ template "kube-prometheus-stack.namespace" . }}:{{ .Values.thanosRuler.service.port }} +{{- end }} + nodeSelector: {{ include "linux-node-selector" . | nindent 4 }} +{{- if .Values.thanosRuler.thanosRulerSpec.additionalArgs }} + additionalArgs: +{{ toYaml .Values.thanosRuler.thanosRulerSpec.additionalArgs | indent 4 }} +{{- end }} +{{- if .Values.thanosRuler.thanosRulerSpec.nodeSelector }} +{{ toYaml .Values.thanosRuler.thanosRulerSpec.nodeSelector | indent 4 }} +{{- end }} + paused: {{ .Values.thanosRuler.thanosRulerSpec.paused }} + logFormat: {{ .Values.thanosRuler.thanosRulerSpec.logFormat | quote }} + logLevel: {{ .Values.thanosRuler.thanosRulerSpec.logLevel | quote }} + retention: {{ .Values.thanosRuler.thanosRulerSpec.retention | quote }} +{{- if .Values.thanosRuler.thanosRulerSpec.evaluationInterval }} + evaluationInterval: {{ .Values.thanosRuler.thanosRulerSpec.evaluationInterval }} +{{- end }} +{{- if .Values.thanosRuler.thanosRulerSpec.ruleNamespaceSelector }} + ruleNamespaceSelector: +{{ tpl (toYaml .Values.thanosRuler.thanosRulerSpec.ruleNamespaceSelector | indent 4) . }} +{{ else }} + ruleNamespaceSelector: {} +{{- end }} +{{- if .Values.thanosRuler.thanosRulerSpec.ruleSelector }} + ruleSelector: +{{ tpl (toYaml .Values.thanosRuler.thanosRulerSpec.ruleSelector | indent 4) .}} +{{- else if .Values.thanosRuler.thanosRulerSpec.ruleSelectorNilUsesHelmValues }} + ruleSelector: + matchLabels: + release: {{ $.Release.Name | quote }} +{{ else }} + ruleSelector: {} +{{- end }} +{{- if .Values.thanosRuler.thanosRulerSpec.alertQueryUrl }} + alertQueryUrl: "{{ .Values.thanosRuler.thanosRulerSpec.alertQueryUrl }}" +{{- end}} +{{- if .Values.thanosRuler.thanosRulerSpec.alertmanagersUrl }} + alertmanagersUrl: +{{ toYaml .Values.thanosRuler.thanosRulerSpec.alertmanagersUrl | indent 4 }} +{{- end }} +{{- if .Values.thanosRuler.thanosRulerSpec.alertmanagersConfig.existingSecret }} + alertmanagersConfig: + key: "{{.Values.thanosRuler.thanosRulerSpec.alertmanagersConfig.existingSecret.key }}" + name: "{{.Values.thanosRuler.thanosRulerSpec.alertmanagersConfig.existingSecret.name }}" +{{- else if .Values.thanosRuler.thanosRulerSpec.alertmanagersConfig.secret }} + alertmanagersConfig: + key: alertmanager-configs.yaml + name: {{ template "kube-prometheus-stack.thanosRuler.name" . }} +{{- end }} +{{- if .Values.thanosRuler.thanosRulerSpec.queryEndpoints }} + queryEndpoints: +{{ toYaml .Values.thanosRuler.thanosRulerSpec.queryEndpoints | indent 4 }} +{{- end }} +{{- if .Values.thanosRuler.thanosRulerSpec.queryConfig.existingSecret }} + queryConfig: + key: "{{.Values.thanosRuler.thanosRulerSpec.queryConfig.existingSecret.key }}" + name: "{{.Values.thanosRuler.thanosRulerSpec.queryConfig.existingSecret.name }}" +{{- else if .Values.thanosRuler.thanosRulerSpec.queryConfig.secret }} + queryConfig: + key: query-configs.yaml + name: {{ template "kube-prometheus-stack.thanosRuler.name" . }} +{{- end }} +{{- if .Values.thanosRuler.thanosRulerSpec.resources }} + resources: +{{ toYaml .Values.thanosRuler.thanosRulerSpec.resources | indent 4 }} +{{- end }} +{{- if .Values.thanosRuler.thanosRulerSpec.routePrefix }} + routePrefix: "{{ .Values.thanosRuler.thanosRulerSpec.routePrefix }}" +{{- end }} +{{- if .Values.thanosRuler.thanosRulerSpec.securityContext }} + securityContext: +{{ toYaml .Values.thanosRuler.thanosRulerSpec.securityContext | indent 4 }} +{{- end }} +{{- if .Values.thanosRuler.thanosRulerSpec.storage }} + storage: +{{ toYaml .Values.thanosRuler.thanosRulerSpec.storage | indent 4 }} +{{- end }} +{{- if .Values.thanosRuler.thanosRulerSpec.objectStorageConfig.existingSecret }} + objectStorageConfig: + key: "{{.Values.thanosRuler.thanosRulerSpec.objectStorageConfig.existingSecret.key }}" + name: "{{.Values.thanosRuler.thanosRulerSpec.objectStorageConfig.existingSecret.name }}" +{{- else if .Values.thanosRuler.thanosRulerSpec.objectStorageConfig.secret }} + objectStorageConfig: + key: object-storage-configs.yaml + name: {{ template "kube-prometheus-stack.thanosRuler.name" . }} +{{- end }} +{{- if .Values.thanosRuler.thanosRulerSpec.labels }} + labels: +{{ toYaml .Values.thanosRuler.thanosRulerSpec.labels | indent 4 }} +{{- end }} +{{- if .Values.thanosRuler.thanosRulerSpec.podMetadata }} + podMetadata: +{{ toYaml .Values.thanosRuler.thanosRulerSpec.podMetadata | indent 4 }} +{{- end }} +{{- if or .Values.thanosRuler.thanosRulerSpec.podAntiAffinity .Values.thanosRuler.thanosRulerSpec.affinity }} + affinity: +{{- end }} +{{- if .Values.thanosRuler.thanosRulerSpec.affinity }} +{{ toYaml .Values.thanosRuler.thanosRulerSpec.affinity | indent 4 }} +{{- end }} +{{- if eq .Values.thanosRuler.thanosRulerSpec.podAntiAffinity "hard" }} + podAntiAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - topologyKey: {{ .Values.thanosRuler.thanosRulerSpec.podAntiAffinityTopologyKey }} + labelSelector: + matchExpressions: + - {key: app.kubernetes.io/name, operator: In, values: [thanos-ruler]} + - {key: thanos-ruler, operator: In, values: [{{ template "kube-prometheus-stack.thanosRuler.name" . }}]} +{{- else if eq .Values.thanosRuler.thanosRulerSpec.podAntiAffinity "soft" }} + podAntiAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 100 + podAffinityTerm: + topologyKey: {{ .Values.thanosRuler.thanosRulerSpec.podAntiAffinityTopologyKey }} + labelSelector: + matchExpressions: + - {key: app.kubernetes.io/name, operator: In, values: [thanos-ruler]} + - {key: thanos-ruler, operator: In, values: [{{ template "kube-prometheus-stack.thanosRuler.name" . }}]} +{{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 4 }} +{{- if .Values.thanosRuler.thanosRulerSpec.tolerations }} +{{ toYaml .Values.thanosRuler.thanosRulerSpec.tolerations | indent 4 }} +{{- end }} +{{- if .Values.thanosRuler.thanosRulerSpec.topologySpreadConstraints }} + topologySpreadConstraints: +{{ toYaml .Values.thanosRuler.thanosRulerSpec.topologySpreadConstraints | indent 4 }} +{{- end }} +{{- if .Values.global.imagePullSecrets }} + imagePullSecrets: +{{ toYaml .Values.global.imagePullSecrets | indent 4 }} +{{- end }} +{{- if .Values.thanosRuler.thanosRulerSpec.containers }} + containers: +{{ toYaml .Values.thanosRuler.thanosRulerSpec.containers | indent 4 }} +{{- end }} +{{- if .Values.thanosRuler.thanosRulerSpec.initContainers }} + initContainers: +{{ toYaml .Values.thanosRuler.thanosRulerSpec.initContainers | indent 4 }} +{{- end }} +{{- if .Values.thanosRuler.thanosRulerSpec.priorityClassName }} + priorityClassName: {{.Values.thanosRuler.thanosRulerSpec.priorityClassName }} +{{- end }} +{{- if .Values.thanosRuler.thanosRulerSpec.volumes }} + volumes: +{{ toYaml .Values.thanosRuler.thanosRulerSpec.volumes | indent 4 }} +{{- end }} +{{- if .Values.thanosRuler.thanosRulerSpec.volumeMounts }} + volumeMounts: +{{ toYaml .Values.thanosRuler.thanosRulerSpec.volumeMounts | indent 4 }} +{{- end }} + portName: {{ .Values.thanosRuler.thanosRulerSpec.portName }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/thanos-ruler/secret.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/thanos-ruler/secret.yaml new file mode 100644 index 00000000000..acab7fd9aed --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/thanos-ruler/secret.yaml @@ -0,0 +1,26 @@ +{{- if .Values.thanosRuler.enabled }} +apiVersion: v1 +kind: Secret +metadata: + name: {{ template "kube-prometheus-stack.thanosRuler.name" . }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ include "kube-prometheus-stack.thanosRuler.name" . }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +data: + {{- with .Values.thanosRuler.thanosRulerSpec.alertmanagersConfig }} + {{- if and .secret (not .existingSecret) }} + alertmanager-configs.yaml: {{ toYaml .secret | b64enc | quote }} + {{- end }} + {{- end }} + {{- with .Values.thanosRuler.thanosRulerSpec.objectStorageConfig }} + {{- if and .secret (not .existingSecret) }} + object-storage-configs.yaml: {{ toYaml .secret | b64enc | quote }} + {{- end }} + {{- end }} + {{- with .Values.thanosRuler.thanosRulerSpec.queryConfig }} + {{- if and .secret (not .existingSecret) }} + query-configs.yaml: {{ toYaml .secret | b64enc | quote }} + {{- end }} + {{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/thanos-ruler/service.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/thanos-ruler/service.yaml new file mode 100644 index 00000000000..be0c8445910 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/thanos-ruler/service.yaml @@ -0,0 +1,53 @@ +{{- if .Values.thanosRuler.enabled }} +apiVersion: v1 +kind: Service +metadata: + name: {{ template "kube-prometheus-stack.thanosRuler.name" . }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.thanosRuler.name" . }} + self-monitor: {{ .Values.thanosRuler.serviceMonitor.selfMonitor | quote }} +{{- include "kube-prometheus-stack.labels" . | indent 4 -}} +{{- if .Values.thanosRuler.service.labels }} +{{ toYaml .Values.thanosRuler.service.labels | indent 4 }} +{{- end }} +{{- if .Values.thanosRuler.service.annotations }} + annotations: +{{ toYaml .Values.thanosRuler.service.annotations | indent 4 }} +{{- end }} +spec: +{{- if .Values.thanosRuler.service.clusterIP }} + clusterIP: {{ .Values.thanosRuler.service.clusterIP }} +{{- end }} +{{- if .Values.thanosRuler.service.externalIPs }} + externalIPs: +{{ toYaml .Values.thanosRuler.service.externalIPs | indent 4 }} +{{- end }} +{{- if .Values.thanosRuler.service.loadBalancerIP }} + loadBalancerIP: {{ .Values.thanosRuler.service.loadBalancerIP }} +{{- end }} +{{- if .Values.thanosRuler.service.loadBalancerSourceRanges }} + loadBalancerSourceRanges: + {{- range $cidr := .Values.thanosRuler.service.loadBalancerSourceRanges }} + - {{ $cidr }} + {{- end }} +{{- end }} +{{- if ne .Values.thanosRuler.service.type "ClusterIP" }} + externalTrafficPolicy: {{ .Values.thanosRuler.service.externalTrafficPolicy }} +{{- end }} + ports: + - name: {{ .Values.thanosRuler.thanosRulerSpec.portName }} + {{- if eq .Values.thanosRuler.service.type "NodePort" }} + nodePort: {{ .Values.thanosRuler.service.nodePort }} + {{- end }} + port: {{ .Values.thanosRuler.service.port }} + targetPort: {{ .Values.thanosRuler.service.targetPort }} + protocol: TCP +{{- if .Values.thanosRuler.service.additionalPorts }} +{{ toYaml .Values.thanosRuler.service.additionalPorts | indent 2 }} +{{- end }} + selector: + app.kubernetes.io/name: thanos-ruler + thanos-ruler: {{ template "kube-prometheus-stack.thanosRuler.name" . }} + type: "{{ .Values.thanosRuler.service.type }}" +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/thanos-ruler/serviceaccount.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/thanos-ruler/serviceaccount.yaml new file mode 100644 index 00000000000..b58f1cd4df5 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/thanos-ruler/serviceaccount.yaml @@ -0,0 +1,20 @@ +{{- if and .Values.thanosRuler.enabled .Values.thanosRuler.serviceAccount.create }} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "kube-prometheus-stack.thanosRuler.serviceAccountName" . }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.thanosRuler.name" . }} + app.kubernetes.io/name: {{ template "kube-prometheus-stack.thanosRuler.name" . }} + app.kubernetes.io/component: thanos-ruler +{{- include "kube-prometheus-stack.labels" . | indent 4 -}} +{{- if .Values.thanosRuler.serviceAccount.annotations }} + annotations: +{{ toYaml .Values.thanosRuler.serviceAccount.annotations | indent 4 }} +{{- end }} +{{- if .Values.global.imagePullSecrets }} +imagePullSecrets: +{{ toYaml .Values.global.imagePullSecrets | indent 2 }} +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/thanos-ruler/servicemonitor.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/thanos-ruler/servicemonitor.yaml new file mode 100644 index 00000000000..b2b138b4986 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/thanos-ruler/servicemonitor.yaml @@ -0,0 +1,82 @@ +{{- if and .Values.thanosRuler.enabled .Values.thanosRuler.serviceMonitor.selfMonitor }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ template "kube-prometheus-stack.thanosRuler.name" . }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.thanosRuler.name" . }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- with .Values.thanosRuler.serviceMonitor.additionalLabels }} +{{- toYaml . | nindent 4 }} +{{- end }} +spec: + {{- include "servicemonitor.scrapeLimits" .Values.thanosRuler.serviceMonitor | nindent 2 }} + selector: + matchLabels: + app: {{ template "kube-prometheus-stack.thanosRuler.name" . }} + release: {{ $.Release.Name | quote }} + self-monitor: {{ .Values.thanosRuler.serviceMonitor.selfMonitor | quote }} + namespaceSelector: + matchNames: + - {{ printf "%s" (include "kube-prometheus-stack.namespace" .) | quote }} + endpoints: + - port: {{ .Values.thanosRuler.thanosRulerSpec.portName }} + {{- if .Values.thanosRuler.serviceMonitor.interval }} + interval: {{ .Values.thanosRuler.serviceMonitor.interval }} + {{- end }} + {{- if .Values.thanosRuler.serviceMonitor.proxyUrl }} + proxyUrl: {{ .Values.thanosRuler.serviceMonitor.proxyUrl}} + {{- end }} + {{- if .Values.thanosRuler.serviceMonitor.scheme }} + scheme: {{ .Values.thanosRuler.serviceMonitor.scheme }} + {{- end }} + {{- if .Values.thanosRuler.serviceMonitor.bearerTokenFile }} + bearerTokenFile: {{ .Values.thanosRuler.serviceMonitor.bearerTokenFile }} + {{- end }} + {{- if .Values.thanosRuler.serviceMonitor.tlsConfig }} + tlsConfig: {{- toYaml .Values.thanosRuler.serviceMonitor.tlsConfig | nindent 6 }} + {{- end }} + path: "{{ trimSuffix "/" .Values.thanosRuler.thanosRulerSpec.routePrefix }}/metrics" + {{- if .Values.thanosRuler.serviceMonitor.metricRelabelings }} + metricRelabelings: {{- tpl (toYaml .Values.thanosRuler.serviceMonitor.metricRelabelings | nindent 6) . }} + {{- end }} + {{ if .Values.global.cattle.clusterId }} + - sourceLabels: [__address__] + targetLabel: cluster_id + replacement: {{ .Values.global.cattle.clusterId }} + {{- end }} + {{ if .Values.global.cattle.clusterName}} + - sourceLabels: [__address__] + targetLabel: cluster_name + replacement: {{ .Values.global.cattle.clusterName }} + {{- end }} + {{- if .Values.thanosRuler.serviceMonitor.relabelings }} + relabelings: {{- toYaml .Values.thanosRuler.serviceMonitor.relabelings | nindent 6 }} + {{- end }} + {{- range .Values.thanosRuler.serviceMonitor.additionalEndpoints }} + - port: {{ .port }} + {{- if or $.Values.thanosRuler.serviceMonitor.interval .interval }} + interval: {{ default $.Values.thanosRuler.serviceMonitor.interval .interval }} + {{- end }} + {{- if or $.Values.thanosRuler.serviceMonitor.proxyUrl .proxyUrl }} + proxyUrl: {{ default $.Values.thanosRuler.serviceMonitor.proxyUrl .proxyUrl }} + {{- end }} + {{- if or $.Values.thanosRuler.serviceMonitor.scheme .scheme }} + scheme: {{ default $.Values.thanosRuler.serviceMonitor.scheme .scheme }} + {{- end }} + {{- if or $.Values.thanosRuler.serviceMonitor.bearerTokenFile .bearerTokenFile }} + bearerTokenFile: {{ default $.Values.thanosRuler.serviceMonitor.bearerTokenFile .bearerTokenFile }} + {{- end }} + {{- if or $.Values.thanosRuler.serviceMonitor.tlsConfig .tlsConfig }} + tlsConfig: {{- default $.Values.thanosRuler.serviceMonitor.tlsConfig .tlsConfig | toYaml | nindent 6 }} + {{- end }} + path: {{ .path }} + {{- if or $.Values.thanosRuler.serviceMonitor.metricRelabelings .metricRelabelings }} + metricRelabelings: {{- tpl (default $.Values.thanosRuler.serviceMonitor.metricRelabelings .metricRelabelings | toYaml | nindent 6) . }} + {{- end }} + {{- if or $.Values.thanosRuler.serviceMonitor.relabelings .relabelings }} + relabelings: {{- default $.Values.thanosRuler.serviceMonitor.relabelings .relabelings | toYaml | nindent 6 }} + {{- end }} + {{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/validate-install-crd.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/validate-install-crd.yaml new file mode 100644 index 00000000000..6fcb8b3a691 --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/validate-install-crd.yaml @@ -0,0 +1,23 @@ +#{{- if gt (len (lookup "rbac.authorization.k8s.io/v1" "ClusterRole" "" "")) 0 -}} +# {{- $found := dict -}} +# {{- set $found "monitoring.coreos.com/v1alpha1/AlertmanagerConfig" false -}} +# {{- set $found "monitoring.coreos.com/v1/Alertmanager" false -}} +# {{- set $found "monitoring.coreos.com/v1/PodMonitor" false -}} +# {{- set $found "monitoring.coreos.com/v1/Probe" false -}} +# {{- set $found "monitoring.coreos.com/v1alpha1/PrometheusAgent" false -}} +# {{- set $found "monitoring.coreos.com/v1/Prometheus" false -}} +# {{- set $found "monitoring.coreos.com/v1/PrometheusRule" false -}} +# {{- set $found "monitoring.coreos.com/v1alpha1/ScrapeConfig" false -}} +# {{- set $found "monitoring.coreos.com/v1/ServiceMonitor" false -}} +# {{- set $found "monitoring.coreos.com/v1/ThanosRuler" false -}} +# {{- range .Capabilities.APIVersions -}} +# {{- if hasKey $found (toString .) -}} +# {{- set $found (toString .) true -}} +# {{- end -}} +# {{- end -}} +# {{- range $_, $exists := $found -}} +# {{- if (eq $exists false) -}} +# {{- required "Required CRDs are missing. Please install the corresponding CRD chart before installing this chart." "" -}} +# {{- end -}} +# {{- end -}} +#{{- end -}} \ No newline at end of file diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/templates/validate-psp-install.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/validate-psp-install.yaml new file mode 100644 index 00000000000..a30c59d3b7c --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/templates/validate-psp-install.yaml @@ -0,0 +1,7 @@ +#{{- if gt (len (lookup "rbac.authorization.k8s.io/v1" "ClusterRole" "" "")) 0 -}} +#{{- if .Values.global.cattle.psp.enabled }} +#{{- if not (.Capabilities.APIVersions.Has "policy/v1beta1/PodSecurityPolicy") }} +#{{- fail "The target cluster does not have the PodSecurityPolicy API resource. Please disable PSPs in this chart before proceeding." -}} +#{{- end }} +#{{- end }} +#{{- end }} diff --git a/charts/rancher-monitoring/103.2.2+up57.0.3/values.yaml b/charts/rancher-monitoring/103.2.2+up57.0.3/values.yaml new file mode 100644 index 00000000000..40f676556ae --- /dev/null +++ b/charts/rancher-monitoring/103.2.2+up57.0.3/values.yaml @@ -0,0 +1,5433 @@ +# Default values for kube-prometheus-stack. +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. + +# Rancher Monitoring Configuration + +## Configuration for prometheus-adapter +## ref: https://github.com/prometheus-community/helm-charts/tree/main/charts/prometheus-adapter +## +prometheus-adapter: + enabled: true + prometheus: + # Change this if you change the namespaceOverride or nameOverride of prometheus-operator + url: http://rancher-monitoring-prometheus.cattle-monitoring-system.svc + port: 9090 + +## RKE PushProx Monitoring +## ref: https://github.com/rancher/charts/tree/dev-v2.9/packages/rancher-monitoring/rancher-pushprox +## +rkeControllerManager: + enabled: false + metricsPort: 10257 # default to secure port as of k8s >= 1.22 + component: kube-controller-manager + clients: + https: + enabled: true + insecureSkipVerify: true + useServiceAccountCredentials: true + port: 10011 + useLocalhost: true + nodeSelector: + node-role.kubernetes.io/controlplane: "true" + tolerations: + - effect: "NoExecute" + operator: "Exists" + - effect: "NoSchedule" + operator: "Exists" + kubeVersionOverrides: + - constraint: "< 1.22" + values: + metricsPort: 10252 # default to insecure port in k8s < 1.22 + clients: + https: + enabled: false + insecureSkipVerify: false + useServiceAccountCredentials: false + +rkeScheduler: + enabled: false + metricsPort: 10259 + component: kube-scheduler + clients: + https: + enabled: true + insecureSkipVerify: true + useServiceAccountCredentials: true + port: 10012 + useLocalhost: true + nodeSelector: + node-role.kubernetes.io/controlplane: "true" + tolerations: + - effect: "NoExecute" + operator: "Exists" + - effect: "NoSchedule" + operator: "Exists" + kubeVersionOverrides: + - constraint: "< 1.23" + values: + metricsPort: 10251 # default to insecure port in k8s < 1.23 + clients: + https: + enabled: false + insecureSkipVerify: false + useServiceAccountCredentials: false + +rkeProxy: + enabled: false + metricsPort: 10249 + component: kube-proxy + clients: + port: 10013 + useLocalhost: true + tolerations: + - effect: "NoExecute" + operator: "Exists" + - effect: "NoSchedule" + operator: "Exists" + +rkeEtcd: + enabled: false + metricsPort: 2379 + component: kube-etcd + clients: + port: 10014 + https: + enabled: true + certDir: /etc/kubernetes/ssl + certFile: kube-etcd-*.pem + keyFile: kube-etcd-*-key.pem + caCertFile: kube-ca.pem + seLinuxOptions: + # Gives rkeEtcd permissions to read files in /etc/kubernetes/* + # Type is defined in https://github.com/rancher/rancher-selinux + type: rke_kubereader_t + nodeSelector: + node-role.kubernetes.io/etcd: "true" + tolerations: + - effect: "NoExecute" + operator: "Exists" + - effect: "NoSchedule" + operator: "Exists" + +rkeIngressNginx: + enabled: false + metricsPort: 10254 + component: ingress-nginx + clients: + port: 10015 + useLocalhost: true + tolerations: + - effect: "NoExecute" + operator: "Exists" + - effect: "NoSchedule" + operator: "Exists" + nodeSelector: + node-role.kubernetes.io/worker: "true" + +## k3s PushProx Monitoring +## ref: https://github.com/rancher/charts/tree/dev-v2.9/packages/rancher-monitoring/rancher-pushprox +## +k3sServer: + enabled: false + metricsPort: 10250 + component: k3s-server + clients: + port: 10013 + useLocalhost: true + https: + enabled: true + useServiceAccountCredentials: true + insecureSkipVerify: true + rbac: + additionalRules: + - nonResourceURLs: ["/metrics/cadvisor"] + verbs: ["get"] + - apiGroups: [""] + resources: ["nodes/metrics"] + verbs: ["get"] + tolerations: + - effect: "NoExecute" + operator: "Exists" + - effect: "NoSchedule" + operator: "Exists" + serviceMonitor: + endpoints: + - port: metrics + honorLabels: true + relabelings: + - sourceLabels: [__metrics_path__] + targetLabel: metrics_path + - port: metrics + path: /metrics/cadvisor + honorLabels: true + relabelings: + - sourceLabels: [__metrics_path__] + targetLabel: metrics_path + - port: metrics + path: /metrics/probes + honorLabels: true + relabelings: + - sourceLabels: [__metrics_path__] + targetLabel: metrics_path + +hardened: + k3s: + networkPolicy: + enabled: true + +## KubeADM PushProx Monitoring +## ref: https://github.com/rancher/charts/tree/dev-v2.9/packages/rancher-monitoring/rancher-pushprox +## +kubeAdmControllerManager: + enabled: false + metricsPort: 10257 + component: kube-controller-manager + clients: + port: 10011 + useLocalhost: true + https: + enabled: true + useServiceAccountCredentials: true + insecureSkipVerify: true + nodeSelector: + node-role.kubernetes.io/master: "" + tolerations: + - effect: "NoExecute" + operator: "Exists" + - effect: "NoSchedule" + operator: "Exists" + +kubeAdmScheduler: + enabled: false + metricsPort: 10259 + component: kube-scheduler + clients: + port: 10012 + useLocalhost: true + https: + enabled: true + useServiceAccountCredentials: true + insecureSkipVerify: true + nodeSelector: + node-role.kubernetes.io/master: "" + tolerations: + - effect: "NoExecute" + operator: "Exists" + - effect: "NoSchedule" + operator: "Exists" + +kubeAdmProxy: + enabled: false + metricsPort: 10249 + component: kube-proxy + clients: + port: 10013 + useLocalhost: true + tolerations: + - effect: "NoExecute" + operator: "Exists" + - effect: "NoSchedule" + operator: "Exists" + +kubeAdmEtcd: + enabled: false + metricsPort: 2381 + component: kube-etcd + clients: + port: 10014 + useLocalhost: true + nodeSelector: + node-role.kubernetes.io/master: "" + tolerations: + - effect: "NoExecute" + operator: "Exists" + - effect: "NoSchedule" + operator: "Exists" + +## rke2 PushProx Monitoring +## ref: https://github.com/rancher/charts/tree/dev-v2.9/packages/rancher-monitoring/rancher-pushprox +## +rke2ControllerManager: + enabled: false + metricsPort: 10257 # default to secure port as of k8s >= 1.22 + component: kube-controller-manager + clients: + https: + enabled: true + insecureSkipVerify: true + useServiceAccountCredentials: true + port: 10011 + useLocalhost: true + nodeSelector: + node-role.kubernetes.io/master: "true" + tolerations: + - effect: "NoExecute" + operator: "Exists" + - effect: "NoSchedule" + operator: "Exists" + kubeVersionOverrides: + - constraint: "< 1.22" + values: + metricsPort: 10252 # default to insecure port in k8s < 1.22 + clients: + https: + enabled: false + insecureSkipVerify: false + useServiceAccountCredentials: false + +rke2Scheduler: + enabled: false + metricsPort: 10259 # default to secure port as of k8s >= 1.22 + component: kube-scheduler + clients: + https: + enabled: true + insecureSkipVerify: true + useServiceAccountCredentials: true + port: 10012 + useLocalhost: true + nodeSelector: + node-role.kubernetes.io/master: "true" + tolerations: + - effect: "NoExecute" + operator: "Exists" + - effect: "NoSchedule" + operator: "Exists" + kubeVersionOverrides: + - constraint: "< 1.22" + values: + metricsPort: 10251 # default to insecure port in k8s < 1.22 + clients: + https: + enabled: false + insecureSkipVerify: false + useServiceAccountCredentials: false + +rke2Proxy: + enabled: false + metricsPort: 10249 + component: kube-proxy + clients: + port: 10013 + useLocalhost: true + tolerations: + - effect: "NoExecute" + operator: "Exists" + - effect: "NoSchedule" + operator: "Exists" + +rke2Etcd: + enabled: false + metricsPort: 2381 + component: kube-etcd + clients: + port: 10014 + useLocalhost: true + nodeSelector: + node-role.kubernetes.io/etcd: "true" + tolerations: + - effect: "NoExecute" + operator: "Exists" + - effect: "NoSchedule" + operator: "Exists" + +rke2IngressNginx: + enabled: false + networkPolicy: + enabled: false + metricsPort: 10254 + component: ingress-nginx + # in the RKE2 cluster, the ingress-nginx-controller is deployed + # as a non-hostNetwork workload starting at the following versions + # - >= v1.22.12+rke2r1 < 1.23.0-0 + # - >= v1.23.9+rke2r1 < 1.24.0-0 + # - >= v1.24.3+rke2r1 < 1.25.0-0 + # - >= v1.25.0+rke2r1 + # As a result we do not need clients and proxies as we can directly create + # a service that targets the workload with the given app name + namespaceOverride: kube-system + clients: + enabled: false + proxy: + enabled: false + service: + selector: + app.kubernetes.io/name: rke2-ingress-nginx + kubeVersionOverrides: + - constraint: "< 1.21.0-0" + values: + namespaceOverride: "" + clients: + enabled: true + port: 10015 + useLocalhost: true + tolerations: + - effect: "NoExecute" + operator: "Exists" + - effect: "NoSchedule" + operator: "Exists" + affinity: + podAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchExpressions: + - key: "app.kubernetes.io/component" + operator: "In" + values: + - "controller" + topologyKey: "kubernetes.io/hostname" + namespaces: + - "kube-system" + # in the RKE2 cluster, the ingress-nginx-controller is deployed as + # a DaemonSet with 1 pod when RKE2 version is < 1.21.0-0 + deployment: + enabled: false + proxy: + enabled: true + service: + selector: false + - constraint: ">= 1.21.0-0 < 1.22.12-0" + values: + namespaceOverride: "" + clients: + enabled: true + port: 10015 + useLocalhost: true + tolerations: + - effect: "NoExecute" + operator: "Exists" + - effect: "NoSchedule" + operator: "Exists" + affinity: + podAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchExpressions: + - key: "app.kubernetes.io/component" + operator: "In" + values: + - "controller" + topologyKey: "kubernetes.io/hostname" + namespaces: + - "kube-system" + # in the RKE2 cluster, the ingress-nginx-controller is deployed as + # a hostNetwork Deployment with 1 pod when RKE2 version is >= 1.21.0-0 + deployment: + enabled: true + replicas: 1 + proxy: + enabled: true + service: + selector: false + - constraint: ">= 1.23.0-0 < v1.23.9-0" + values: + namespaceOverride: "" + clients: + enabled: true + port: 10015 + useLocalhost: true + tolerations: + - effect: "NoExecute" + operator: "Exists" + - effect: "NoSchedule" + operator: "Exists" + affinity: + podAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchExpressions: + - key: "app.kubernetes.io/component" + operator: "In" + values: + - "controller" + topologyKey: "kubernetes.io/hostname" + namespaces: + - "kube-system" + # in the RKE2 cluster, the ingress-nginx-controller is deployed as + # a hostNetwork Deployment with 1 pod when RKE2 version is >= 1.20.0-0 + deployment: + enabled: true + replicas: 1 + proxy: + enabled: true + service: + selector: false + - constraint: ">= 1.24.0-0 < v1.24.3-0" + values: + namespaceOverride: "" + clients: + enabled: true + port: 10015 + useLocalhost: true + tolerations: + - effect: "NoExecute" + operator: "Exists" + - effect: "NoSchedule" + operator: "Exists" + affinity: + podAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchExpressions: + - key: "app.kubernetes.io/component" + operator: "In" + values: + - "controller" + topologyKey: "kubernetes.io/hostname" + namespaces: + - "kube-system" + # in the RKE2 cluster, the ingress-nginx-controller is deployed as + # a hostNetwork Deployment with 1 pod when RKE2 version is >= 1.20.0-0 + deployment: + enabled: true + replicas: 1 + proxy: + enabled: true + service: + selector: false + + + +## Additional PushProx Monitoring +## ref: https://github.com/rancher/charts/tree/dev-v2.9/packages/rancher-monitoring/rancher-pushprox +## + +# hardenedKubelet can only be deployed if kubelet.enabled=true +# If enabled, it replaces the ServiceMonitor deployed by the default kubelet option with a +# PushProx-based exporter that does not require a host port to be open to scrape metrics. +hardenedKubelet: + enabled: false + metricsPort: 10250 + component: kubelet + clients: + port: 10015 + useLocalhost: true + https: + enabled: true + useServiceAccountCredentials: true + insecureSkipVerify: true + rbac: + additionalRules: + - nonResourceURLs: ["/metrics/cadvisor"] + verbs: ["get"] + - apiGroups: [""] + resources: ["nodes/metrics"] + verbs: ["get"] + tolerations: + - effect: "NoExecute" + operator: "Exists" + - effect: "NoSchedule" + operator: "Exists" + serviceMonitor: + endpoints: + - port: metrics + honorLabels: true + relabelings: + - sourceLabels: [__metrics_path__] + targetLabel: metrics_path + - port: metrics + path: /metrics/cadvisor + honorLabels: true + relabelings: + - sourceLabels: [__metrics_path__] + targetLabel: metrics_path + - port: metrics + path: /metrics/probes + honorLabels: true + relabelings: + - sourceLabels: [__metrics_path__] + targetLabel: metrics_path + +# hardenedNodeExporter can only be deployed if nodeExporter.enabled=true +# If enabled, it replaces the ServiceMonitor deployed by the default nodeExporter with a +# PushProx-based exporter that does not require a host port to be open to scrape metrics. +hardenedNodeExporter: + enabled: false + metricsPort: 9796 + component: node-exporter + clients: + port: 10016 + useLocalhost: true + tolerations: + - effect: "NoExecute" + operator: "Exists" + - effect: "NoSchedule" + operator: "Exists" + +## Upgrades +upgrade: + ## Run upgrade scripts before an upgrade or rollback via a Job hook + enabled: true + ## Image to use to run the scripts + image: + repository: rancher/shell + tag: v0.2.1 + +## Rancher Monitoring +## + +rancherMonitoring: + enabled: true + + ## A namespaceSelector to identify the namespace to find the Rancher deployment + ## + namespaceSelector: + matchNames: + - cattle-system + + ## A selector to identify the Rancher deployment + ## If not set, the chart will try to search for the Rancher deployment in the cattle-system namespace and infer the selector values from it + ## If the Rancher deployment does not exist, no resources will be deployed. + ## + selector: {} + +## Component scraping nginx-ingress-controller +## +ingressNginx: + enabled: false + + ## The namespace to search for your nginx-ingress-controller + ## + namespace: ingress-nginx + + service: + port: 9913 + targetPort: 10254 + # selector: + # app: ingress-nginx + serviceMonitor: + ## Scrape interval. If not set, the Prometheus default scrape interval is used. + ## + interval: "30s" + + ## proxyUrl: URL of a proxy that should be used for scraping. + ## + proxyUrl: "" + + ## metric relabel configs to apply to samples before ingestion. + ## + metricRelabelings: [] + # - action: keep + # regex: 'kube_(daemonset|deployment|pod|namespace|node|statefulset).+' + # sourceLabels: [__name__] + + # relabel configs to apply to samples before ingestion. + ## + relabelings: [] + # - sourceLabels: [__meta_kubernetes_pod_node_name] + # separator: ; + # regex: ^(.*)$ + # targetLabel: nodename + # replacement: $1 + # action: replace + +# Prometheus Operator Configuration + +## Provide a name in place of kube-prometheus-stack for `app:` labels +## NOTE: If you change this value, you must update the prometheus-adapter.prometheus.url +## +nameOverride: "rancher-monitoring" + +## Override the deployment namespace +## NOTE: If you change this value, you must update the prometheus-adapter.prometheus.url +## +namespaceOverride: "cattle-monitoring-system" + +## Provide a k8s version to auto dashboard import script example: kubeTargetVersionOverride: 1.26.6 +## +kubeTargetVersionOverride: "" + +## Allow kubeVersion to be overridden while creating the ingress +## +kubeVersionOverride: "" + +## Provide a name to substitute for the full names of resources +## +fullnameOverride: "" + +## Labels to apply to all resources +## +commonLabels: {} +# scmhash: abc123 +# myLabel: aakkmd + +## Install Prometheus Operator CRDs +## +crds: + enabled: true + +## custom Rules to override "for" and "severity" in defaultRules +## +customRules: {} + # AlertmanagerFailedReload: + # for: 3m + # AlertmanagerMembersInconsistent: + # for: 5m + # severity: "warning" + +## Create default rules for monitoring the cluster +## +defaultRules: + create: true + rules: + alertmanager: true + etcd: true + configReloaders: true + general: true + k8sContainerCpuUsageSecondsTotal: true + k8sContainerMemoryCache: true + k8sContainerMemoryRss: true + k8sContainerMemorySwap: true + k8sContainerResource: true + k8sContainerMemoryWorkingSetBytes: true + k8sPodOwner: true + kubeApiserverAvailability: true + kubeApiserverBurnrate: true + kubeApiserverHistogram: true + kubeApiserverSlos: true + kubeControllerManager: true + kubelet: true + kubeProxy: true + kubePrometheusGeneral: true + kubePrometheusNodeRecording: true + kubernetesApps: true + kubernetesResources: true + kubernetesStorage: true + kubernetesSystem: true + kubeSchedulerAlerting: true + kubeSchedulerRecording: true + kubeStateMetrics: true + network: true + node: true + nodeExporterAlerting: true + nodeExporterRecording: true + prometheus: true + prometheusOperator: true + windows: true + + ## Reduce app namespace alert scope + appNamespacesTarget: ".*" + + ## Set keep_firing_for for all alerts + keepFiringFor: "" + + ## Labels for default rules + labels: {} + ## Annotations for default rules + annotations: {} + + ## Additional labels for PrometheusRule alerts + additionalRuleLabels: {} + + ## Additional annotations for PrometheusRule alerts + additionalRuleAnnotations: {} + + ## Additional labels for specific PrometheusRule alert groups + additionalRuleGroupLabels: + alertmanager: {} + etcd: {} + configReloaders: {} + general: {} + k8sContainerCpuUsageSecondsTotal: {} + k8sContainerMemoryCache: {} + k8sContainerMemoryRss: {} + k8sContainerMemorySwap: {} + k8sContainerResource: {} + k8sPodOwner: {} + kubeApiserverAvailability: {} + kubeApiserverBurnrate: {} + kubeApiserverHistogram: {} + kubeApiserverSlos: {} + kubeControllerManager: {} + kubelet: {} + kubeProxy: {} + kubePrometheusGeneral: {} + kubePrometheusNodeRecording: {} + kubernetesApps: {} + kubernetesResources: {} + kubernetesStorage: {} + kubernetesSystem: {} + kubeSchedulerAlerting: {} + kubeSchedulerRecording: {} + kubeStateMetrics: {} + network: {} + node: {} + nodeExporterAlerting: {} + nodeExporterRecording: {} + prometheus: {} + prometheusOperator: {} + + ## Additional annotations for specific PrometheusRule alerts groups + additionalRuleGroupAnnotations: + alertmanager: {} + etcd: {} + configReloaders: {} + general: {} + k8sContainerCpuUsageSecondsTotal: {} + k8sContainerMemoryCache: {} + k8sContainerMemoryRss: {} + k8sContainerMemorySwap: {} + k8sContainerResource: {} + k8sPodOwner: {} + kubeApiserverAvailability: {} + kubeApiserverBurnrate: {} + kubeApiserverHistogram: {} + kubeApiserverSlos: {} + kubeControllerManager: {} + kubelet: {} + kubeProxy: {} + kubePrometheusGeneral: {} + kubePrometheusNodeRecording: {} + kubernetesApps: {} + kubernetesResources: {} + kubernetesStorage: {} + kubernetesSystem: {} + kubeSchedulerAlerting: {} + kubeSchedulerRecording: {} + kubeStateMetrics: {} + network: {} + node: {} + nodeExporterAlerting: {} + nodeExporterRecording: {} + prometheus: {} + prometheusOperator: {} + + additionalAggregationLabels: [] + + ## Prefix for runbook URLs. Use this to override the first part of the runbookURLs that is common to all rules. + runbookUrl: "https://runbooks.prometheus-operator.dev/runbooks" + + ## Disabled PrometheusRule alerts + disabled: {} + # KubeAPIDown: true + # NodeRAIDDegraded: true + +## Deprecated way to provide custom recording or alerting rules to be deployed into the cluster. +## +# additionalPrometheusRules: [] +# - name: my-rule-file +# groups: +# - name: my_group +# rules: +# - record: my_record +# expr: 100 * my_record + +## Provide custom recording or alerting rules to be deployed into the cluster. +## +additionalPrometheusRulesMap: {} +# rule-name: +# groups: +# - name: my_group +# rules: +# - record: my_record +# expr: 100 * my_record + +## +global: + cattle: + psp: + enabled: false + + systemDefaultRegistry: "" + ## Windows Monitoring + ## ref: https://github.com/rancher/charts/tree/dev-v2.5-source/packages/rancher-windows-exporter + ## + ## Deploys a DaemonSet of Prometheus exporters based on https://github.com/prometheus-community/windows_exporter. + ## Every Windows host must have a wins version of 0.1.0+ to use this chart (default as of Rancher 2.5.8). + ## To upgrade wins versions on Windows hosts, see https://github.com/rancher/wins/tree/master/charts/rancher-wins-upgrader. + ## + windows: + enabled: false + seLinux: + enabled: false + kubectl: + repository: rancher/kubectl + tag: v1.20.2 + pullPolicy: IfNotPresent + rbac: + ## Create RBAC resources for ServiceAccounts and users + ## + create: true + + userRoles: + ## Create default user ClusterRoles to allow users to interact with Prometheus CRs, ConfigMaps, and Secrets + create: true + ## Aggregate default user ClusterRoles into default k8s ClusterRoles + aggregateToDefaultRoles: true + + pspAnnotations: {} + ## Specify pod annotations + ## Ref: https://kubernetes.io/docs/concepts/policy/pod-security-policy/#apparmor + ## Ref: https://kubernetes.io/docs/concepts/policy/pod-security-policy/#seccomp + ## Ref: https://kubernetes.io/docs/concepts/policy/pod-security-policy/#sysctl + ## + # seccomp.security.alpha.kubernetes.io/allowedProfileNames: '*' + # seccomp.security.alpha.kubernetes.io/defaultProfileName: 'docker/default' + # apparmor.security.beta.kubernetes.io/defaultProfileName: 'runtime/default' + + ## Global image registry to use if it needs to be overriden for some specific use cases (e.g local registries, custom images, ...) + ## + imageRegistry: docker.io + + ## Reference to one or more secrets to be used when pulling images + ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/ + ## + imagePullSecrets: [] + # - name: "image-pull-secret" + # or + # - "image-pull-secret" + +windowsMonitoring: + ## Deploys the windows-exporter and Windows-specific dashboards and rules (job name must be 'windows-exporter') + enabled: false + +## Configuration for prometheus-windows-exporter +## ref: https://github.com/prometheus-community/helm-charts/tree/main/charts/prometheus-windows-exporter +## +prometheus-windows-exporter: + ## Enable ServiceMonitor and set Kubernetes label to use as a job label + ## + prometheus: + monitor: + enabled: true + jobLabel: jobLabel + + releaseLabel: true + + ## Set job label to 'windows-exporter' as required by the default Prometheus rules and Grafana dashboards + ## + podLabels: + jobLabel: windows-exporter + + ## Enable memory and container metrics as required by the default Prometheus rules and Grafana dashboards + ## + config: |- + collectors: + enabled: '[defaults],memory,container' + +## Configuration for alertmanager +## ref: https://prometheus.io/docs/alerting/alertmanager/ +## +alertmanager: + + ## Deploy alertmanager + ## + enabled: true + + ## Annotations for Alertmanager + ## + annotations: {} + + ## Api that prometheus will use to communicate with alertmanager. Possible values are v1, v2 + ## + apiVersion: v2 + + ## Service account for Alertmanager to use. + ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/configure-service-account/ + ## + serviceAccount: + create: true + name: "" + annotations: {} + automountServiceAccountToken: true + + ## Configure pod disruption budgets for Alertmanager + ## ref: https://kubernetes.io/docs/tasks/run-application/configure-pdb/#specifying-a-poddisruptionbudget + ## This configuration is immutable once created and will require the PDB to be deleted to be changed + ## https://github.com/kubernetes/kubernetes/issues/45398 + ## + podDisruptionBudget: + enabled: false + minAvailable: 1 + maxUnavailable: "" + + ## Alertmanager configuration directives + ## ref: https://prometheus.io/docs/alerting/configuration/#configuration-file + ## https://prometheus.io/webtools/alerting/routing-tree-editor/ + ## + config: + global: + resolve_timeout: 5m + inhibit_rules: + - source_matchers: + - 'severity = critical' + target_matchers: + - 'severity =~ warning|info' + equal: + - 'namespace' + - 'alertname' + - source_matchers: + - 'severity = warning' + target_matchers: + - 'severity = info' + equal: + - 'namespace' + - 'alertname' + - source_matchers: + - 'alertname = InfoInhibitor' + target_matchers: + - 'severity = info' + equal: + - 'namespace' + - target_matchers: + - 'alertname = InfoInhibitor' + route: + group_by: ['namespace'] + group_wait: 30s + group_interval: 5m + repeat_interval: 12h + receiver: 'null' + routes: + - receiver: 'null' + matchers: + - alertname = "Watchdog" + receivers: + - name: 'null' + templates: + - '/etc/alertmanager/config/*.tmpl' + + ## Alertmanager configuration directives (as string type, preferred over the config hash map) + ## stringConfig will be used only, if tplConfig is true + ## ref: https://prometheus.io/docs/alerting/configuration/#configuration-file + ## https://prometheus.io/webtools/alerting/routing-tree-editor/ + ## + stringConfig: "" + + ## Pass the Alertmanager configuration directives through Helm's templating + ## engine. If the Alertmanager configuration contains Alertmanager templates, + ## they'll need to be properly escaped so that they are not interpreted by + ## Helm + ## ref: https://helm.sh/docs/developing_charts/#using-the-tpl-function + ## https://prometheus.io/docs/alerting/configuration/#tmpl_string + ## https://prometheus.io/docs/alerting/notifications/ + ## https://prometheus.io/docs/alerting/notification_examples/ + tplConfig: false + + ## Alertmanager template files to format alerts + ## By default, templateFiles are placed in /etc/alertmanager/config/ and if + ## they have a .tmpl file suffix will be loaded. See config.templates above + ## to change, add other suffixes. If adding other suffixes, be sure to update + ## config.templates above to include those suffixes. + ## ref: https://prometheus.io/docs/alerting/notifications/ + ## https://prometheus.io/docs/alerting/notification_examples/ + ## + + templateFiles: + rancher_defaults.tmpl: |- + {{- define "slack.rancher.text" -}} + {{ template "rancher.text_multiple" . }} + {{- end -}} + + {{- define "rancher.text_multiple" -}} + *[GROUP - Details]* + One or more alarms in this group have triggered a notification. + + {{- if gt (len .GroupLabels.Values) 0 }} + *Group Labels:* + {{- range .GroupLabels.SortedPairs }} + • *{{ .Name }}:* `{{ .Value }}` + {{- end }} + {{- end }} + {{- if .ExternalURL }} + *Link to AlertManager:* {{ .ExternalURL }} + {{- end }} + + {{- range .Alerts }} + {{ template "rancher.text_single" . }} + {{- end }} + {{- end -}} + + {{- define "rancher.text_single" -}} + {{- if .Labels.alertname }} + *[ALERT - {{ .Labels.alertname }}]* + {{- else }} + *[ALERT]* + {{- end }} + {{- if .Labels.severity }} + *Severity:* `{{ .Labels.severity }}` + {{- end }} + {{- if .Labels.cluster }} + *Cluster:* {{ .Labels.cluster }} + {{- end }} + {{- if .Annotations.summary }} + *Summary:* {{ .Annotations.summary }} + {{- end }} + {{- if .Annotations.message }} + *Message:* {{ .Annotations.message }} + {{- end }} + {{- if .Annotations.description }} + *Description:* {{ .Annotations.description }} + {{- end }} + {{- if .Annotations.runbook_url }} + *Runbook URL:* <{{ .Annotations.runbook_url }}|:spiral_note_pad:> + {{- end }} + {{- with .Labels }} + {{- with .Remove (stringSlice "alertname" "severity" "cluster") }} + {{- if gt (len .) 0 }} + *Additional Labels:* + {{- range .SortedPairs }} + • *{{ .Name }}:* `{{ .Value }}` + {{- end }} + {{- end }} + {{- end }} + {{- end }} + {{- with .Annotations }} + {{- with .Remove (stringSlice "summary" "message" "description" "runbook_url") }} + {{- if gt (len .) 0 }} + *Additional Annotations:* + {{- range .SortedPairs }} + • *{{ .Name }}:* `{{ .Value }}` + {{- end }} + {{- end }} + {{- end }} + {{- end }} + {{- end -}} + + ingress: + enabled: false + + # For Kubernetes >= 1.18 you should specify the ingress-controller via the field ingressClassName + # See https://kubernetes.io/blog/2020/04/02/improvements-to-the-ingress-api-in-kubernetes-1.18/#specifying-the-class-of-an-ingress + # ingressClassName: nginx + + annotations: {} + + labels: {} + + ## Override ingress to a different defined port on the service + # servicePort: 8081 + ## Override ingress to a different service then the default, this is useful if you need to + ## point to a specific instance of the alertmanager (eg kube-prometheus-stack-alertmanager-0) + # serviceName: kube-prometheus-stack-alertmanager-0 + + ## Hosts must be provided if Ingress is enabled. + ## + hosts: [] + # - alertmanager.domain.com + + ## Paths to use for ingress rules - one path should match the alertmanagerSpec.routePrefix + ## + paths: [] + # - / + + ## For Kubernetes >= 1.18 you should specify the pathType (determines how Ingress paths should be matched) + ## See https://kubernetes.io/blog/2020/04/02/improvements-to-the-ingress-api-in-kubernetes-1.18/#better-path-matching-with-path-types + # pathType: ImplementationSpecific + + ## TLS configuration for Alertmanager Ingress + ## Secret must be manually created in the namespace + ## + tls: [] + # - secretName: alertmanager-general-tls + # hosts: + # - alertmanager.example.com + + ## Configuration for Alertmanager secret + ## + secret: + annotations: {} + + # by default the alertmanager secret is not overwritten if it already exists + recreateIfExists: false + + ## Configuration for creating an Ingress that will map to each Alertmanager replica service + ## alertmanager.servicePerReplica must be enabled + ## + ingressPerReplica: + enabled: false + + # For Kubernetes >= 1.18 you should specify the ingress-controller via the field ingressClassName + # See https://kubernetes.io/blog/2020/04/02/improvements-to-the-ingress-api-in-kubernetes-1.18/#specifying-the-class-of-an-ingress + # ingressClassName: nginx + + annotations: {} + labels: {} + + ## Final form of the hostname for each per replica ingress is + ## {{ ingressPerReplica.hostPrefix }}-{{ $replicaNumber }}.{{ ingressPerReplica.hostDomain }} + ## + ## Prefix for the per replica ingress that will have `-$replicaNumber` + ## appended to the end + hostPrefix: "" + ## Domain that will be used for the per replica ingress + hostDomain: "" + + ## Paths to use for ingress rules + ## + paths: [] + # - / + + ## For Kubernetes >= 1.18 you should specify the pathType (determines how Ingress paths should be matched) + ## See https://kubernetes.io/blog/2020/04/02/improvements-to-the-ingress-api-in-kubernetes-1.18/#better-path-matching-with-path-types + # pathType: ImplementationSpecific + + ## Secret name containing the TLS certificate for alertmanager per replica ingress + ## Secret must be manually created in the namespace + tlsSecretName: "" + + ## Separated secret for each per replica Ingress. Can be used together with cert-manager + ## + tlsSecretPerReplica: + enabled: false + ## Final form of the secret for each per replica ingress is + ## {{ tlsSecretPerReplica.prefix }}-{{ $replicaNumber }} + ## + prefix: "alertmanager" + + ## Configuration for Alertmanager service + ## + service: + annotations: {} + labels: {} + clusterIP: "" + + ## Port for Alertmanager Service to listen on + ## + port: 9093 + ## To be used with a proxy extraContainer port + ## + targetPort: 9093 + ## Port to expose on each node + ## Only used if service.type is 'NodePort' + ## + nodePort: 30903 + ## List of IP addresses at which the Prometheus server service is available + ## Ref: https://kubernetes.io/docs/user-guide/services/#external-ips + ## + + ## Additional ports to open for Alertmanager service + ## + additionalPorts: [] + # - name: oauth-proxy + # port: 8081 + # targetPort: 8081 + # - name: oauth-metrics + # port: 8082 + # targetPort: 8082 + + externalIPs: [] + loadBalancerIP: "" + loadBalancerSourceRanges: [] + + ## Denotes if this Service desires to route external traffic to node-local or cluster-wide endpoints + ## + externalTrafficPolicy: Cluster + + ## If you want to make sure that connections from a particular client are passed to the same Pod each time + ## Accepts 'ClientIP' or 'None' + ## + sessionAffinity: None + + ## If you want to modify the ClientIP sessionAffinity timeout + ## The value must be >0 && <=86400(for 1 day) if ServiceAffinity == "ClientIP" + ## + sessionAffinityConfig: + clientIP: + timeoutSeconds: 10800 + + ## Service type + ## + type: ClusterIP + + ## Configuration for creating a separate Service for each statefulset Alertmanager replica + ## + servicePerReplica: + enabled: false + annotations: {} + + ## Port for Alertmanager Service per replica to listen on + ## + port: 9093 + + ## To be used with a proxy extraContainer port + targetPort: 9093 + + ## Port to expose on each node + ## Only used if servicePerReplica.type is 'NodePort' + ## + nodePort: 30904 + + ## Loadbalancer source IP ranges + ## Only used if servicePerReplica.type is "LoadBalancer" + loadBalancerSourceRanges: [] + + ## Denotes if this Service desires to route external traffic to node-local or cluster-wide endpoints + ## + externalTrafficPolicy: Cluster + + ## Service type + ## + type: ClusterIP + + ## Configuration for creating a ServiceMonitor for AlertManager + ## + serviceMonitor: + ## If true, a ServiceMonitor will be created for the AlertManager service. + ## + selfMonitor: true + + ## Scrape interval. If not set, the Prometheus default scrape interval is used. + ## + interval: "" + + ## Additional labels + ## + additionalLabels: {} + + ## SampleLimit defines per-scrape limit on number of scraped samples that will be accepted. + ## + sampleLimit: 0 + + ## TargetLimit defines a limit on the number of scraped targets that will be accepted. + ## + targetLimit: 0 + + ## Per-scrape limit on number of labels that will be accepted for a sample. Only valid in Prometheus versions 2.27.0 and newer. + ## + labelLimit: 0 + + ## Per-scrape limit on length of labels name that will be accepted for a sample. Only valid in Prometheus versions 2.27.0 and newer. + ## + labelNameLengthLimit: 0 + + ## Per-scrape limit on length of labels value that will be accepted for a sample. Only valid in Prometheus versions 2.27.0 and newer. + ## + labelValueLengthLimit: 0 + + ## proxyUrl: URL of a proxy that should be used for scraping. + ## + proxyUrl: "" + + ## scheme: HTTP scheme to use for scraping. Can be used with `tlsConfig` for example if using istio mTLS. + scheme: "" + + ## enableHttp2: Whether to enable HTTP2. + ## See https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#endpoint + enableHttp2: true + + ## tlsConfig: TLS configuration to use when scraping the endpoint. For example if using istio mTLS. + ## Of type: https://github.com/coreos/prometheus-operator/blob/main/Documentation/api.md#tlsconfig + tlsConfig: {} + + bearerTokenFile: + + ## MetricRelabelConfigs to apply to samples after scraping, but before ingestion. + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#relabelconfig + ## + metricRelabelings: [] + # - action: keep + # regex: 'kube_(daemonset|deployment|pod|namespace|node|statefulset).+' + # sourceLabels: [__name__] + + ## RelabelConfigs to apply to samples before scraping + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#relabelconfig + ## + relabelings: [] + # - sourceLabels: [__meta_kubernetes_pod_node_name] + # separator: ; + # regex: ^(.*)$ + # targetLabel: nodename + # replacement: $1 + # action: replace + + ## Additional Endpoints + ## + additionalEndpoints: [] + # - port: oauth-metrics + # path: /metrics + + ## Settings affecting alertmanagerSpec + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#alertmanagerspec + ## + alertmanagerSpec: + ## Standard object's metadata. More info: https://github.com/kubernetes/community/blob/master/contributors/devel/sig-architecture/api-conventions.md#metadata + ## Metadata Labels and Annotations gets propagated to the Alertmanager pods. + ## + podMetadata: {} + + ## Image of Alertmanager + ## + image: + repository: rancher/mirrored-prometheus-alertmanager + tag: v0.27.0 + sha: "" + + ## If true then the user will be responsible to provide a secret with alertmanager configuration + ## So when true the config part will be ignored (including templateFiles) and the one in the secret will be used + ## + useExistingSecret: false + + ## Secrets is a list of Secrets in the same namespace as the Alertmanager object, which shall be mounted into the + ## Alertmanager Pods. The Secrets are mounted into /etc/alertmanager/secrets/. + ## + secrets: [] + + ## If false then the user will opt out of automounting API credentials. + ## + automountServiceAccountToken: true + + ## ConfigMaps is a list of ConfigMaps in the same namespace as the Alertmanager object, which shall be mounted into the Alertmanager Pods. + ## The ConfigMaps are mounted into /etc/alertmanager/configmaps/. + ## + configMaps: [] + + ## ConfigSecret is the name of a Kubernetes Secret in the same namespace as the Alertmanager object, which contains configuration for + ## this Alertmanager instance. Defaults to 'alertmanager-' The secret is mounted into /etc/alertmanager/config. + ## + # configSecret: + + ## WebTLSConfig defines the TLS parameters for HTTPS + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#alertmanagerwebspec + web: {} + + ## AlertmanagerConfigs to be selected to merge and configure Alertmanager with. + ## + alertmanagerConfigSelector: {} + ## Example which selects all alertmanagerConfig resources + ## with label "alertconfig" with values any of "example-config" or "example-config-2" + # alertmanagerConfigSelector: + # matchExpressions: + # - key: alertconfig + # operator: In + # values: + # - example-config + # - example-config-2 + # + ## Example which selects all alertmanagerConfig resources with label "role" set to "example-config" + # alertmanagerConfigSelector: + # matchLabels: + # role: example-config + + ## Namespaces to be selected for AlertmanagerConfig discovery. If nil, only check own namespace. + ## + alertmanagerConfigNamespaceSelector: {} + ## Example which selects all namespaces + ## with label "alertmanagerconfig" with values any of "example-namespace" or "example-namespace-2" + # alertmanagerConfigNamespaceSelector: + # matchExpressions: + # - key: alertmanagerconfig + # operator: In + # values: + # - example-namespace + # - example-namespace-2 + + ## Example which selects all namespaces with label "alertmanagerconfig" set to "enabled" + # alertmanagerConfigNamespaceSelector: + # matchLabels: + # alertmanagerconfig: enabled + + ## AlermanagerConfig to be used as top level configuration + ## + alertmanagerConfiguration: {} + ## Example with select a global alertmanagerconfig + # alertmanagerConfiguration: + # name: global-alertmanager-Configuration + + ## Defines the strategy used by AlertmanagerConfig objects to match alerts. eg: + ## + alertmanagerConfigMatcherStrategy: {} + ## Example with use OnNamespace strategy + # alertmanagerConfigMatcherStrategy: + # type: OnNamespace + + ## Define Log Format + # Use logfmt (default) or json logging + logFormat: logfmt + + ## Log level for Alertmanager to be configured with. + ## + logLevel: info + + ## Size is the expected size of the alertmanager cluster. The controller will eventually make the size of the + ## running cluster equal to the expected size. + replicas: 1 + + ## Time duration Alertmanager shall retain data for. Default is '120h', and must match the regular expression + ## [0-9]+(ms|s|m|h) (milliseconds seconds minutes hours). + ## + retention: 120h + + ## Storage is the definition of how storage will be used by the Alertmanager instances. + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/user-guides/storage.md + ## + storage: {} + # volumeClaimTemplate: + # spec: + # storageClassName: gluster + # accessModes: ["ReadWriteOnce"] + # resources: + # requests: + # storage: 50Gi + # selector: {} + + + ## The external URL the Alertmanager instances will be available under. This is necessary to generate correct URLs. This is necessary if Alertmanager is not served from root of a DNS name. string false + ## + externalUrl: + + ## The route prefix Alertmanager registers HTTP handlers for. This is useful, if using ExternalURL and a proxy is rewriting HTTP routes of a request, and the actual ExternalURL is still true, + ## but the server serves requests under a different route prefix. For example for use with kubectl proxy. + ## + routePrefix: / + + ## scheme: HTTP scheme to use. Can be used with `tlsConfig` for example if using istio mTLS. + scheme: "" + + ## tlsConfig: TLS configuration to use when connect to the endpoint. For example if using istio mTLS. + ## Of type: https://github.com/coreos/prometheus-operator/blob/main/Documentation/api.md#tlsconfig + tlsConfig: {} + + ## If set to true all actions on the underlying managed objects are not going to be performed, except for delete actions. + ## + paused: false + + ## Define which Nodes the Pods are scheduled on. + ## ref: https://kubernetes.io/docs/user-guide/node-selection/ + ## + nodeSelector: {} + + ## Define resources requests and limits for single Pods. + ## ref: https://kubernetes.io/docs/user-guide/compute-resources/ + ## + resources: + limits: + memory: 500Mi + cpu: 1000m + requests: + memory: 100Mi + cpu: 100m + + ## Pod anti-affinity can prevent the scheduler from placing Prometheus replicas on the same node. + ## The default value "soft" means that the scheduler should *prefer* to not schedule two replica pods onto the same node but no guarantee is provided. + ## The value "hard" means that the scheduler is *required* to not schedule two replica pods onto the same node. + ## The value "" will disable pod anti-affinity so that no anti-affinity rules will be configured. + ## + podAntiAffinity: "" + + ## If anti-affinity is enabled sets the topologyKey to use for anti-affinity. + ## This can be changed to, for example, failure-domain.beta.kubernetes.io/zone + ## + podAntiAffinityTopologyKey: kubernetes.io/hostname + + ## Assign custom affinity rules to the alertmanager instance + ## ref: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/ + ## + affinity: {} + # nodeAffinity: + # requiredDuringSchedulingIgnoredDuringExecution: + # nodeSelectorTerms: + # - matchExpressions: + # - key: kubernetes.io/e2e-az-name + # operator: In + # values: + # - e2e-az1 + # - e2e-az2 + + ## If specified, the pod's tolerations. + ## ref: https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/ + ## + tolerations: [] + # - key: "key" + # operator: "Equal" + # value: "value" + # effect: "NoSchedule" + + ## If specified, the pod's topology spread constraints. + ## ref: https://kubernetes.io/docs/concepts/workloads/pods/pod-topology-spread-constraints/ + ## + topologySpreadConstraints: [] + # - maxSkew: 1 + # topologyKey: topology.kubernetes.io/zone + # whenUnsatisfiable: DoNotSchedule + # labelSelector: + # matchLabels: + # app: alertmanager + + ## SecurityContext holds pod-level security attributes and common container settings. + ## This defaults to non root user with uid 1000 and gid 2000. *v1.PodSecurityContext false + ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/ + ## + securityContext: + runAsGroup: 2000 + runAsNonRoot: true + runAsUser: 1000 + fsGroup: 2000 + seccompProfile: + type: RuntimeDefault + + ## ListenLocal makes the Alertmanager server listen on loopback, so that it does not bind against the Pod IP. + ## Note this is only for the Alertmanager UI, not the gossip communication. + ## + listenLocal: false + + ## Containers allows injecting additional containers. This is meant to allow adding an authentication proxy to an Alertmanager pod. + ## + containers: [] + # containers: + # - name: oauth-proxy + # image: quay.io/oauth2-proxy/oauth2-proxy:v7.5.1 + # args: + # - --upstream=http://127.0.0.1:9093 + # - --http-address=0.0.0.0:8081 + # - --metrics-address=0.0.0.0:8082 + # - ... + # ports: + # - containerPort: 8081 + # name: oauth-proxy + # protocol: TCP + # - containerPort: 8082 + # name: oauth-metrics + # protocol: TCP + # resources: {} + + # Additional volumes on the output StatefulSet definition. + volumes: [] + + # Additional VolumeMounts on the output StatefulSet definition. + volumeMounts: [] + + ## InitContainers allows injecting additional initContainers. This is meant to allow doing some changes + ## (permissions, dir tree) on mounted volumes before starting prometheus + initContainers: [] + + ## Priority class assigned to the Pods + ## + priorityClassName: "" + + ## AdditionalPeers allows injecting a set of additional Alertmanagers to peer with to form a highly available cluster. + ## + additionalPeers: [] + + ## PortName to use for Alert Manager. + ## + portName: "http-web" + + ## ClusterAdvertiseAddress is the explicit address to advertise in cluster. Needs to be provided for non RFC1918 [1] (public) addresses. [1] RFC1918: https://tools.ietf.org/html/rfc1918 + ## + clusterAdvertiseAddress: false + + ## clusterGossipInterval determines interval between gossip attempts. + ## Needs to be specified as GoDuration, a time duration that can be parsed by Go’s time.ParseDuration() (e.g. 45ms, 30s, 1m, 1h20m15s) + clusterGossipInterval: "" + + ## clusterPeerTimeout determines timeout for cluster peering. + ## Needs to be specified as GoDuration, a time duration that can be parsed by Go’s time.ParseDuration() (e.g. 45ms, 30s, 1m, 1h20m15s) + clusterPeerTimeout: "" + + ## clusterPushpullInterval determines interval between pushpull attempts. + ## Needs to be specified as GoDuration, a time duration that can be parsed by Go’s time.ParseDuration() (e.g. 45ms, 30s, 1m, 1h20m15s) + clusterPushpullInterval: "" + + ## ForceEnableClusterMode ensures Alertmanager does not deactivate the cluster mode when running with a single replica. + ## Use case is e.g. spanning an Alertmanager cluster across Kubernetes clusters with a single replica in each. + forceEnableClusterMode: false + + ## Minimum number of seconds for which a newly created pod should be ready without any of its container crashing for it to + ## be considered available. Defaults to 0 (pod will be considered available as soon as it is ready). + minReadySeconds: 0 + + ## Additional configuration which is not covered by the properties above. (passed through tpl) + additionalConfig: {} + + ## Additional configuration which is not covered by the properties above. + ## Useful, if you need advanced templating inside alertmanagerSpec. + ## Otherwise, use alertmanager.alertmanagerSpec.additionalConfig (passed through tpl) + additionalConfigString: "" + + ## ExtraSecret can be used to store various data in an extra secret + ## (use it for example to store hashed basic auth credentials) + extraSecret: + ## if not set, name will be auto generated + # name: "" + annotations: {} + data: {} + # auth: | + # foo:$apr1$OFG3Xybp$ckL0FHDAkoXYIlH9.cysT0 + # someoneelse:$apr1$DMZX2Z4q$6SbQIfyuLQd.xmo/P0m2c. + +## Using default values from https://github.com/grafana/helm-charts/blob/main/charts/grafana/values.yaml +## +grafana: + enabled: true + namespaceOverride: "" + + ## Grafana's primary configuration + ## NOTE: values in map will be converted to ini format + ## ref: http://docs.grafana.org/installation/configuration/ + ## + grafana.ini: + users: + auto_assign_org_role: Viewer + auth: + disable_login_form: false + auth.anonymous: + enabled: true + org_role: Viewer + auth.basic: + enabled: false + dashboards: + # Modify this value to change the default dashboard shown on the main Grafana page + default_home_dashboard_path: /tmp/dashboards/rancher-default-home.json + security: + # Required to embed dashboards in Rancher Cluster Overview Dashboard on Cluster Explorer + allow_embedding: true + + deploymentStrategy: + type: Recreate + + ## ForceDeployDatasources Create datasource configmap even if grafana deployment has been disabled + ## + forceDeployDatasources: false + + ## ForceDeployDashboard Create dashboard configmap even if grafana deployment has been disabled + ## + forceDeployDashboards: false + + ## Deploy default dashboards + ## + defaultDashboardsEnabled: true + + # Additional options for defaultDashboards + defaultDashboards: + # The default namespace to place defaultDashboards within + namespace: cattle-dashboards + # Whether to create the default namespace as a Helm managed namespace or use an existing namespace + # If false, the defaultDashboards.namespace will be created as a Helm managed namespace + useExistingNamespace: false + # Whether the Helm managed namespace created by this chart should be left behind on a Helm uninstall + # If you place other dashboards in this namespace, then they will be deleted on a helm uninstall + # Ignore if useExistingNamespace is true + cleanupOnUninstall: false + + ## Timezone for the default dashboards + ## Other options are: browser or a specific timezone, i.e. Europe/Luxembourg + ## + defaultDashboardsTimezone: utc + + ## Editable flag for the default dashboards + ## + defaultDashboardsEditable: true + + adminPassword: prom-operator + + ingress: + ## If true, Grafana Ingress will be created + ## + enabled: false + + ## IngressClassName for Grafana Ingress. + ## Should be provided if Ingress is enable. + ## + # ingressClassName: nginx + + ## Annotations for Grafana Ingress + ## + annotations: {} + # kubernetes.io/ingress.class: nginx + # kubernetes.io/tls-acme: "true" + + ## Labels to be added to the Ingress + ## + labels: {} + + ## Hostnames. + ## Must be provided if Ingress is enable. + ## + # hosts: + # - grafana.domain.com + hosts: [] + + ## Path for grafana ingress + path: / + + ## TLS configuration for grafana Ingress + ## Secret must be manually created in the namespace + ## + tls: [] + # - secretName: grafana-general-tls + # hosts: + # - grafana.example.com + + # # To make Grafana persistent (Using Statefulset) + # # + # persistence: + # enabled: true + # type: sts + # storageClassName: "storageClassName" + # accessModes: + # - ReadWriteOnce + # size: 20Gi + # finalizers: + # - kubernetes.io/pvc-protection + + serviceAccount: + create: true + autoMount: true + + sidecar: + dashboards: + enabled: true + label: grafana_dashboard + searchNamespace: cattle-dashboards + labelValue: "1" + + # Support for new table panels, when enabled grafana auto migrates the old table panels to newer table panels + enableNewTablePanelSyntax: false + + ## Annotations for Grafana dashboard configmaps + ## + annotations: {} + multicluster: + global: + enabled: false + etcd: + enabled: false + provider: + allowUiUpdates: false + datasources: + enabled: true + defaultDatasourceEnabled: true + isDefaultDatasource: true + + uid: prometheus + + ## URL of prometheus datasource + ## + # url: http://prometheus-stack-prometheus:9090/ + + ## Prometheus request timeout in seconds + # timeout: 30 + + # If not defined, will use prometheus.prometheusSpec.scrapeInterval or its default + # defaultDatasourceScrapeInterval: 15s + + ## Annotations for Grafana datasource configmaps + ## + annotations: {} + + ## Set method for HTTP to send query to datasource + httpMethod: POST + + ## Create datasource for each Pod of Prometheus StatefulSet; + ## this uses headless service `prometheus-operated` which is + ## created by Prometheus Operator + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/0fee93e12dc7c2ea1218f19ae25ec6b893460590/pkg/prometheus/statefulset.go#L255-L286 + createPrometheusReplicasDatasources: false + label: grafana_datasource + labelValue: "1" + + ## Field with internal link pointing to existing data source in Grafana. + ## Can be provisioned via additionalDataSources + exemplarTraceIdDestinations: {} + # datasourceUid: Jaeger + # traceIdLabelName: trace_id + alertmanager: + enabled: true + uid: alertmanager + handleGrafanaManagedAlerts: false + implementation: prometheus + + extraConfigmapMounts: [] + # - name: certs-configmap + # mountPath: /etc/grafana/ssl/ + # configMap: certs-configmap + # readOnly: true + + deleteDatasources: [] + # - name: example-datasource + # orgId: 1 + + ## Configure additional grafana datasources (passed through tpl) + ## ref: http://docs.grafana.org/administration/provisioning/#datasources + additionalDataSources: [] + # - name: prometheus-sample + # access: proxy + # basicAuth: true + # basicAuthPassword: pass + # basicAuthUser: daco + # editable: false + # jsonData: + # tlsSkipVerify: true + # orgId: 1 + # type: prometheus + # url: https://{{ printf "%s-prometheus.svc" .Release.Name }}:9090 + # version: 1 + + ## Passed to grafana subchart and used by servicemonitor below + ## + service: + portName: nginx-http + ## Port for Grafana Service to listen on + ## + port: 80 + ## To be used with a proxy extraContainer port + ## + targetPort: 8080 + ## Port to expose on each node + ## Only used if service.type is 'NodePort' + ## + nodePort: 30950 + ## Service type + ## + type: ClusterIP + + proxy: + image: + repository: rancher/mirrored-library-nginx + tag: 1.24.0-alpine + + ## Enable an Specify container in extraContainers. This is meant to allow adding an authentication proxy to a grafana pod + extraContainers: | + - name: grafana-proxy + args: + - nginx + - -g + - daemon off; + - -c + - /nginx/nginx.conf + image: "{{ template "system_default_registry" . }}{{ .Values.proxy.image.repository }}:{{ .Values.proxy.image.tag }}" + ports: + - containerPort: 8080 + name: nginx-http + protocol: TCP + volumeMounts: + - mountPath: /nginx + name: grafana-nginx + - mountPath: /var/cache/nginx + name: nginx-home + securityContext: + runAsUser: 101 + runAsGroup: 101 + + ## Volumes that can be used in containers + extraContainerVolumes: + - name: nginx-home + emptyDir: {} + - name: grafana-nginx + configMap: + name: grafana-nginx-proxy-config + items: + - key: nginx.conf + mode: 438 + path: nginx.conf + + ## If true, create a serviceMonitor for grafana + ## + serviceMonitor: + # If true, a ServiceMonitor CRD is created for a prometheus operator + # https://github.com/coreos/prometheus-operator + # + enabled: true + + # Path to use for scraping metrics. Might be different if server.root_url is set + # in grafana.ini + path: "/metrics" + + # namespace: monitoring (defaults to use the namespace this chart is deployed to) + + # labels for the ServiceMonitor + labels: {} + + # Scrape interval. If not set, the Prometheus default scrape interval is used. + # + interval: "" + scheme: http + tlsConfig: {} + scrapeTimeout: 30s + + ## RelabelConfigs to apply to samples before scraping + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#relabelconfig + ## + relabelings: [] + # - sourceLabels: [__meta_kubernetes_pod_node_name] + # separator: ; + # regex: ^(.*)$ + # targetLabel: nodename + # replacement: $1 + # action: replace + + resources: + limits: + memory: 200Mi + cpu: 200m + requests: + memory: 100Mi + cpu: 100m + + testFramework: + enabled: false + +## Flag to disable all the kubernetes component scrapers +## +kubernetesServiceMonitors: + enabled: true + +## Component scraping the kube api server +## +kubeApiServer: + enabled: true + tlsConfig: + serverName: kubernetes + insecureSkipVerify: false + serviceMonitor: + ## Scrape interval. If not set, the Prometheus default scrape interval is used. + ## + interval: "" + + ## SampleLimit defines per-scrape limit on number of scraped samples that will be accepted. + ## + sampleLimit: 0 + + ## TargetLimit defines a limit on the number of scraped targets that will be accepted. + ## + targetLimit: 0 + + ## Per-scrape limit on number of labels that will be accepted for a sample. Only valid in Prometheus versions 2.27.0 and newer. + ## + labelLimit: 0 + + ## Per-scrape limit on length of labels name that will be accepted for a sample. Only valid in Prometheus versions 2.27.0 and newer. + ## + labelNameLengthLimit: 0 + + ## Per-scrape limit on length of labels value that will be accepted for a sample. Only valid in Prometheus versions 2.27.0 and newer. + ## + labelValueLengthLimit: 0 + + ## proxyUrl: URL of a proxy that should be used for scraping. + ## + proxyUrl: "" + + jobLabel: component + selector: + matchLabels: + component: apiserver + provider: kubernetes + + ## MetricRelabelConfigs to apply to samples after scraping, but before ingestion. + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#relabelconfig + ## + metricRelabelings: + # Drop excessively noisy apiserver buckets. + - action: drop + regex: apiserver_request_duration_seconds_bucket;(0.15|0.2|0.3|0.35|0.4|0.45|0.6|0.7|0.8|0.9|1.25|1.5|1.75|2|3|3.5|4|4.5|6|7|8|9|15|25|40|50) + sourceLabels: + - __name__ + - le + # - action: keep + # regex: 'kube_(daemonset|deployment|pod|namespace|node|statefulset).+' + # sourceLabels: [__name__] + + ## RelabelConfigs to apply to samples before scraping + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#relabelconfig + ## + relabelings: [] + # - sourceLabels: + # - __meta_kubernetes_namespace + # - __meta_kubernetes_service_name + # - __meta_kubernetes_endpoint_port_name + # action: keep + # regex: default;kubernetes;https + # - targetLabel: __address__ + # replacement: kubernetes.default.svc:443 + + ## Additional labels + ## + additionalLabels: {} + # foo: bar + +## Component scraping the kubelet and kubelet-hosted cAdvisor +## +kubelet: + enabled: true + namespace: kube-system + + serviceMonitor: + ## Attach metadata to discovered targets. Requires Prometheus v2.45 for endpoints created by the operator. + ## + attachMetadata: + node: false + + ## Scrape interval. If not set, the Prometheus default scrape interval is used. + ## + interval: "" + + ## If true, Prometheus use (respect) labels provided by exporter. + ## + honorLabels: true + + ## If true, Prometheus ingests metrics with timestamp provided by exporter. If false, Prometheus ingests metrics with timestamp of scrape. + ## + honorTimestamps: true + + ## SampleLimit defines per-scrape limit on number of scraped samples that will be accepted. + ## + sampleLimit: 0 + + ## TargetLimit defines a limit on the number of scraped targets that will be accepted. + ## + targetLimit: 0 + + ## Per-scrape limit on number of labels that will be accepted for a sample. Only valid in Prometheus versions 2.27.0 and newer. + ## + labelLimit: 0 + + ## Per-scrape limit on length of labels name that will be accepted for a sample. Only valid in Prometheus versions 2.27.0 and newer. + ## + labelNameLengthLimit: 0 + + ## Per-scrape limit on length of labels value that will be accepted for a sample. Only valid in Prometheus versions 2.27.0 and newer. + ## + labelValueLengthLimit: 0 + + ## proxyUrl: URL of a proxy that should be used for scraping. + ## + proxyUrl: "" + + ## Enable scraping the kubelet over https. For requirements to enable this see + ## https://github.com/prometheus-operator/prometheus-operator/issues/926 + ## + https: true + + ## Enable scraping /metrics/cadvisor from kubelet's service + ## + cAdvisor: true + + ## Enable scraping /metrics/probes from kubelet's service + ## + probes: true + + ## Enable scraping /metrics/resource from kubelet's service + ## This is disabled by default because container metrics are already exposed by cAdvisor + ## + resource: false + # From kubernetes 1.18, /metrics/resource/v1alpha1 renamed to /metrics/resource + resourcePath: "/metrics/resource/v1alpha1" + + ## MetricRelabelConfigs to apply to samples after scraping, but before ingestion. + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#relabelconfig + ## + cAdvisorMetricRelabelings: + # Drop less useful container CPU metrics. + - sourceLabels: [__name__] + action: drop + regex: 'container_cpu_(cfs_throttled_seconds_total|load_average_10s|system_seconds_total|user_seconds_total)' + # Drop less useful container / always zero filesystem metrics. + - sourceLabels: [__name__] + action: drop + regex: 'container_fs_(io_current|io_time_seconds_total|io_time_weighted_seconds_total|reads_merged_total|sector_reads_total|sector_writes_total|writes_merged_total)' + # Drop less useful / always zero container memory metrics. + - sourceLabels: [__name__] + action: drop + regex: 'container_memory_(mapped_file|swap)' + # Drop less useful container process metrics. + - sourceLabels: [__name__] + action: drop + regex: 'container_(file_descriptors|tasks_state|threads_max)' + # Drop container spec metrics that overlap with kube-state-metrics. + - sourceLabels: [__name__] + action: drop + regex: 'container_spec.*' + # Drop cgroup metrics with no pod. + - sourceLabels: [id, pod] + action: drop + regex: '.+;' + # - sourceLabels: [__name__, image] + # separator: ; + # regex: container_([a-z_]+); + # replacement: $1 + # action: drop + # - sourceLabels: [__name__] + # separator: ; + # regex: container_(network_tcp_usage_total|network_udp_usage_total|tasks_state|cpu_load_average_10s) + # replacement: $1 + # action: drop + + ## MetricRelabelConfigs to apply to samples after scraping, but before ingestion. + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#relabelconfig + ## + probesMetricRelabelings: [] + # - sourceLabels: [__name__, image] + # separator: ; + # regex: container_([a-z_]+); + # replacement: $1 + # action: drop + # - sourceLabels: [__name__] + # separator: ; + # regex: container_(network_tcp_usage_total|network_udp_usage_total|tasks_state|cpu_load_average_10s) + # replacement: $1 + # action: drop + + ## RelabelConfigs to apply to samples before scraping + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#relabelconfig + ## + ## metrics_path is required to match upstream rules and charts + cAdvisorRelabelings: + - action: replace + sourceLabels: [__metrics_path__] + targetLabel: metrics_path + # - sourceLabels: [__meta_kubernetes_pod_node_name] + # separator: ; + # regex: ^(.*)$ + # targetLabel: nodename + # replacement: $1 + # action: replace + + ## RelabelConfigs to apply to samples before scraping + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#relabelconfig + ## + probesRelabelings: + - action: replace + sourceLabels: [__metrics_path__] + targetLabel: metrics_path + # - sourceLabels: [__meta_kubernetes_pod_node_name] + # separator: ; + # regex: ^(.*)$ + # targetLabel: nodename + # replacement: $1 + # action: replace + + ## RelabelConfigs to apply to samples before scraping + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#relabelconfig + ## + resourceRelabelings: + - action: replace + sourceLabels: [__metrics_path__] + targetLabel: metrics_path + # - sourceLabels: [__meta_kubernetes_pod_node_name] + # separator: ; + # regex: ^(.*)$ + # targetLabel: nodename + # replacement: $1 + # action: replace + + ## MetricRelabelConfigs to apply to samples after scraping, but before ingestion. + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#relabelconfig + ## + metricRelabelings: [] + # - sourceLabels: [__name__, image] + # separator: ; + # regex: container_([a-z_]+); + # replacement: $1 + # action: drop + # - sourceLabels: [__name__] + # separator: ; + # regex: container_(network_tcp_usage_total|network_udp_usage_total|tasks_state|cpu_load_average_10s) + # replacement: $1 + # action: drop + + ## RelabelConfigs to apply to samples before scraping + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#relabelconfig + ## + ## metrics_path is required to match upstream rules and charts + relabelings: + - action: replace + sourceLabels: [__metrics_path__] + targetLabel: metrics_path + # - sourceLabels: [__meta_kubernetes_pod_node_name] + # separator: ; + # regex: ^(.*)$ + # targetLabel: nodename + # replacement: $1 + # action: replace + + ## Additional labels + ## + additionalLabels: {} + # foo: bar + +## Component scraping the kube controller manager +## +kubeControllerManager: + enabled: false + + ## If your kube controller manager is not deployed as a pod, specify IPs it can be found on + ## + endpoints: [] + # - 10.141.4.22 + # - 10.141.4.23 + # - 10.141.4.24 + + ## If using kubeControllerManager.endpoints only the port and targetPort are used + ## + service: + enabled: true + ## If null or unset, the value is determined dynamically based on target Kubernetes version due to change + ## of default port in Kubernetes 1.22. + ## + port: null + targetPort: null + # selector: + # component: kube-controller-manager + + serviceMonitor: + enabled: true + ## Scrape interval. If not set, the Prometheus default scrape interval is used. + ## + interval: "" + + ## SampleLimit defines per-scrape limit on number of scraped samples that will be accepted. + ## + sampleLimit: 0 + + ## TargetLimit defines a limit on the number of scraped targets that will be accepted. + ## + targetLimit: 0 + + ## Per-scrape limit on number of labels that will be accepted for a sample. Only valid in Prometheus versions 2.27.0 and newer. + ## + labelLimit: 0 + + ## Per-scrape limit on length of labels name that will be accepted for a sample. Only valid in Prometheus versions 2.27.0 and newer. + ## + labelNameLengthLimit: 0 + + ## Per-scrape limit on length of labels value that will be accepted for a sample. Only valid in Prometheus versions 2.27.0 and newer. + ## + labelValueLengthLimit: 0 + + ## proxyUrl: URL of a proxy that should be used for scraping. + ## + proxyUrl: "" + + ## port: Name of the port the metrics will be scraped from + ## + port: http-metrics + + jobLabel: jobLabel + selector: {} + # matchLabels: + # component: kube-controller-manager + + ## Enable scraping kube-controller-manager over https. + ## Requires proper certs (not self-signed) and delegated authentication/authorization checks. + ## If null or unset, the value is determined dynamically based on target Kubernetes version. + ## + https: null + + # Skip TLS certificate validation when scraping + insecureSkipVerify: null + + # Name of the server to use when validating TLS certificate + serverName: null + + ## MetricRelabelConfigs to apply to samples after scraping, but before ingestion. + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#relabelconfig + ## + metricRelabelings: [] + # - action: keep + # regex: 'kube_(daemonset|deployment|pod|namespace|node|statefulset).+' + # sourceLabels: [__name__] + + ## RelabelConfigs to apply to samples before scraping + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#relabelconfig + ## + relabelings: [] + # - sourceLabels: [__meta_kubernetes_pod_node_name] + # separator: ; + # regex: ^(.*)$ + # targetLabel: nodename + # replacement: $1 + # action: replace + + ## Additional labels + ## + additionalLabels: {} + # foo: bar + +## Component scraping coreDns. Use either this or kubeDns +## +coreDns: + enabled: true + service: + enabled: true + port: 9153 + targetPort: 9153 + # selector: + # k8s-app: kube-dns + serviceMonitor: + enabled: true + ## Scrape interval. If not set, the Prometheus default scrape interval is used. + ## + interval: "" + + ## SampleLimit defines per-scrape limit on number of scraped samples that will be accepted. + ## + sampleLimit: 0 + + ## TargetLimit defines a limit on the number of scraped targets that will be accepted. + ## + targetLimit: 0 + + ## Per-scrape limit on number of labels that will be accepted for a sample. Only valid in Prometheus versions 2.27.0 and newer. + ## + labelLimit: 0 + + ## Per-scrape limit on length of labels name that will be accepted for a sample. Only valid in Prometheus versions 2.27.0 and newer. + ## + labelNameLengthLimit: 0 + + ## Per-scrape limit on length of labels value that will be accepted for a sample. Only valid in Prometheus versions 2.27.0 and newer. + ## + labelValueLengthLimit: 0 + + ## proxyUrl: URL of a proxy that should be used for scraping. + ## + proxyUrl: "" + + ## port: Name of the port the metrics will be scraped from + ## + port: http-metrics + + jobLabel: jobLabel + selector: {} + # matchLabels: + # k8s-app: kube-dns + + ## MetricRelabelConfigs to apply to samples after scraping, but before ingestion. + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#relabelconfig + ## + metricRelabelings: [] + # - action: keep + # regex: 'kube_(daemonset|deployment|pod|namespace|node|statefulset).+' + # sourceLabels: [__name__] + + ## RelabelConfigs to apply to samples before scraping + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#relabelconfig + ## + relabelings: [] + # - sourceLabels: [__meta_kubernetes_pod_node_name] + # separator: ; + # regex: ^(.*)$ + # targetLabel: nodename + # replacement: $1 + # action: replace + + ## Additional labels + ## + additionalLabels: {} + # foo: bar + +## Component scraping kubeDns. Use either this or coreDns +## +kubeDns: + enabled: false + service: + dnsmasq: + port: 10054 + targetPort: 10054 + skydns: + port: 10055 + targetPort: 10055 + # selector: + # k8s-app: kube-dns + serviceMonitor: + ## Scrape interval. If not set, the Prometheus default scrape interval is used. + ## + interval: "" + + ## SampleLimit defines per-scrape limit on number of scraped samples that will be accepted. + ## + sampleLimit: 0 + + ## TargetLimit defines a limit on the number of scraped targets that will be accepted. + ## + targetLimit: 0 + + ## Per-scrape limit on number of labels that will be accepted for a sample. Only valid in Prometheus versions 2.27.0 and newer. + ## + labelLimit: 0 + + ## Per-scrape limit on length of labels name that will be accepted for a sample. Only valid in Prometheus versions 2.27.0 and newer. + ## + labelNameLengthLimit: 0 + + ## Per-scrape limit on length of labels value that will be accepted for a sample. Only valid in Prometheus versions 2.27.0 and newer. + ## + labelValueLengthLimit: 0 + + ## proxyUrl: URL of a proxy that should be used for scraping. + ## + proxyUrl: "" + + jobLabel: jobLabel + selector: {} + # matchLabels: + # k8s-app: kube-dns + + ## MetricRelabelConfigs to apply to samples after scraping, but before ingestion. + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#relabelconfig + ## + metricRelabelings: [] + # - action: keep + # regex: 'kube_(daemonset|deployment|pod|namespace|node|statefulset).+' + # sourceLabels: [__name__] + + ## RelabelConfigs to apply to samples before scraping + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#relabelconfig + ## + relabelings: [] + # - sourceLabels: [__meta_kubernetes_pod_node_name] + # separator: ; + # regex: ^(.*)$ + # targetLabel: nodename + # replacement: $1 + # action: replace + + ## MetricRelabelConfigs to apply to samples after scraping, but before ingestion. + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#relabelconfig + ## + dnsmasqMetricRelabelings: [] + # - action: keep + # regex: 'kube_(daemonset|deployment|pod|namespace|node|statefulset).+' + # sourceLabels: [__name__] + + ## RelabelConfigs to apply to samples before scraping + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#relabelconfig + ## + dnsmasqRelabelings: [] + # - sourceLabels: [__meta_kubernetes_pod_node_name] + # separator: ; + # regex: ^(.*)$ + # targetLabel: nodename + # replacement: $1 + # action: replace + + ## Additional labels + ## + additionalLabels: {} + # foo: bar + +## Component scraping etcd +## +kubeEtcd: + enabled: false + + ## If your etcd is not deployed as a pod, specify IPs it can be found on + ## + endpoints: [] + # - 10.141.4.22 + # - 10.141.4.23 + # - 10.141.4.24 + + ## Etcd service. If using kubeEtcd.endpoints only the port and targetPort are used + ## + service: + enabled: true + port: 2381 + targetPort: 2381 + # selector: + # component: etcd + + ## Configure secure access to the etcd cluster by loading a secret into prometheus and + ## specifying security configuration below. For example, with a secret named etcd-client-cert + ## + ## serviceMonitor: + ## scheme: https + ## insecureSkipVerify: false + ## serverName: localhost + ## caFile: /etc/prometheus/secrets/etcd-client-cert/etcd-ca + ## certFile: /etc/prometheus/secrets/etcd-client-cert/etcd-client + ## keyFile: /etc/prometheus/secrets/etcd-client-cert/etcd-client-key + ## + serviceMonitor: + enabled: true + ## Scrape interval. If not set, the Prometheus default scrape interval is used. + ## + interval: "" + + ## SampleLimit defines per-scrape limit on number of scraped samples that will be accepted. + ## + sampleLimit: 0 + + ## TargetLimit defines a limit on the number of scraped targets that will be accepted. + ## + targetLimit: 0 + + ## Per-scrape limit on number of labels that will be accepted for a sample. Only valid in Prometheus versions 2.27.0 and newer. + ## + labelLimit: 0 + + ## Per-scrape limit on length of labels name that will be accepted for a sample. Only valid in Prometheus versions 2.27.0 and newer. + ## + labelNameLengthLimit: 0 + + ## Per-scrape limit on length of labels value that will be accepted for a sample. Only valid in Prometheus versions 2.27.0 and newer. + ## + labelValueLengthLimit: 0 + + ## proxyUrl: URL of a proxy that should be used for scraping. + ## + proxyUrl: "" + scheme: http + insecureSkipVerify: false + serverName: "" + caFile: "" + certFile: "" + keyFile: "" + + ## port: Name of the port the metrics will be scraped from + ## + port: http-metrics + + jobLabel: jobLabel + selector: {} + # matchLabels: + # component: etcd + + ## MetricRelabelConfigs to apply to samples after scraping, but before ingestion. + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#relabelconfig + ## + metricRelabelings: [] + # - action: keep + # regex: 'kube_(daemonset|deployment|pod|namespace|node|statefulset).+' + # sourceLabels: [__name__] + + ## RelabelConfigs to apply to samples before scraping + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#relabelconfig + ## + relabelings: [] + # - sourceLabels: [__meta_kubernetes_pod_node_name] + # separator: ; + # regex: ^(.*)$ + # targetLabel: nodename + # replacement: $1 + # action: replace + + ## Additional labels + ## + additionalLabels: {} + # foo: bar + +## Component scraping kube scheduler +## +kubeScheduler: + enabled: false + + ## If your kube scheduler is not deployed as a pod, specify IPs it can be found on + ## + endpoints: [] + # - 10.141.4.22 + # - 10.141.4.23 + # - 10.141.4.24 + + ## If using kubeScheduler.endpoints only the port and targetPort are used + ## + service: + enabled: true + ## If null or unset, the value is determined dynamically based on target Kubernetes version due to change + ## of default port in Kubernetes 1.23. + ## + port: null + targetPort: null + # selector: + # component: kube-scheduler + + serviceMonitor: + enabled: true + ## Scrape interval. If not set, the Prometheus default scrape interval is used. + ## + interval: "" + + ## SampleLimit defines per-scrape limit on number of scraped samples that will be accepted. + ## + sampleLimit: 0 + + ## TargetLimit defines a limit on the number of scraped targets that will be accepted. + ## + targetLimit: 0 + + ## Per-scrape limit on number of labels that will be accepted for a sample. Only valid in Prometheus versions 2.27.0 and newer. + ## + labelLimit: 0 + + ## Per-scrape limit on length of labels name that will be accepted for a sample. Only valid in Prometheus versions 2.27.0 and newer. + ## + labelNameLengthLimit: 0 + + ## Per-scrape limit on length of labels value that will be accepted for a sample. Only valid in Prometheus versions 2.27.0 and newer. + ## + labelValueLengthLimit: 0 + + ## proxyUrl: URL of a proxy that should be used for scraping. + ## + proxyUrl: "" + ## Enable scraping kube-scheduler over https. + ## Requires proper certs (not self-signed) and delegated authentication/authorization checks. + ## If null or unset, the value is determined dynamically based on target Kubernetes version. + ## + https: null + + ## port: Name of the port the metrics will be scraped from + ## + port: http-metrics + + jobLabel: jobLabel + selector: {} + # matchLabels: + # component: kube-scheduler + + ## Skip TLS certificate validation when scraping + insecureSkipVerify: null + + ## Name of the server to use when validating TLS certificate + serverName: null + + ## MetricRelabelConfigs to apply to samples after scraping, but before ingestion. + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#relabelconfig + ## + metricRelabelings: [] + # - action: keep + # regex: 'kube_(daemonset|deployment|pod|namespace|node|statefulset).+' + # sourceLabels: [__name__] + + ## RelabelConfigs to apply to samples before scraping + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#relabelconfig + ## + relabelings: [] + # - sourceLabels: [__meta_kubernetes_pod_node_name] + # separator: ; + # regex: ^(.*)$ + # targetLabel: nodename + # replacement: $1 + # action: replace + + ## Additional labels + ## + additionalLabels: {} + # foo: bar + +## Component scraping kube proxy +## +kubeProxy: + enabled: false + + ## If your kube proxy is not deployed as a pod, specify IPs it can be found on + ## + endpoints: [] + # - 10.141.4.22 + # - 10.141.4.23 + # - 10.141.4.24 + + service: + enabled: true + port: 10249 + targetPort: 10249 + # selector: + # k8s-app: kube-proxy + + serviceMonitor: + enabled: true + ## Scrape interval. If not set, the Prometheus default scrape interval is used. + ## + interval: "" + + ## SampleLimit defines per-scrape limit on number of scraped samples that will be accepted. + ## + sampleLimit: 0 + + ## TargetLimit defines a limit on the number of scraped targets that will be accepted. + ## + targetLimit: 0 + + ## Per-scrape limit on number of labels that will be accepted for a sample. Only valid in Prometheus versions 2.27.0 and newer. + ## + labelLimit: 0 + + ## Per-scrape limit on length of labels name that will be accepted for a sample. Only valid in Prometheus versions 2.27.0 and newer. + ## + labelNameLengthLimit: 0 + + ## Per-scrape limit on length of labels value that will be accepted for a sample. Only valid in Prometheus versions 2.27.0 and newer. + ## + labelValueLengthLimit: 0 + + ## proxyUrl: URL of a proxy that should be used for scraping. + ## + proxyUrl: "" + + ## port: Name of the port the metrics will be scraped from + ## + port: http-metrics + + jobLabel: jobLabel + selector: {} + # matchLabels: + # k8s-app: kube-proxy + + ## Enable scraping kube-proxy over https. + ## Requires proper certs (not self-signed) and delegated authentication/authorization checks + ## + https: false + + ## MetricRelabelConfigs to apply to samples after scraping, but before ingestion. + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#relabelconfig + ## + metricRelabelings: [] + # - action: keep + # regex: 'kube_(daemonset|deployment|pod|namespace|node|statefulset).+' + # sourceLabels: [__name__] + + ## RelabelConfigs to apply to samples before scraping + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#relabelconfig + ## + relabelings: [] + # - action: keep + # regex: 'kube_(daemonset|deployment|pod|namespace|node|statefulset).+' + # sourceLabels: [__name__] + + ## Additional labels + ## + additionalLabels: {} + # foo: bar + +## Component scraping kube state metrics +## +kubeStateMetrics: + enabled: true + +## Configuration for kube-state-metrics subchart +## +kube-state-metrics: + namespaceOverride: "" + rbac: + create: true + releaseLabel: true + prometheus: + monitor: + enabled: true + + ## Scrape interval. If not set, the Prometheus default scrape interval is used. + ## + interval: "" + + ## SampleLimit defines per-scrape limit on number of scraped samples that will be accepted. + ## + sampleLimit: 0 + + ## TargetLimit defines a limit on the number of scraped targets that will be accepted. + ## + targetLimit: 0 + + ## Per-scrape limit on number of labels that will be accepted for a sample. Only valid in Prometheus versions 2.27.0 and newer. + ## + labelLimit: 0 + + ## Per-scrape limit on length of labels name that will be accepted for a sample. Only valid in Prometheus versions 2.27.0 and newer. + ## + labelNameLengthLimit: 0 + + ## Per-scrape limit on length of labels value that will be accepted for a sample. Only valid in Prometheus versions 2.27.0 and newer. + ## + labelValueLengthLimit: 0 + + ## Scrape Timeout. If not set, the Prometheus default scrape timeout is used. + ## + scrapeTimeout: "" + + ## proxyUrl: URL of a proxy that should be used for scraping. + ## + proxyUrl: "" + + # Keep labels from scraped data, overriding server-side labels + ## + honorLabels: true + + ## MetricRelabelConfigs to apply to samples after scraping, but before ingestion. + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#relabelconfig + ## + metricRelabelings: [] + # - action: keep + # regex: 'kube_(daemonset|deployment|pod|namespace|node|statefulset).+' + # sourceLabels: [__name__] + + ## RelabelConfigs to apply to samples before scraping + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#relabelconfig + ## + relabelings: [] + # - sourceLabels: [__meta_kubernetes_pod_node_name] + # separator: ; + # regex: ^(.*)$ + # targetLabel: nodename + # replacement: $1 + # action: replace + + selfMonitor: + enabled: false + +## Deploy node exporter as a daemonset to all nodes +## +nodeExporter: + enabled: true + operatingSystems: + linux: + enabled: true + darwin: + enabled: true + + ## ForceDeployDashboard Create dashboard configmap even if nodeExporter deployment has been disabled + ## + forceDeployDashboards: false + +## Configuration for prometheus-node-exporter subchart +## +prometheus-node-exporter: + namespaceOverride: "" + podLabels: + ## Add the 'node-exporter' label to be used by serviceMonitor to match standard common usage in rules and grafana dashboards + ## + jobLabel: node-exporter + releaseLabel: true + extraArgs: + - --collector.filesystem.mount-points-exclude=^/(dev|proc|sys|var/lib/docker/.+|var/lib/kubelet/.+)($|/) + - --collector.filesystem.fs-types-exclude=^(autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs)$ + service: + portName: http-metrics + prometheus: + monitor: + enabled: true + + jobLabel: jobLabel + + ## Scrape interval. If not set, the Prometheus default scrape interval is used. + ## + interval: "" + + ## SampleLimit defines per-scrape limit on number of scraped samples that will be accepted. + ## + sampleLimit: 0 + + ## TargetLimit defines a limit on the number of scraped targets that will be accepted. + ## + targetLimit: 0 + + ## Per-scrape limit on number of labels that will be accepted for a sample. Only valid in Prometheus versions 2.27.0 and newer. + ## + labelLimit: 0 + + ## Per-scrape limit on length of labels name that will be accepted for a sample. Only valid in Prometheus versions 2.27.0 and newer. + ## + labelNameLengthLimit: 0 + + ## Per-scrape limit on length of labels value that will be accepted for a sample. Only valid in Prometheus versions 2.27.0 and newer. + ## + labelValueLengthLimit: 0 + + ## How long until a scrape request times out. If not set, the Prometheus default scape timeout is used. + ## + scrapeTimeout: "" + + ## proxyUrl: URL of a proxy that should be used for scraping. + ## + proxyUrl: "" + + ## MetricRelabelConfigs to apply to samples after scraping, but before ingestion. + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#relabelconfig + ## + metricRelabelings: [] + # - sourceLabels: [__name__] + # separator: ; + # regex: ^node_mountstats_nfs_(event|operations|transport)_.+ + # replacement: $1 + # action: drop + + ## RelabelConfigs to apply to samples before scraping + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#relabelconfig + ## + relabelings: [] + # - sourceLabels: [__meta_kubernetes_pod_node_name] + # separator: ; + # regex: ^(.*)$ + # targetLabel: nodename + # replacement: $1 + # action: replace + +## Manages Prometheus and Alertmanager components +## +prometheusOperator: + enabled: true + + ## Use '{{ template "kube-prometheus-stack.fullname" . }}-operator' by default + fullnameOverride: "" + + ## Number of old replicasets to retain ## + ## The default value is 10, 0 will garbage-collect old replicasets ## + revisionHistoryLimit: 10 + + ## Strategy of the deployment + ## + strategy: {} + + ## Prometheus-Operator v0.39.0 and later support TLS natively. + ## + tls: + enabled: true + # Value must match version names from https://golang.org/pkg/crypto/tls/#pkg-constants + tlsMinVersion: VersionTLS13 + # Users who are deploying this chart in GKE private clusters will need to add firewall rules to expose this port for admissions webhooks + internalPort: 8443 + + ## Admission webhook support for PrometheusRules resources added in Prometheus Operator 0.30 can be enabled to prevent incorrectly formatted + ## rules from making their way into prometheus and potentially preventing the container from starting + admissionWebhooks: + ## Valid values: Fail, Ignore, IgnoreOnInstallOnly + ## IgnoreOnInstallOnly - If Release.IsInstall returns "true", set "Ignore" otherwise "Fail" + failurePolicy: "" + ## The default timeoutSeconds is 10 and the maximum value is 30. + timeoutSeconds: 10 + enabled: true + ## A PEM encoded CA bundle which will be used to validate the webhook's server certificate. + ## If unspecified, system trust roots on the apiserver are used. + caBundle: "" + ## If enabled, generate a self-signed certificate, then patch the webhook configurations with the generated data. + ## On chart upgrades (or if the secret exists) the cert will not be re-generated. You can use this to provide your own + ## certs ahead of time if you wish. + ## + annotations: {} + # argocd.argoproj.io/hook: PreSync + # argocd.argoproj.io/hook-delete-policy: HookSucceeded + + namespaceSelector: {} + + deployment: + enabled: false + + ## Number of replicas + ## + replicas: 1 + + ## Strategy of the deployment + ## + strategy: {} + + # Ref: https://kubernetes.io/docs/tasks/run-application/configure-pdb/ + podDisruptionBudget: {} + # maxUnavailable: 1 + # minAvailable: 1 + + ## Number of old replicasets to retain ## + ## The default value is 10, 0 will garbage-collect old replicasets ## + revisionHistoryLimit: 10 + + ## Prometheus-Operator v0.39.0 and later support TLS natively. + ## + tls: + enabled: true + # Value must match version names from https://golang.org/pkg/crypto/tls/#pkg-constants + tlsMinVersion: VersionTLS13 + # The default webhook port is 10250 in order to work out-of-the-box in GKE private clusters and avoid adding firewall rules. + internalPort: 10250 + + ## Service account for Prometheus Operator Webhook to use. + ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/configure-service-account/ + ## + serviceAccount: + automountServiceAccountToken: false + create: true + name: "" + + ## Configuration for Prometheus operator Webhook service + ## + service: + annotations: {} + labels: {} + clusterIP: "" + + ## Port to expose on each node + ## Only used if service.type is 'NodePort' + ## + nodePort: 31080 + + nodePortTls: 31443 + + ## Additional ports to open for Prometheus operator Webhook service + ## ref: https://kubernetes.io/docs/concepts/services-networking/service/#multi-port-services + ## + additionalPorts: [] + + ## Loadbalancer IP + ## Only use if service.type is "LoadBalancer" + ## + loadBalancerIP: "" + loadBalancerSourceRanges: [] + + ## Denotes if this Service desires to route external traffic to node-local or cluster-wide endpoints + ## + externalTrafficPolicy: Cluster + + ## Service type + ## NodePort, ClusterIP, LoadBalancer + ## + type: ClusterIP + + ## List of IP addresses at which the Prometheus server service is available + ## Ref: https://kubernetes.io/docs/user-guide/services/#external-ips + ## + externalIPs: [] + + # ## Labels to add to the operator webhook deployment + # ## + labels: {} + + ## Annotations to add to the operator webhook deployment + ## + annotations: {} + + ## Labels to add to the operator webhook pod + ## + podLabels: {} + + ## Annotations to add to the operator webhook pod + ## + podAnnotations: {} + + ## Assign a PriorityClassName to pods if set + # priorityClassName: "" + + ## Define Log Format + # Use logfmt (default) or json logging + # logFormat: logfmt + + ## Decrease log verbosity to errors only + # logLevel: error + + ## Prometheus-operator webhook image + ## + image: + registry: quay.io + repository: rancher/mirrored-prometheus-operator-admission-webhook + # if not set appVersion field from Chart.yaml is used + tag: v0.72.0 + sha: "" + pullPolicy: IfNotPresent + + ## Define Log Format + # Use logfmt (default) or json logging + # logFormat: logfmt + + ## Decrease log verbosity to errors only + # logLevel: error + + + ## Liveness probe + ## + livenessProbe: + enabled: true + failureThreshold: 3 + initialDelaySeconds: 30 + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 1 + + ## Readiness probe + ## + readinessProbe: + enabled: true + failureThreshold: 3 + initialDelaySeconds: 5 + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 1 + + ## Resource limits & requests + ## + resources: {} + # limits: + # cpu: 200m + # memory: 200Mi + # requests: + # cpu: 100m + # memory: 100Mi + + # Required for use in managed kubernetes clusters (such as AWS EKS) with custom CNI (such as calico), + # because control-plane managed by AWS cannot communicate with pods' IP CIDR and admission webhooks are not working + ## + hostNetwork: false + + ## Define which Nodes the Pods are scheduled on. + ## ref: https://kubernetes.io/docs/user-guide/node-selection/ + ## + nodeSelector: {} + + ## Tolerations for use with node taints + ## ref: https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/ + ## + tolerations: [] + # - key: "key" + # operator: "Equal" + # value: "value" + # effect: "NoSchedule" + + ## Assign custom affinity rules to the prometheus operator + ## ref: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/ + ## + affinity: {} + # nodeAffinity: + # requiredDuringSchedulingIgnoredDuringExecution: + # nodeSelectorTerms: + # - matchExpressions: + # - key: kubernetes.io/e2e-az-name + # operator: In + # values: + # - e2e-az1 + # - e2e-az2 + dnsConfig: {} + # nameservers: + # - 1.2.3.4 + # searches: + # - ns1.svc.cluster-domain.example + # - my.dns.search.suffix + # options: + # - name: ndots + # value: "2" + # - name: edns0 + securityContext: + fsGroup: 65534 + runAsGroup: 65534 + runAsNonRoot: true + runAsUser: 65534 + seccompProfile: + type: RuntimeDefault + + ## Container-specific security context configuration + ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/ + ## + containerSecurityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: + drop: + - ALL + + ## If false then the user will opt out of automounting API credentials. + ## + automountServiceAccountToken: true + + patch: + enabled: true + image: + repository: rancher/mirrored-ingress-nginx-kube-webhook-certgen + tag: v1.4.3 + sha: "" + pullPolicy: IfNotPresent + resources: {} + ## Provide a priority class name to the webhook patching job + ## + priorityClassName: "" + annotations: {} + # argocd.argoproj.io/hook: PreSync + # argocd.argoproj.io/hook-delete-policy: HookSucceeded + podAnnotations: {} + nodeSelector: {} + affinity: {} + tolerations: [] + + ## SecurityContext holds pod-level security attributes and common container settings. + ## This defaults to non root user with uid 2000 and gid 2000. *v1.PodSecurityContext false + ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/ + ## + securityContext: + runAsGroup: 2000 + runAsNonRoot: true + runAsUser: 2000 + seccompProfile: + type: RuntimeDefault + + # Security context for create job container + createSecretJob: + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: + drop: + - ALL + + # Security context for patch job container + patchWebhookJob: + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: + drop: + - ALL + + # Use certmanager to generate webhook certs + certManager: + enabled: false + # self-signed root certificate + rootCert: + duration: "" # default to be 5y + admissionCert: + duration: "" # default to be 1y + # issuerRef: + # name: "issuer" + # kind: "ClusterIssuer" + + ## Namespaces to scope the interaction of the Prometheus Operator and the apiserver (allow list). + ## This is mutually exclusive with denyNamespaces. Setting this to an empty object will disable the configuration + ## + namespaces: {} + # releaseNamespace: true + # additional: + # - kube-system + + ## Namespaces not to scope the interaction of the Prometheus Operator (deny list). + ## + denyNamespaces: [] + + ## Filter namespaces to look for prometheus-operator custom resources + ## + alertmanagerInstanceNamespaces: [] + alertmanagerConfigNamespaces: [] + prometheusInstanceNamespaces: [] + thanosRulerInstanceNamespaces: [] + + ## The clusterDomain value will be added to the cluster.peer option of the alertmanager. + ## Without this specified option cluster.peer will have value alertmanager-monitoring-alertmanager-0.alertmanager-operated:9094 (default value) + ## With this specified option cluster.peer will have value alertmanager-monitoring-alertmanager-0.alertmanager-operated.namespace.svc.cluster-domain:9094 + ## + # clusterDomain: "cluster.local" + + networkPolicy: + ## Enable creation of NetworkPolicy resources. + ## + enabled: false + + ## Flavor of the network policy to use. + # Can be: + # * kubernetes for networking.k8s.io/v1/NetworkPolicy + # * cilium for cilium.io/v2/CiliumNetworkPolicy + flavor: kubernetes + + # cilium: + # egress: + + ## match labels used in selector + # matchLabels: {} + + ## Service account for Prometheus Operator to use. + ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/configure-service-account/ + ## + serviceAccount: + create: true + name: "" + automountServiceAccountToken: true + + ## Configuration for Prometheus operator service + ## + service: + annotations: {} + labels: {} + clusterIP: "" + + ## Port to expose on each node + ## Only used if service.type is 'NodePort' + ## + nodePort: 30080 + + nodePortTls: 30443 + + ## Additional ports to open for Prometheus operator service + ## ref: https://kubernetes.io/docs/concepts/services-networking/service/#multi-port-services + ## + additionalPorts: [] + + ## Loadbalancer IP + ## Only use if service.type is "LoadBalancer" + ## + loadBalancerIP: "" + loadBalancerSourceRanges: [] + + ## Denotes if this Service desires to route external traffic to node-local or cluster-wide endpoints + ## + externalTrafficPolicy: Cluster + + ## Service type + ## NodePort, ClusterIP, LoadBalancer + ## + type: ClusterIP + + ## List of IP addresses at which the Prometheus server service is available + ## Ref: https://kubernetes.io/docs/user-guide/services/#external-ips + ## + externalIPs: [] + + # ## Labels to add to the operator deployment + # ## + labels: {} + + ## Annotations to add to the operator deployment + ## + annotations: {} + + ## Labels to add to the operator pod + ## + podLabels: {} + + ## Annotations to add to the operator pod + ## + podAnnotations: {} + + ## Assign a PriorityClassName to pods if set + # priorityClassName: "" + + ## Define Log Format + # Use logfmt (default) or json logging + # logFormat: logfmt + + ## Decrease log verbosity to errors only + # logLevel: error + + kubeletService: + ## If true, the operator will create and maintain a service for scraping kubelets + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/helm/prometheus-operator/README.md + ## + enabled: true + namespace: kube-system + ## Use '{{ template "kube-prometheus-stack.fullname" . }}-kubelet' by default + name: "" + + ## Create a servicemonitor for the operator + ## + serviceMonitor: + ## If true, create a serviceMonitor for prometheus operator + ## + selfMonitor: true + + ## Labels for ServiceMonitor + additionalLabels: {} + + ## Scrape interval. If not set, the Prometheus default scrape interval is used. + ## + interval: "" + + ## SampleLimit defines per-scrape limit on number of scraped samples that will be accepted. + ## + sampleLimit: 0 + + ## TargetLimit defines a limit on the number of scraped targets that will be accepted. + ## + targetLimit: 0 + + ## Per-scrape limit on number of labels that will be accepted for a sample. Only valid in Prometheus versions 2.27.0 and newer. + ## + labelLimit: 0 + + ## Per-scrape limit on length of labels name that will be accepted for a sample. Only valid in Prometheus versions 2.27.0 and newer. + ## + labelNameLengthLimit: 0 + + ## Per-scrape limit on length of labels value that will be accepted for a sample. Only valid in Prometheus versions 2.27.0 and newer. + ## + labelValueLengthLimit: 0 + + ## Scrape timeout. If not set, the Prometheus default scrape timeout is used. + scrapeTimeout: "" + + ## Metric relabel configs to apply to samples before ingestion. + ## + metricRelabelings: [] + # - action: keep + # regex: 'kube_(daemonset|deployment|pod|namespace|node|statefulset).+' + # sourceLabels: [__name__] + + # relabel configs to apply to samples before ingestion. + ## + relabelings: [] + # - sourceLabels: [__meta_kubernetes_pod_node_name] + # separator: ; + # regex: ^(.*)$ + # targetLabel: nodename + # replacement: $1 + # action: replace + + ## Resource limits & requests + ## + resources: + limits: + cpu: 200m + memory: 500Mi + requests: + cpu: 100m + memory: 100Mi + + ## Operator Environment + ## env: + ## VARIABLE: value + env: + GOGC: "30" + + # Required for use in managed kubernetes clusters (such as AWS EKS) with custom CNI (such as calico), + # because control-plane managed by AWS cannot communicate with pods' IP CIDR and admission webhooks are not working + ## + hostNetwork: false + + ## Define which Nodes the Pods are scheduled on. + ## ref: https://kubernetes.io/docs/user-guide/node-selection/ + ## + nodeSelector: {} + + ## Tolerations for use with node taints + ## ref: https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/ + ## + tolerations: [] + # - key: "key" + # operator: "Equal" + # value: "value" + # effect: "NoSchedule" + + ## Assign custom affinity rules to the prometheus operator + ## ref: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/ + ## + affinity: {} + # nodeAffinity: + # requiredDuringSchedulingIgnoredDuringExecution: + # nodeSelectorTerms: + # - matchExpressions: + # - key: kubernetes.io/e2e-az-name + # operator: In + # values: + # - e2e-az1 + # - e2e-az2 + dnsConfig: {} + # nameservers: + # - 1.2.3.4 + # searches: + # - ns1.svc.cluster-domain.example + # - my.dns.search.suffix + # options: + # - name: ndots + # value: "2" + # - name: edns0 + securityContext: + fsGroup: 65534 + runAsGroup: 65534 + runAsNonRoot: true + runAsUser: 65534 + seccompProfile: + type: RuntimeDefault + + ## Container-specific security context configuration + ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/ + ## + containerSecurityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: + drop: + - ALL + + # Enable vertical pod autoscaler support for prometheus-operator + verticalPodAutoscaler: + enabled: false + + # Recommender responsible for generating recommendation for the object. + # List should be empty (then the default recommender will generate the recommendation) + # or contain exactly one recommender. + # recommenders: + # - name: custom-recommender-performance + + # List of resources that the vertical pod autoscaler can control. Defaults to cpu and memory + controlledResources: [] + # Specifies which resource values should be controlled: RequestsOnly or RequestsAndLimits. + # controlledValues: RequestsAndLimits + + # Define the max allowed resources for the pod + maxAllowed: {} + # cpu: 200m + # memory: 100Mi + # Define the min allowed resources for the pod + minAllowed: {} + # cpu: 200m + # memory: 100Mi + + updatePolicy: + # Specifies minimal number of replicas which need to be alive for VPA Updater to attempt pod eviction + # minReplicas: 1 + # Specifies whether recommended updates are applied when a Pod is started and whether recommended updates + # are applied during the life of a Pod. Possible values are "Off", "Initial", "Recreate", and "Auto". + updateMode: Auto + + ## Prometheus-operator image + ## + image: + repository: rancher/mirrored-prometheus-operator-prometheus-operator + tag: v0.72.0 + sha: "" + pullPolicy: IfNotPresent + + ## Prometheus image to use for prometheuses managed by the operator + ## + # prometheusDefaultBaseImage: prometheus/prometheus + + ## Prometheus image registry to use for prometheuses managed by the operator + ## + # prometheusDefaultBaseImageRegistry: quay.io + + ## Alertmanager image to use for alertmanagers managed by the operator + ## + # alertmanagerDefaultBaseImage: prometheus/alertmanager + + ## Alertmanager image registry to use for alertmanagers managed by the operator + ## + # alertmanagerDefaultBaseImageRegistry: quay.io + + ## Prometheus-config-reloader + ## + prometheusConfigReloader: + image: + repository: rancher/mirrored-prometheus-operator-prometheus-config-reloader + tag: v0.72.0 + sha: "" + + # add prometheus config reloader liveness and readiness probe. Default: false + enableProbe: false + + # resource config for prometheusConfigReloader + resources: {} + # requests: + # cpu: 200m + # memory: 50Mi + # limits: + # cpu: 200m + # memory: 50Mi + + ## Thanos side-car image when configured + ## + thanosImage: + repository: rancher/mirrored-thanos-thanos + tag: v0.34.1 + sha: "" + + ## Set a Label Selector to filter watched prometheus and prometheusAgent + ## + prometheusInstanceSelector: "" + + ## Set a Label Selector to filter watched alertmanager + ## + alertmanagerInstanceSelector: "" + + ## Set a Label Selector to filter watched thanosRuler + thanosRulerInstanceSelector: "" + + ## Set a Field Selector to filter watched secrets + ## + secretFieldSelector: "type!=kubernetes.io/dockercfg,type!=kubernetes.io/service-account-token,type!=helm.sh/release.v1" + + ## If false then the user will opt out of automounting API credentials. + ## + automountServiceAccountToken: true + + ## Additional volumes + ## + extraVolumes: [] + + ## Additional volume mounts + ## + extraVolumeMounts: [] + +## Deploy a Prometheus instance +## +prometheus: + enabled: true + + ## Toggle prometheus into agent mode + ## Note many of features described below (e.g. rules, query, alerting, remote read, thanos) will not work in agent mode. + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/designs/prometheus-agent.md + ## + agentMode: false + + ## Annotations for Prometheus + ## + annotations: {} + + ## Configure network policy for the prometheus + networkPolicy: + enabled: false + + ## Flavor of the network policy to use. + # Can be: + # * kubernetes for networking.k8s.io/v1/NetworkPolicy + # * cilium for cilium.io/v2/CiliumNetworkPolicy + flavor: kubernetes + + # cilium: + # endpointSelector: + # egress: + # ingress: + + # egress: + # - {} + # ingress: + # - {} + # podSelector: + # matchLabels: + # app: prometheus + + ## Service account for Prometheuses to use. + ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/configure-service-account/ + ## + serviceAccount: + create: true + name: "" + annotations: {} + automountServiceAccountToken: true + + # Service for thanos service discovery on sidecar + # Enable this can make Thanos Query can use + # `--store=dnssrv+_grpc._tcp.${kube-prometheus-stack.fullname}-thanos-discovery.${namespace}.svc.cluster.local` to discovery + # Thanos sidecar on prometheus nodes + # (Please remember to change ${kube-prometheus-stack.fullname} and ${namespace}. Not just copy and paste!) + thanosService: + enabled: false + annotations: {} + labels: {} + + ## Denotes if this Service desires to route external traffic to node-local or cluster-wide endpoints + ## + externalTrafficPolicy: Cluster + + ## Service type + ## + type: ClusterIP + + ## gRPC port config + portName: grpc + port: 10901 + targetPort: "grpc" + + ## HTTP port config (for metrics) + httpPortName: http + httpPort: 10902 + targetHttpPort: "http" + + ## ClusterIP to assign + # Default is to make this a headless service ("None") + clusterIP: "None" + + ## Port to expose on each node, if service type is NodePort + ## + nodePort: 30901 + httpNodePort: 30902 + + # ServiceMonitor to scrape Sidecar metrics + # Needs thanosService to be enabled as well + thanosServiceMonitor: + enabled: false + interval: "" + + ## Additional labels + ## + additionalLabels: {} + + ## scheme: HTTP scheme to use for scraping. Can be used with `tlsConfig` for example if using istio mTLS. + scheme: "" + + ## tlsConfig: TLS configuration to use when scraping the endpoint. For example if using istio mTLS. + ## Of type: https://github.com/coreos/prometheus-operator/blob/main/Documentation/api.md#tlsconfig + tlsConfig: {} + + bearerTokenFile: + + ## Metric relabel configs to apply to samples before ingestion. + metricRelabelings: [] + + ## relabel configs to apply to samples before ingestion. + relabelings: [] + + # Service for external access to sidecar + # Enabling this creates a service to expose thanos-sidecar outside the cluster. + thanosServiceExternal: + enabled: false + annotations: {} + labels: {} + loadBalancerIP: "" + loadBalancerSourceRanges: [] + + ## gRPC port config + portName: grpc + port: 10901 + targetPort: "grpc" + + ## HTTP port config (for metrics) + httpPortName: http + httpPort: 10902 + targetHttpPort: "http" + + ## Denotes if this Service desires to route external traffic to node-local or cluster-wide endpoints + ## + externalTrafficPolicy: Cluster + + ## Service type + ## + type: LoadBalancer + + ## Port to expose on each node + ## + nodePort: 30901 + httpNodePort: 30902 + + ## Configuration for Prometheus service + ## + service: + annotations: {} + labels: {} + clusterIP: "" + + ## Port for Prometheus Service to listen on + ## + port: 9090 + + ## To be used with a proxy extraContainer port + targetPort: 8081 + + ## Port for Prometheus Reloader to listen on + ## + reloaderWebPort: 8080 + + ## List of IP addresses at which the Prometheus server service is available + ## Ref: https://kubernetes.io/docs/user-guide/services/#external-ips + ## + externalIPs: [] + + ## Port to expose on each node + ## Only used if service.type is 'NodePort' + ## + nodePort: 30090 + + ## Loadbalancer IP + ## Only use if service.type is "LoadBalancer" + loadBalancerIP: "" + loadBalancerSourceRanges: [] + + ## Denotes if this Service desires to route external traffic to node-local or cluster-wide endpoints + ## + externalTrafficPolicy: Cluster + + ## Service type + ## + type: ClusterIP + + ## Additional ports to open for Prometheus service + ## + additionalPorts: [] + # additionalPorts: + # - name: oauth-proxy + # port: 8081 + # targetPort: 8081 + # - name: oauth-metrics + # port: 8082 + # targetPort: 8082 + + ## Consider that all endpoints are considered "ready" even if the Pods themselves are not + ## Ref: https://kubernetes.io/docs/reference/kubernetes-api/service-resources/service-v1/#ServiceSpec + publishNotReadyAddresses: false + + ## If you want to make sure that connections from a particular client are passed to the same Pod each time + ## Accepts 'ClientIP' or 'None' + ## + sessionAffinity: None + + ## If you want to modify the ClientIP sessionAffinity timeout + ## The value must be >0 && <=86400(for 1 day) if ServiceAffinity == "ClientIP" + ## + sessionAffinityConfig: + clientIP: + timeoutSeconds: 10800 + + ## Configuration for creating a separate Service for each statefulset Prometheus replica + ## + servicePerReplica: + enabled: false + annotations: {} + + ## Port for Prometheus Service per replica to listen on + ## + port: 9090 + + ## To be used with a proxy extraContainer port + targetPort: 9090 + + ## Port to expose on each node + ## Only used if servicePerReplica.type is 'NodePort' + ## + nodePort: 30091 + + ## Loadbalancer source IP ranges + ## Only used if servicePerReplica.type is "LoadBalancer" + loadBalancerSourceRanges: [] + + ## Denotes if this Service desires to route external traffic to node-local or cluster-wide endpoints + ## + externalTrafficPolicy: Cluster + + ## Service type + ## + type: ClusterIP + + ## Configure pod disruption budgets for Prometheus + ## ref: https://kubernetes.io/docs/tasks/run-application/configure-pdb/#specifying-a-poddisruptionbudget + ## This configuration is immutable once created and will require the PDB to be deleted to be changed + ## https://github.com/kubernetes/kubernetes/issues/45398 + ## + podDisruptionBudget: + enabled: false + minAvailable: 1 + maxUnavailable: "" + + # Ingress exposes thanos sidecar outside the cluster + thanosIngress: + enabled: false + + # For Kubernetes >= 1.18 you should specify the ingress-controller via the field ingressClassName + # See https://kubernetes.io/blog/2020/04/02/improvements-to-the-ingress-api-in-kubernetes-1.18/#specifying-the-class-of-an-ingress + # ingressClassName: nginx + + annotations: {} + labels: {} + servicePort: 10901 + + ## Port to expose on each node + ## Only used if service.type is 'NodePort' + ## + nodePort: 30901 + + ## Hosts must be provided if Ingress is enabled. + ## + hosts: [] + # - thanos-gateway.domain.com + + ## Paths to use for ingress rules + ## + paths: [] + # - / + + ## For Kubernetes >= 1.18 you should specify the pathType (determines how Ingress paths should be matched) + ## See https://kubernetes.io/blog/2020/04/02/improvements-to-the-ingress-api-in-kubernetes-1.18/#better-path-matching-with-path-types + # pathType: ImplementationSpecific + + ## TLS configuration for Thanos Ingress + ## Secret must be manually created in the namespace + ## + tls: [] + # - secretName: thanos-gateway-tls + # hosts: + # - thanos-gateway.domain.com + # + + ## ExtraSecret can be used to store various data in an extra secret + ## (use it for example to store hashed basic auth credentials) + extraSecret: + ## if not set, name will be auto generated + # name: "" + annotations: {} + data: {} + # auth: | + # foo:$apr1$OFG3Xybp$ckL0FHDAkoXYIlH9.cysT0 + # someoneelse:$apr1$DMZX2Z4q$6SbQIfyuLQd.xmo/P0m2c. + + ingress: + enabled: false + + # For Kubernetes >= 1.18 you should specify the ingress-controller via the field ingressClassName + # See https://kubernetes.io/blog/2020/04/02/improvements-to-the-ingress-api-in-kubernetes-1.18/#specifying-the-class-of-an-ingress + # ingressClassName: nginx + + annotations: {} + labels: {} + + ## Redirect ingress to an additional defined port on the service + # servicePort: 8081 + + ## Hostnames. + ## Must be provided if Ingress is enabled. + ## + # hosts: + # - prometheus.domain.com + hosts: [] + + ## Paths to use for ingress rules - one path should match the prometheusSpec.routePrefix + ## + paths: [] + # - / + + ## For Kubernetes >= 1.18 you should specify the pathType (determines how Ingress paths should be matched) + ## See https://kubernetes.io/blog/2020/04/02/improvements-to-the-ingress-api-in-kubernetes-1.18/#better-path-matching-with-path-types + # pathType: ImplementationSpecific + + ## TLS configuration for Prometheus Ingress + ## Secret must be manually created in the namespace + ## + tls: [] + # - secretName: prometheus-general-tls + # hosts: + # - prometheus.example.com + + ## Configuration for creating an Ingress that will map to each Prometheus replica service + ## prometheus.servicePerReplica must be enabled + ## + ingressPerReplica: + enabled: false + + # For Kubernetes >= 1.18 you should specify the ingress-controller via the field ingressClassName + # See https://kubernetes.io/blog/2020/04/02/improvements-to-the-ingress-api-in-kubernetes-1.18/#specifying-the-class-of-an-ingress + # ingressClassName: nginx + + annotations: {} + labels: {} + + ## Final form of the hostname for each per replica ingress is + ## {{ ingressPerReplica.hostPrefix }}-{{ $replicaNumber }}.{{ ingressPerReplica.hostDomain }} + ## + ## Prefix for the per replica ingress that will have `-$replicaNumber` + ## appended to the end + hostPrefix: "" + ## Domain that will be used for the per replica ingress + hostDomain: "" + + ## Paths to use for ingress rules + ## + paths: [] + # - / + + ## For Kubernetes >= 1.18 you should specify the pathType (determines how Ingress paths should be matched) + ## See https://kubernetes.io/blog/2020/04/02/improvements-to-the-ingress-api-in-kubernetes-1.18/#better-path-matching-with-path-types + # pathType: ImplementationSpecific + + ## Secret name containing the TLS certificate for Prometheus per replica ingress + ## Secret must be manually created in the namespace + tlsSecretName: "" + + ## Separated secret for each per replica Ingress. Can be used together with cert-manager + ## + tlsSecretPerReplica: + enabled: false + ## Final form of the secret for each per replica ingress is + ## {{ tlsSecretPerReplica.prefix }}-{{ $replicaNumber }} + ## + prefix: "prometheus" + + ## Configure additional options for default pod security policy for Prometheus + ## ref: https://kubernetes.io/docs/concepts/policy/pod-security-policy/ + podSecurityPolicy: + allowedCapabilities: [] + allowedHostPaths: [] + volumes: [] + + serviceMonitor: + ## If true, create a serviceMonitor for prometheus + ## + selfMonitor: true + + ## Scrape interval. If not set, the Prometheus default scrape interval is used. + ## + interval: "" + + ## Additional labels + ## + additionalLabels: {} + + ## SampleLimit defines per-scrape limit on number of scraped samples that will be accepted. + ## + sampleLimit: 0 + + ## TargetLimit defines a limit on the number of scraped targets that will be accepted. + ## + targetLimit: 0 + + ## Per-scrape limit on number of labels that will be accepted for a sample. Only valid in Prometheus versions 2.27.0 and newer. + ## + labelLimit: 0 + + ## Per-scrape limit on length of labels name that will be accepted for a sample. Only valid in Prometheus versions 2.27.0 and newer. + ## + labelNameLengthLimit: 0 + + ## Per-scrape limit on length of labels value that will be accepted for a sample. Only valid in Prometheus versions 2.27.0 and newer. + ## + labelValueLengthLimit: 0 + + ## scheme: HTTP scheme to use for scraping. Can be used with `tlsConfig` for example if using istio mTLS. + scheme: "" + + ## tlsConfig: TLS configuration to use when scraping the endpoint. For example if using istio mTLS. + ## Of type: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#tlsconfig + tlsConfig: {} + + bearerTokenFile: + + ## Metric relabel configs to apply to samples before ingestion. + ## + metricRelabelings: [] + # - action: keep + # regex: 'kube_(daemonset|deployment|pod|namespace|node|statefulset).+' + # sourceLabels: [__name__] + + # relabel configs to apply to samples before ingestion. + ## + relabelings: [] + # - sourceLabels: [__meta_kubernetes_pod_node_name] + # separator: ; + # regex: ^(.*)$ + # targetLabel: nodename + # replacement: $1 + # action: replace + + ## Additional Endpoints + ## + additionalEndpoints: [] + # - port: oauth-metrics + # path: /metrics + + ## Settings affecting prometheusSpec + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#prometheusspec + ## + prometheusSpec: + ## If true, pass --storage.tsdb.max-block-duration=2h to prometheus. This is already done if using Thanos + ## + disableCompaction: false + ## APIServerConfig + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#apiserverconfig + ## + apiserverConfig: {} + + ## Allows setting additional arguments for the Prometheus container + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#monitoring.coreos.com/v1.Prometheus + additionalArgs: [] + + ## Interval between consecutive scrapes. + ## Defaults to 30s. + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/release-0.44/pkg/prometheus/promcfg.go#L180-L183 + ## + scrapeInterval: "30s" + + ## Number of seconds to wait for target to respond before erroring + ## + # scrapeTimeout: "30s" + + ## Interval between consecutive evaluations. + ## + evaluationInterval: "30s" + + ## ListenLocal makes the Prometheus server listen on loopback, so that it does not bind against the Pod IP. + ## + listenLocal: false + + ## EnableAdminAPI enables Prometheus the administrative HTTP API which includes functionality such as deleting time series. + ## This is disabled by default. + ## ref: https://prometheus.io/docs/prometheus/latest/querying/api/#tsdb-admin-apis + ## + enableAdminAPI: false + + ## Sets version of Prometheus overriding the Prometheus version as derived + ## from the image tag. Useful in cases where the tag does not follow semver v2. + version: "" + + ## WebTLSConfig defines the TLS parameters for HTTPS + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#webtlsconfig + web: {} + + ## Exemplars related settings that are runtime reloadable. + ## It requires to enable the exemplar storage feature to be effective. + exemplars: "" + ## Maximum number of exemplars stored in memory for all series. + ## If not set, Prometheus uses its default value. + ## A value of zero or less than zero disables the storage. + # maxSize: 100000 + + # EnableFeatures API enables access to Prometheus disabled features. + # ref: https://prometheus.io/docs/prometheus/latest/disabled_features/ + enableFeatures: [] + # - exemplar-storage + + ## Image of Prometheus. + ## + image: + repository: rancher/mirrored-prometheus-prometheus + tag: v2.50.1 + sha: "" + + ## Tolerations for use with node taints + ## ref: https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/ + ## + tolerations: [] + # - key: "key" + # operator: "Equal" + # value: "value" + # effect: "NoSchedule" + + ## If specified, the pod's topology spread constraints. + ## ref: https://kubernetes.io/docs/concepts/workloads/pods/pod-topology-spread-constraints/ + ## + topologySpreadConstraints: [] + # - maxSkew: 1 + # topologyKey: topology.kubernetes.io/zone + # whenUnsatisfiable: DoNotSchedule + # labelSelector: + # matchLabels: + # app: prometheus + + ## Alertmanagers to which alerts will be sent + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#alertmanagerendpoints + ## + ## Default configuration will connect to the alertmanager deployed as part of this release + ## + alertingEndpoints: [] + # - name: "" + # namespace: "" + # port: http + # scheme: http + # pathPrefix: "" + # tlsConfig: {} + # bearerTokenFile: "" + # apiVersion: v2 + + ## External labels to add to any time series or alerts when communicating with external systems + ## + externalLabels: {} + + ## enable --web.enable-remote-write-receiver flag on prometheus-server + ## + enableRemoteWriteReceiver: false + + ## Name of the external label used to denote replica name + ## + replicaExternalLabelName: "" + + ## If true, the Operator won't add the external label used to denote replica name + ## + replicaExternalLabelNameClear: false + + ## Name of the external label used to denote Prometheus instance name + ## + prometheusExternalLabelName: "" + + ## If true, the Operator won't add the external label used to denote Prometheus instance name + ## + prometheusExternalLabelNameClear: false + + ## External URL at which Prometheus will be reachable. + ## + externalUrl: "" + + ## Define which Nodes the Pods are scheduled on. + ## ref: https://kubernetes.io/docs/user-guide/node-selection/ + ## + nodeSelector: {} + + ## Secrets is a list of Secrets in the same namespace as the Prometheus object, which shall be mounted into the Prometheus Pods. + ## The Secrets are mounted into /etc/prometheus/secrets/. Secrets changes after initial creation of a Prometheus object are not + ## reflected in the running Pods. To change the secrets mounted into the Prometheus Pods, the object must be deleted and recreated + ## with the new list of secrets. + ## + secrets: [] + + ## ConfigMaps is a list of ConfigMaps in the same namespace as the Prometheus object, which shall be mounted into the Prometheus Pods. + ## The ConfigMaps are mounted into /etc/prometheus/configmaps/. + ## + configMaps: [] + + ## QuerySpec defines the query command line flags when starting Prometheus. + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#queryspec + ## + query: {} + + ## If nil, select own namespace. Namespaces to be selected for PrometheusRules discovery. + ruleNamespaceSelector: {} + ## Example which selects PrometheusRules in namespaces with label "prometheus" set to "somelabel" + # ruleNamespaceSelector: + # matchLabels: + # prometheus: somelabel + + ## If true, a nil or {} value for prometheus.prometheusSpec.ruleSelector will cause the + ## prometheus resource to be created with selectors based on values in the helm deployment, + ## which will also match the PrometheusRule resources created + ## + ruleSelectorNilUsesHelmValues: false + + ## PrometheusRules to be selected for target discovery. + ## If {}, select all PrometheusRules + ## + ruleSelector: {} + ## Example which select all PrometheusRules resources + ## with label "prometheus" with values any of "example-rules" or "example-rules-2" + # ruleSelector: + # matchExpressions: + # - key: prometheus + # operator: In + # values: + # - example-rules + # - example-rules-2 + # + ## Example which select all PrometheusRules resources with label "role" set to "example-rules" + # ruleSelector: + # matchLabels: + # role: example-rules + + ## If true, a nil or {} value for prometheus.prometheusSpec.serviceMonitorSelector will cause the + ## prometheus resource to be created with selectors based on values in the helm deployment, + ## which will also match the servicemonitors created + ## + serviceMonitorSelectorNilUsesHelmValues: false + + ## ServiceMonitors to be selected for target discovery. + ## If {}, select all ServiceMonitors + ## + serviceMonitorSelector: {} + ## Example which selects ServiceMonitors with label "prometheus" set to "somelabel" + # serviceMonitorSelector: + # matchLabels: + # prometheus: somelabel + + ## Namespaces to be selected for ServiceMonitor discovery. + ## + serviceMonitorNamespaceSelector: {} + ## Example which selects ServiceMonitors in namespaces with label "prometheus" set to "somelabel" + # serviceMonitorNamespaceSelector: + # matchLabels: + # prometheus: somelabel + + ## If true, a nil or {} value for prometheus.prometheusSpec.podMonitorSelector will cause the + ## prometheus resource to be created with selectors based on values in the helm deployment, + ## which will also match the podmonitors created + ## + podMonitorSelectorNilUsesHelmValues: false + + ## PodMonitors to be selected for target discovery. + ## If {}, select all PodMonitors + ## + podMonitorSelector: {} + ## Example which selects PodMonitors with label "prometheus" set to "somelabel" + # podMonitorSelector: + # matchLabels: + # prometheus: somelabel + + ## If nil, select own namespace. Namespaces to be selected for PodMonitor discovery. + podMonitorNamespaceSelector: {} + ## Example which selects PodMonitor in namespaces with label "prometheus" set to "somelabel" + # podMonitorNamespaceSelector: + # matchLabels: + # prometheus: somelabel + + ## If true, a nil or {} value for prometheus.prometheusSpec.probeSelector will cause the + ## prometheus resource to be created with selectors based on values in the helm deployment, + ## which will also match the probes created + ## + probeSelectorNilUsesHelmValues: true + + ## Probes to be selected for target discovery. + ## If {}, select all Probes + ## + probeSelector: {} + ## Example which selects Probes with label "prometheus" set to "somelabel" + # probeSelector: + # matchLabels: + # prometheus: somelabel + + ## If nil, select own namespace. Namespaces to be selected for Probe discovery. + probeNamespaceSelector: {} + ## Example which selects Probe in namespaces with label "prometheus" set to "somelabel" + # probeNamespaceSelector: + # matchLabels: + # prometheus: somelabel + + ## If true, a nil or {} value for prometheus.prometheusSpec.scrapeConfigSelector will cause the + ## prometheus resource to be created with selectors based on values in the helm deployment, + ## which will also match the scrapeConfigs created + ## + scrapeConfigSelectorNilUsesHelmValues: true + + ## scrapeConfigs to be selected for target discovery. + ## If {}, select all scrapeConfigs + ## + scrapeConfigSelector: {} + ## Example which selects scrapeConfigs with label "prometheus" set to "somelabel" + # scrapeConfigSelector: + # matchLabels: + # prometheus: somelabel + + ## If nil, select own namespace. Namespaces to be selected for scrapeConfig discovery. + scrapeConfigNamespaceSelector: {} + ## Example which selects scrapeConfig in namespaces with label "prometheus" set to "somelabel" + # scrapeConfigNamespaceSelector: + # matchLabels: + # prometheus: somelabel + + ## How long to retain metrics + ## + retention: 10d + + ## Maximum size of metrics + ## + retentionSize: "" + + ## Allow out-of-order/out-of-bounds samples ingested into Prometheus for a specified duration + ## See https://prometheus.io/docs/prometheus/latest/configuration/configuration/#tsdb + tsdb: + outOfOrderTimeWindow: 0s + + ## Enable compression of the write-ahead log using Snappy. + ## + walCompression: true + + ## If true, the Operator won't process any Prometheus configuration changes + ## + paused: false + + ## Number of replicas of each shard to deploy for a Prometheus deployment. + ## Number of replicas multiplied by shards is the total number of Pods created. + ## + replicas: 1 + + ## EXPERIMENTAL: Number of shards to distribute targets onto. + ## Number of replicas multiplied by shards is the total number of Pods created. + ## Note that scaling down shards will not reshard data onto remaining instances, it must be manually moved. + ## Increasing shards will not reshard data either but it will continue to be available from the same instances. + ## To query globally use Thanos sidecar and Thanos querier or remote write data to a central location. + ## Sharding is done on the content of the `__address__` target meta-label. + ## + shards: 1 + + ## Log level for Prometheus be configured in + ## + logLevel: info + + ## Log format for Prometheus be configured in + ## + logFormat: logfmt + + ## Prefix used to register routes, overriding externalUrl route. + ## Useful for proxies that rewrite URLs. + ## + routePrefix: / + + ## Standard object's metadata. More info: https://github.com/kubernetes/community/blob/master/contributors/devel/sig-architecture/api-conventions.md#metadata + ## Metadata Labels and Annotations gets propagated to the prometheus pods. + ## + podMetadata: {} + # labels: + # app: prometheus + # k8s-app: prometheus + + ## Pod anti-affinity can prevent the scheduler from placing Prometheus replicas on the same node. + ## The default value "soft" means that the scheduler should *prefer* to not schedule two replica pods onto the same node but no guarantee is provided. + ## The value "hard" means that the scheduler is *required* to not schedule two replica pods onto the same node. + ## The value "" will disable pod anti-affinity so that no anti-affinity rules will be configured. + podAntiAffinity: "" + + ## If anti-affinity is enabled sets the topologyKey to use for anti-affinity. + ## This can be changed to, for example, failure-domain.beta.kubernetes.io/zone + ## + podAntiAffinityTopologyKey: kubernetes.io/hostname + + ## Assign custom affinity rules to the prometheus instance + ## ref: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/ + ## + affinity: {} + # nodeAffinity: + # requiredDuringSchedulingIgnoredDuringExecution: + # nodeSelectorTerms: + # - matchExpressions: + # - key: kubernetes.io/e2e-az-name + # operator: In + # values: + # - e2e-az1 + # - e2e-az2 + + ## The remote_read spec configuration for Prometheus. + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#remotereadspec + remoteRead: [] + # - url: http://remote1/read + ## additionalRemoteRead is appended to remoteRead + additionalRemoteRead: [] + + ## The remote_write spec configuration for Prometheus. + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#remotewritespec + remoteWrite: [] + # - url: http://remote1/push + ## additionalRemoteWrite is appended to remoteWrite + additionalRemoteWrite: [] + + ## Enable/Disable Grafana dashboards provisioning for prometheus remote write feature + remoteWriteDashboards: false + + ## Resource limits & requests + ## + resources: + limits: + memory: 3000Mi + cpu: 1000m + requests: + memory: 750Mi + cpu: 750m + + ## Prometheus StorageSpec for persistent data + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/user-guides/storage.md + ## + storageSpec: {} + ## Using PersistentVolumeClaim + ## + # volumeClaimTemplate: + # spec: + # storageClassName: gluster + # accessModes: ["ReadWriteOnce"] + # resources: + # requests: + # storage: 50Gi + # selector: {} + + ## Using tmpfs volume + ## + # emptyDir: + # medium: Memory + + # Additional volumes on the output StatefulSet definition. + volumes: + - name: nginx-home + emptyDir: {} + - name: prometheus-nginx + configMap: + name: prometheus-nginx-proxy-config + defaultMode: 438 + + # Additional VolumeMounts on the output StatefulSet definition. + volumeMounts: [] + + ## AdditionalScrapeConfigs allows specifying additional Prometheus scrape configurations. Scrape configurations + ## are appended to the configurations generated by the Prometheus Operator. Job configurations must have the form + ## as specified in the official Prometheus documentation: + ## https://prometheus.io/docs/prometheus/latest/configuration/configuration/#scrape_config. As scrape configs are + ## appended, the user is responsible to make sure it is valid. Note that using this feature may expose the possibility + ## to break upgrades of Prometheus. It is advised to review Prometheus release notes to ensure that no incompatible + ## scrape configs are going to break Prometheus after the upgrade. + ## AdditionalScrapeConfigs can be defined as a list or as a templated string. + ## + ## The scrape configuration example below will find master nodes, provided they have the name .*mst.*, relabel the + ## port to 2379 and allow etcd scraping provided it is running on all Kubernetes master nodes + ## + additionalScrapeConfigs: [] + # - job_name: kube-etcd + # kubernetes_sd_configs: + # - role: node + # scheme: https + # tls_config: + # ca_file: /etc/prometheus/secrets/etcd-client-cert/etcd-ca + # cert_file: /etc/prometheus/secrets/etcd-client-cert/etcd-client + # key_file: /etc/prometheus/secrets/etcd-client-cert/etcd-client-key + # relabel_configs: + # - action: labelmap + # regex: __meta_kubernetes_node_label_(.+) + # - source_labels: [__address__] + # action: replace + # targetLabel: __address__ + # regex: ([^:;]+):(\d+) + # replacement: ${1}:2379 + # - source_labels: [__meta_kubernetes_node_name] + # action: keep + # regex: .*mst.* + # - source_labels: [__meta_kubernetes_node_name] + # action: replace + # targetLabel: node + # regex: (.*) + # replacement: ${1} + # metric_relabel_configs: + # - regex: (kubernetes_io_hostname|failure_domain_beta_kubernetes_io_region|beta_kubernetes_io_os|beta_kubernetes_io_arch|beta_kubernetes_io_instance_type|failure_domain_beta_kubernetes_io_zone) + # action: labeldrop + # + ## If scrape config contains a repetitive section, you may want to use a template. + ## In the following example, you can see how to define `gce_sd_configs` for multiple zones + # additionalScrapeConfigs: | + # - job_name: "node-exporter" + # gce_sd_configs: + # {{range $zone := .Values.gcp_zones}} + # - project: "project1" + # zone: "{{$zone}}" + # port: 9100 + # {{end}} + # relabel_configs: + # ... + + + ## If additional scrape configurations are already deployed in a single secret file you can use this section. + ## Expected values are the secret name and key + ## Cannot be used with additionalScrapeConfigs + additionalScrapeConfigsSecret: {} + # enabled: false + # name: + # key: + + ## additionalPrometheusSecretsAnnotations allows to add annotations to the kubernetes secret. This can be useful + ## when deploying via spinnaker to disable versioning on the secret, strategy.spinnaker.io/versioned: 'false' + additionalPrometheusSecretsAnnotations: {} + + ## AdditionalAlertManagerConfigs allows for manual configuration of alertmanager jobs in the form as specified + ## in the official Prometheus documentation https://prometheus.io/docs/prometheus/latest/configuration/configuration/#. + ## AlertManager configurations specified are appended to the configurations generated by the Prometheus Operator. + ## As AlertManager configs are appended, the user is responsible to make sure it is valid. Note that using this + ## feature may expose the possibility to break upgrades of Prometheus. It is advised to review Prometheus release + ## notes to ensure that no incompatible AlertManager configs are going to break Prometheus after the upgrade. + ## + additionalAlertManagerConfigs: [] + # - consul_sd_configs: + # - server: consul.dev.test:8500 + # scheme: http + # datacenter: dev + # tag_separator: ',' + # services: + # - metrics-prometheus-alertmanager + + ## If additional alertmanager configurations are already deployed in a single secret, or you want to manage + ## them separately from the helm deployment, you can use this section. + ## Expected values are the secret name and key + ## Cannot be used with additionalAlertManagerConfigs + additionalAlertManagerConfigsSecret: {} + # name: + # key: + # optional: false + + ## AdditionalAlertRelabelConfigs allows specifying Prometheus alert relabel configurations. Alert relabel configurations specified are appended + ## to the configurations generated by the Prometheus Operator. Alert relabel configurations specified must have the form as specified in the + ## official Prometheus documentation: https://prometheus.io/docs/prometheus/latest/configuration/configuration/#alert_relabel_configs. + ## As alert relabel configs are appended, the user is responsible to make sure it is valid. Note that using this feature may expose the + ## possibility to break upgrades of Prometheus. It is advised to review Prometheus release notes to ensure that no incompatible alert relabel + ## configs are going to break Prometheus after the upgrade. + ## + additionalAlertRelabelConfigs: [] + # - separator: ; + # regex: prometheus_replica + # replacement: $1 + # action: labeldrop + + ## If additional alert relabel configurations are already deployed in a single secret, or you want to manage + ## them separately from the helm deployment, you can use this section. + ## Expected values are the secret name and key + ## Cannot be used with additionalAlertRelabelConfigs + additionalAlertRelabelConfigsSecret: {} + # name: + # key: + + ## SecurityContext holds pod-level security attributes and common container settings. + ## This defaults to non root user with uid 1000 and gid 2000. + ## https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md + ## + securityContext: + runAsGroup: 2000 + runAsNonRoot: true + runAsUser: 1000 + fsGroup: 2000 + seccompProfile: + type: RuntimeDefault + + ## Priority class assigned to the Pods + ## + priorityClassName: "" + + ## Thanos configuration allows configuring various aspects of a Prometheus server in a Thanos environment. + ## This section is experimental, it may change significantly without deprecation notice in any release. + ## This is experimental and may change significantly without backward compatibility in any release. + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#thanosspec + ## + thanos: {} + # secretProviderClass: + # provider: gcp + # parameters: + # secrets: | + # - resourceName: "projects/$PROJECT_ID/secrets/testsecret/versions/latest" + # fileName: "objstore.yaml" + ## ObjectStorageConfig configures object storage in Thanos. + # objectStorageConfig: + # # use existing secret, if configured, objectStorageConfig.secret will not be used + # existingSecret: {} + # # name: "" + # # key: "" + # # will render objectStorageConfig secret data and configure it to be used by Thanos custom resource, + # # ignored when prometheusspec.thanos.objectStorageConfig.existingSecret is set + # # https://thanos.io/tip/thanos/storage.md/#s3 + # secret: {} + # # type: S3 + # # config: + # # bucket: "" + # # endpoint: "" + # # region: "" + # # access_key: "" + # # secret_key: "" + + proxy: + image: + repository: rancher/mirrored-library-nginx + tag: 1.24.0-alpine + + ## Containers allows injecting additional containers. This is meant to allow adding an authentication proxy to a Prometheus pod. + ## if using proxy extraContainer update targetPort with proxy container port + containers: | + - name: prometheus-proxy + args: + - nginx + - -g + - daemon off; + - -c + - /nginx/nginx.conf + image: "{{ template "system_default_registry" . }}{{ .Values.prometheus.prometheusSpec.proxy.image.repository }}:{{ .Values.prometheus.prometheusSpec.proxy.image.tag }}" + ports: + - containerPort: 8081 + name: nginx-http + protocol: TCP + volumeMounts: + - mountPath: /nginx + name: prometheus-nginx + - mountPath: /var/cache/nginx + name: nginx-home + securityContext: + runAsUser: 101 + runAsGroup: 101 + + ## InitContainers allows injecting additional initContainers. This is meant to allow doing some changes + ## (permissions, dir tree) on mounted volumes before starting prometheus + initContainers: [] + + ## PortName to use for Prometheus. + ## + portName: "http-web" + + ## ArbitraryFSAccessThroughSMs configures whether configuration based on a service monitor can access arbitrary files + ## on the file system of the Prometheus container e.g. bearer token files. + arbitraryFSAccessThroughSMs: false + + ## OverrideHonorLabels if set to true overrides all user configured honor_labels. If HonorLabels is set in ServiceMonitor + ## or PodMonitor to true, this overrides honor_labels to false. + overrideHonorLabels: false + + ## OverrideHonorTimestamps allows to globally enforce honoring timestamps in all scrape configs. + overrideHonorTimestamps: false + + ## When ignoreNamespaceSelectors is set to true, namespaceSelector from all PodMonitor, ServiceMonitor and Probe objects will be ignored, + ## they will only discover targets within the namespace of the PodMonitor, ServiceMonitor and Probe object, + ## and servicemonitors will be installed in the default service namespace. + ## Defaults to false. + ignoreNamespaceSelectors: true + + ## EnforcedNamespaceLabel enforces adding a namespace label of origin for each alert and metric that is user created. + ## The label value will always be the namespace of the object that is being created. + ## Disabled by default + enforcedNamespaceLabel: "" + + ## PrometheusRulesExcludedFromEnforce - list of prometheus rules to be excluded from enforcing of adding namespace labels. + ## Works only if enforcedNamespaceLabel set to true. Make sure both ruleNamespace and ruleName are set for each pair + ## Deprecated, use `excludedFromEnforcement` instead + prometheusRulesExcludedFromEnforce: [] + + ## ExcludedFromEnforcement - list of object references to PodMonitor, ServiceMonitor, Probe and PrometheusRule objects + ## to be excluded from enforcing a namespace label of origin. + ## Works only if enforcedNamespaceLabel set to true. + ## See https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#objectreference + excludedFromEnforcement: [] + + ## QueryLogFile specifies the file to which PromQL queries are logged. Note that this location must be writable, + ## and can be persisted using an attached volume. Alternatively, the location can be set to a stdout location such + ## as /dev/stdout to log querie information to the default Prometheus log stream. This is only available in versions + ## of Prometheus >= 2.16.0. For more details, see the Prometheus docs (https://prometheus.io/docs/guides/query-log/) + queryLogFile: false + + # Use to set global sample_limit for Prometheus. This act as default SampleLimit for ServiceMonitor or/and PodMonitor. + # Set to 'false' to disable global sample_limit. or set to a number to override the default value. + sampleLimit: false + + # EnforcedKeepDroppedTargetsLimit defines on the number of targets dropped by relabeling that will be kept in memory. + # The value overrides any spec.keepDroppedTargets set by ServiceMonitor, PodMonitor, Probe objects unless spec.keepDroppedTargets + # is greater than zero and less than spec.enforcedKeepDroppedTargets. 0 means no limit. + enforcedKeepDroppedTargets: 0 + + ## EnforcedSampleLimit defines global limit on number of scraped samples that will be accepted. This overrides any SampleLimit + ## set per ServiceMonitor or/and PodMonitor. It is meant to be used by admins to enforce the SampleLimit to keep overall + ## number of samples/series under the desired limit. Note that if SampleLimit is lower that value will be taken instead. + enforcedSampleLimit: false + + ## EnforcedTargetLimit defines a global limit on the number of scraped targets. This overrides any TargetLimit set + ## per ServiceMonitor or/and PodMonitor. It is meant to be used by admins to enforce the TargetLimit to keep the overall + ## number of targets under the desired limit. Note that if TargetLimit is lower, that value will be taken instead, except + ## if either value is zero, in which case the non-zero value will be used. If both values are zero, no limit is enforced. + enforcedTargetLimit: false + + + ## Per-scrape limit on number of labels that will be accepted for a sample. If more than this number of labels are present + ## post metric-relabeling, the entire scrape will be treated as failed. 0 means no limit. Only valid in Prometheus versions + ## 2.27.0 and newer. + enforcedLabelLimit: false + + ## Per-scrape limit on length of labels name that will be accepted for a sample. If a label name is longer than this number + ## post metric-relabeling, the entire scrape will be treated as failed. 0 means no limit. Only valid in Prometheus versions + ## 2.27.0 and newer. + enforcedLabelNameLengthLimit: false + + ## Per-scrape limit on length of labels value that will be accepted for a sample. If a label value is longer than this + ## number post metric-relabeling, the entire scrape will be treated as failed. 0 means no limit. Only valid in Prometheus + ## versions 2.27.0 and newer. + enforcedLabelValueLengthLimit: false + + ## AllowOverlappingBlocks enables vertical compaction and vertical query merge in Prometheus. This is still experimental + ## in Prometheus so it may change in any upcoming release. + allowOverlappingBlocks: false + + ## Minimum number of seconds for which a newly created pod should be ready without any of its container crashing for it to + ## be considered available. Defaults to 0 (pod will be considered available as soon as it is ready). + minReadySeconds: 0 + + # Required for use in managed kubernetes clusters (such as AWS EKS) with custom CNI (such as calico), + # because control-plane managed by AWS cannot communicate with pods' IP CIDR and admission webhooks are not working + # Use the host's network namespace if true. Make sure to understand the security implications if you want to enable it. + # When hostNetwork is enabled, this will set dnsPolicy to ClusterFirstWithHostNet automatically. + hostNetwork: false + + # HostAlias holds the mapping between IP and hostnames that will be injected + # as an entry in the pod’s hosts file. + hostAliases: [] + # - ip: 10.10.0.100 + # hostnames: + # - a1.app.local + # - b1.app.local + + ## TracingConfig configures tracing in Prometheus. + ## See https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#prometheustracingconfig + tracingConfig: {} + + ## Additional configuration which is not covered by the properties above. (passed through tpl) + additionalConfig: {} + + ## Additional configuration which is not covered by the properties above. + ## Useful, if you need advanced templating inside alertmanagerSpec. + ## Otherwise, use prometheus.prometheusSpec.additionalConfig (passed through tpl) + additionalConfigString: "" + + ## Defines the maximum time that the `prometheus` container's startup probe + ## will wait before being considered failed. The startup probe will return + ## success after the WAL replay is complete. If set, the value should be + ## greater than 60 (seconds). Otherwise it will be equal to 600 seconds (15 + ## minutes). + maximumStartupDurationSeconds: 0 + + additionalRulesForClusterRole: [] + # - apiGroups: [ "" ] + # resources: + # - nodes/proxy + # verbs: [ "get", "list", "watch" ] + + additionalServiceMonitors: [] + ## Name of the ServiceMonitor to create + ## + # - name: "" + + ## Additional labels to set used for the ServiceMonitorSelector. Together with standard labels from + ## the chart + ## + # additionalLabels: {} + + ## Service label for use in assembling a job name of the form