From 0f33f0e9e9eb8a59d1b0263a1dcc923a8664132e Mon Sep 17 00:00:00 2001 From: Ishaan Mittal Date: Sun, 6 Oct 2024 22:29:57 +0530 Subject: [PATCH 1/9] Add kubeRBACProxy property to support querying in cluster prometheus in openshift --- charts/opencost/templates/_helpers.tpl | 9 +++++++ .../templates/clusterrolebinding.yaml | 20 +++++++++++++++ charts/opencost/templates/deployment.yaml | 4 +++ charts/opencost/values.yaml | 25 +++++++++++-------- 4 files changed, 47 insertions(+), 11 deletions(-) diff --git a/charts/opencost/templates/_helpers.tpl b/charts/opencost/templates/_helpers.tpl index ec38d3d..d62d501 100644 --- a/charts/opencost/templates/_helpers.tpl +++ b/charts/opencost/templates/_helpers.tpl @@ -120,6 +120,15 @@ Check that either thanos external or internal is defined {{- end -}} {{- end -}} +{{/* + Verify if both kube-rbac-proxy and bearer token are set +}} +{{- define "kubeRBACProxyBearerTokenCheck" -}} +{{- if and (.Values.opencost.prometheus.kubeRBACProxy .Values.opencost.prometheus.bearer_token) }} + {{- fail "\n\nBoth kubeRBACProxy and bearer_token are set. Please specify only one." -}} +{{- end -}} +{{- end -}} + {{/* Check that the config is valid */}} diff --git a/charts/opencost/templates/clusterrolebinding.yaml b/charts/opencost/templates/clusterrolebinding.yaml index 56f4e00..fc431c7 100644 --- a/charts/opencost/templates/clusterrolebinding.yaml +++ b/charts/opencost/templates/clusterrolebinding.yaml @@ -15,4 +15,24 @@ subjects: - kind: ServiceAccount name: {{ template "opencost.serviceAccountName" . }} namespace: {{ include "opencost.namespace" . }} +--- {{- end }} +{{- if .Values.opencost.prometheus.createPrometheusClusterRoleBinding }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ include "opencost.fullname" . }}-operator + labels: {{- include "opencost.labels" . | nindent 4 }} + {{- with .Values.annotations }} + annotations: {{- toYaml . | nindent 4 }} + {{- end }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: cluster-monitoring-operator +subjects: + - kind: ServiceAccount + name: {{ template "opencost.serviceAccountName" . }} + namespace: {{ include "opencost.namespace" . }} +{{- end }} +--- \ No newline at end of file diff --git a/charts/opencost/templates/deployment.yaml b/charts/opencost/templates/deployment.yaml index 1d8e291..e543457 100644 --- a/charts/opencost/templates/deployment.yaml +++ b/charts/opencost/templates/deployment.yaml @@ -166,6 +166,10 @@ spec: name: {{ .Values.opencost.prometheus.existingSecretName | default (include "opencost.prometheus.secretname" .) }} key: {{ .Values.opencost.prometheus.bearer_token_key }} {{- end }} + {{- if .Values.opencost.prometheus.kubeRbacProxy }} + - name: ENABLE_KUBE_RBAC_PROXY + value: {{ (quote .Values.opencost.prometheus.kubeRbacProxy) }} + {{- end }} {{- if and .Values.opencost.exporter.persistence.enabled .Values.opencost.exporter.csv_path }} - name: EXPORT_CSV_FILE value: {{ .Values.opencost.exporter.csv_path | quote }} diff --git a/charts/opencost/values.yaml b/charts/opencost/values.yaml index ed1ce77..05c3efa 100644 --- a/charts/opencost/values.yaml +++ b/charts/opencost/values.yaml @@ -126,11 +126,11 @@ opencost: defaultClusterId: 'default-cluster' image: # -- Exporter container image registry - registry: ghcr.io + registry: docker.io # -- Exporter container image name - repository: opencost/opencost + repository: ishaanmittal/opencost # -- Exporter container image tag - tag: "1.111.0@sha256:6aa68e52a24b14ba41f23db08d1b9db1429a1c0300f4c0381ecc2c61fc311a97" + tag: "test-amd64" # -- Exporter container image pull policy pullPolicy: IfNotPresent # -- Override the full image name for development purposes @@ -333,6 +333,10 @@ opencost: # -- Prometheus Bearer token bearer_token: "" bearer_token_key: DB_BEARER_TOKEN + # -- If true, opencost will use kube-rbac-proxy to authenticate with in cluster Prometheus for openshift + kubeRBACProxy: false + # -- If true, opencost will create a ClusterRoleBinding to allow using in-cluster Prometheus for openshift + createPrometheusClusterRoleBinding: false external: # -- Use external Prometheus (eg. Grafana Cloud) enabled: false @@ -369,15 +373,14 @@ opencost: # -- Enable OpenCost UI enabled: true image: - # -- UI container image registry - registry: ghcr.io - # -- UI container image name - repository: opencost/opencost-ui - # -- UI container image tag - # @default -- `""` (use appVersion in Chart.yaml) - tag: "1.111.0@sha256:f7221e7a708d71663f5eca6c238268757eb4352f3e9f46b1029d33ab4e53fd8a" + # -- Exporter container image registry + registry: docker.io + # -- Exporter container image name + repository: ishaanmittal/opencost-ui + # -- Exporter container image tag + tag: "test-amd64" # -- UI container image pull policy - pullPolicy: IfNotPresent + pullPolicy: Always # -- Override the full image name for development purposes fullImageName: null resources: From e22f874fb4efd8610852ce7be888a4ff5830a615 Mon Sep 17 00:00:00 2001 From: Ishaan Mittal Date: Fri, 11 Oct 2024 03:28:20 +0530 Subject: [PATCH 2/9] add rrequired role and rindings option --- charts/opencost/templates/deployment.yaml | 4 +-- .../monitoring-role-binding-template.yaml | 15 ++++++++++ .../templates/monitoring-role-template.yaml | 17 +++++++++++ charts/opencost/values.yaml | 28 +++++++++++-------- 4 files changed, 51 insertions(+), 13 deletions(-) create mode 100644 charts/opencost/templates/monitoring-role-binding-template.yaml create mode 100644 charts/opencost/templates/monitoring-role-template.yaml diff --git a/charts/opencost/templates/deployment.yaml b/charts/opencost/templates/deployment.yaml index e543457..9bb84a8 100644 --- a/charts/opencost/templates/deployment.yaml +++ b/charts/opencost/templates/deployment.yaml @@ -166,9 +166,9 @@ spec: name: {{ .Values.opencost.prometheus.existingSecretName | default (include "opencost.prometheus.secretname" .) }} key: {{ .Values.opencost.prometheus.bearer_token_key }} {{- end }} - {{- if .Values.opencost.prometheus.kubeRbacProxy }} + {{- if .Values.opencost.prometheus.kubeRBACProxy }} - name: ENABLE_KUBE_RBAC_PROXY - value: {{ (quote .Values.opencost.prometheus.kubeRbacProxy) }} + value: {{ (quote .Values.opencost.prometheus.kubeRBACProxy) }} {{- end }} {{- if and .Values.opencost.exporter.persistence.enabled .Values.opencost.exporter.csv_path }} - name: EXPORT_CSV_FILE diff --git a/charts/opencost/templates/monitoring-role-binding-template.yaml b/charts/opencost/templates/monitoring-role-binding-template.yaml new file mode 100644 index 0000000..ef71c99 --- /dev/null +++ b/charts/opencost/templates/monitoring-role-binding-template.yaml @@ -0,0 +1,15 @@ +{{- if (.Values.opencost.prometheus.createMonitoringResourceReaderRoleBinding) }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + namespace: {{ include "opencost.namespace" . }} + name: {{ include "opencost.fullname" . }}-reader +subjects: +- kind: ServiceAccount + name: {{ .Values.opencost.prometheus.monitoringServiceAccountName | quote }} + namespace: openshift-monitoring +roleRef: + kind: Role + name: {{ include "opencost.fullname" . }}-reader + apiGroup: rbac.authorization.k8s.io +{{- end -}} diff --git a/charts/opencost/templates/monitoring-role-template.yaml b/charts/opencost/templates/monitoring-role-template.yaml new file mode 100644 index 0000000..9f193a0 --- /dev/null +++ b/charts/opencost/templates/monitoring-role-template.yaml @@ -0,0 +1,17 @@ +{{- if (.Values.opencost.prometheus.createMonitoringResourceReaderRoleBinding) }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + namespace: {{ include "opencost.namespace" . }} + name: {{ include "opencost.fullname" . }}-reader +rules: + - apiGroups: + - '' + resources: + - "pods" + - "services" + - "endpoints" + verbs: + - list + - watch +{{- end -}} \ No newline at end of file diff --git a/charts/opencost/values.yaml b/charts/opencost/values.yaml index 05c3efa..47ed3e1 100644 --- a/charts/opencost/values.yaml +++ b/charts/opencost/values.yaml @@ -126,11 +126,11 @@ opencost: defaultClusterId: 'default-cluster' image: # -- Exporter container image registry - registry: docker.io + registry: ghcr.io # -- Exporter container image name - repository: ishaanmittal/opencost + repository: opencost/opencost # -- Exporter container image tag - tag: "test-amd64" + tag: "1.111.0@sha256:6aa68e52a24b14ba41f23db08d1b9db1429a1c0300f4c0381ecc2c61fc311a97" # -- Exporter container image pull policy pullPolicy: IfNotPresent # -- Override the full image name for development purposes @@ -336,7 +336,12 @@ opencost: # -- If true, opencost will use kube-rbac-proxy to authenticate with in cluster Prometheus for openshift kubeRBACProxy: false # -- If true, opencost will create a ClusterRoleBinding to allow using in-cluster Prometheus for openshift - createPrometheusClusterRoleBinding: false + createMonitoringClusterRoleBinding: false + # -- If true, opencost will create a role and role binding to allow in-cluster prometheus to list and watch resources + # This will be necessary if you are not using bundled prometheus and need to add scrape config for resources. + createMonitoringResourceReaderRoleBinding: false + # -- Name of the service account to bind to the ClusterRoleBinding + monitoringServiceAccountName: prometheus-k8s external: # -- Use external Prometheus (eg. Grafana Cloud) enabled: false @@ -373,14 +378,15 @@ opencost: # -- Enable OpenCost UI enabled: true image: - # -- Exporter container image registry - registry: docker.io - # -- Exporter container image name - repository: ishaanmittal/opencost-ui - # -- Exporter container image tag - tag: "test-amd64" + # -- UI container image registry + registry: ghcr.io + # -- UI container image name + repository: opencost/opencost-ui + # -- UI container image tag + # @default -- `""` (use appVersion in Chart.yaml) + tag: "1.111.0@sha256:f7221e7a708d71663f5eca6c238268757eb4352f3e9f46b1029d33ab4e53fd8a" # -- UI container image pull policy - pullPolicy: Always + pullPolicy: IfNotPresent # -- Override the full image name for development purposes fullImageName: null resources: From 3de1688986251518fe153902efe2634d318f77cd Mon Sep 17 00:00:00 2001 From: Ishaan Mittal Date: Fri, 11 Oct 2024 03:44:54 +0530 Subject: [PATCH 3/9] nit fix --- charts/opencost/templates/clusterrolebinding.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/charts/opencost/templates/clusterrolebinding.yaml b/charts/opencost/templates/clusterrolebinding.yaml index fc431c7..88ca128 100644 --- a/charts/opencost/templates/clusterrolebinding.yaml +++ b/charts/opencost/templates/clusterrolebinding.yaml @@ -17,7 +17,7 @@ subjects: namespace: {{ include "opencost.namespace" . }} --- {{- end }} -{{- if .Values.opencost.prometheus.createPrometheusClusterRoleBinding }} +{{- if .Values.opencost.prometheus.createMonitoringClusterRoleBinding }} apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding metadata: From 7d8771b449f8af78f32c297b954fb69ab3bca9b3 Mon Sep 17 00:00:00 2001 From: Ishaan Mittal Date: Thu, 24 Oct 2024 01:42:06 +0530 Subject: [PATCH 4/9] update env var name --- charts/opencost/templates/deployment.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/charts/opencost/templates/deployment.yaml b/charts/opencost/templates/deployment.yaml index 9bb84a8..666e56a 100644 --- a/charts/opencost/templates/deployment.yaml +++ b/charts/opencost/templates/deployment.yaml @@ -167,7 +167,7 @@ spec: key: {{ .Values.opencost.prometheus.bearer_token_key }} {{- end }} {{- if .Values.opencost.prometheus.kubeRBACProxy }} - - name: ENABLE_KUBE_RBAC_PROXY + - name: KUBE_RBAC_PROXY_ENABLED value: {{ (quote .Values.opencost.prometheus.kubeRBACProxy) }} {{- end }} {{- if and .Values.opencost.exporter.persistence.enabled .Values.opencost.exporter.csv_path }} From 899264130eb56086518ca26c7dec3a927e2ef6e1 Mon Sep 17 00:00:00 2001 From: Ishaan Mittal Date: Thu, 5 Dec 2024 17:32:56 +0530 Subject: [PATCH 5/9] shift monitoring namespace to helm config and nit fixes --- charts/opencost/templates/_helpers.tpl | 2 +- charts/opencost/templates/clusterrolebinding.yaml | 5 ++++- charts/opencost/templates/deployment.yaml | 1 + .../templates/monitoring-role-binding-template.yaml | 2 +- charts/opencost/values.yaml | 10 ++++++---- 5 files changed, 13 insertions(+), 7 deletions(-) diff --git a/charts/opencost/templates/_helpers.tpl b/charts/opencost/templates/_helpers.tpl index d62d501..a9b5044 100644 --- a/charts/opencost/templates/_helpers.tpl +++ b/charts/opencost/templates/_helpers.tpl @@ -121,7 +121,7 @@ Check that either thanos external or internal is defined {{- end -}} {{/* - Verify if both kube-rbac-proxy and bearer token are set + Fail if both kube-rbac-proxy and bearer token are set }} {{- define "kubeRBACProxyBearerTokenCheck" -}} {{- if and (.Values.opencost.prometheus.kubeRBACProxy .Values.opencost.prometheus.bearer_token) }} diff --git a/charts/opencost/templates/clusterrolebinding.yaml b/charts/opencost/templates/clusterrolebinding.yaml index 88ca128..a8e5023 100644 --- a/charts/opencost/templates/clusterrolebinding.yaml +++ b/charts/opencost/templates/clusterrolebinding.yaml @@ -27,9 +27,12 @@ metadata: annotations: {{- toYaml . | nindent 4 }} {{- end }} roleRef: + # Grant the kubecost service account the cluster-monitoring-view role to enable it to query OpenShift Prometheus. + # This is necessary for Kubecost to get access and query the in-cluster Prometheus instance using its service account token. + # https://docs.redhat.com/en/documentation/openshift_container_platform/4.2/html/monitoring/cluster-monitoring#monitoring-accessing-prometheus-alerting-ui-grafana-using-the-web-console_accessing-prometheus apiGroup: rbac.authorization.k8s.io kind: ClusterRole - name: cluster-monitoring-operator + name: cluster-monitoring-view subjects: - kind: ServiceAccount name: {{ template "opencost.serviceAccountName" . }} diff --git a/charts/opencost/templates/deployment.yaml b/charts/opencost/templates/deployment.yaml index 666e56a..efa828e 100644 --- a/charts/opencost/templates/deployment.yaml +++ b/charts/opencost/templates/deployment.yaml @@ -1,4 +1,5 @@ {{- include "isPrometheusConfigValid" . }} +{{- include "kubeRBACProxyBearerTokenCheck" . }} apiVersion: apps/v1 kind: Deployment metadata: diff --git a/charts/opencost/templates/monitoring-role-binding-template.yaml b/charts/opencost/templates/monitoring-role-binding-template.yaml index ef71c99..45141e8 100644 --- a/charts/opencost/templates/monitoring-role-binding-template.yaml +++ b/charts/opencost/templates/monitoring-role-binding-template.yaml @@ -7,7 +7,7 @@ metadata: subjects: - kind: ServiceAccount name: {{ .Values.opencost.prometheus.monitoringServiceAccountName | quote }} - namespace: openshift-monitoring + namespace: {{ .Values.opencost.prometheus.monitoringServiceAccountNamespace | quote }} roleRef: kind: Role name: {{ include "opencost.fullname" . }}-reader diff --git a/charts/opencost/values.yaml b/charts/opencost/values.yaml index 47ed3e1..64be53b 100644 --- a/charts/opencost/values.yaml +++ b/charts/opencost/values.yaml @@ -335,13 +335,15 @@ opencost: bearer_token_key: DB_BEARER_TOKEN # -- If true, opencost will use kube-rbac-proxy to authenticate with in cluster Prometheus for openshift kubeRBACProxy: false - # -- If true, opencost will create a ClusterRoleBinding to allow using in-cluster Prometheus for openshift + # OPTIONAL. The following configs only to be enabled when using a Prometheus instance already installed in the cluster. + # -- If true, opencost will create a ClusterRoleBinding to grant the Kubecost serviceaccount access to query Prometheus. createMonitoringClusterRoleBinding: false - # -- If true, opencost will create a role and role binding to allow in-cluster prometheus to list and watch resources - # This will be necessary if you are not using bundled prometheus and need to add scrape config for resources. + # -- If true, opencost will rreate a Role and Role Binding to allow Prometheus to list and watch Kubecost resources. createMonitoringResourceReaderRoleBinding: false - # -- Name of the service account to bind to the ClusterRoleBinding + # -- Name of the Prometheus serviceaccount to bind to the Resource Reader Role Binding. monitoringServiceAccountName: prometheus-k8s + # -- Namespace of the Prometheus serviceaccount to bind to the Resource Reader Role Binding. + monitoringServiceAccountNamespace: openshift-monitoring external: # -- Use external Prometheus (eg. Grafana Cloud) enabled: false From 9559c736c7a49872c35470ed0de862103903f903 Mon Sep 17 00:00:00 2001 From: Ishaan Mittal Date: Fri, 6 Dec 2024 22:14:26 +0530 Subject: [PATCH 6/9] fix check errors --- charts/opencost/templates/_helpers.tpl | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/charts/opencost/templates/_helpers.tpl b/charts/opencost/templates/_helpers.tpl index a9b5044..8e5e9a5 100644 --- a/charts/opencost/templates/_helpers.tpl +++ b/charts/opencost/templates/_helpers.tpl @@ -122,10 +122,10 @@ Check that either thanos external or internal is defined {{/* Fail if both kube-rbac-proxy and bearer token are set -}} +*/}} {{- define "kubeRBACProxyBearerTokenCheck" -}} -{{- if and (.Values.opencost.prometheus.kubeRBACProxy .Values.opencost.prometheus.bearer_token) }} - {{- fail "\n\nBoth kubeRBACProxy and bearer_token are set. Please specify only one." -}} +{{- if and .Values.opencost.prometheus.kubeRBACProxy .Values.opencost.prometheus.bearer_token }} + {{- fail "Both kubeRBACProxy and bearer_token are set. Please specify only one." -}} {{- end -}} {{- end -}} From 8bcbf0a14177ff56cf0dead84f652387a4e0ec6f Mon Sep 17 00:00:00 2001 From: Ishaan Mittal Date: Fri, 13 Dec 2024 11:37:24 +0530 Subject: [PATCH 7/9] Apply suggestions from code review Co-authored-by: Thomas Nguyen Signed-off-by: Ishaan Mittal --- charts/opencost/templates/clusterrolebinding.yaml | 4 ++-- charts/opencost/values.yaml | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/charts/opencost/templates/clusterrolebinding.yaml b/charts/opencost/templates/clusterrolebinding.yaml index a8e5023..7cdd7f0 100644 --- a/charts/opencost/templates/clusterrolebinding.yaml +++ b/charts/opencost/templates/clusterrolebinding.yaml @@ -27,8 +27,8 @@ metadata: annotations: {{- toYaml . | nindent 4 }} {{- end }} roleRef: - # Grant the kubecost service account the cluster-monitoring-view role to enable it to query OpenShift Prometheus. - # This is necessary for Kubecost to get access and query the in-cluster Prometheus instance using its service account token. + # Grant the OpenCost ServiceAccount the cluster-monitoring-view role to enable it to query a KUBE_RBAC_PROXY enabled Prometheus. + # This is necessary for OpenCost to get access and query the in-cluster Prometheus instance using its service account token. # https://docs.redhat.com/en/documentation/openshift_container_platform/4.2/html/monitoring/cluster-monitoring#monitoring-accessing-prometheus-alerting-ui-grafana-using-the-web-console_accessing-prometheus apiGroup: rbac.authorization.k8s.io kind: ClusterRole diff --git a/charts/opencost/values.yaml b/charts/opencost/values.yaml index 385923c..920395b 100644 --- a/charts/opencost/values.yaml +++ b/charts/opencost/values.yaml @@ -336,9 +336,9 @@ opencost: # -- If true, opencost will use kube-rbac-proxy to authenticate with in cluster Prometheus for openshift kubeRBACProxy: false # OPTIONAL. The following configs only to be enabled when using a Prometheus instance already installed in the cluster. - # -- If true, opencost will create a ClusterRoleBinding to grant the Kubecost serviceaccount access to query Prometheus. + # -- If true, the helm chart will create a ClusterRoleBinding to grant the OpenCost ServiceAccount access to query Prometheus. createMonitoringClusterRoleBinding: false - # -- If true, opencost will rreate a Role and Role Binding to allow Prometheus to list and watch Kubecost resources. + # -- If true, create a Role and RoleBinding to allow Prometheus to list and watch OpenCost resources. createMonitoringResourceReaderRoleBinding: false # -- Name of the Prometheus serviceaccount to bind to the Resource Reader Role Binding. monitoringServiceAccountName: prometheus-k8s From ab0b4cf15f7b09bcefea7d313f80648e36ad8bbf Mon Sep 17 00:00:00 2001 From: Ishaan Mittal Date: Mon, 16 Dec 2024 10:10:56 +0530 Subject: [PATCH 8/9] bump chart version --- charts/opencost/Chart.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/charts/opencost/Chart.yaml b/charts/opencost/Chart.yaml index 55a492f..49cde2a 100755 --- a/charts/opencost/Chart.yaml +++ b/charts/opencost/Chart.yaml @@ -9,7 +9,7 @@ keywords: - finops - monitoring - opencost -version: 1.42.3 +version: 1.42.4 maintainers: - name: mattray url: https://mattray.dev From 2d4865477a6d55d9cc3d3b88d3d62e3dccdb256a Mon Sep 17 00:00:00 2001 From: Ishaan Mittal Date: Thu, 19 Dec 2024 02:35:44 +0530 Subject: [PATCH 9/9] bump minor version instead --- charts/opencost/Chart.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/charts/opencost/Chart.yaml b/charts/opencost/Chart.yaml index 49cde2a..2b5e2cd 100755 --- a/charts/opencost/Chart.yaml +++ b/charts/opencost/Chart.yaml @@ -9,7 +9,7 @@ keywords: - finops - monitoring - opencost -version: 1.42.4 +version: 1.43.0 maintainers: - name: mattray url: https://mattray.dev