Merge pull request #6 from hasura/ndc-bigquery

Add ndc-bigquery Helm chart

CONNECTORS

@@ -1,3 +1,4 @@
+ndc-bigquery
 ndc-connector-oracle
 ndc-connector-phoenix
 ndc-elasticsearch

charts/ndc-bigquery/Chart.yaml

@@ -0,0 +1,29 @@
+apiVersion: v2
+name: ndc-bigquery
+description: (DDN) A Helm chart for deploying ndc-bigquery
+
+# A chart can be either an 'application' or a 'library' chart.
+#
+# Application charts are a collection of templates that can be packaged into versioned archives
+# to be deployed.
+#
+# Library charts provide useful utilities or functions for the chart developer. They're included as
+# a dependency of application charts to inject those utilities and functions into the rendering
+# pipeline. Library charts do not define any templates and therefore cannot be deployed.
+type: application
+
+# This is the chart version. This version number should be incremented each time you make changes
+# to the chart and its templates, including the app version.
+# Versions are expected to follow Semantic Versioning (https://semver.org/)
+version: v2024.12.05
+
+# This is the version number of the application being deployed. This version number should be
+# incremented each time you make changes to the application. Versions are not expected to
+# follow Semantic Versioning. They should reflect the version the application is using.
+# It is recommended to use it with quotes.
+appVersion: "3.0.0"
+
+dependencies:
+- name: common
+  version: 0.0.8
+  repository: oci://us-west1-docker.pkg.dev/hasura-ee/helm-charts

charts/ndc-bigquery/README.md

@@ -0,0 +1,77 @@
+# Ndc-bigquery Helm Chart
+
+This chart deploys the ndc-bigquery connector. Refer to the pre-requisites section [here](../../README.md#get-started)
+
+## Install Chart
+
+See all [configuration](#parameters) below.
+
+```bash
+# EXAMPLES:
+
+# helm template and apply manifests via kubectl (example)
+helm template <release-name> \
+  --set namespace="default" \
+  --set image.repository="my_repo/ndc-bigquery" \
+  --set image.tag="my_custom_image_tag" \
+  --set connectorEnvVars.HASURA_BIGQUERY_SERVICE_KEY="bq_service_key" \
+  --set connectorEnvVars.HASURA_BIGQUERY_PROJECT_ID="bq_project_id" \
+  --set connectorEnvVars.HASURA_BIGQUERY_DATASET_ID="bq_dataset_id" \
+  --set connectorEnvVars.HASURA_SERVICE_TOKEN_SECRET="token" \
+  --set dataPlane.id="data_plane_id" \
+  --set dataPlane.key="data_plane_key" \
+  hasura-ddn/ndc-bigquery | kubectl apply -f-
+
+# helm upgrade --install (pass configuration via command line)
+helm upgrade --install <release-name> \
+  --set namespace="default" \
+  --set image.repository="my_repo/ndc-bigquery" \
+  --set image.tag="my_custom_image_tag" \
+  --set connectorEnvVars.HASURA_BIGQUERY_SERVICE_KEY="bq_service_key" \
+  --set connectorEnvVars.HASURA_BIGQUERY_PROJECT_ID="bq_project_id" \
+  --set connectorEnvVars.HASURA_BIGQUERY_DATASET_ID="bq_dataset_id" \
+  --set connectorEnvVars.HASURA_SERVICE_TOKEN_SECRET="token" \
+  --set dataPlane.id="data_plane_id" \
+  --set dataPlane.key="data_plane_key" \
+  hasura-ddn/ndc-bigquery
+```
+
+## Connector ENV Inputs
+
+| Name                                              | Description                                                                                                | Value                           |
+| ------------------------------------------------- | ---------------------------------------------------------------------------------------------------------- | ------------------------------- |
+| `connectorEnvVars.HASURA_SERVICE_TOKEN_SECRET`    | Hasura Service Token Secret (Required)                                                                     | `""`                            |
+| `connectorEnvVars.HASURA_BIGQUERY_SERVICE_KEY`    | The BigQuery service key (Required)                                                                        | `""`                            |
+| `connectorEnvVars.HASURA_BIGQUERY_PROJECT_ID`     | The BigQuery project ID/name (Required)                                                                    | `""`                            |
+| `connectorEnvVars.HASURA_BIGQUERY_DATASET_ID`     | The BigQuery dataset ID/name (Required)                                                                    | `""`                            |
+
+## Additional Parameters
+
+| Name                                              | Description                                                                                                | Value                           |
+| ------------------------------------------------- | ---------------------------------------------------------------------------------------------------------- | ------------------------------- |
+| `namespace`                                       | Namespace to deploy to                                                                                     | `"default"`                     |
+| `image.repository`                                | Image repository containing custom created ndc-bigquery                                                    | `""`                            |
+| `image.tag`                                       | Image tag to use for custom created ndc-bigquery                                                           | `""`                            |
+| `image.pullPolicy`                                | Image pull policy                                                                                          | `Always`                        |
+| `image.otelCollectorRepository`                   | OTEL collector image repository                                                                            | `otel/opentelemetry-collector`  |
+| `image.otelCollectorTag`                          | OTEL collector image tag                                                                                   | `0.104.0`                       |
+| `observability.enabled`                           | Deploy OTEL collector as sidecar                                                                           | `true`                          |
+| `dataPlane.id`                                    | Data Plane ID (Required when observability.enabled is set to true)                                         | `""`                         |
+| `dataPlane.key`                                   | Data Plane Key (Required when observability.enabled is set to true)                                        | `""`                         |
+| `controlPlane.otlpEndpoint`                       | OTEL endpoint under Hasura                                                                                 | `"https://gateway.otlp.hasura.io:443"`                         |
+| `controlPlane.oauthTokenEndpoint`                 | Oauth Token URL                                                                                            | `"https://ddn-oauth.pro.hasura.io/oauth2/token"`                         |
+| `extraVolumes`                                    | Optionally specify extra list of additional volumes for the ndc-bigquery pod                               | `[]`                            |
+| `extraContainers`                                 | Optionally specify extra list of additional containers for the ndc-bigquery pod                            | `[]`                               |                               |
+| `resources`                                       | Resource requests and limits of ndc-bigquery container                                                     | `{}`                            |
+| `env`                                             | Env variable section for ndc-bigquery                                                                      | `[]`                            |
+| `replicas`                                        | Replicas setting for pod                                                                                   | `1`                             |
+| `wsInactiveExpiryMins`                            | To be documented                                                                                           | `1`                             |
+| `securityContext`                                 | Define privilege and access control settings for a Pod or Container                                        | `{}`                            |
+| `healthChecks.enabled`                            | Enable health check for ndc-bigquery container                                                             | `false`                         |
+| `healthChecks.livenessProbePath`                  | Health check liveness Probe path ndc-bigquery container                                                    | `"/healthz"`                    |
+| `healthChecks.readinessProbePath`                 | Health check readiness Probe path ndc-bigquery container                                                   | `"/healthz"`                    |
+| `hpa.enabled`                                     | Enable HPA for ndc-bigquery.  Ensure metrics cluster is configured when enabling                           | `false`                         |
+| `hpa.minReplicas`                                 | minReplicas setting for HPA                                                                                | `2`                             |
+| `hpa.maxReplicas`                                 | maxReplicas setting for HPA                                                                                | `4`                             |
+| `hpa.metrics.resource.name`                       | Resource name to autoscale on                                                                              | ``                              |
+| `hpa.metrics.resource.target.averageUtilization`  | Utilization target on specific resource type                                                               | ``                              |

charts/ndc-bigquery/templates/NOTES.txt

@@ -0,0 +1,22 @@
+Ndc-bigquery Helm Chart Deployment
+
+1. Deployment Information:
+   - Release Name: {{ .Release.Name }}
+   - Namespace: {{ template "common.namespace" . }}
+   - Chart Name: {{ .Chart.Name }}
+   - Chart Version: {{ .Chart.Version }}
+
+2. Service Information:
+   - Service Name: {{ template "common.name" . }}
+   - Service Port: {{ .Values.httpPort }}
+
+3. Useful Commands:
+   - Check the Deployment Status:
+     helm status {{ .Release.Name }}
+
+   - Get Detailed Information about the Deployment:
+     helm get all {{ .Release.Name }}
+
+4. Clean Up:
+   - To uninstall/delete the deployment, run:
+     helm uninstall {{ .Release.Name }}

charts/ndc-bigquery/templates/deployment.yaml

@@ -0,0 +1,2 @@
+# deployment.yaml
+{{- template "common.deployment" . -}}

charts/ndc-bigquery/templates/hpa.yaml

@@ -0,0 +1,2 @@
+# hpa.yaml
+{{- template "common.hpa" . -}}

charts/ndc-bigquery/templates/imagepullsecret.yaml

@@ -0,0 +1,16 @@
+{{- if ((.Values.global).dataPlane).deployImagePullSecret -}}
+{{- with .Values.secrets }}
+{{- if .imagePullSecret -}}
+---
+apiVersion: v1
+kind: Secret
+metadata:
+  name: hasura-image-pull
+  namespace: {{ template "common.namespace" $ }}
+type: kubernetes.io/dockerconfigjson
+data:
+  .dockerconfigjson: |
+{{- toJson .imagePullSecret | b64enc | nindent 4 }}
+{{- end }}
+{{- end }}
+{{- end -}}

charts/ndc-bigquery/templates/secret.yaml

@@ -0,0 +1,13 @@
+apiVersion: v1
+kind: Secret
+metadata:
+  name: {{ printf "%s-secret" (include "common.name" .) }}
+  namespace: {{ template "common.namespace" $ }}
+data:
+  HASURA_SERVICE_TOKEN_SECRET: {{ required "Error: .Values.connectorEnvVars.HASURA_SERVICE_TOKEN_SECRET is required!" .Values.connectorEnvVars.HASURA_SERVICE_TOKEN_SECRET | b64enc | quote }}
+  HASURA_BIGQUERY_SERVICE_KEY: {{ required "Error: .Values.connectorEnvVars.HASURA_BIGQUERY_SERVICE_KEY is required!" .Values.connectorEnvVars.HASURA_BIGQUERY_SERVICE_KEY | b64enc | quote }}
+  HASURA_BIGQUERY_PROJECT_ID: {{ required "Error: .Values.connectorEnvVars.HASURA_BIGQUERY_PROJECT_ID is required!" .Values.connectorEnvVars.HASURA_BIGQUERY_PROJECT_ID | b64enc | quote }}
+  HASURA_BIGQUERY_DATASET_ID: {{ required "Error: .Values.connectorEnvVars.HASURA_BIGQUERY_DATASET_ID is required!" .Values.connectorEnvVars.HASURA_BIGQUERY_DATASET_ID | b64enc | quote }}
+  {{- if .Values.observability.enabled }}
+  otel-collector-config.yaml: {{ (tpl .Values.otel.config .) | b64enc | quote }}
+  {{- end }}

charts/ndc-bigquery/templates/service.yaml

@@ -0,0 +1,2 @@
+# service.yaml
+{{- template "common.service" . -}}

charts/ndc-bigquery/templates/serviceaccount.yaml

@@ -0,0 +1,2 @@
+# serviceaccount.yaml
+{{- template "common.serviceaccount" . -}}

charts/ndc-bigquery/values.yaml

@@ -0,0 +1,170 @@
+namespace: "default"
+useReleaseName: true
+
+additionalAnnotations: |
+  checksum/config: {{ include (print $.Template.BasePath "/secret.yaml") . | sha256sum }}
+
+# Container Configs
+image:
+  repository: ""
+  tag: ""
+  pullPolicy: Always
+  otelCollectorRepository: otel/opentelemetry-collector-contrib
+  otelCollectorTag: 0.104.0
+replicas: "1"
+wsInactiveExpiryMins: "1"
+securityContext:
+  runAsNonRoot: true
+  runAsGroup: 1000
+  runAsUser: 100
+  fsGroup: 1000
+
+healthChecks:
+  enabled: false
+  livenessProbePath: "/healthz"
+  readinessProbePath: "/healthz"
+
+hpa:
+  enabled: false
+  minReplicas: 2
+  maxReplicas: 4
+  metrics:
+    - type: Resource
+      resource:
+        name: cpu
+        target:
+          type: Utilization
+          averageUtilization: 50  # Target 50% CPU utilization per pod
+    - type: Resource
+      resource:
+        name: memory
+        target:
+          type: Utilization
+          averageUtilization: 80  # Target 80% memory utilization per pod
+
+# Observability defaults are tuned for Hasura hosted Control Plane
+# Enable observability
+observability:
+  enabled: true
+
+# Required (when observability.enabled is set to true)
+dataPlane:
+  id: ""
+  key: ""
+
+controlPlane:
+  otlpEndpoint: https://gateway.otlp.hasura.io:443
+  oauthTokenEndpoint: https://ddn-oauth.pro.hasura.io/oauth2/token
+
+otel:
+  config: |
+    extensions:
+      oauth2client:
+        client_id: {{ required "Error: .Values.dataPlane.id is required!" .Values.dataPlane.id }}
+        client_secret: {{ required "Error: .Values.dataPlane.key is required!" .Values.dataPlane.key }}
+        token_url: {{ .Values.controlPlane.oauthTokenEndpoint }}
+        scopes: ["opentelemetry:write"]
+        endpoint_params:
+          audience: https://ddn.hasura.io
+    exporters:
+      otlp/oauth:
+        endpoint: {{ .Values.controlPlane.otlpEndpoint }}
+        auth:
+          authenticator: oauth2client
+    processors:
+      batch: {}
+    receivers:
+      otlp:
+        protocols:
+          grpc:
+            endpoint: 0.0.0.0:4317
+          http:
+            endpoint: 0.0.0.0:4318
+    service:
+      extensions: [oauth2client]
+      pipelines:
+        logs:
+          exporters:
+            - otlp/oauth
+          processors:
+            - batch
+          receivers:
+            - otlp
+        metrics:
+          exporters:
+            - otlp/oauth
+          processors:
+            - batch
+          receivers:
+            - otlp
+        traces:
+          exporters:
+            - otlp/oauth
+          processors:
+            - batch
+          receivers:
+            - otlp
+
+extraVolumes: |
+  {{- if .Values.observability.enabled }}
+  - name: otel-config
+    secret:
+      secretName: {{ printf "%s-secret" (include "common.name" .) }}
+      items:
+        - key: otel-collector-config.yaml
+          path: otel-collector-config.yaml
+  {{- end }}
+
+extraContainers: |
+  {{- if .Values.observability.enabled }}
+  - name: "otel-collector"
+    command:
+      - --config=/etc/otel-collector-config.yaml
+    command:
+    image: {{ template "common.image" (dict "Values" $.Values "repository" .Values.image.otelCollectorRepository "tag" .Values.image.otelCollectorTag) }}
+    env:
+    - name: "OTEL_ENDPOINT"
+      value: "{{ .Values.controlPlane.otlpEndpoint }}"
+    volumeMounts:
+      - name: otel-config
+        mountPath: "/etc/otelcol-contrib/config.yaml"
+        subPath: otel-collector-config.yaml
+  {{- end }}
+
+resources: |
+  requests:
+    cpu: "500m"
+    memory: "500Mi"
+  limits:
+    cpu: "1"
+    memory: "1Gi"
+
+connectorEnvVars:
+  HASURA_SERVICE_TOKEN_SECRET: ""
+  HASURA_BIGQUERY_SERVICE_KEY: ""
+  HASURA_BIGQUERY_PROJECT_ID: ""
+  HASURA_BIGQUERY_DATASET_ID: ""
+
+env: |
+  - name: HASURA_SERVICE_TOKEN_SECRET
+    valueFrom:
+      secretKeyRef:
+        key: HASURA_SERVICE_TOKEN_SECRET
+        name: {{ printf "%s-secret" (include "common.name" .) }}
+  - name: HASURA_BIGQUERY_SERVICE_KEY
+    valueFrom:
+      secretKeyRef:
+        key: HASURA_BIGQUERY_SERVICE_KEY
+        name: {{ printf "%s-secret" (include "common.name" .) }}
+  - name: HASURA_BIGQUERY_PROJECT_ID
+    valueFrom:
+      secretKeyRef:
+        key: HASURA_BIGQUERY_PROJECT_ID
+        name: {{ printf "%s-secret" (include "common.name" .) }}
+  - name: HASURA_BIGQUERY_DATASET_ID
+    valueFrom:
+      secretKeyRef:
+        key: HASURA_BIGQUERY_DATASET_ID
+        name: {{ printf "%s-secret" (include "common.name" .) }}
+  - name: OTEL_EXPORTER_OTLP_ENDPOINT
+    value: http://localhost:4317