diff --git a/CONNECTORS b/CONNECTORS index fa97d6f..bc5f2d0 100644 --- a/CONNECTORS +++ b/CONNECTORS @@ -1,3 +1,4 @@ +ndc-bigquery ndc-connector-oracle ndc-connector-phoenix ndc-elasticsearch diff --git a/charts/ndc-bigquery/Chart.yaml b/charts/ndc-bigquery/Chart.yaml new file mode 100644 index 0000000..b7500dd --- /dev/null +++ b/charts/ndc-bigquery/Chart.yaml @@ -0,0 +1,29 @@ +apiVersion: v2 +name: ndc-bigquery +description: (DDN) A Helm chart for deploying ndc-bigquery + +# A chart can be either an 'application' or a 'library' chart. +# +# Application charts are a collection of templates that can be packaged into versioned archives +# to be deployed. +# +# Library charts provide useful utilities or functions for the chart developer. They're included as +# a dependency of application charts to inject those utilities and functions into the rendering +# pipeline. Library charts do not define any templates and therefore cannot be deployed. +type: application + +# This is the chart version. This version number should be incremented each time you make changes +# to the chart and its templates, including the app version. +# Versions are expected to follow Semantic Versioning (https://semver.org/) +version: v2024.12.05 + +# This is the version number of the application being deployed. This version number should be +# incremented each time you make changes to the application. Versions are not expected to +# follow Semantic Versioning. They should reflect the version the application is using. +# It is recommended to use it with quotes. +appVersion: "3.0.0" + +dependencies: +- name: common + version: 0.0.8 + repository: oci://us-west1-docker.pkg.dev/hasura-ee/helm-charts diff --git a/charts/ndc-bigquery/README.md b/charts/ndc-bigquery/README.md new file mode 100644 index 0000000..8327e27 --- /dev/null +++ b/charts/ndc-bigquery/README.md @@ -0,0 +1,77 @@ +# Ndc-bigquery Helm Chart + +This chart deploys the ndc-bigquery connector. Refer to the pre-requisites section [here](../../README.md#get-started) + +## Install Chart + +See all [configuration](#parameters) below. + +```bash +# EXAMPLES: + +# helm template and apply manifests via kubectl (example) +helm template \ + --set namespace="default" \ + --set image.repository="my_repo/ndc-bigquery" \ + --set image.tag="my_custom_image_tag" \ + --set connectorEnvVars.HASURA_BIGQUERY_SERVICE_KEY="bq_service_key" \ + --set connectorEnvVars.HASURA_BIGQUERY_PROJECT_ID="bq_project_id" \ + --set connectorEnvVars.HASURA_BIGQUERY_DATASET_ID="bq_dataset_id" \ + --set connectorEnvVars.HASURA_SERVICE_TOKEN_SECRET="token" \ + --set dataPlane.id="data_plane_id" \ + --set dataPlane.key="data_plane_key" \ + hasura-ddn/ndc-bigquery | kubectl apply -f- + +# helm upgrade --install (pass configuration via command line) +helm upgrade --install \ + --set namespace="default" \ + --set image.repository="my_repo/ndc-bigquery" \ + --set image.tag="my_custom_image_tag" \ + --set connectorEnvVars.HASURA_BIGQUERY_SERVICE_KEY="bq_service_key" \ + --set connectorEnvVars.HASURA_BIGQUERY_PROJECT_ID="bq_project_id" \ + --set connectorEnvVars.HASURA_BIGQUERY_DATASET_ID="bq_dataset_id" \ + --set connectorEnvVars.HASURA_SERVICE_TOKEN_SECRET="token" \ + --set dataPlane.id="data_plane_id" \ + --set dataPlane.key="data_plane_key" \ + hasura-ddn/ndc-bigquery +``` + +## Connector ENV Inputs + +| Name | Description | Value | +| ------------------------------------------------- | ---------------------------------------------------------------------------------------------------------- | ------------------------------- | +| `connectorEnvVars.HASURA_SERVICE_TOKEN_SECRET` | Hasura Service Token Secret (Required) | `""` | +| `connectorEnvVars.HASURA_BIGQUERY_SERVICE_KEY` | The BigQuery service key (Required) | `""` | +| `connectorEnvVars.HASURA_BIGQUERY_PROJECT_ID` | The BigQuery project ID/name (Required) | `""` | +| `connectorEnvVars.HASURA_BIGQUERY_DATASET_ID` | The BigQuery dataset ID/name (Required) | `""` | + +## Additional Parameters + +| Name | Description | Value | +| ------------------------------------------------- | ---------------------------------------------------------------------------------------------------------- | ------------------------------- | +| `namespace` | Namespace to deploy to | `"default"` | +| `image.repository` | Image repository containing custom created ndc-bigquery | `""` | +| `image.tag` | Image tag to use for custom created ndc-bigquery | `""` | +| `image.pullPolicy` | Image pull policy | `Always` | +| `image.otelCollectorRepository` | OTEL collector image repository | `otel/opentelemetry-collector` | +| `image.otelCollectorTag` | OTEL collector image tag | `0.104.0` | +| `observability.enabled` | Deploy OTEL collector as sidecar | `true` | +| `dataPlane.id` | Data Plane ID (Required when observability.enabled is set to true) | `""` | +| `dataPlane.key` | Data Plane Key (Required when observability.enabled is set to true) | `""` | +| `controlPlane.otlpEndpoint` | OTEL endpoint under Hasura | `"https://gateway.otlp.hasura.io:443"` | +| `controlPlane.oauthTokenEndpoint` | Oauth Token URL | `"https://ddn-oauth.pro.hasura.io/oauth2/token"` | +| `extraVolumes` | Optionally specify extra list of additional volumes for the ndc-bigquery pod | `[]` | +| `extraContainers` | Optionally specify extra list of additional containers for the ndc-bigquery pod | `[]` | | +| `resources` | Resource requests and limits of ndc-bigquery container | `{}` | +| `env` | Env variable section for ndc-bigquery | `[]` | +| `replicas` | Replicas setting for pod | `1` | +| `wsInactiveExpiryMins` | To be documented | `1` | +| `securityContext` | Define privilege and access control settings for a Pod or Container | `{}` | +| `healthChecks.enabled` | Enable health check for ndc-bigquery container | `false` | +| `healthChecks.livenessProbePath` | Health check liveness Probe path ndc-bigquery container | `"/healthz"` | +| `healthChecks.readinessProbePath` | Health check readiness Probe path ndc-bigquery container | `"/healthz"` | +| `hpa.enabled` | Enable HPA for ndc-bigquery. Ensure metrics cluster is configured when enabling | `false` | +| `hpa.minReplicas` | minReplicas setting for HPA | `2` | +| `hpa.maxReplicas` | maxReplicas setting for HPA | `4` | +| `hpa.metrics.resource.name` | Resource name to autoscale on | `` | +| `hpa.metrics.resource.target.averageUtilization` | Utilization target on specific resource type | `` | \ No newline at end of file diff --git a/charts/ndc-bigquery/templates/NOTES.txt b/charts/ndc-bigquery/templates/NOTES.txt new file mode 100644 index 0000000..cf27fb6 --- /dev/null +++ b/charts/ndc-bigquery/templates/NOTES.txt @@ -0,0 +1,22 @@ +Ndc-bigquery Helm Chart Deployment + +1. Deployment Information: + - Release Name: {{ .Release.Name }} + - Namespace: {{ template "common.namespace" . }} + - Chart Name: {{ .Chart.Name }} + - Chart Version: {{ .Chart.Version }} + +2. Service Information: + - Service Name: {{ template "common.name" . }} + - Service Port: {{ .Values.httpPort }} + +3. Useful Commands: + - Check the Deployment Status: + helm status {{ .Release.Name }} + + - Get Detailed Information about the Deployment: + helm get all {{ .Release.Name }} + +4. Clean Up: + - To uninstall/delete the deployment, run: + helm uninstall {{ .Release.Name }} \ No newline at end of file diff --git a/charts/ndc-bigquery/templates/deployment.yaml b/charts/ndc-bigquery/templates/deployment.yaml new file mode 100644 index 0000000..96fb6e0 --- /dev/null +++ b/charts/ndc-bigquery/templates/deployment.yaml @@ -0,0 +1,2 @@ +# deployment.yaml +{{- template "common.deployment" . -}} diff --git a/charts/ndc-bigquery/templates/hpa.yaml b/charts/ndc-bigquery/templates/hpa.yaml new file mode 100644 index 0000000..f40ea32 --- /dev/null +++ b/charts/ndc-bigquery/templates/hpa.yaml @@ -0,0 +1,2 @@ +# hpa.yaml +{{- template "common.hpa" . -}} \ No newline at end of file diff --git a/charts/ndc-bigquery/templates/imagepullsecret.yaml b/charts/ndc-bigquery/templates/imagepullsecret.yaml new file mode 100644 index 0000000..679ba77 --- /dev/null +++ b/charts/ndc-bigquery/templates/imagepullsecret.yaml @@ -0,0 +1,16 @@ +{{- if ((.Values.global).dataPlane).deployImagePullSecret -}} +{{- with .Values.secrets }} +{{- if .imagePullSecret -}} +--- +apiVersion: v1 +kind: Secret +metadata: + name: hasura-image-pull + namespace: {{ template "common.namespace" $ }} +type: kubernetes.io/dockerconfigjson +data: + .dockerconfigjson: | +{{- toJson .imagePullSecret | b64enc | nindent 4 }} +{{- end }} +{{- end }} +{{- end -}} \ No newline at end of file diff --git a/charts/ndc-bigquery/templates/secret.yaml b/charts/ndc-bigquery/templates/secret.yaml new file mode 100644 index 0000000..d574cce --- /dev/null +++ b/charts/ndc-bigquery/templates/secret.yaml @@ -0,0 +1,13 @@ +apiVersion: v1 +kind: Secret +metadata: + name: {{ printf "%s-secret" (include "common.name" .) }} + namespace: {{ template "common.namespace" $ }} +data: + HASURA_SERVICE_TOKEN_SECRET: {{ required "Error: .Values.connectorEnvVars.HASURA_SERVICE_TOKEN_SECRET is required!" .Values.connectorEnvVars.HASURA_SERVICE_TOKEN_SECRET | b64enc | quote }} + HASURA_BIGQUERY_SERVICE_KEY: {{ required "Error: .Values.connectorEnvVars.HASURA_BIGQUERY_SERVICE_KEY is required!" .Values.connectorEnvVars.HASURA_BIGQUERY_SERVICE_KEY | b64enc | quote }} + HASURA_BIGQUERY_PROJECT_ID: {{ required "Error: .Values.connectorEnvVars.HASURA_BIGQUERY_PROJECT_ID is required!" .Values.connectorEnvVars.HASURA_BIGQUERY_PROJECT_ID | b64enc | quote }} + HASURA_BIGQUERY_DATASET_ID: {{ required "Error: .Values.connectorEnvVars.HASURA_BIGQUERY_DATASET_ID is required!" .Values.connectorEnvVars.HASURA_BIGQUERY_DATASET_ID | b64enc | quote }} + {{- if .Values.observability.enabled }} + otel-collector-config.yaml: {{ (tpl .Values.otel.config .) | b64enc | quote }} + {{- end }} \ No newline at end of file diff --git a/charts/ndc-bigquery/templates/service.yaml b/charts/ndc-bigquery/templates/service.yaml new file mode 100644 index 0000000..7356445 --- /dev/null +++ b/charts/ndc-bigquery/templates/service.yaml @@ -0,0 +1,2 @@ +# service.yaml +{{- template "common.service" . -}} \ No newline at end of file diff --git a/charts/ndc-bigquery/templates/serviceaccount.yaml b/charts/ndc-bigquery/templates/serviceaccount.yaml new file mode 100644 index 0000000..030a351 --- /dev/null +++ b/charts/ndc-bigquery/templates/serviceaccount.yaml @@ -0,0 +1,2 @@ +# serviceaccount.yaml +{{- template "common.serviceaccount" . -}} \ No newline at end of file diff --git a/charts/ndc-bigquery/values.yaml b/charts/ndc-bigquery/values.yaml new file mode 100644 index 0000000..ad21acb --- /dev/null +++ b/charts/ndc-bigquery/values.yaml @@ -0,0 +1,170 @@ +namespace: "default" +useReleaseName: true + +additionalAnnotations: | + checksum/config: {{ include (print $.Template.BasePath "/secret.yaml") . | sha256sum }} + +# Container Configs +image: + repository: "" + tag: "" + pullPolicy: Always + otelCollectorRepository: otel/opentelemetry-collector-contrib + otelCollectorTag: 0.104.0 +replicas: "1" +wsInactiveExpiryMins: "1" +securityContext: + runAsNonRoot: true + runAsGroup: 1000 + runAsUser: 100 + fsGroup: 1000 + +healthChecks: + enabled: false + livenessProbePath: "/healthz" + readinessProbePath: "/healthz" + +hpa: + enabled: false + minReplicas: 2 + maxReplicas: 4 + metrics: + - type: Resource + resource: + name: cpu + target: + type: Utilization + averageUtilization: 50 # Target 50% CPU utilization per pod + - type: Resource + resource: + name: memory + target: + type: Utilization + averageUtilization: 80 # Target 80% memory utilization per pod + +# Observability defaults are tuned for Hasura hosted Control Plane +# Enable observability +observability: + enabled: true + +# Required (when observability.enabled is set to true) +dataPlane: + id: "" + key: "" + +controlPlane: + otlpEndpoint: https://gateway.otlp.hasura.io:443 + oauthTokenEndpoint: https://ddn-oauth.pro.hasura.io/oauth2/token + +otel: + config: | + extensions: + oauth2client: + client_id: {{ required "Error: .Values.dataPlane.id is required!" .Values.dataPlane.id }} + client_secret: {{ required "Error: .Values.dataPlane.key is required!" .Values.dataPlane.key }} + token_url: {{ .Values.controlPlane.oauthTokenEndpoint }} + scopes: ["opentelemetry:write"] + endpoint_params: + audience: https://ddn.hasura.io + exporters: + otlp/oauth: + endpoint: {{ .Values.controlPlane.otlpEndpoint }} + auth: + authenticator: oauth2client + processors: + batch: {} + receivers: + otlp: + protocols: + grpc: + endpoint: 0.0.0.0:4317 + http: + endpoint: 0.0.0.0:4318 + service: + extensions: [oauth2client] + pipelines: + logs: + exporters: + - otlp/oauth + processors: + - batch + receivers: + - otlp + metrics: + exporters: + - otlp/oauth + processors: + - batch + receivers: + - otlp + traces: + exporters: + - otlp/oauth + processors: + - batch + receivers: + - otlp + +extraVolumes: | + {{- if .Values.observability.enabled }} + - name: otel-config + secret: + secretName: {{ printf "%s-secret" (include "common.name" .) }} + items: + - key: otel-collector-config.yaml + path: otel-collector-config.yaml + {{- end }} + +extraContainers: | + {{- if .Values.observability.enabled }} + - name: "otel-collector" + command: + - --config=/etc/otel-collector-config.yaml + command: + image: {{ template "common.image" (dict "Values" $.Values "repository" .Values.image.otelCollectorRepository "tag" .Values.image.otelCollectorTag) }} + env: + - name: "OTEL_ENDPOINT" + value: "{{ .Values.controlPlane.otlpEndpoint }}" + volumeMounts: + - name: otel-config + mountPath: "/etc/otelcol-contrib/config.yaml" + subPath: otel-collector-config.yaml + {{- end }} + +resources: | + requests: + cpu: "500m" + memory: "500Mi" + limits: + cpu: "1" + memory: "1Gi" + +connectorEnvVars: + HASURA_SERVICE_TOKEN_SECRET: "" + HASURA_BIGQUERY_SERVICE_KEY: "" + HASURA_BIGQUERY_PROJECT_ID: "" + HASURA_BIGQUERY_DATASET_ID: "" + +env: | + - name: HASURA_SERVICE_TOKEN_SECRET + valueFrom: + secretKeyRef: + key: HASURA_SERVICE_TOKEN_SECRET + name: {{ printf "%s-secret" (include "common.name" .) }} + - name: HASURA_BIGQUERY_SERVICE_KEY + valueFrom: + secretKeyRef: + key: HASURA_BIGQUERY_SERVICE_KEY + name: {{ printf "%s-secret" (include "common.name" .) }} + - name: HASURA_BIGQUERY_PROJECT_ID + valueFrom: + secretKeyRef: + key: HASURA_BIGQUERY_PROJECT_ID + name: {{ printf "%s-secret" (include "common.name" .) }} + - name: HASURA_BIGQUERY_DATASET_ID + valueFrom: + secretKeyRef: + key: HASURA_BIGQUERY_DATASET_ID + name: {{ printf "%s-secret" (include "common.name" .) }} + - name: OTEL_EXPORTER_OTLP_ENDPOINT + value: http://localhost:4317