feat(bootstrapMCPs): follow default globals (#520)

bootstrapMCPs * use default timezone * use default CLI version * simplify overrides
acryldata · Nov 12, 2024 · 6fe188a · 6fe188a
1 parent 6849c2c
commit 6fe188a
Show file tree

Hide file tree

Showing 4 changed files with 44 additions and 17 deletions.
diff --git a/charts/datahub/Chart.yaml b/charts/datahub/Chart.yaml
@@ -4,7 +4,7 @@ description: A Helm chart for DataHub
 type: application
 # This is the chart version. This version number should be incremented each time you make changes
 # to the chart and its templates, including the app version.
-version: 0.4.35
+version: 0.4.36
 # This is the version number of the application being deployed. This version number should be
 # incremented each time you make changes to the application.
 appVersion: 0.14.1

diff --git a/charts/datahub/templates/datahub-upgrade/_upgrade.tpl b/charts/datahub/templates/datahub-upgrade/_upgrade.tpl
@@ -192,19 +192,27 @@ Return the env variables for upgrade jobs
 datahubGC cron daily custom scheduling
 */}}
 {{- define "datahub.systemUpdate.datahubGC.dailyCronWindow" -}}
-{{- if .Values.datahubSystemUpdate.bootstrapMCPs.datahubGC.dailyCronWindow.enabled -}}
+{{- if hasKey (index .Values.datahubSystemUpdate.bootstrapMCPs.datahubGC.values "schedule" | default dict) "interval" -}}
 schedule:
-  interval: {{ printf "%d %s * * * " (mod (randNumeric 2) 60) (include "randomHourInRange" (list .Values.datahubSystemUpdate.bootstrapMCPs.datahubGC.dailyCronWindow.startHour .Values.datahubSystemUpdate.bootstrapMCPs.datahubGC.dailyCronWindow.endHour)) }}
+  interval: {{ .Values.datahubSystemUpdate.bootstrapMCPs.datahubGC.values.schedule.interval | quote }}
 {{- else }}
 schedule:
-  interval: {{ .Values.datahubSystemUpdate.bootstrapMCPs.datahubGC.values.schedule.interval | quote }}
+  interval: {{ printf "%d %s * * * " (mod (randNumeric 2) 60) (include "randomHourInRange" (list .Values.datahubSystemUpdate.bootstrapMCPs.datahubGC.dailyCronWindow.startHour .Values.datahubSystemUpdate.bootstrapMCPs.datahubGC.dailyCronWindow.endHour)) }}
 {{- end }}
 {{- end -}}
 
 {{/*
-datahubGC timezone
+  timezone
 */}}
-{{- define "datahub.systemUpdate.datahubGC.timezone" -}}
+{{- define "datahub.bootstrapMCPs.default.schedule.timezone" -}}
 schedule:
-  timezone: {{ .Values.global.datahub.timezone | default .Values.datahubSystemUpdate.bootstrapMCPs.datahubGC.values.schedule.timezone | quote }}
+  timezone: {{ .Values.global.datahub.timezone | quote }}
+{{- end -}}
+
+{{/*
+  default cli version
+*/}}
+{{- define "datahub.bootstrapMCPs.default.ingestion.version" -}}
+ingestion:
+  version: {{ .Values.global.datahub.managed_ingestion.defaultCliVersion | quote }}
 {{- end -}}
diff --git a/charts/datahub/templates/datahub-upgrade/datahub-system-update-job.yml b/charts/datahub/templates/datahub-upgrade/datahub-system-update-job.yml
@@ -135,7 +135,14 @@ spec:
               value: {{ . | quote }}
             {{- end }}
             {{- range $k, $v := .Values.datahubSystemUpdate.bootstrapMCPs }}
-            {{- $result := deepCopy $v.values }}
+            {{- if ne $k "default" }}
+            {{- $result := dict }}
+            {{- range $.Values.datahubSystemUpdate.bootstrapMCPs.default.value_configs }}
+              {{- $funcOutput := include . $ | fromYaml }}
+              {{- $result = include "deepMerge" (dict "dst" $result "src" $funcOutput) | fromYaml }}
+            {{- end }}
+            {{- $valuesCopy := deepCopy $v.values }}
+            {{- $result = include "deepMerge" (dict "dst" $result "src" $valuesCopy) | fromYaml }}
             {{- range $v.values_generated_configs }}
               {{- $funcOutput := include . $ | fromYaml }}
               {{- $result = include "deepMerge" (dict "dst" $result "src" $funcOutput) | fromYaml }}
@@ -147,6 +154,7 @@ spec:
               value: '{"version":"{{ $.Chart.Version }}-{{ $.Release.Revision }}"}'
             {{- end }}
             {{- end }}
+            {{- end }}
           {{- with .Values.datahubSystemUpdate.extraEnvs }}
             {{- toYaml . | nindent 12 }}
           {{- end }}
@@ -310,7 +318,14 @@ spec:
               value: {{ . | quote }}
             {{- end }}
             {{- range $k, $v := .Values.datahubSystemUpdate.bootstrapMCPs }}
-            {{- $result := deepCopy $v.values }}
+            {{- if ne $k "default" }}
+            {{- $result := dict }}
+            {{- range $.Values.datahubSystemUpdate.bootstrapMCPs.default.value_configs }}
+              {{- $funcOutput := include . $ | fromYaml }}
+              {{- $result = include "deepMerge" (dict "dst" $result "src" $funcOutput) | fromYaml }}
+            {{- end }}
+            {{- $valuesCopy := deepCopy $v.values }}
+            {{- $result = include "deepMerge" (dict "dst" $result "src" $valuesCopy) | fromYaml }}
             {{- range $v.values_generated_configs }}
               {{- $funcOutput := include . $ | fromYaml }}
               {{- $result = include "deepMerge" (dict "dst" $result "src" $funcOutput) | fromYaml }}
@@ -322,6 +337,7 @@ spec:
               value: '{"version":"{{ $.Chart.Version }}-{{ $.Release.Revision }}"}'
             {{- end }}
             {{- end }}
+            {{- end }}
           {{- with .Values.datahubSystemUpdate.extraEnvs }}
             {{- toYaml . | nindent 12 }}
           {{- end }}

diff --git a/charts/datahub/values.yaml b/charts/datahub/values.yaml
@@ -395,30 +395,33 @@ datahubSystemUpdate:
       # args: []
   # Depends on v0.14.2 or greater
   bootstrapMCPs:
+    default:
+      value_configs:
+        - "datahub.bootstrapMCPs.default.ingestion.version"
+        - "datahub.bootstrapMCPs.default.schedule.timezone"
     datahubGC:
       # For information about this recipe https://datahubproject.io/docs/0.14.0/generated/ingestion/sources/datahubgc/#install-the-plugin
-      # Overrides values.schedule.interval below with a jitter window using a generated config
+      # Generate values.schedule.interval with a jitter window using a generated config
       dailyCronWindow:
-        enabled: true
         startHour: 18
         endHour: 5
       # dynamic overrides, the output to each function is deepMerged with values
       values_generated_configs:
         - "datahub.systemUpdate.datahubGC.dailyCronWindow"
-        - "datahub.systemUpdate.datahubGC.timezone"
       # Environment variable containing the json value for the template mcp
       values_env: DATAHUB_GC_BOOTSTRAP_VALUES
       revision_env: DATAHUB_GC_BOOTSTRAP_REVISION
       # Base values for the template mcp
       values:
         ingestion:
           name: datahub-gc
+          # override global.datahub.managed_ingestion.defaultCliVersion
           version: "0.14.1.7rc2"
-        schedule:
-          # overridden by global.datahub.timezone
-          timezone: "UTC"
-          # overridden if dynamic.dailyCronWindow.enable is true
-          interval: "0 1 * * *"
+        # schedule:
+          # # override global.datahub.timezone
+          # timezone: "UTC"
+          # # override dailyCronWindow
+          # interval: "0 1 * * *"
         cleanup_expired_tokens: "false"
         truncate_indices: "true"
         dataprocess_cleanup: