0.14.2 datahub upgrade (#255)

* update helm versions for dev deploy workflow * update values file for datahub 0.14.2 * actually update main helm chart verison * add cron for reindex so not out of hours default * update other envs helm versions * change reindex cron for weekdays only
ministryofjustice · Sep 3, 2024 · 49d2c3f · 49d2c3f
1 parent 003a113
commit 49d2c3f
Show file tree

Hide file tree

Showing 3 changed files with 60 additions and 15 deletions.
diff --git a/.github/workflows/deploy-dev.yml b/.github/workflows/deploy-dev.yml
@@ -14,15 +14,15 @@ jobs:
     uses: ./.github/workflows/deploy-workflow.yml
     with:
       env: dev
-      datahub_helm_version: "0.4.9"
-      datahub_prereqs_helm_version: "0.1.10"
+      datahub_helm_version: "0.4.23"
+      datahub_prereqs_helm_version: "0.1.13"
     secrets:
       kube_namespace: "${{ secrets.KUBE_NAMESPACE }}"
       kube_cert: "${{ secrets.KUBE_CERT }}"
       kube_cluster: "${{ secrets.KUBE_CLUSTER }}"
       kube_token: "${{ secrets.KUBE_TOKEN }}"
       postgres_host: ${{ secrets.POSTGRES_HOST}}
-      postgres_client_host: ${{ secrets.POSTGRES_CLIENT_HOST }} 
+      postgres_client_host: ${{ secrets.POSTGRES_CLIENT_HOST }}
       postgres_url: ${{ secrets.POSTGRES_URL }}
       opensearch_proxy_host: ${{ secrets.OPENSEARCH_PROXY_HOST }}
       azure_client_secret: ${{ secrets.AZURE_CLIENT_SECRET }}
diff --git a/.github/workflows/deploy-staged.yml b/.github/workflows/deploy-staged.yml
@@ -7,23 +7,23 @@ on:
       - '.github/workflows/**deploy**'
       - '!helm_deploy/**test**'
       - '!.github/workflows/**test**'
-    branches: 
+    branches:
       - 'main'
 
 jobs:
   deploy-test:
     uses: ./.github/workflows/deploy-workflow.yml
     with:
       env: test
-      datahub_helm_version: "0.4.9"
-      datahub_prereqs_helm_version: "0.1.10"
+      datahub_helm_version: "0.4.23"
+      datahub_prereqs_helm_version: "0.1.13"
     secrets:
       kube_namespace: "${{ secrets.KUBE_NAMESPACE }}"
       kube_cert: "${{ secrets.KUBE_CERT }}"
       kube_cluster: "${{ secrets.KUBE_CLUSTER }}"
       kube_token: "${{ secrets.KUBE_TOKEN }}"
       postgres_host: ${{ secrets.POSTGRES_HOST}}
-      postgres_client_host: ${{ secrets.POSTGRES_CLIENT_HOST }} 
+      postgres_client_host: ${{ secrets.POSTGRES_CLIENT_HOST }}
       postgres_url: ${{ secrets.POSTGRES_URL }}
       opensearch_proxy_host: ${{ secrets.OPENSEARCH_PROXY_HOST }}
       azure_client_secret: ${{ secrets.AZURE_CLIENT_SECRET }}
@@ -33,15 +33,15 @@ jobs:
     needs: [deploy-test]
     with:
       env: preprod
-      datahub_helm_version: "0.4.9"
-      datahub_prereqs_helm_version: "0.1.10"
+      datahub_helm_version: "0.4.23"
+      datahub_prereqs_helm_version: "0.1.13"
     secrets:
       kube_namespace: "${{ secrets.KUBE_NAMESPACE }}"
       kube_cert: "${{ secrets.KUBE_CERT }}"
       kube_cluster: "${{ secrets.KUBE_CLUSTER }}"
       kube_token: "${{ secrets.KUBE_TOKEN }}"
       postgres_host: ${{ secrets.POSTGRES_HOST}}
-      postgres_client_host: ${{ secrets.POSTGRES_CLIENT_HOST }} 
+      postgres_client_host: ${{ secrets.POSTGRES_CLIENT_HOST }}
       postgres_url: ${{ secrets.POSTGRES_URL }}
       opensearch_proxy_host: ${{ secrets.OPENSEARCH_PROXY_HOST }}
       azure_client_secret: ${{ secrets.AZURE_CLIENT_SECRET }}
@@ -51,15 +51,15 @@ jobs:
     needs: [deploy-preprod]
     with:
       env: prod
-      datahub_helm_version: "0.4.9"
-      datahub_prereqs_helm_version: "0.1.10"
+      datahub_helm_version: "0.4.23"
+      datahub_prereqs_helm_version: "0.1.13"
     secrets:
       kube_namespace: "${{ secrets.KUBE_NAMESPACE }}"
       kube_cert: "${{ secrets.KUBE_CERT }}"
       kube_cluster: "${{ secrets.KUBE_CLUSTER }}"
       kube_token: "${{ secrets.KUBE_TOKEN }}"
       postgres_host: ${{ secrets.POSTGRES_HOST}}
-      postgres_client_host: ${{ secrets.POSTGRES_CLIENT_HOST }} 
+      postgres_client_host: ${{ secrets.POSTGRES_CLIENT_HOST }}
       postgres_url: ${{ secrets.POSTGRES_URL }}
       opensearch_proxy_host: ${{ secrets.OPENSEARCH_PROXY_HOST }}
       azure_client_secret: ${{ secrets.AZURE_CLIENT_SECRET }}
diff --git a/helm_deploy/values-base.yaml b/helm_deploy/values-base.yaml
@@ -159,6 +159,8 @@ elasticsearchSetupJob:
     capabilities:
       drop: ["ALL"]
       add: ["NET_BIND_SERVICE"]
+    # Schedule of CronJob when enabled
+    schedule: "0 8 * * 1-5"
 
 kafkaSetupJob:
   enabled: true
@@ -454,6 +456,26 @@ global:
       # glue:
       #   region: us-east-1
       #   registry: datahub
+    metadataChangeLog:
+      hooks:
+        siblings:
+          enabled: true
+          consumerGroupSuffix: ''
+        updateIndices:
+          enabled: true
+          consumerGroupSuffix: ''
+        ingestionScheduler:
+          enabled: true
+          consumerGroupSuffix: ''
+        incidents:
+          enabled: true
+          consumerGroupSuffix: ''
+        entityChangeEvents:
+          enabled: true
+          consumerGroupSuffix: ''
+        forms:
+          enabled: true
+          consumerGroupSuffix: ''
 
   sql:
     datasource:
@@ -478,7 +500,7 @@ global:
               key: database_name
 
   datahub:
-    version: v0.13.3
+    version: v0.14.0.2
     gms:
       port: "8080"
       nodePort: "30001"
@@ -509,7 +531,7 @@ global:
 
     managed_ingestion:
       enabled: true
-      defaultCliVersion: "0.13.3"
+      defaultCliVersion: "0.14.0"
 
     metadata_service_authentication:
       enabled: true
@@ -541,3 +563,26 @@ global:
       show_browse_v2: true
       # If on, run the backfill upgrade job that generates default browse paths for relevant entities
       backfill_browse_v2: true
+
+    ## v0.13.4+
+    mcp:
+      throttle:
+        # updateIntervalMs: 60000
+        ## Versioned MCL topic
+        versioned:
+          ## Whether to throttle MCP processing based on MCL backlog
+          enabled: false
+        #  threshold: 4000
+        #  maxAttempts: 1000
+        #  initialIntervalMs: 100
+        #  multiplier: 10
+        #  maxIntervalMs:  30000
+        # Timeseries MCL topic
+        timeseries:
+          ## Whether to throttle MCP processing based on MCL backlog
+          enabled: false
+        #  threshold: 4000
+        #  maxAttempts: 1000
+        #  initialIntervalMs: 100
+        #  multiplier: 10
+        #  maxIntervalMs: 30000