diff --git a/.github/workflows/deploy-dev.yml b/.github/workflows/deploy-dev.yml index 9508c10..f4c6836 100644 --- a/.github/workflows/deploy-dev.yml +++ b/.github/workflows/deploy-dev.yml @@ -14,15 +14,15 @@ jobs: uses: ./.github/workflows/deploy-workflow.yml with: env: dev - datahub_helm_version: "0.4.9" - datahub_prereqs_helm_version: "0.1.10" + datahub_helm_version: "0.4.23" + datahub_prereqs_helm_version: "0.1.13" secrets: kube_namespace: "${{ secrets.KUBE_NAMESPACE }}" kube_cert: "${{ secrets.KUBE_CERT }}" kube_cluster: "${{ secrets.KUBE_CLUSTER }}" kube_token: "${{ secrets.KUBE_TOKEN }}" postgres_host: ${{ secrets.POSTGRES_HOST}} - postgres_client_host: ${{ secrets.POSTGRES_CLIENT_HOST }} + postgres_client_host: ${{ secrets.POSTGRES_CLIENT_HOST }} postgres_url: ${{ secrets.POSTGRES_URL }} opensearch_proxy_host: ${{ secrets.OPENSEARCH_PROXY_HOST }} azure_client_secret: ${{ secrets.AZURE_CLIENT_SECRET }} diff --git a/.github/workflows/deploy-staged.yml b/.github/workflows/deploy-staged.yml index a258b7e..1a38aca 100644 --- a/.github/workflows/deploy-staged.yml +++ b/.github/workflows/deploy-staged.yml @@ -7,7 +7,7 @@ on: - '.github/workflows/**deploy**' - '!helm_deploy/**test**' - '!.github/workflows/**test**' - branches: + branches: - 'main' jobs: @@ -15,15 +15,15 @@ jobs: uses: ./.github/workflows/deploy-workflow.yml with: env: test - datahub_helm_version: "0.4.9" - datahub_prereqs_helm_version: "0.1.10" + datahub_helm_version: "0.4.23" + datahub_prereqs_helm_version: "0.1.13" secrets: kube_namespace: "${{ secrets.KUBE_NAMESPACE }}" kube_cert: "${{ secrets.KUBE_CERT }}" kube_cluster: "${{ secrets.KUBE_CLUSTER }}" kube_token: "${{ secrets.KUBE_TOKEN }}" postgres_host: ${{ secrets.POSTGRES_HOST}} - postgres_client_host: ${{ secrets.POSTGRES_CLIENT_HOST }} + postgres_client_host: ${{ secrets.POSTGRES_CLIENT_HOST }} postgres_url: ${{ secrets.POSTGRES_URL }} opensearch_proxy_host: ${{ secrets.OPENSEARCH_PROXY_HOST }} azure_client_secret: ${{ secrets.AZURE_CLIENT_SECRET }} @@ -33,15 +33,15 @@ jobs: needs: [deploy-test] with: env: preprod - datahub_helm_version: "0.4.9" - datahub_prereqs_helm_version: "0.1.10" + datahub_helm_version: "0.4.23" + datahub_prereqs_helm_version: "0.1.13" secrets: kube_namespace: "${{ secrets.KUBE_NAMESPACE }}" kube_cert: "${{ secrets.KUBE_CERT }}" kube_cluster: "${{ secrets.KUBE_CLUSTER }}" kube_token: "${{ secrets.KUBE_TOKEN }}" postgres_host: ${{ secrets.POSTGRES_HOST}} - postgres_client_host: ${{ secrets.POSTGRES_CLIENT_HOST }} + postgres_client_host: ${{ secrets.POSTGRES_CLIENT_HOST }} postgres_url: ${{ secrets.POSTGRES_URL }} opensearch_proxy_host: ${{ secrets.OPENSEARCH_PROXY_HOST }} azure_client_secret: ${{ secrets.AZURE_CLIENT_SECRET }} @@ -51,15 +51,15 @@ jobs: needs: [deploy-preprod] with: env: prod - datahub_helm_version: "0.4.9" - datahub_prereqs_helm_version: "0.1.10" + datahub_helm_version: "0.4.23" + datahub_prereqs_helm_version: "0.1.13" secrets: kube_namespace: "${{ secrets.KUBE_NAMESPACE }}" kube_cert: "${{ secrets.KUBE_CERT }}" kube_cluster: "${{ secrets.KUBE_CLUSTER }}" kube_token: "${{ secrets.KUBE_TOKEN }}" postgres_host: ${{ secrets.POSTGRES_HOST}} - postgres_client_host: ${{ secrets.POSTGRES_CLIENT_HOST }} + postgres_client_host: ${{ secrets.POSTGRES_CLIENT_HOST }} postgres_url: ${{ secrets.POSTGRES_URL }} opensearch_proxy_host: ${{ secrets.OPENSEARCH_PROXY_HOST }} azure_client_secret: ${{ secrets.AZURE_CLIENT_SECRET }} diff --git a/helm_deploy/values-base.yaml b/helm_deploy/values-base.yaml index 816bc36..a920ae0 100644 --- a/helm_deploy/values-base.yaml +++ b/helm_deploy/values-base.yaml @@ -159,6 +159,8 @@ elasticsearchSetupJob: capabilities: drop: ["ALL"] add: ["NET_BIND_SERVICE"] + # Schedule of CronJob when enabled + schedule: "0 8 * * 1-5" kafkaSetupJob: enabled: true @@ -454,6 +456,26 @@ global: # glue: # region: us-east-1 # registry: datahub + metadataChangeLog: + hooks: + siblings: + enabled: true + consumerGroupSuffix: '' + updateIndices: + enabled: true + consumerGroupSuffix: '' + ingestionScheduler: + enabled: true + consumerGroupSuffix: '' + incidents: + enabled: true + consumerGroupSuffix: '' + entityChangeEvents: + enabled: true + consumerGroupSuffix: '' + forms: + enabled: true + consumerGroupSuffix: '' sql: datasource: @@ -478,7 +500,7 @@ global: key: database_name datahub: - version: v0.13.3 + version: v0.14.0.2 gms: port: "8080" nodePort: "30001" @@ -509,7 +531,7 @@ global: managed_ingestion: enabled: true - defaultCliVersion: "0.13.3" + defaultCliVersion: "0.14.0" metadata_service_authentication: enabled: true @@ -541,3 +563,26 @@ global: show_browse_v2: true # If on, run the backfill upgrade job that generates default browse paths for relevant entities backfill_browse_v2: true + + ## v0.13.4+ + mcp: + throttle: + # updateIntervalMs: 60000 + ## Versioned MCL topic + versioned: + ## Whether to throttle MCP processing based on MCL backlog + enabled: false + # threshold: 4000 + # maxAttempts: 1000 + # initialIntervalMs: 100 + # multiplier: 10 + # maxIntervalMs: 30000 + # Timeseries MCL topic + timeseries: + ## Whether to throttle MCP processing based on MCL backlog + enabled: false + # threshold: 4000 + # maxAttempts: 1000 + # initialIntervalMs: 100 + # multiplier: 10 + # maxIntervalMs: 30000