From 159f6b02674228d5dc84a1c9d4d8b79194995358 Mon Sep 17 00:00:00 2001 From: Akash Jain <15995028+akash-jain-10@users.noreply.github.com> Date: Fri, 10 Nov 2023 15:18:59 +0530 Subject: [PATCH 1/5] feat: Migrate to OpenSearch 2.7 as Chart's default search dependency (#194) * feat: Add OS Deps; update mysql charts dep version * fix(docs): Update README.md --- charts/deps/Chart.lock | 14 +++++++------- charts/deps/Chart.yaml | 12 ++++++------ charts/deps/values.yaml | 28 ++++++++++++++-------------- charts/openmetadata/README.md | 4 ++-- charts/openmetadata/values.yaml | 4 ++-- 5 files changed, 31 insertions(+), 31 deletions(-) diff --git a/charts/deps/Chart.lock b/charts/deps/Chart.lock index eab4b4c3..69d7c7d1 100644 --- a/charts/deps/Chart.lock +++ b/charts/deps/Chart.lock @@ -1,12 +1,12 @@ dependencies: - name: mysql - repository: https://raw.githubusercontent.com/bitnami/charts/archive-full-index/bitnami - version: 9.7.1 -- name: elasticsearch - repository: https://helm.elastic.co - version: 8.5.1 + repository: https://charts.bitnami.com/bitnami + version: 9.7.2 - name: airflow repository: https://airflow-helm.github.io/charts version: 8.8.0 -digest: sha256:c4a68ed6f7480bda021d99bb39daf90ed686edeae3605389158b2fc5b9d05f7c -generated: "2023-10-26T15:20:32.348887+05:30" +- name: opensearch + repository: https://opensearch-project.github.io/helm-charts/ + version: 2.12.1 +digest: sha256:ef5c9862e7107bdf2ba81ffb30b9da74b6072dec41800a80723eb08bcdac6022 +generated: "2023-11-07T16:06:28.907894+05:30" diff --git a/charts/deps/Chart.yaml b/charts/deps/Chart.yaml index 2f1fd3b8..a5860515 100644 --- a/charts/deps/Chart.yaml +++ b/charts/deps/Chart.yaml @@ -60,14 +60,14 @@ icon: https://open-metadata.org/images/favicon.png # Add Dependencies of other charts dependencies: - name: mysql - version: "9.7.1" - repository: "https://raw.githubusercontent.com/bitnami/charts/archive-full-index/bitnami" + version: "9.7.2" + repository: "https://charts.bitnami.com/bitnami" condition: mysql.enabled -- name: elasticsearch - version: "8.5.1" - repository: "https://helm.elastic.co" - condition: elasticsearch.enabled - name: airflow version: "8.8.0" repository: "https://airflow-helm.github.io/charts" condition: airflow.enabled +- name: opensearch + version: "2.12.1" + repository: "https://opensearch-project.github.io/helm-charts/" + condition: opensearch.enabled diff --git a/charts/deps/values.yaml b/charts/deps/values.yaml index c00c96b2..d8b715db 100644 --- a/charts/deps/values.yaml +++ b/charts/deps/values.yaml @@ -30,22 +30,22 @@ mysql: GRANT ALL PRIVILEGES ON airflow_db.* TO 'airflow_user'@'%' WITH GRANT OPTION; commit; -# ElasticSearch configurations for helm dependency -# you can find more details about this here https://artifacthub.io/packages/helm/elastic/elasticsearch -elasticsearch: +# OpenSearch Helm Dependency +# you can find more details about this here https://artifacthub.io/packages/helm/opensearch-project-helm-charts/opensearch/2.12.2 +opensearch: enabled: true - replicas: 1 - minimumMasterNodes: 1 - fullnameOverride: "elasticsearch" - imagePullPolicy: "Always" - esJavaOpts: "-Xmx1024m -Xms1024m" + clusterName: opensearch + fullnameOverride: opensearch + nodeGroup: "" + imagePullPolicy: Always + opensearchJavaOpts: "-Xmx1g -Xms1g" + persistence: + size: 30Gi protocol: http - createCert: false - esConfig: - elasticsearch.yml: | - xpack.security.enabled: false - maxUnavailable: 0 - clusterHealthCheckParams: "wait_for_status=yellow&timeout=20s" + config: + opensearch.yml: | + plugins.security.disabled: true + singleNode: true resources: requests: cpu: "100m" diff --git a/charts/openmetadata/README.md b/charts/openmetadata/README.md index d5947474..b53bebad 100644 --- a/charts/openmetadata/README.md +++ b/charts/openmetadata/README.md @@ -117,10 +117,10 @@ helm install openmetadata open-metadata/openmetadata --values < Date: Mon, 13 Nov 2023 17:40:01 +0530 Subject: [PATCH 2/5] feat: Prepare 1.2.1 Application Release (#195) * fix: remove `MIGRATION_EXTENSION_PATH` * fix: remove basic login extended configurations * chore: bump versions * fix(CI): Update helm repo add step with the changes * fix(ci): Chart lint fixes --- .github/workflows/lint-test.yaml | 4 ++-- .github/workflows/release.yaml | 4 ++-- charts/deps/Chart.yaml | 4 ++-- charts/deps/values.yaml | 2 +- charts/openmetadata/Chart.yaml | 4 ++-- charts/openmetadata/README.md | 5 +---- charts/openmetadata/templates/_helpers.tpl | 6 ------ charts/openmetadata/values.schema.json | 21 --------------------- charts/openmetadata/values.yaml | 6 ------ 9 files changed, 10 insertions(+), 46 deletions(-) diff --git a/.github/workflows/lint-test.yaml b/.github/workflows/lint-test.yaml index 7010cacb..363193c8 100644 --- a/.github/workflows/lint-test.yaml +++ b/.github/workflows/lint-test.yaml @@ -35,8 +35,8 @@ jobs: - name: Add Helm repos run: | - helm repo add elasticsearch https://helm.elastic.co - helm repo add mysql https://raw.githubusercontent.com/bitnami/charts/archive-full-index/bitnami + helm repo add opensearch https://opensearch-project.github.io/helm-charts/ + helm repo add mysql https://charts.bitnami.com/bitnami helm repo add airflow https://airflow-helm.github.io/charts - uses: actions/setup-python@v2 diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index ae6c0332..33243829 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -24,8 +24,8 @@ jobs: - name: Add Helm repos run: | - helm repo add elasticsearch https://helm.elastic.co - helm repo add mysql https://raw.githubusercontent.com/bitnami/charts/archive-full-index/bitnami + helm repo add opensearch https://opensearch-project.github.io/helm-charts/ + helm repo add mysql https://charts.bitnami.com/bitnami helm repo add airflow https://airflow-helm.github.io/charts - name: Run chart-releaser diff --git a/charts/deps/Chart.yaml b/charts/deps/Chart.yaml index a5860515..88a338ac 100644 --- a/charts/deps/Chart.yaml +++ b/charts/deps/Chart.yaml @@ -16,13 +16,13 @@ type: application # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 1.2.1 +version: 1.2.2 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. Versions are not expected to # follow Semantic Versioning. They should reflect the version the application is using. # It is recommended to use it with quotes. -appVersion: "1.2.0" +appVersion: "1.2.1" home: https://open-metadata.org/ diff --git a/charts/deps/values.yaml b/charts/deps/values.yaml index d8b715db..dc76902c 100644 --- a/charts/deps/values.yaml +++ b/charts/deps/values.yaml @@ -61,7 +61,7 @@ airflow: airflow: image: repository: docker.getcollate.io/openmetadata/ingestion - tag: 1.2.0 + tag: 1.2.1 pullPolicy: "IfNotPresent" executor: "KubernetesExecutor" config: diff --git a/charts/openmetadata/Chart.yaml b/charts/openmetadata/Chart.yaml index 30f62bb5..c3a693d8 100644 --- a/charts/openmetadata/Chart.yaml +++ b/charts/openmetadata/Chart.yaml @@ -16,13 +16,13 @@ type: application # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 1.2.1 +version: 1.2.2 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. Versions are not expected to # follow Semantic Versioning. They should reflect the version the application is using. # It is recommended to use it with quotes. -appVersion: "1.2.0" +appVersion: "1.2.1" home: https://open-metadata.org/ diff --git a/charts/openmetadata/README.md b/charts/openmetadata/README.md index b53bebad..607c0109 100644 --- a/charts/openmetadata/README.md +++ b/charts/openmetadata/README.md @@ -101,8 +101,6 @@ helm install openmetadata open-metadata/openmetadata --values < Date: Mon, 20 Nov 2023 17:30:15 +0530 Subject: [PATCH 3/5] chore: Version Bump to 1.2.2 Application Release (#198) --- charts/deps/Chart.yaml | 4 ++-- charts/deps/values.yaml | 2 +- charts/openmetadata/Chart.yaml | 4 ++-- charts/openmetadata/README.md | 2 +- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/charts/deps/Chart.yaml b/charts/deps/Chart.yaml index 88a338ac..7cdb45cd 100644 --- a/charts/deps/Chart.yaml +++ b/charts/deps/Chart.yaml @@ -16,13 +16,13 @@ type: application # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 1.2.2 +version: 1.2.3 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. Versions are not expected to # follow Semantic Versioning. They should reflect the version the application is using. # It is recommended to use it with quotes. -appVersion: "1.2.1" +appVersion: "1.2.2" home: https://open-metadata.org/ diff --git a/charts/deps/values.yaml b/charts/deps/values.yaml index dc76902c..e2d7634e 100644 --- a/charts/deps/values.yaml +++ b/charts/deps/values.yaml @@ -61,7 +61,7 @@ airflow: airflow: image: repository: docker.getcollate.io/openmetadata/ingestion - tag: 1.2.1 + tag: 1.2.2 pullPolicy: "IfNotPresent" executor: "KubernetesExecutor" config: diff --git a/charts/openmetadata/Chart.yaml b/charts/openmetadata/Chart.yaml index c3a693d8..8b2a188b 100644 --- a/charts/openmetadata/Chart.yaml +++ b/charts/openmetadata/Chart.yaml @@ -16,13 +16,13 @@ type: application # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 1.2.2 +version: 1.2.3 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. Versions are not expected to # follow Semantic Versioning. They should reflect the version the application is using. # It is recommended to use it with quotes. -appVersion: "1.2.1" +appVersion: "1.2.2" home: https://open-metadata.org/ diff --git a/charts/openmetadata/README.md b/charts/openmetadata/README.md index 607c0109..c0b3aed3 100644 --- a/charts/openmetadata/README.md +++ b/charts/openmetadata/README.md @@ -206,7 +206,7 @@ helm install openmetadata open-metadata/openmetadata --values < Date: Tue, 21 Nov 2023 10:53:31 +0530 Subject: [PATCH 4/5] Introduction of poddisruptionbudget for openmetadata server (#197) * Introduction of poddisruptionbudget for openmetadata server * Updated PR as suggested in previous commit --- .../templates/poddisruptionbudget.yaml | 20 +++++++++++++++++++ charts/openmetadata/values.schema.json | 17 ++++++++++++++++ charts/openmetadata/values.yaml | 8 +++++++- 3 files changed, 44 insertions(+), 1 deletion(-) create mode 100644 charts/openmetadata/templates/poddisruptionbudget.yaml diff --git a/charts/openmetadata/templates/poddisruptionbudget.yaml b/charts/openmetadata/templates/poddisruptionbudget.yaml new file mode 100644 index 00000000..f0f16681 --- /dev/null +++ b/charts/openmetadata/templates/poddisruptionbudget.yaml @@ -0,0 +1,20 @@ +{{- if .Values.podDisruptionBudget.enabled }} +apiVersion: policy/v1 +kind: PodDisruptionBudget +metadata: + name: {{ include "OpenMetadata.fullname" . }}-poddisruptionbudget + labels: + {{- include "OpenMetadata.labels" . | indent 4 }} +spec: + {{- with .Values.podDisruptionBudget.config }} + {{- if .minAvailable }} + minAvailable: {{ .minAvailable }} + {{- end }} + {{- if .maxUnavailable }} + maxUnavailable: {{ .maxUnavailable }} + {{- end }} + {{- end }} + selector: + matchLabels: + {{- include "OpenMetadata.selectorLabels" . | nindent 6 }} +{{- end }} \ No newline at end of file diff --git a/charts/openmetadata/values.schema.json b/charts/openmetadata/values.schema.json index bb04ced4..ffb84ac4 100644 --- a/charts/openmetadata/values.schema.json +++ b/charts/openmetadata/values.schema.json @@ -1052,6 +1052,23 @@ "commonLabels": { "type": "object" }, + "podDisruptionBudget": { + "type": "object", + "additionalProperties": false, + "properties": { + "config": { + "type": "object", + "properties": { + "maxUnavailable": { + "type": "string" + }, + "minAvailable": { + "type": "string" + } + } + } + } + }, "podAnnotations": { "type": "object" }, diff --git a/charts/openmetadata/values.yaml b/charts/openmetadata/values.yaml index 194057a6..7c791e14 100644 --- a/charts/openmetadata/values.yaml +++ b/charts/openmetadata/values.yaml @@ -389,5 +389,11 @@ startupProbe: path: /healthcheck port: http-admin +podDisruptionBudget: + enabled: false + config: + maxUnavailable: 1 + minAvailable: 1 + commonLabels: {} -podAnnotations: {} +podAnnotations: {} \ No newline at end of file From 3d830a45aa84917592ad1e61a85ff8190fbd8d7a Mon Sep 17 00:00:00 2001 From: Dhruvin Maniar Date: Tue, 21 Nov 2023 19:20:17 +0530 Subject: [PATCH 5/5] [Fix] Airflow Auth (#199) * [Fix] Airflow Auth * [Fix] Airflow Auth * [Fix] Airflow Auth * [Fix] Airflow Auth * [Fix] Updating charts * [Fix] Updating charts --- charts/openmetadata/Chart.yaml | 2 +- charts/openmetadata/templates/_helpers.tpl | 2 ++ charts/openmetadata/values.schema.json | 8 +++++++- charts/openmetadata/values.yaml | 7 ++++--- 4 files changed, 14 insertions(+), 5 deletions(-) diff --git a/charts/openmetadata/Chart.yaml b/charts/openmetadata/Chart.yaml index 8b2a188b..12aa43fe 100644 --- a/charts/openmetadata/Chart.yaml +++ b/charts/openmetadata/Chart.yaml @@ -16,7 +16,7 @@ type: application # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 1.2.3 +version: 1.2.4 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. Versions are not expected to diff --git a/charts/openmetadata/templates/_helpers.tpl b/charts/openmetadata/templates/_helpers.tpl index c840c4d8..23b79447 100644 --- a/charts/openmetadata/templates/_helpers.tpl +++ b/charts/openmetadata/templates/_helpers.tpl @@ -359,6 +359,7 @@ OpenMetadata Configurations Environment Variables*/}} value: "{{ .Values.openmetadata.config.pipelineServiceClientConfig.ingestionIpInfoEnabled }}" - name: PIPELINE_SERVICE_CLIENT_ENDPOINT value: "{{ .Values.openmetadata.config.pipelineServiceClientConfig.apiEndpoint }}" +{{ if .Values.openmetadata.config.pipelineServiceClientConfig.auth.enabled }} {{- with .Values.openmetadata.config.pipelineServiceClientConfig.auth }} - name: AIRFLOW_USERNAME value: "{{ .username }}" @@ -368,6 +369,7 @@ OpenMetadata Configurations Environment Variables*/}} name: {{ .password.secretRef }} key: {{ .password.secretKey }} {{- end }} +{{- end }} - name: PIPELINE_SERVICE_CLIENT_VERIFY_SSL value: "{{ .Values.openmetadata.config.pipelineServiceClientConfig.verifySsl }}" - name: PIPELINE_SERVICE_CLIENT_HOST_IP diff --git a/charts/openmetadata/values.schema.json b/charts/openmetadata/values.schema.json index ffb84ac4..3169a681 100644 --- a/charts/openmetadata/values.schema.json +++ b/charts/openmetadata/values.schema.json @@ -166,6 +166,9 @@ "type": "object", "additionalProperties": false, "properties": { + "enabled" : { + "type": "boolean" + }, "password": { "type": "object", "additionalProperties": false, @@ -1056,6 +1059,9 @@ "type": "object", "additionalProperties": false, "properties": { + "enabled" : { + "type": "boolean" + }, "config": { "type": "object", "properties": { @@ -1082,4 +1088,4 @@ } } } -} \ No newline at end of file +} diff --git a/charts/openmetadata/values.yaml b/charts/openmetadata/values.yaml index 7c791e14..3e8b6d53 100644 --- a/charts/openmetadata/values.yaml +++ b/charts/openmetadata/values.yaml @@ -70,6 +70,7 @@ openmetadata: # local path in Airflow Pod sslCertificatePath: "/no/path" auth: + enabled: true username: admin password: secretRef: airflow-secrets @@ -392,8 +393,8 @@ startupProbe: podDisruptionBudget: enabled: false config: - maxUnavailable: 1 - minAvailable: 1 + maxUnavailable: "1" + minAvailable: "1" commonLabels: {} -podAnnotations: {} \ No newline at end of file +podAnnotations: {}