Merge branch 'main' into feat/unify-collecting-from-secrets

open-metadata · Dec 5, 2023 · c85de4f · c85de4f
2 parents 01e5d3f + 3d830a4
commit c85de4f
Show file tree

Hide file tree

Showing 11 changed files with 94 additions and 79 deletions.
diff --git a/.github/workflows/lint-test.yaml b/.github/workflows/lint-test.yaml
@@ -35,8 +35,8 @@ jobs:
 
       - name: Add Helm repos
         run: |
-          helm repo add elasticsearch https://helm.elastic.co
-          helm repo add mysql https://raw.githubusercontent.com/bitnami/charts/archive-full-index/bitnami
+          helm repo add opensearch https://opensearch-project.github.io/helm-charts/
+          helm repo add mysql https://charts.bitnami.com/bitnami
           helm repo add airflow https://airflow-helm.github.io/charts
 
       - uses: actions/setup-python@v2

diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml
@@ -24,8 +24,8 @@ jobs:
 
       - name: Add Helm repos
         run: |
-          helm repo add elasticsearch https://helm.elastic.co
-          helm repo add mysql https://raw.githubusercontent.com/bitnami/charts/archive-full-index/bitnami
+          helm repo add opensearch https://opensearch-project.github.io/helm-charts/
+          helm repo add mysql https://charts.bitnami.com/bitnami
           helm repo add airflow https://airflow-helm.github.io/charts
 
       - name: Run chart-releaser

diff --git a/charts/deps/Chart.lock b/charts/deps/Chart.lock
@@ -1,12 +1,12 @@
 dependencies:
 - name: mysql
-  repository: https://raw.githubusercontent.com/bitnami/charts/archive-full-index/bitnami
-  version: 9.7.1
-- name: elasticsearch
-  repository: https://helm.elastic.co
-  version: 8.5.1
+  repository: https://charts.bitnami.com/bitnami
+  version: 9.7.2
 - name: airflow
   repository: https://airflow-helm.github.io/charts
   version: 8.8.0
-digest: sha256:c4a68ed6f7480bda021d99bb39daf90ed686edeae3605389158b2fc5b9d05f7c
-generated: "2023-10-26T15:20:32.348887+05:30"
+- name: opensearch
+  repository: https://opensearch-project.github.io/helm-charts/
+  version: 2.12.1
+digest: sha256:ef5c9862e7107bdf2ba81ffb30b9da74b6072dec41800a80723eb08bcdac6022
+generated: "2023-11-07T16:06:28.907894+05:30"
diff --git a/charts/deps/Chart.yaml b/charts/deps/Chart.yaml
@@ -16,13 +16,13 @@ type: application
 # to the chart and its templates, including the app version.
 # Versions are expected to follow Semantic Versioning (https://semver.org/)
 
-version: 1.2.1
+version: 1.2.3
 
 # This is the version number of the application being deployed. This version number should be
 # incremented each time you make changes to the application. Versions are not expected to
 # follow Semantic Versioning. They should reflect the version the application is using.
 # It is recommended to use it with quotes.
-appVersion: "1.2.0"
+appVersion: "1.2.2"
 
 home: https://open-metadata.org/
 
@@ -60,14 +60,14 @@ icon: https://open-metadata.org/images/favicon.png
 # Add Dependencies of other charts
 dependencies:
 - name: mysql
-  version: "9.7.1"
-  repository: "https://raw.githubusercontent.com/bitnami/charts/archive-full-index/bitnami"
+  version: "9.7.2"
+  repository: "https://charts.bitnami.com/bitnami"
   condition: mysql.enabled
-- name: elasticsearch
-  version: "8.5.1"
-  repository: "https://helm.elastic.co"
-  condition: elasticsearch.enabled
 - name: airflow
   version: "8.8.0"
   repository: "https://airflow-helm.github.io/charts"
   condition: airflow.enabled
+- name: opensearch
+  version: "2.12.1"
+  repository: "https://opensearch-project.github.io/helm-charts/"
+  condition: opensearch.enabled
diff --git a/charts/deps/values.yaml b/charts/deps/values.yaml
@@ -30,22 +30,22 @@ mysql:
       GRANT ALL PRIVILEGES ON airflow_db.* TO 'airflow_user'@'%' WITH GRANT OPTION;
       commit;
 
-# ElasticSearch configurations for helm dependency
-# you can find more details about this here https://artifacthub.io/packages/helm/elastic/elasticsearch
-elasticsearch:
+# OpenSearch Helm Dependency
+# you can find more details about this here https://artifacthub.io/packages/helm/opensearch-project-helm-charts/opensearch/2.12.2
+opensearch:
   enabled: true
-  replicas: 1
-  minimumMasterNodes: 1
-  fullnameOverride: "elasticsearch"
-  imagePullPolicy: "Always"
-  esJavaOpts: "-Xmx1024m -Xms1024m"
+  clusterName: opensearch
+  fullnameOverride: opensearch
+  nodeGroup: ""
+  imagePullPolicy: Always
+  opensearchJavaOpts: "-Xmx1g -Xms1g"
+  persistence:
+    size: 30Gi
   protocol: http
-  createCert: false
-  esConfig:
-    elasticsearch.yml: |
-      xpack.security.enabled: false
-  maxUnavailable: 0
-  clusterHealthCheckParams: "wait_for_status=yellow&timeout=20s"
+  config:
+    opensearch.yml: |
+      plugins.security.disabled: true
+  singleNode: true
   resources:
     requests:
       cpu: "100m"
@@ -61,7 +61,7 @@ airflow:
   airflow:
     image:
       repository: docker.getcollate.io/openmetadata/ingestion
-      tag: 1.2.0
+      tag: 1.2.2
       pullPolicy: "IfNotPresent"
     executor: "KubernetesExecutor"
     config:

diff --git a/charts/openmetadata/Chart.yaml b/charts/openmetadata/Chart.yaml
@@ -16,13 +16,13 @@ type: application
 # to the chart and its templates, including the app version.
 # Versions are expected to follow Semantic Versioning (https://semver.org/)
 
-version: 1.2.1
+version: 1.2.4
 
 # This is the version number of the application being deployed. This version number should be
 # incremented each time you make changes to the application. Versions are not expected to
 # follow Semantic Versioning. They should reflect the version the application is using.
 # It is recommended to use it with quotes.
-appVersion: "1.2.0"
+appVersion: "1.2.2"
 
 home: https://open-metadata.org/
 

diff --git a/charts/openmetadata/README.md b/charts/openmetadata/README.md
@@ -101,8 +101,6 @@ helm install openmetadata open-metadata/openmetadata --values <<path-to-values-f
 | openmetadata.config.airflow.openmetadata.serverHostApiUrl | string | `http://openmetadata:8585/api` | SERVER_HOST_API_URL |
 | openmetadata.config.airflow.sslCertificatePath | string | `/no/path` | PIPELINE_SERVICE_CLIENT_SSL_CERT_PATH |
 | openmetadata.config.airflow.verifySsl | string | `no-ssl` | PIPELINE_SERVICE_CLIENT_VERIFY_SSL |
-| openmetadata.config.basicLogin.maxLoginFailAttempts | int | 3 | OM_MAX_FAILED_LOGIN_ATTEMPTS |
-| openmetadata.config.basicLogin.accessBlockTime | int | 600 | OM_LOGIN_ACCESS_BLOCK_TIME |
 | openmetadata.config.clusterName | string | `openmetadata` | OPENMETADATA_CLUSTER_NAME |
 | openmetadata.config.database.auth.password.secretRef | string | `mysql-secrets` | DB_USER_PASSWORD |
 | openmetadata.config.database.auth.password.secretKey | string | `openmetadata-mysql-password` | DB_USER_PASSWORD |
@@ -117,10 +115,10 @@ helm install openmetadata open-metadata/openmetadata --values <<path-to-values-f
 | openmetadata.config.elasticsearch.auth.username | string | `elasticsearch` | ELASTICSEARCH_USER |
 | openmetadata.config.elasticsearch.auth.password.secretRef | string | `elasticsearch-secrets` | ELASTICSEARCH_PASSWORD |
 | openmetadata.config.elasticsearch.auth.password.secretKey | string | `openmetadata-elasticsearch-password` | ELASTICSEARCH_PASSWORD |
-| openmetadata.config.elasticsearch.host | string | `elasticsearch` | ELASTICSEARCH_HOST |
+| openmetadata.config.elasticsearch.host | string | `opensearch` | ELASTICSEARCH_HOST |
 | openmetadata.config.elasticsearch.keepAliveTimeoutSecs | int | `600` | ELASTICSEARCH_KEEP_ALIVE_TIMEOUT_SECS |
 | openmetadata.config.elasticsearch.port | int | 9200 | ELASTICSEARCH_PORT |
-| openmetadata.config.elasticsearch.searchType | string | `elasticsearch` | SEARCH_TYPE |
+| openmetadata.config.elasticsearch.searchType | string | `opensearch` | SEARCH_TYPE |
 | openmetadata.config.elasticsearch.scheme | string | `http` | ELASTICSEARCH_SCHEME |
 | openmetadata.config.elasticsearch.searchIndexMappingLanguage | string | `EN`| ELASTICSEARCH_INDEX_MAPPING_LANG |
 | openmetadata.config.elasticsearch.trustStore.enabled | bool | `false` | |
@@ -141,7 +139,6 @@ helm install openmetadata open-metadata/openmetadata --values <<path-to-values-f
 | openmetadata.config.jwtTokenConfiguration.keyId | string | `Gb389a-9f76-gdjs-a92j-0242bk94356` | JWT_KEY_ID |
 | openmetadata.config.logLevel | string | `INFO` | LOG_LEVEL |
 | openmetadata.config.maskPasswordsApi | bool | `false` | MASK_PASSWORDS_API |
-| openmetadata.config.migrationConfigs.extensionPath | string | `Empty string` | MIGRATION_EXTENSION_PATH |
 | openmetadata.config.openmetadata.adminPort | int | 8586 | SERVER_ADMIN_PORT |
 | openmetadata.config.openmetadata.host | string | `openmetadata` | OPENMETADATA_SERVER_URL |
 | openmetadata.config.openmetadata.port | int | 8585 | SERVER_PORT |
@@ -209,7 +206,7 @@ helm install openmetadata open-metadata/openmetadata --values <<path-to-values-f
 | fullnameOverride | string | `"openmetadata"` |
 | image.pullPolicy | string | `"Always"` |
 | image.repository | string | `"docker.getcollate.io/openmetadata/server"` |
-| image.tag | string | `1.2.0` |
+| image.tag | string | `1.2.2` |
 | imagePullSecrets | list | `[]` |
 | ingress.annotations | object | `{}` |
 | ingress.className | string | `""` |

diff --git a/charts/openmetadata/templates/_helpers.tpl b/charts/openmetadata/templates/_helpers.tpl
@@ -112,8 +112,6 @@ OpenMetadata Configurations Environment Variables*/}}
       name: {{ include "OpenMetadata.fullname" . }}-secret
       key: FERNET_KEY
 {{- end }}
-- name: MIGRATION_EXTENSION_PATH
-  value: "{{ .Values.openmetadata.config.migrationConfigs.extensionPath }}"
 - name: EVENT_MONITOR
   value: "{{ .Values.openmetadata.config.eventMonitor.type }}"
 - name: EVENT_MONITOR_BATCH_SIZE
@@ -150,10 +148,6 @@ OpenMetadata Configurations Environment Variables*/}}
   value: '[{{ include "OpenMetadata.commaJoinedQuotedList" (dict "value" .Values.openmetadata.config.authentication.jwtPrincipalClaims) }}]'
 - name: AUTHENTICATION_ENABLE_SELF_SIGNUP
   value: "{{ .Values.openmetadata.config.authentication.enableSelfSignup }}"
-- name: OM_MAX_FAILED_LOGIN_ATTEMPTS
-  value: "{{ .Values.openmetadata.config.basicLogin.maxLoginFailAttempts }}"
-- name: OM_LOGIN_ACCESS_BLOCK_TIME
-  value: "{{ .Values.openmetadata.config.basicLogin.accessBlockTime }}"
 - name: AUTHORIZER_CLASS_NAME
   value: "{{ .Values.openmetadata.config.authorizer.className }}"
 - name: AUTHORIZER_REQUEST_FILTER
@@ -384,7 +378,8 @@ OpenMetadata Configurations Environment Variables*/}}
 - name: PIPELINE_SERVICE_IP_INFO_ENABLED
   value: "{{ .Values.openmetadata.config.pipelineServiceClientConfig.ingestionIpInfoEnabled }}"
 - name: PIPELINE_SERVICE_CLIENT_ENDPOINT
-  value: "{{ tpl .Values.openmetadata.config.pipelineServiceClientConfig.apiEndpoint . }}"
+   value: "{{ tpl .Values.openmetadata.config.pipelineServiceClientConfig.apiEndpoint . }}"
+{{ if .Values.openmetadata.config.pipelineServiceClientConfig.auth.enabled }}
 {{- with .Values.openmetadata.config.pipelineServiceClientConfig.auth }}
 - name: AIRFLOW_USERNAME
   value: "{{ .username }}"

diff --git a/charts/openmetadata/templates/poddisruptionbudget.yaml b/charts/openmetadata/templates/poddisruptionbudget.yaml
@@ -0,0 +1,20 @@
+{{- if .Values.podDisruptionBudget.enabled }}
+apiVersion: policy/v1
+kind: PodDisruptionBudget
+metadata:
+  name: {{ include "OpenMetadata.fullname" . }}-poddisruptionbudget
+  labels:
+    {{- include "OpenMetadata.labels" . | indent 4 }}
+spec:
+  {{- with .Values.podDisruptionBudget.config }}
+  {{- if .minAvailable }}
+  minAvailable: {{ .minAvailable }}
+  {{- end }}
+  {{- if .maxUnavailable }}
+  maxUnavailable: {{ .maxUnavailable }}
+  {{- end }}
+  {{- end }}
+  selector:
+    matchLabels:
+      {{- include "OpenMetadata.selectorLabels" . | nindent 6 }}
+{{- end }}
diff --git a/charts/openmetadata/values.schema.json b/charts/openmetadata/values.schema.json
@@ -39,15 +39,6 @@
                         "errorMessage": "Global keyword has been replaced by openmetadata.config"
                     },
                     "properties": {
-                        "migrationConfigs": {
-                            "type": "object",
-                            "additionalProperties": false,
-                            "properties": {
-                                "extensionPath": {
-                                    "type": "string"
-                                }
-                            }
-                        },
                         "web": {
                             "type": "object",
                             "additionalProperties": false,
@@ -175,6 +166,9 @@
                                     "type": "object",
                                     "additionalProperties": false,
                                     "properties": {
+                                        "enabled" : {
+                                            "type": "boolean"
+                                        },
                                         "password": {
                                             "type": "object",
                                             "additionalProperties": false,
@@ -549,18 +543,6 @@
                                 }
                             }
                         },
-                        "basicLogin": {
-                            "type": "object",
-                            "additionalProperties": false,
-                            "properties": {
-                                "accessBlockTime": {
-                                    "type": "integer"
-                                },
-                                "maxLoginFailAttempts": {
-                                    "type": "integer"
-                                }
-                            }
-                        },
                         "clusterName": {
                             "type": "string"
                         },
@@ -1073,6 +1055,26 @@
         "commonLabels": {
             "type": "object"
         },
+        "podDisruptionBudget": {
+            "type": "object",
+            "additionalProperties": false,
+            "properties": {
+                "enabled" : {
+                    "type": "boolean"
+                },
+                "config": {
+                    "type": "object",
+                    "properties": {
+                        "maxUnavailable": {
+                            "type": "string"
+                        },
+                        "minAvailable": {
+                            "type": "string"
+                        }
+                    }
+                }
+            }
+        },
         "podAnnotations": {
             "type": "object"
         },
@@ -1086,4 +1088,4 @@
             }
         }
     }
-}
+}
diff --git a/charts/openmetadata/values.yaml b/charts/openmetadata/values.yaml
@@ -7,8 +7,6 @@ replicaCount: 1
 # Below are defaults as per openmetadata-dependencies Helm Chart Values
 openmetadata:
   config:
-    migrationConfigs:
-      extensionPath: ""
     upgradeMigrationConfigs:
       force: false
       migrationLimitParam: 1200
@@ -23,8 +21,8 @@ openmetadata:
       port: 8585
       adminPort: 8586
     elasticsearch:
-      host: elasticsearch
-      searchType: elasticsearch
+      host: opensearch
+      searchType: opensearch
       port: 9200
       scheme: http
       connectionTimeoutSecs: 5
@@ -72,6 +70,7 @@ openmetadata:
       # local path in Airflow Pod
       sslCertificatePath: "/no/path"
       auth:
+        enabled: true
         username: admin
         password:
           secretRef: airflow-secrets
@@ -162,10 +161,6 @@ openmetadata:
             secretRef: ""
             secretKey: ""
 
-    basicLogin:
-      maxLoginFailAttempts: 3
-      # in seconds
-      accessBlockTime: 600
     jwtTokenConfiguration:
       enabled: true
       # File Path on Airflow Container
@@ -395,5 +390,11 @@ startupProbe:
     path: /healthcheck
     port: http-admin
 
+podDisruptionBudget:
+  enabled: false
+  config:
+    maxUnavailable: "1"
+    minAvailable: "1"
+
 commonLabels: {}
 podAnnotations: {}