43 add support ckan (#44)

* add: required data catalog * add: config requiredb by ui * add: data server s3 * fix: ct config * release: 3.3.0
mintproject · Aug 31, 2024 · f36c981 · f36c981
1 parent b0b5686
commit f36c981
Show file tree

Hide file tree

Showing 8 changed files with 194 additions and 122 deletions.
diff --git a/charts/mint/Chart.yaml b/charts/mint/Chart.yaml
@@ -15,7 +15,7 @@ type: application
 # This is the chart version. This version number should be incremented each time you make changes
 # to the chart and its templates, including the app version.
 # Versions are expected to follow Semantic Versioning (https://semver.org/)
-version: 3.2.0
+version: 3.3.0
 # This is the version number of the application being deployed. This version number should be
 # incremented each time you make changes to the application. Versions are not expected to
 # follow Semantic Versioning. They should reflect the version the application is using.

diff --git a/charts/mint/templates/ensemble-manager-config.yaml b/charts/mint/templates/ensemble-manager-config.yaml
@@ -48,13 +48,40 @@ data:
          {{- end }}
 
 
-         {{- with .Values.components.ensemble_manager }}
-         {{- if .execution_engine.type }}
-         "execution_engine": {{ .execution_engine.type | quote }},
-         "{{ .execution_engine.type }}":
-            {{ .execution_engine.config  | toRawJson | nindent 12}},
+         {{- with .Values.components.ensemble_manager.config }}
+         {{- if .type }}
+         "execution_engine": {{ .type | quote }},
+         "{{ .type }}":
+            {{ .execution_engine | toRawJson | nindent 12}},
          {{- end }}
          {{- end }}
+
+
+         {{- with .Values.components.ensemble_manager.config }}
+         {{- if eq .data_catalog.type "CKAN"}}
+         "data_catalog_api": {{ .data_catalog.api | quote }},
+         "data_catalog_type": {{ .data_catalog.type | quote }},
+         "data_catalog_extra": {
+            "owner_organization_id": {{ .data_catalog.extra.owner_organization_id | quote }},
+            "owner_provenance_id": {{ .data_catalog.extra.owner_provenance_id | quote }}
+         },
+         {{- end}}
+         {{- end}}
+
+
+
+         {{- if eq .Values.components.ensemble_manager.config.data_server.type "S3" }}
+         {{- with .Values.components.ensemble_manager.config }}
+         "data_server_type": {{ .data_server.type | quote }},
+         "data_server": {
+            "region": {{ .data_server.region | quote }},
+            "bucket": {{ .data_server.bucket | quote }},
+         {{- end}}
+            "access_key": {{ .Values.secrets.data_server.s3.access_key | quote }},
+            "secret_access_key": {{ .Values.secrets.data_server.s3.secret_key | quote }}
+         },
+         {{- end}}
+
          "auth_server": {{ .Values.auth.url | quote }},
          "auth_realm": {{ .Values.auth.realm | quote }},
          "auth_client_id": {{ .Values.auth.ui_client_id | quote }},

diff --git a/charts/mint/templates/ui-config.yaml b/charts/mint/templates/ui-config.yaml
@@ -38,35 +38,35 @@ data:
     {{ if  .enabled }}
     window.REACT_APP_ENSEMBLE_MANAGER_API = "http{{ if .ingress.tls }}s{{ end }}://{{ with (first .ingress.hosts ) }}{{ .host }}{{ end }}/{{ .api_version }}";
     //Execution - Local
-    window.REACT_APP_EXECUTION_ENGINE = "{{ .execution_engine.type }}";
+    window.REACT_APP_EXECUTION_ENGINE = "{{ .config.type }}";
 
-    {{ if eq .execution_engine.type "localex" }}
-    window.REACT_APP_LOCALEX_CODEDIR = "{{ .execution_engine.config.data_dir }}/code";
-    window.REACT_APP_LOCALEX_DATADIR = "{{ .execution_engine.config.data_dir }}/data";
-    window.REACT_APP_LOCALEX_TEMPDIR = "{{ .execution_engine.config.data_dir }}/temp";
-    window.REACT_APP_LOCALEX_LOGDIR = "{{ .execution_engine.config.data_dir }}/logs";
-    window.REACT_APP_LOCALEX_DATAURL = "{{ .execution_engine.config.data_url }}/data";
-    window.REACT_APP_LOCALEX_LOGURL = "{{ .execution_engine.config.data_url }}/logs";
-    window.REACT_APP_LOCALEX_PARALLEL = "{{ .execution_engine.config.parallel }}";
+    {{ if eq .config.type "localex" }}
+    window.REACT_APP_LOCALEX_CODEDIR = "{{ .config.data_dir }}/code";
+    window.REACT_APP_LOCALEX_DATADIR = "{{ .config.data_dir }}/data";
+    window.REACT_APP_LOCALEX_TEMPDIR = "{{ .config.data_dir }}/temp";
+    window.REACT_APP_LOCALEX_LOGDIR = "{{ .config.data_dir }}/logs";
+    window.REACT_APP_LOCALEX_DATAURL = "{{ .config.data_url }}/data";
+    window.REACT_APP_LOCALEX_LOGURL = "{{ .config.data_url }}/logs";
+    window.REACT_APP_LOCALEX_PARALLEL = "{{ .config.parallel }}";
     {{ end }}
 
-    {{ if eq .execution_engine.type "tapis" }}
-    window.REACT_APP_LOCALEX_CODEDIR = "{{ .execution_engine.config.data_dir }}/code";
-    window.REACT_APP_LOCALEX_DATADIR = "{{ .execution_engine.config.data_dir }}/data";
-    window.REACT_APP_LOCALEX_TEMPDIR = "{{ .execution_engine.config.data_dir }}/temp";
-    window.REACT_APP_LOCALEX_LOGDIR = "{{ .execution_engine.config.data_dir }}/logs";
-    window.REACT_APP_LOCALEX_DATAURL = "{{ .execution_engine.config.data_url }}/data";
-    window.REACT_APP_LOCALEX_LOGURL = "{{ .execution_engine.config.data_url }}/logs";
-    window.REACT_APP_LOCALEX_PARALLEL = "{{ .execution_engine.config.parallel }}";
+    {{ if eq .config.type "tapis" }}
+    window.REACT_APP_LOCALEX_CODEDIR = "{{ .config.data_dir }}/code";
+    window.REACT_APP_LOCALEX_DATADIR = "{{ .config.data_dir }}/data";
+    window.REACT_APP_LOCALEX_TEMPDIR = "{{ .config.data_dir }}/temp";
+    window.REACT_APP_LOCALEX_LOGDIR = "{{ .config.data_dir }}/logs";
+    window.REACT_APP_LOCALEX_DATAURL = "{{ .config.data_url }}/data";
+    window.REACT_APP_LOCALEX_LOGURL = "{{ .config.data_url }}/logs";
+    window.REACT_APP_LOCALEX_PARALLEL = "{{ .config.parallelism }}";
     {{ end }}
 
 
-    {{ if eq .execution_engine.type "wings" }}
+    {{ if eq .config.type "wings" }}
     //Execution - Wings
-    window.REACT_APP_WINGS_EXPORT_URL = "{{ .execution_engine.config.wings_export_url }}";
-    window.REACT_APP_WINGS_STORAGE = "{{ .execution_engine.config.wings_storage }}";
-    window.REACT_APP_WINGS_DOTPATH = "{{ .execution_engine.config.wings_dot_path }}";
-    window.REACT_APP_WINGS_ONTURL = "{{ .execution_engine.config.wings_ont_url }}";
+    window.REACT_APP_WINGS_EXPORT_URL = "{{ .config.wings_export_url }}";
+    window.REACT_APP_WINGS_STORAGE = "{{ .config.wings_storage }}";
+    window.REACT_APP_WINGS_DOTPATH = "{{ .config.wings_dot_path }}";
+    window.REACT_APP_WINGS_ONTURL = "{{ .config.wings_ont_url }}";
     {{ end }}
 
     {{ end }}
@@ -87,4 +87,10 @@ data:
     window.REACT_APP_INGESTION_API = "http://node1.ingestion.mint.isi.edu/v1.3.0";
     window.REACT_APP_AIRFLOW_API = "https://airflow.mint.isi.edu/api/v1";
     window.REACT_APP_AIRFLOW_DAG_DOWNLOAD_THREAD_ID = "download_thread"
+
+    {{ if  and .Values.components.ensemble_manager.config.data_catalog (eq .Values.components.ensemble_manager.config.data_catalog.type "CKAN") }}
+    window.REACT_APP_DATA_CATALOG_TYPE = "CKAN";
+    window.REACT_APP_DATA_CATALOG_KEY = {{ .Values.secrets.data_catalog.ckan.api_key | quote }};
+    {{ end }}
+
 {{- end}}
diff --git a/charts/mint/values.yaml b/charts/mint/values.yaml
@@ -263,17 +263,33 @@ components:
     serviceAccountName: default
     environment:
       data_dir: /var/mint
-      data_url: http://localhost:30010/data
-      parallel: 1
-    execution_engine:
-      type: localex
-      config:
-        codedir: /home/node/app/data/code
-        datadir: /home/node/app/data/data
-        tempdir: /home/node/app/data/temp
-        logdir: /home/node/app/data/logs
-        dataurl: http://localhost:30010/data
-        parallelism: 1
+    config:
+      data_catalog:
+        api: "http://localhost:5000"
+        type: "CKAN"
+        extra:
+          owner_organization_id: "21033e7d-484d-4974-9340-8c5755242c1f"
+          owner_provenance_id: "9ef60317-5da5-4050-8bbc-7d6826fee49f"
+      data_server:
+        type: "S3"
+        region: "ap-south-1"
+        bucket: "mintdata"
+
+      graphql:
+        endpoint: "localhost:30003/v1/graphql"
+        enable_ssl: false
+        use_secret: true
+
+      execution_engine:
+        type: "localex"
+        code_dir: "/home/node/app/data/code"
+        data_dir: "/home/node/app/data/data"
+        temp_dir: "/home/node/app/data/temp"
+        log_dir: "/home/node/app/data/logs"
+        data_url: "s3://mintdata"
+        log_url: "s3://mintdata"
+        parallelism: 2
+
     ingress:
       enabled: true
       className: ""
@@ -293,10 +309,19 @@ components:
       selector: {}
       annotations:
         helm.sh/resource-policy: keep
+
 google:
   maps:
     key: AIzaSyAkRnERo4F4dy9AhdrWHAN5vdJWs0vZCgM
+
 secrets:
+  data_catalog:
+    ckan:
+      api_key: CHANGEME
+  data_server:
+    s3:
+      access_key: CHANGEME
+      secret_key: CHANGEME
   database:
     data_catalog:
       username: datacatalog

diff --git a/ct-arm-64.yaml b/ct-arm-64.yaml
@@ -1,6 +1,6 @@
 remote: origin
 target-branch: main
 chart-dirs:
-  - charts/mint
+  - charts
 helm-extra-args: --timeout 600s
 helm-extra-set-args: --set=arm_support=true --set components.data_catalog.enabled=false
diff --git a/ct.yaml b/ct.yaml
@@ -1,6 +1,6 @@
 remote: origin
 target-branch: main
 chart-dirs:
-  - charts/mint
+  - charts
 helm-extra-args: --timeout 600s
 #helm-extra-set-args: --set=arm_support=true --set components.data_catalog.enabled=false