Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

add: support ckan #44

Merged
merged 6 commits into from
Aug 31, 2024
Merged

add: support ckan #44

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
f38bd42
add: required data catalog
mosoriob Aug 31, 2024
a566a09
add: config requiredb by ui
mosoriob Aug 31, 2024
fe80c7e
add: data server s3
mosoriob Aug 31, 2024
cf16523
Merge branch 'main' into 43-add-support-ckan
mosoriob Aug 31, 2024
7936ad8
fix: ct config
mosoriob Aug 31, 2024
2297de4
release: 3.3.0
mosoriob Aug 31, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion charts/mint/Chart.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@ type: application
# This is the chart version. This version number should be incremented each time you make changes
# to the chart and its templates, including the app version.
# Versions are expected to follow Semantic Versioning (https://semver.org/)
version: 3.2.0
version: 3.3.0
# This is the version number of the application being deployed. This version number should be
# incremented each time you make changes to the application. Versions are not expected to
# follow Semantic Versioning. They should reflect the version the application is using.
Expand Down
37 changes: 32 additions & 5 deletions charts/mint/templates/ensemble-manager-config.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -48,13 +48,40 @@ data:
{{- end }}


{{- with .Values.components.ensemble_manager }}
{{- if .execution_engine.type }}
"execution_engine": {{ .execution_engine.type | quote }},
"{{ .execution_engine.type }}":
{{ .execution_engine.config | toRawJson | nindent 12}},
{{- with .Values.components.ensemble_manager.config }}
{{- if .type }}
"execution_engine": {{ .type | quote }},
"{{ .type }}":
{{ .execution_engine | toRawJson | nindent 12}},
{{- end }}
{{- end }}


{{- with .Values.components.ensemble_manager.config }}
{{- if eq .data_catalog.type "CKAN"}}
"data_catalog_api": {{ .data_catalog.api | quote }},
"data_catalog_type": {{ .data_catalog.type | quote }},
"data_catalog_extra": {
"owner_organization_id": {{ .data_catalog.extra.owner_organization_id | quote }},
"owner_provenance_id": {{ .data_catalog.extra.owner_provenance_id | quote }}
},
{{- end}}
{{- end}}



{{- if eq .Values.components.ensemble_manager.config.data_server.type "S3" }}
{{- with .Values.components.ensemble_manager.config }}
"data_server_type": {{ .data_server.type | quote }},
"data_server": {
"region": {{ .data_server.region | quote }},
"bucket": {{ .data_server.bucket | quote }},
{{- end}}
"access_key": {{ .Values.secrets.data_server.s3.access_key | quote }},
"secret_access_key": {{ .Values.secrets.data_server.s3.secret_key | quote }}
},
{{- end}}

"auth_server": {{ .Values.auth.url | quote }},
"auth_realm": {{ .Values.auth.realm | quote }},
"auth_client_id": {{ .Values.auth.ui_client_id | quote }},
Expand Down
50 changes: 28 additions & 22 deletions charts/mint/templates/ui-config.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -38,35 +38,35 @@ data:
{{ if .enabled }}
window.REACT_APP_ENSEMBLE_MANAGER_API = "http{{ if .ingress.tls }}s{{ end }}://{{ with (first .ingress.hosts ) }}{{ .host }}{{ end }}/{{ .api_version }}";
//Execution - Local
window.REACT_APP_EXECUTION_ENGINE = "{{ .execution_engine.type }}";
window.REACT_APP_EXECUTION_ENGINE = "{{ .config.type }}";

{{ if eq .execution_engine.type "localex" }}
window.REACT_APP_LOCALEX_CODEDIR = "{{ .execution_engine.config.data_dir }}/code";
window.REACT_APP_LOCALEX_DATADIR = "{{ .execution_engine.config.data_dir }}/data";
window.REACT_APP_LOCALEX_TEMPDIR = "{{ .execution_engine.config.data_dir }}/temp";
window.REACT_APP_LOCALEX_LOGDIR = "{{ .execution_engine.config.data_dir }}/logs";
window.REACT_APP_LOCALEX_DATAURL = "{{ .execution_engine.config.data_url }}/data";
window.REACT_APP_LOCALEX_LOGURL = "{{ .execution_engine.config.data_url }}/logs";
window.REACT_APP_LOCALEX_PARALLEL = "{{ .execution_engine.config.parallel }}";
{{ if eq .config.type "localex" }}
window.REACT_APP_LOCALEX_CODEDIR = "{{ .config.data_dir }}/code";
window.REACT_APP_LOCALEX_DATADIR = "{{ .config.data_dir }}/data";
window.REACT_APP_LOCALEX_TEMPDIR = "{{ .config.data_dir }}/temp";
window.REACT_APP_LOCALEX_LOGDIR = "{{ .config.data_dir }}/logs";
window.REACT_APP_LOCALEX_DATAURL = "{{ .config.data_url }}/data";
window.REACT_APP_LOCALEX_LOGURL = "{{ .config.data_url }}/logs";
window.REACT_APP_LOCALEX_PARALLEL = "{{ .config.parallel }}";
{{ end }}

{{ if eq .execution_engine.type "tapis" }}
window.REACT_APP_LOCALEX_CODEDIR = "{{ .execution_engine.config.data_dir }}/code";
window.REACT_APP_LOCALEX_DATADIR = "{{ .execution_engine.config.data_dir }}/data";
window.REACT_APP_LOCALEX_TEMPDIR = "{{ .execution_engine.config.data_dir }}/temp";
window.REACT_APP_LOCALEX_LOGDIR = "{{ .execution_engine.config.data_dir }}/logs";
window.REACT_APP_LOCALEX_DATAURL = "{{ .execution_engine.config.data_url }}/data";
window.REACT_APP_LOCALEX_LOGURL = "{{ .execution_engine.config.data_url }}/logs";
window.REACT_APP_LOCALEX_PARALLEL = "{{ .execution_engine.config.parallel }}";
{{ if eq .config.type "tapis" }}
window.REACT_APP_LOCALEX_CODEDIR = "{{ .config.data_dir }}/code";
window.REACT_APP_LOCALEX_DATADIR = "{{ .config.data_dir }}/data";
window.REACT_APP_LOCALEX_TEMPDIR = "{{ .config.data_dir }}/temp";
window.REACT_APP_LOCALEX_LOGDIR = "{{ .config.data_dir }}/logs";
window.REACT_APP_LOCALEX_DATAURL = "{{ .config.data_url }}/data";
window.REACT_APP_LOCALEX_LOGURL = "{{ .config.data_url }}/logs";
window.REACT_APP_LOCALEX_PARALLEL = "{{ .config.parallelism }}";
{{ end }}


{{ if eq .execution_engine.type "wings" }}
{{ if eq .config.type "wings" }}
//Execution - Wings
window.REACT_APP_WINGS_EXPORT_URL = "{{ .execution_engine.config.wings_export_url }}";
window.REACT_APP_WINGS_STORAGE = "{{ .execution_engine.config.wings_storage }}";
window.REACT_APP_WINGS_DOTPATH = "{{ .execution_engine.config.wings_dot_path }}";
window.REACT_APP_WINGS_ONTURL = "{{ .execution_engine.config.wings_ont_url }}";
window.REACT_APP_WINGS_EXPORT_URL = "{{ .config.wings_export_url }}";
window.REACT_APP_WINGS_STORAGE = "{{ .config.wings_storage }}";
window.REACT_APP_WINGS_DOTPATH = "{{ .config.wings_dot_path }}";
window.REACT_APP_WINGS_ONTURL = "{{ .config.wings_ont_url }}";
{{ end }}

{{ end }}
Expand All @@ -87,4 +87,10 @@ data:
window.REACT_APP_INGESTION_API = "http://node1.ingestion.mint.isi.edu/v1.3.0";
window.REACT_APP_AIRFLOW_API = "https://airflow.mint.isi.edu/api/v1";
window.REACT_APP_AIRFLOW_DAG_DOWNLOAD_THREAD_ID = "download_thread"

{{ if and .Values.components.ensemble_manager.config.data_catalog (eq .Values.components.ensemble_manager.config.data_catalog.type "CKAN") }}
window.REACT_APP_DATA_CATALOG_TYPE = "CKAN";
window.REACT_APP_DATA_CATALOG_KEY = {{ .Values.secrets.data_catalog.ckan.api_key | quote }};
{{ end }}

{{- end}}
47 changes: 36 additions & 11 deletions charts/mint/values.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -263,17 +263,33 @@ components:
serviceAccountName: default
environment:
data_dir: /var/mint
data_url: http://localhost:30010/data
parallel: 1
execution_engine:
type: localex
config:
codedir: /home/node/app/data/code
datadir: /home/node/app/data/data
tempdir: /home/node/app/data/temp
logdir: /home/node/app/data/logs
dataurl: http://localhost:30010/data
parallelism: 1
config:
data_catalog:
api: "http://localhost:5000"
type: "CKAN"
extra:
owner_organization_id: "21033e7d-484d-4974-9340-8c5755242c1f"
owner_provenance_id: "9ef60317-5da5-4050-8bbc-7d6826fee49f"
data_server:
type: "S3"
region: "ap-south-1"
bucket: "mintdata"

graphql:
endpoint: "localhost:30003/v1/graphql"
enable_ssl: false
use_secret: true

execution_engine:
type: "localex"
code_dir: "/home/node/app/data/code"
data_dir: "/home/node/app/data/data"
temp_dir: "/home/node/app/data/temp"
log_dir: "/home/node/app/data/logs"
data_url: "s3://mintdata"
log_url: "s3://mintdata"
parallelism: 2

ingress:
enabled: true
className: ""
Expand All @@ -293,10 +309,19 @@ components:
selector: {}
annotations:
helm.sh/resource-policy: keep

google:
maps:
key: AIzaSyAkRnERo4F4dy9AhdrWHAN5vdJWs0vZCgM

secrets:
data_catalog:
ckan:
api_key: CHANGEME
data_server:
s3:
access_key: CHANGEME
secret_key: CHANGEME
Comment on lines +318 to +324
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@varunratnakar 👀

database:
data_catalog:
username: datacatalog
Expand Down
2 changes: 1 addition & 1 deletion ct-arm-64.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
remote: origin
target-branch: main
chart-dirs:
- charts/mint
- charts
helm-extra-args: --timeout 600s
helm-extra-set-args: --set=arm_support=true --set components.data_catalog.enabled=false
2 changes: 1 addition & 1 deletion ct.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
remote: origin
target-branch: main
chart-dirs:
- charts/mint
- charts
helm-extra-args: --timeout 600s
#helm-extra-set-args: --set=arm_support=true --set components.data_catalog.enabled=false
Loading
Loading