From 8b1c65b9b79a5022e6b89aee73b9d2552ec6e97f Mon Sep 17 00:00:00 2001 From: sriniarul <125246441+arulsrinivaasan-everbee@users.noreply.github.com> Date: Sun, 1 Oct 2023 16:02:17 +0530 Subject: [PATCH 01/11] redis support support redis standalone scheduler and webserver option --- charts/mageai/Chart.lock | 6 ++ charts/mageai/Chart.yaml | 6 ++ charts/mageai/templates/deployment.yaml | 21 ++++ charts/mageai/templates/scheduler.yaml | 128 ++++++++++++++++++++++++ charts/mageai/templates/webservice.yaml | 128 ++++++++++++++++++++++++ charts/mageai/values.yaml | 15 ++- 6 files changed, 303 insertions(+), 1 deletion(-) create mode 100644 charts/mageai/Chart.lock create mode 100644 charts/mageai/templates/scheduler.yaml create mode 100644 charts/mageai/templates/webservice.yaml diff --git a/charts/mageai/Chart.lock b/charts/mageai/Chart.lock new file mode 100644 index 0000000..50e1b2f --- /dev/null +++ b/charts/mageai/Chart.lock @@ -0,0 +1,6 @@ +dependencies: +- name: redis + repository: https://charts.bitnami.com/bitnami + version: 18.1.1 +digest: sha256:79d2d6d43f93313876bfd130ef9b5908377fde03b01819759daff0a82c7afdc7 +generated: "2023-10-01T14:08:47.871298+05:30" diff --git a/charts/mageai/Chart.yaml b/charts/mageai/Chart.yaml index 6dba14a..f598f11 100644 --- a/charts/mageai/Chart.yaml +++ b/charts/mageai/Chart.yaml @@ -18,6 +18,12 @@ maintainers: email: eng@mage.ai name: mageai +dependencies: + - name: redis + version: 18.1.1 + repository: https://charts.bitnami.com/bitnami + condition: redis.enabled + sources: - https://github.com/mage-ai/helm-charts/tree/master/charts/mageai diff --git a/charts/mageai/templates/deployment.yaml b/charts/mageai/templates/deployment.yaml index 0337420..7c9098c 100644 --- a/charts/mageai/templates/deployment.yaml +++ b/charts/mageai/templates/deployment.yaml @@ -1,3 +1,4 @@ +{{- if not .Values.seperateScheduler }} apiVersion: apps/v1 kind: Deployment metadata: @@ -25,6 +26,18 @@ spec: serviceAccountName: {{ include "mageai.serviceAccountName" . }} securityContext: {{- toYaml .Values.podSecurityContext | nindent 8 }} + initContainers: + - name: wait-for-redis + image: alpine + env: + {{- if .Values.redis.enabled }} + - name: REDIS_URL + value: redis://mage-chart-redis-headless:6379/0 + {{- else if .Values.redis.url}} + - name: REDIS_URL + values: {{.Values.redis.url}} + {{- end }} + command: ["sh", "-c", "until nc -z -v $(echo $REDIS_URL | cut -d'/' -f3 | cut -d':' -f1) $(echo $REDIS_URL | cut -d'/' -f3 | cut -d':' -f2); do sleep 1; done"] containers: - name: {{ .Chart.Name }} securityContext: @@ -80,6 +93,13 @@ spec: {{- else if .Values.extraEnvs }} {{- toYaml .Values.extraEnvs | nindent 12 }} {{- end }} + {{- if .Values.redis.enabled }} + - name: REDIS_URL + value: redis://mage-chart-redis-headless:6379/0 + {{- else if .Values.redis.url}} + - name: REDIS_URL + values: {{.Values.redis.url}} + {{- end }} volumeMounts: {{- if .Values.volumes }} - name: mage-fs @@ -105,3 +125,4 @@ spec: {{- else if .Values.extraVolumes -}} {{ toYaml .Values.extraVolumes | nindent 8 }} {{- end }} +{{- end }} diff --git a/charts/mageai/templates/scheduler.yaml b/charts/mageai/templates/scheduler.yaml new file mode 100644 index 0000000..e37c8cd --- /dev/null +++ b/charts/mageai/templates/scheduler.yaml @@ -0,0 +1,128 @@ +{{- if .Values.seperateScheduler }} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "mageai.fullname" . }} + labels: + {{- include "mageai.labels" . | nindent 4 }} +spec: + replicas: {{ .Values.scheduler.replicaCount }} + selector: + matchLabels: + {{- include "mageai.selectorLabels" . | nindent 6 }} + template: + metadata: + {{- with .Values.podAnnotations }} + annotations: + {{- toYaml . | nindent 8 }} + {{- end }} + labels: + {{- include "mageai.selectorLabels" . | nindent 8 }} + spec: + {{- with .Values.imagePullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 8 }} + {{- end }} + serviceAccountName: {{ include "mageai.serviceAccountName" . }} + securityContext: + {{- toYaml .Values.podSecurityContext | nindent 8 }} + initContainers: + - name: wait-for-redis + image: alpine + env: + {{- if .Values.redis.enabled }} + - name: REDIS_URL + value: redis://mage-chart-redis-headless:6379/0 + {{- else if .Values.redis.url}} + - name: REDIS_URL + values: {{.Values.redis.url}} + {{- end }} + command: ["sh", "-c", "until nc -z -v $(echo $REDIS_URL | cut -d'/' -f3 | cut -d':' -f1) $(echo $REDIS_URL | cut -d'/' -f3 | cut -d':' -f2); do sleep 1; done"] + containers: + - name: {{ .Chart.Name }} + securityContext: + {{- toYaml .Values.securityContext | nindent 12 }} + image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}" + imagePullPolicy: {{ .Values.image.pullPolicy }} + ports: + - name: http + containerPort: {{ .Values.service.port }} + protocol: TCP + {{- if .Values.customLivenessProbe }} + livenessProbe: {{- toYaml .Values.customLivenessProbe | nindent 12 }} + {{- else if .Values.livenessProbe.enabled }} + livenessProbe: + httpGet: + path: {{ .Values.livenessProbe.path }} + port: {{ .Values.livenessProbe.port }} + initialDelaySeconds: {{ .Values.livenessProbe.initialDelaySeconds }} + failureThreshold: {{ .Values.livenessProbe.failureThreshold }} + periodSeconds: {{ .Values.livenessProbe.periodSeconds }} + successThreshold: {{ .Values.livenessProbe.successThreshold }} + terminationGracePeriodSeconds: {{ .Values.livenessProbe.terminationGracePeriodSeconds }} + timeoutSeconds: {{ .Values.livenessProbe.timeoutSeconds }} + {{- end }} + {{- if .Values.customReadinessProbe }} + readinessProbe: {{- toYaml .Values.customReadinessProbe | nindent 12 }} + {{- else if .Values.readinessProbe.enabled }} + readinessProbe: + httpGet: + path: {{ .Values.readinessProbe.path }} + port: {{ .Values.readinessProbe.port }} + initialDelaySeconds: {{ .Values.readinessProbe.initialDelaySeconds }} + failureThreshold: {{ .Values.readinessProbe.failureThreshold }} + periodSeconds: {{ .Values.readinessProbe.periodSeconds }} + successThreshold: {{ .Values.readinessProbe.successThreshold }} + terminationGracePeriodSeconds: {{ .Values.readinessProbe.terminationGracePeriodSeconds }} + timeoutSeconds: {{ .Values.readinessProbe.timeoutSeconds }} + {{- end }} + resources: + {{- toYaml .Values.resources | nindent 12 }} + envFrom: + {{- if .Values.config }} + - configMapRef: + name: {{ include "mageai.fullname" . }}-env + {{- end }} + {{- if or (.Values.existingSecret) (.Values.secrets) }} + - secretRef: + name: {{ include "mageai.secretName" . }} + {{- end }} + env: + {{- if .Values.env }} + {{- toYaml .Values.env | nindent 12 }} + {{- else if .Values.extraEnvs }} + {{- toYaml .Values.extraEnvs | nindent 12 }} + {{- end }} + {{- if .Values.redis.enabled }} + - name: REDIS_URL + value: redis://mage-chart-redis-headless:6379/0 + {{- else if .Values.redis.url}} + - name: REDIS_URL + values: {{.Values.redis.url}} + {{- end }} + volumeMounts: + {{- if .Values.volumes }} + - name: mage-fs + mountPath: /home/src + {{- else if .Values.extraVolumeMounts }} + {{- toYaml .Values.extraVolumeMounts | nindent 12 }} + {{- end }} + {{- with .Values.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} + volumes: + {{- if .Values.volumes }} + {{- toYaml .Values.volumes | nindent 8 }} + {{- else if .Values.extraVolumes -}} + {{ toYaml .Values.extraVolumes | nindent 8 }} + {{- end }} +{{- end }} diff --git a/charts/mageai/templates/webservice.yaml b/charts/mageai/templates/webservice.yaml new file mode 100644 index 0000000..022ea55 --- /dev/null +++ b/charts/mageai/templates/webservice.yaml @@ -0,0 +1,128 @@ +{{- if .Values.seperateScheduler }} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "mageai.fullname" . }} + labels: + {{- include "mageai.labels" . | nindent 4 }} +spec: + replicas: {{ .Values.webService.replicaCount }} + selector: + matchLabels: + {{- include "mageai.selectorLabels" . | nindent 6 }} + template: + metadata: + {{- with .Values.podAnnotations }} + annotations: + {{- toYaml . | nindent 8 }} + {{- end }} + labels: + {{- include "mageai.selectorLabels" . | nindent 8 }} + spec: + {{- with .Values.imagePullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 8 }} + {{- end }} + serviceAccountName: {{ include "mageai.serviceAccountName" . }} + securityContext: + {{- toYaml .Values.podSecurityContext | nindent 8 }} + initContainers: + - name: wait-for-redis + image: alpine + env: + {{- if .Values.redis.enabled }} + - name: REDIS_URL + value: redis://mage-chart-redis-headless:6379/0 + {{- else if .Values.redis.url}} + - name: REDIS_URL + values: {{.Values.redis.url}} + {{- end }} + command: ["sh", "-c", "until nc -z -v $(echo $REDIS_URL | cut -d'/' -f3 | cut -d':' -f1) $(echo $REDIS_URL | cut -d'/' -f3 | cut -d':' -f2); do sleep 1; done"] + containers: + - name: {{ .Chart.Name }} + securityContext: + {{- toYaml .Values.securityContext | nindent 12 }} + image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}" + imagePullPolicy: {{ .Values.image.pullPolicy }} + ports: + - name: http + containerPort: {{ .Values.service.port }} + protocol: TCP + {{- if .Values.customLivenessProbe }} + livenessProbe: {{- toYaml .Values.customLivenessProbe | nindent 12 }} + {{- else if .Values.livenessProbe.enabled }} + livenessProbe: + httpGet: + path: {{ .Values.livenessProbe.path }} + port: {{ .Values.livenessProbe.port }} + initialDelaySeconds: {{ .Values.livenessProbe.initialDelaySeconds }} + failureThreshold: {{ .Values.livenessProbe.failureThreshold }} + periodSeconds: {{ .Values.livenessProbe.periodSeconds }} + successThreshold: {{ .Values.livenessProbe.successThreshold }} + terminationGracePeriodSeconds: {{ .Values.livenessProbe.terminationGracePeriodSeconds }} + timeoutSeconds: {{ .Values.livenessProbe.timeoutSeconds }} + {{- end }} + {{- if .Values.customReadinessProbe }} + readinessProbe: {{- toYaml .Values.customReadinessProbe | nindent 12 }} + {{- else if .Values.readinessProbe.enabled }} + readinessProbe: + httpGet: + path: {{ .Values.readinessProbe.path }} + port: {{ .Values.readinessProbe.port }} + initialDelaySeconds: {{ .Values.readinessProbe.initialDelaySeconds }} + failureThreshold: {{ .Values.readinessProbe.failureThreshold }} + periodSeconds: {{ .Values.readinessProbe.periodSeconds }} + successThreshold: {{ .Values.readinessProbe.successThreshold }} + terminationGracePeriodSeconds: {{ .Values.readinessProbe.terminationGracePeriodSeconds }} + timeoutSeconds: {{ .Values.readinessProbe.timeoutSeconds }} + {{- end }} + resources: + {{- toYaml .Values.resources | nindent 12 }} + envFrom: + {{- if .Values.config }} + - configMapRef: + name: {{ include "mageai.fullname" . }}-env + {{- end }} + {{- if or (.Values.existingSecret) (.Values.secrets) }} + - secretRef: + name: {{ include "mageai.secretName" . }} + {{- end }} + env: + {{- if .Values.env }} + {{- toYaml .Values.env | nindent 12 }} + {{- else if .Values.extraEnvs }} + {{- toYaml .Values.extraEnvs | nindent 12 }} + {{- end }} + {{- if .Values.redis.enabled }} + - name: REDIS_URL + value: redis://redis://mage-chart-redis-headless:6379/0 + {{- else if .Values.redis.url}} + - name: REDIS_URL + values: {{.Values.redis.url}} + {{- end }} + volumeMounts: + {{- if .Values.volumes }} + - name: mage-fs + mountPath: /home/src + {{- else if .Values.extraVolumeMounts }} + {{- toYaml .Values.extraVolumeMounts | nindent 12 }} + {{- end }} + {{- with .Values.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} + volumes: + {{- if .Values.volumes }} + {{- toYaml .Values.volumes | nindent 8 }} + {{- else if .Values.extraVolumes -}} + {{ toYaml .Values.extraVolumes | nindent 8 }} + {{- end }} +{{- end }} diff --git a/charts/mageai/values.yaml b/charts/mageai/values.yaml index 1061959..47fd788 100644 --- a/charts/mageai/values.yaml +++ b/charts/mageai/values.yaml @@ -2,7 +2,20 @@ # This is a YAML-formatted file. # Declare variables to be passed into your templates. -replicaCount: 1 +replicaCount: 2 +seperateScheduler: false + +scheduler: + replicaCount: 1 #Effective if seperateScheduler is true +webService: + replicaCount: 1 #Effective if seperateScheduler is true + +redis: + enabled: true #Enable redis if you want more replica + architecture: standalone + auth: + enabled: false + url: redis://redis:6379/0 #Your custom redis url (make sure redis.enabled is set to false) image: repository: mageai/mageai From c4b1dbb1b29944cf605432b24566654b1ea1b792 Mon Sep 17 00:00:00 2001 From: sriniarul <125246441+arulsrinivaasan-everbee@users.noreply.github.com> Date: Sun, 1 Oct 2023 16:28:07 +0530 Subject: [PATCH 02/11] scheduler standalone mode fix scheduler standalone mode fix --- charts/mageai/templates/_helpers.tpl | 10 ++++++++++ charts/mageai/templates/deployment.yaml | 6 +++--- charts/mageai/templates/scheduler.yaml | 12 ++++++------ charts/mageai/templates/webservice.yaml | 10 +++++----- charts/mageai/values.yaml | 8 +++++--- 5 files changed, 29 insertions(+), 17 deletions(-) diff --git a/charts/mageai/templates/_helpers.tpl b/charts/mageai/templates/_helpers.tpl index 7981041..75d861b 100644 --- a/charts/mageai/templates/_helpers.tpl +++ b/charts/mageai/templates/_helpers.tpl @@ -50,6 +50,16 @@ app.kubernetes.io/name: {{ include "mageai.name" . }} app.kubernetes.io/instance: {{ .Release.Name }} {{- end }} + +{{/* +Scheduler Selector labels +*/}} +{{- define "mageai.schedulerSelectorLabels" -}} +app.kubernetes.io/name: {{ .Values.scheduler.name }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end }} + + {{/* Create the name of the service account to use */}} diff --git a/charts/mageai/templates/deployment.yaml b/charts/mageai/templates/deployment.yaml index 7c9098c..a0f8f56 100644 --- a/charts/mageai/templates/deployment.yaml +++ b/charts/mageai/templates/deployment.yaml @@ -33,9 +33,9 @@ spec: {{- if .Values.redis.enabled }} - name: REDIS_URL value: redis://mage-chart-redis-headless:6379/0 - {{- else if .Values.redis.url}} + {{- else if .Values.redis.url }} - name: REDIS_URL - values: {{.Values.redis.url}} + value: {{ .Values.redis.url }} {{- end }} command: ["sh", "-c", "until nc -z -v $(echo $REDIS_URL | cut -d'/' -f3 | cut -d':' -f1) $(echo $REDIS_URL | cut -d'/' -f3 | cut -d':' -f2); do sleep 1; done"] containers: @@ -98,7 +98,7 @@ spec: value: redis://mage-chart-redis-headless:6379/0 {{- else if .Values.redis.url}} - name: REDIS_URL - values: {{.Values.redis.url}} + value: {{.Values.redis.url}} {{- end }} volumeMounts: {{- if .Values.volumes }} diff --git a/charts/mageai/templates/scheduler.yaml b/charts/mageai/templates/scheduler.yaml index e37c8cd..4625d9d 100644 --- a/charts/mageai/templates/scheduler.yaml +++ b/charts/mageai/templates/scheduler.yaml @@ -2,14 +2,14 @@ apiVersion: apps/v1 kind: Deployment metadata: - name: {{ include "mageai.fullname" . }} + name: {{ .Values.scheduler.name }} labels: {{- include "mageai.labels" . | nindent 4 }} spec: replicas: {{ .Values.scheduler.replicaCount }} selector: matchLabels: - {{- include "mageai.selectorLabels" . | nindent 6 }} + {{- include "mageai.schedulerSelectorLabels" . | nindent 6 }} template: metadata: {{- with .Values.podAnnotations }} @@ -17,7 +17,7 @@ spec: {{- toYaml . | nindent 8 }} {{- end }} labels: - {{- include "mageai.selectorLabels" . | nindent 8 }} + {{- include "mageai.schedulerSelectorLabels" . | nindent 8 }} spec: {{- with .Values.imagePullSecrets }} imagePullSecrets: @@ -33,9 +33,9 @@ spec: {{- if .Values.redis.enabled }} - name: REDIS_URL value: redis://mage-chart-redis-headless:6379/0 - {{- else if .Values.redis.url}} + {{- else if .Values.redis.url }} - name: REDIS_URL - values: {{.Values.redis.url}} + value: {{ .Values.redis.url }} {{- end }} command: ["sh", "-c", "until nc -z -v $(echo $REDIS_URL | cut -d'/' -f3 | cut -d':' -f1) $(echo $REDIS_URL | cut -d'/' -f3 | cut -d':' -f2); do sleep 1; done"] containers: @@ -98,7 +98,7 @@ spec: value: redis://mage-chart-redis-headless:6379/0 {{- else if .Values.redis.url}} - name: REDIS_URL - values: {{.Values.redis.url}} + value: {{ .Values.redis.url }} {{- end }} volumeMounts: {{- if .Values.volumes }} diff --git a/charts/mageai/templates/webservice.yaml b/charts/mageai/templates/webservice.yaml index 022ea55..fa1c5c1 100644 --- a/charts/mageai/templates/webservice.yaml +++ b/charts/mageai/templates/webservice.yaml @@ -2,11 +2,11 @@ apiVersion: apps/v1 kind: Deployment metadata: - name: {{ include "mageai.fullname" . }} + name: {{ .Values.webServer.name }} labels: {{- include "mageai.labels" . | nindent 4 }} spec: - replicas: {{ .Values.webService.replicaCount }} + replicas: {{ .Values.webServer.replicaCount }} selector: matchLabels: {{- include "mageai.selectorLabels" . | nindent 6 }} @@ -33,9 +33,9 @@ spec: {{- if .Values.redis.enabled }} - name: REDIS_URL value: redis://mage-chart-redis-headless:6379/0 - {{- else if .Values.redis.url}} + {{- else if .Values.redis.url }} - name: REDIS_URL - values: {{.Values.redis.url}} + value: {{ .Values.redis.url }} {{- end }} command: ["sh", "-c", "until nc -z -v $(echo $REDIS_URL | cut -d'/' -f3 | cut -d':' -f1) $(echo $REDIS_URL | cut -d'/' -f3 | cut -d':' -f2); do sleep 1; done"] containers: @@ -98,7 +98,7 @@ spec: value: redis://redis://mage-chart-redis-headless:6379/0 {{- else if .Values.redis.url}} - name: REDIS_URL - values: {{.Values.redis.url}} + value: {{ .Values.redis.url }} {{- end }} volumeMounts: {{- if .Values.volumes }} diff --git a/charts/mageai/values.yaml b/charts/mageai/values.yaml index 47fd788..115692d 100644 --- a/charts/mageai/values.yaml +++ b/charts/mageai/values.yaml @@ -3,15 +3,17 @@ # Declare variables to be passed into your templates. replicaCount: 2 -seperateScheduler: false +seperateScheduler: true scheduler: replicaCount: 1 #Effective if seperateScheduler is true -webService: + name: mageai-scheduler +webServer: replicaCount: 1 #Effective if seperateScheduler is true + name: mageai-webserver redis: - enabled: true #Enable redis if you want more replica + enabled: false #Enable redis if you want more replica architecture: standalone auth: enabled: false From 38c2a04cba1dbdc084390a4fc54615889d12a3f7 Mon Sep 17 00:00:00 2001 From: sriniarul <125246441+arulsrinivaasan-everbee@users.noreply.github.com> Date: Sun, 1 Oct 2023 17:01:02 +0530 Subject: [PATCH 03/11] redis service name updated --- charts/mageai/templates/deployment.yaml | 4 ++-- charts/mageai/templates/scheduler.yaml | 4 ++-- charts/mageai/templates/webservice.yaml | 4 ++-- charts/mageai/values.yaml | 4 ++-- 4 files changed, 8 insertions(+), 8 deletions(-) diff --git a/charts/mageai/templates/deployment.yaml b/charts/mageai/templates/deployment.yaml index a0f8f56..ea1d08f 100644 --- a/charts/mageai/templates/deployment.yaml +++ b/charts/mageai/templates/deployment.yaml @@ -32,7 +32,7 @@ spec: env: {{- if .Values.redis.enabled }} - name: REDIS_URL - value: redis://mage-chart-redis-headless:6379/0 + value: redis://{{.Release.Name}}-redis-headless:6379/0 {{- else if .Values.redis.url }} - name: REDIS_URL value: {{ .Values.redis.url }} @@ -95,7 +95,7 @@ spec: {{- end }} {{- if .Values.redis.enabled }} - name: REDIS_URL - value: redis://mage-chart-redis-headless:6379/0 + value: redis://{{.Release.Name}}-redis-headless:6379/0 {{- else if .Values.redis.url}} - name: REDIS_URL value: {{.Values.redis.url}} diff --git a/charts/mageai/templates/scheduler.yaml b/charts/mageai/templates/scheduler.yaml index 4625d9d..5094348 100644 --- a/charts/mageai/templates/scheduler.yaml +++ b/charts/mageai/templates/scheduler.yaml @@ -32,7 +32,7 @@ spec: env: {{- if .Values.redis.enabled }} - name: REDIS_URL - value: redis://mage-chart-redis-headless:6379/0 + value: redis://{{.Release.Name }}-redis-headless:6379/0 {{- else if .Values.redis.url }} - name: REDIS_URL value: {{ .Values.redis.url }} @@ -95,7 +95,7 @@ spec: {{- end }} {{- if .Values.redis.enabled }} - name: REDIS_URL - value: redis://mage-chart-redis-headless:6379/0 + value: redis://{{.Release.Name}}-redis-headless:6379/0 {{- else if .Values.redis.url}} - name: REDIS_URL value: {{ .Values.redis.url }} diff --git a/charts/mageai/templates/webservice.yaml b/charts/mageai/templates/webservice.yaml index fa1c5c1..eea8216 100644 --- a/charts/mageai/templates/webservice.yaml +++ b/charts/mageai/templates/webservice.yaml @@ -32,7 +32,7 @@ spec: env: {{- if .Values.redis.enabled }} - name: REDIS_URL - value: redis://mage-chart-redis-headless:6379/0 + value: redis://{{.Release.Name}}-redis-headless:6379/0 {{- else if .Values.redis.url }} - name: REDIS_URL value: {{ .Values.redis.url }} @@ -95,7 +95,7 @@ spec: {{- end }} {{- if .Values.redis.enabled }} - name: REDIS_URL - value: redis://redis://mage-chart-redis-headless:6379/0 + value: redis://{{.Release.Name}}-redis-headless:6379/0 {{- else if .Values.redis.url}} - name: REDIS_URL value: {{ .Values.redis.url }} diff --git a/charts/mageai/values.yaml b/charts/mageai/values.yaml index 115692d..d65a598 100644 --- a/charts/mageai/values.yaml +++ b/charts/mageai/values.yaml @@ -6,14 +6,14 @@ replicaCount: 2 seperateScheduler: true scheduler: - replicaCount: 1 #Effective if seperateScheduler is true + replicaCount: 2 #Effective if seperateScheduler is true name: mageai-scheduler webServer: replicaCount: 1 #Effective if seperateScheduler is true name: mageai-webserver redis: - enabled: false #Enable redis if you want more replica + enabled: true #Enable redis if you want more replica architecture: standalone auth: enabled: false From 95eb275d26ad3b55d940cddc0750316f15e0a620 Mon Sep 17 00:00:00 2001 From: sriniarul <125246441+arulsrinivaasan-everbee@users.noreply.github.com> Date: Mon, 2 Oct 2023 22:49:24 +0530 Subject: [PATCH 04/11] replica count default set to 1 replica count default set to 1 --- charts/mageai/templates/deployment.yaml | 2 +- charts/mageai/templates/scheduler.yaml | 2 +- charts/mageai/templates/webservice.yaml | 2 +- charts/mageai/values.yaml | 13 +++++++------ 4 files changed, 10 insertions(+), 9 deletions(-) diff --git a/charts/mageai/templates/deployment.yaml b/charts/mageai/templates/deployment.yaml index ea1d08f..c3cfb16 100644 --- a/charts/mageai/templates/deployment.yaml +++ b/charts/mageai/templates/deployment.yaml @@ -1,4 +1,4 @@ -{{- if not .Values.seperateScheduler }} +{{- if not .Values.standaloneScheduler }} apiVersion: apps/v1 kind: Deployment metadata: diff --git a/charts/mageai/templates/scheduler.yaml b/charts/mageai/templates/scheduler.yaml index 5094348..f627c33 100644 --- a/charts/mageai/templates/scheduler.yaml +++ b/charts/mageai/templates/scheduler.yaml @@ -1,4 +1,4 @@ -{{- if .Values.seperateScheduler }} +{{- if .Values.standaloneScheduler }} apiVersion: apps/v1 kind: Deployment metadata: diff --git a/charts/mageai/templates/webservice.yaml b/charts/mageai/templates/webservice.yaml index eea8216..2b05033 100644 --- a/charts/mageai/templates/webservice.yaml +++ b/charts/mageai/templates/webservice.yaml @@ -1,4 +1,4 @@ -{{- if .Values.seperateScheduler }} +{{- if .Values.standaloneScheduler }} apiVersion: apps/v1 kind: Deployment metadata: diff --git a/charts/mageai/values.yaml b/charts/mageai/values.yaml index d65a598..e9876f3 100644 --- a/charts/mageai/values.yaml +++ b/charts/mageai/values.yaml @@ -2,22 +2,23 @@ # This is a YAML-formatted file. # Declare variables to be passed into your templates. -replicaCount: 2 -seperateScheduler: true +replicaCount: 1 +standaloneScheduler: true scheduler: - replicaCount: 2 #Effective if seperateScheduler is true + replicaCount: 1 #Effective if standaloneScheduler is true name: mageai-scheduler webServer: - replicaCount: 1 #Effective if seperateScheduler is true + replicaCount: 1 #Effective if standaloneScheduler is true name: mageai-webserver redis: - enabled: true #Enable redis if you want more replica + enabled: false #Enable redis if you want more replica architecture: standalone auth: enabled: false - url: redis://redis:6379/0 #Your custom redis url (make sure redis.enabled is set to false) + #Your custom redis url (make sure redis.enabled is set to false) + # url: redis://redis:6379/0 image: repository: mageai/mageai From 9494b2d55866c06054ca65e4f9ff6bd4edf7e28e Mon Sep 17 00:00:00 2001 From: sriniarul <125246441+arulsrinivaasan-everbee@users.noreply.github.com> Date: Sun, 29 Oct 2023 08:51:13 +0530 Subject: [PATCH 05/11] outdated dependency reference fixed outdated dependency reference fixed --- charts/mageai/Chart.yaml | 5 ----- charts/mageai/requirements.yaml | 5 +++++ 2 files changed, 5 insertions(+), 5 deletions(-) create mode 100644 charts/mageai/requirements.yaml diff --git a/charts/mageai/Chart.yaml b/charts/mageai/Chart.yaml index f598f11..f1a6c1d 100644 --- a/charts/mageai/Chart.yaml +++ b/charts/mageai/Chart.yaml @@ -18,11 +18,6 @@ maintainers: email: eng@mage.ai name: mageai -dependencies: - - name: redis - version: 18.1.1 - repository: https://charts.bitnami.com/bitnami - condition: redis.enabled sources: diff --git a/charts/mageai/requirements.yaml b/charts/mageai/requirements.yaml new file mode 100644 index 0000000..dca8dd9 --- /dev/null +++ b/charts/mageai/requirements.yaml @@ -0,0 +1,5 @@ +dependencies: + - name: redis + version: 18.1.1 + repository: https://charts.bitnami.com/bitnami + condition: redis.enabled From 0d40a125a72e5238755fff3fd68a857b24d78621 Mon Sep 17 00:00:00 2001 From: sriniarul <125246441+arulsrinivaasan-everbee@users.noreply.github.com> Date: Sun, 29 Oct 2023 08:53:49 +0530 Subject: [PATCH 06/11] Revert "outdated dependency reference fixed" This reverts commit 9494b2d55866c06054ca65e4f9ff6bd4edf7e28e. --- charts/mageai/Chart.yaml | 5 +++++ charts/mageai/requirements.yaml | 5 ----- 2 files changed, 5 insertions(+), 5 deletions(-) delete mode 100644 charts/mageai/requirements.yaml diff --git a/charts/mageai/Chart.yaml b/charts/mageai/Chart.yaml index f1a6c1d..f598f11 100644 --- a/charts/mageai/Chart.yaml +++ b/charts/mageai/Chart.yaml @@ -18,6 +18,11 @@ maintainers: email: eng@mage.ai name: mageai +dependencies: + - name: redis + version: 18.1.1 + repository: https://charts.bitnami.com/bitnami + condition: redis.enabled sources: diff --git a/charts/mageai/requirements.yaml b/charts/mageai/requirements.yaml deleted file mode 100644 index dca8dd9..0000000 --- a/charts/mageai/requirements.yaml +++ /dev/null @@ -1,5 +0,0 @@ -dependencies: - - name: redis - version: 18.1.1 - repository: https://charts.bitnami.com/bitnami - condition: redis.enabled From e30c44e7b92118329128957efec4505c1f15c451 Mon Sep 17 00:00:00 2001 From: sriniarul <125246441+arulsrinivaasan-everbee@users.noreply.github.com> Date: Sun, 29 Oct 2023 08:57:05 +0530 Subject: [PATCH 07/11] comment spacing --- charts/mageai/values.yaml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/charts/mageai/values.yaml b/charts/mageai/values.yaml index e9876f3..74fb2e1 100644 --- a/charts/mageai/values.yaml +++ b/charts/mageai/values.yaml @@ -6,18 +6,18 @@ replicaCount: 1 standaloneScheduler: true scheduler: - replicaCount: 1 #Effective if standaloneScheduler is true + replicaCount: 1 # Effective if standaloneScheduler is true name: mageai-scheduler webServer: - replicaCount: 1 #Effective if standaloneScheduler is true + replicaCount: 1 # Effective if standaloneScheduler is true name: mageai-webserver redis: - enabled: false #Enable redis if you want more replica + enabled: false # Enable redis if you want more replica architecture: standalone auth: enabled: false - #Your custom redis url (make sure redis.enabled is set to false) + # Your custom redis url (make sure redis.enabled is set to false) # url: redis://redis:6379/0 image: From c1db0c339e100ffe5a6c23adc172623ae3213f35 Mon Sep 17 00:00:00 2001 From: sriniarul <125246441+arulsrinivaasan-everbee@users.noreply.github.com> Date: Sun, 29 Oct 2023 09:00:21 +0530 Subject: [PATCH 08/11] tab spaces renmoved --- charts/mageai/values.yaml | 47 ++++++++++++++++++++------------------- 1 file changed, 24 insertions(+), 23 deletions(-) diff --git a/charts/mageai/values.yaml b/charts/mageai/values.yaml index 74fb2e1..8b7b927 100644 --- a/charts/mageai/values.yaml +++ b/charts/mageai/values.yaml @@ -8,16 +8,17 @@ standaloneScheduler: true scheduler: replicaCount: 1 # Effective if standaloneScheduler is true name: mageai-scheduler + webServer: replicaCount: 1 # Effective if standaloneScheduler is true name: mageai-webserver redis: - enabled: false # Enable redis if you want more replica + enabled: false # Enable redis if you want more replicas architecture: standalone auth: enabled: false - # Your custom redis url (make sure redis.enabled is set to false) + # Your custom redis URL (make sure redis.enabled is set to false) # url: redis://redis:6379/0 image: @@ -42,15 +43,15 @@ serviceAccount: podAnnotations: {} podSecurityContext: {} - # fsGroup: 2000 +# fsGroup: 2000 securityContext: {} - # capabilities: - # drop: - # - ALL - # readOnlyRootFilesystem: true - # runAsNonRoot: true - # runAsUser: 1000 +# capabilities: +# drop: +# - ALL +# readOnlyRootFilesystem: true +# runAsNonRoot: true +# runAsUser: 1000 service: type: LoadBalancer @@ -87,15 +88,15 @@ readinessProbe: # Custom liveness probe customLivenessProbe: {} -# Custom rediness probe +# Custom readiness probe customReadinessProbe: {} ingress: enabled: false className: "" annotations: {} - # kubernetes.io/ingress.class: nginx - # kubernetes.io/tls-acme: "true" + # kubernetes.io/ingress.class: nginx + # kubernetes.io/tls-acme: "true" hosts: - host: chart-example.local paths: @@ -107,16 +108,16 @@ ingress: # - chart-example.local resources: {} - # We usually recommend not to specify default resources and to leave this as a conscious - # choice for the user. This also increases chances charts run on environments with little - # resources, such as Minikube. If you do want to specify resources, uncomment the following - # lines, adjust them as necessary, and remove the curly braces after 'resources:'. - # limits: - # cpu: 100m - # memory: 128Mi - # requests: - # cpu: 100m - # memory: 128Mi +# We usually recommend not to specify default resources and to leave this as a conscious +# choice for the user. This also increases chances charts run on environments with little +# resources, such as Minikube. If you do want to specify resources, uncomment the following +# lines, adjust them as necessary, and remove the curly braces after 'resources:'. +# limits: +# cpu: 100m +# memory: 128Mi +# requests: +# cpu: 100m +# memory: 128Mi nodeSelector: {} @@ -136,7 +137,7 @@ extraVolumes: # config: Default configuration for mageai as environment variables. These get injected directly in the container. config: {} -# existingSecret: Spcifies an existing secret to be used as environment variables. These get injected directly in the container. +# existingSecret: Specifies an existing secret to be used as environment variables. These get injected directly in the container. existingSecret: "" # secrets: Default secrets for mageai as environment variables. These get injected directly in the container. From 26d776e4615fb1cd2d4d967d622337939add6f03 Mon Sep 17 00:00:00 2001 From: sriniarul <125246441+arulsrinivaasan-everbee@users.noreply.github.com> Date: Sun, 29 Oct 2023 09:03:04 +0530 Subject: [PATCH 09/11] Update values.yaml --- charts/mageai/values.yaml | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/charts/mageai/values.yaml b/charts/mageai/values.yaml index 8b7b927..48be880 100644 --- a/charts/mageai/values.yaml +++ b/charts/mageai/values.yaml @@ -5,20 +5,24 @@ replicaCount: 1 standaloneScheduler: true +# Effective if standaloneScheduler is true scheduler: - replicaCount: 1 # Effective if standaloneScheduler is true + replicaCount: 1 name: mageai-scheduler + +# Effective if standaloneScheduler is true webServer: - replicaCount: 1 # Effective if standaloneScheduler is true + replicaCount: 1 name: mageai-webserver +# Enable redis if you want more replica redis: - enabled: false # Enable redis if you want more replicas + enabled: false architecture: standalone auth: enabled: false - # Your custom redis URL (make sure redis.enabled is set to false) + # Your custom redis url (make sure redis.enabled is set to false) # url: redis://redis:6379/0 image: From 5e4ed6f72b55ac99b8f46eacd32415300734283c Mon Sep 17 00:00:00 2001 From: sriniarul <125246441+arulsrinivaasan-everbee@users.noreply.github.com> Date: Mon, 6 Nov 2023 23:30:35 +0530 Subject: [PATCH 10/11] review issues Patched issued reported on code reviiew --- charts/mageai/templates/scheduler.yaml | 28 ------------------- charts/mageai/values.yaml | 38 +++++++++++++------------- 2 files changed, 19 insertions(+), 47 deletions(-) diff --git a/charts/mageai/templates/scheduler.yaml b/charts/mageai/templates/scheduler.yaml index f627c33..a469d86 100644 --- a/charts/mageai/templates/scheduler.yaml +++ b/charts/mageai/templates/scheduler.yaml @@ -48,34 +48,6 @@ spec: - name: http containerPort: {{ .Values.service.port }} protocol: TCP - {{- if .Values.customLivenessProbe }} - livenessProbe: {{- toYaml .Values.customLivenessProbe | nindent 12 }} - {{- else if .Values.livenessProbe.enabled }} - livenessProbe: - httpGet: - path: {{ .Values.livenessProbe.path }} - port: {{ .Values.livenessProbe.port }} - initialDelaySeconds: {{ .Values.livenessProbe.initialDelaySeconds }} - failureThreshold: {{ .Values.livenessProbe.failureThreshold }} - periodSeconds: {{ .Values.livenessProbe.periodSeconds }} - successThreshold: {{ .Values.livenessProbe.successThreshold }} - terminationGracePeriodSeconds: {{ .Values.livenessProbe.terminationGracePeriodSeconds }} - timeoutSeconds: {{ .Values.livenessProbe.timeoutSeconds }} - {{- end }} - {{- if .Values.customReadinessProbe }} - readinessProbe: {{- toYaml .Values.customReadinessProbe | nindent 12 }} - {{- else if .Values.readinessProbe.enabled }} - readinessProbe: - httpGet: - path: {{ .Values.readinessProbe.path }} - port: {{ .Values.readinessProbe.port }} - initialDelaySeconds: {{ .Values.readinessProbe.initialDelaySeconds }} - failureThreshold: {{ .Values.readinessProbe.failureThreshold }} - periodSeconds: {{ .Values.readinessProbe.periodSeconds }} - successThreshold: {{ .Values.readinessProbe.successThreshold }} - terminationGracePeriodSeconds: {{ .Values.readinessProbe.terminationGracePeriodSeconds }} - timeoutSeconds: {{ .Values.readinessProbe.timeoutSeconds }} - {{- end }} resources: {{- toYaml .Values.resources | nindent 12 }} envFrom: diff --git a/charts/mageai/values.yaml b/charts/mageai/values.yaml index 48be880..9ffb924 100644 --- a/charts/mageai/values.yaml +++ b/charts/mageai/values.yaml @@ -47,15 +47,15 @@ serviceAccount: podAnnotations: {} podSecurityContext: {} -# fsGroup: 2000 + # fsGroup: 2000 securityContext: {} -# capabilities: -# drop: -# - ALL -# readOnlyRootFilesystem: true -# runAsNonRoot: true -# runAsUser: 1000 + # capabilities: + # drop: + # - ALL + # readOnlyRootFilesystem: true + # runAsNonRoot: true + # runAsUser: 1000 service: type: LoadBalancer @@ -99,8 +99,8 @@ ingress: enabled: false className: "" annotations: {} - # kubernetes.io/ingress.class: nginx - # kubernetes.io/tls-acme: "true" + # kubernetes.io/ingress.class: nginx + # kubernetes.io/tls-acme: "true" hosts: - host: chart-example.local paths: @@ -112,16 +112,16 @@ ingress: # - chart-example.local resources: {} -# We usually recommend not to specify default resources and to leave this as a conscious -# choice for the user. This also increases chances charts run on environments with little -# resources, such as Minikube. If you do want to specify resources, uncomment the following -# lines, adjust them as necessary, and remove the curly braces after 'resources:'. -# limits: -# cpu: 100m -# memory: 128Mi -# requests: -# cpu: 100m -# memory: 128Mi + # We usually recommend not to specify default resources and to leave this as a conscious + # choice for the user. This also increases chances charts run on environments with little + # resources, such as Minikube. If you do want to specify resources, uncomment the following + # lines, adjust them as necessary, and remove the curly braces after 'resources:'. + # limits: + # cpu: 100m + # memory: 128Mi + # requests: + # cpu: 100m + # memory: 128Mi nodeSelector: {} From 0079a3c1792e67d801c945948bbd6915080a80e7 Mon Sep 17 00:00:00 2001 From: sriniarul <125246441+arulsrinivaasan-everbee@users.noreply.github.com> Date: Tue, 7 Nov 2023 00:10:49 +0530 Subject: [PATCH 11/11] redis init container loop fix redis init container loop fix --- charts/mageai/templates/deployment.yaml | 12 +++++++----- charts/mageai/templates/scheduler.yaml | 14 ++++++++------ charts/mageai/templates/webservice.yaml | 12 +++++++----- charts/mageai/values.yaml | 2 +- 4 files changed, 23 insertions(+), 17 deletions(-) diff --git a/charts/mageai/templates/deployment.yaml b/charts/mageai/templates/deployment.yaml index c3cfb16..c3442e2 100644 --- a/charts/mageai/templates/deployment.yaml +++ b/charts/mageai/templates/deployment.yaml @@ -26,6 +26,7 @@ spec: serviceAccountName: {{ include "mageai.serviceAccountName" . }} securityContext: {{- toYaml .Values.podSecurityContext | nindent 8 }} + {{- if or .Values.redis.enabled .Values.redis.customRedisURL }} initContainers: - name: wait-for-redis image: alpine @@ -33,11 +34,12 @@ spec: {{- if .Values.redis.enabled }} - name: REDIS_URL value: redis://{{.Release.Name}}-redis-headless:6379/0 - {{- else if .Values.redis.url }} + {{- else if .Values.redis.customRedisURL }} - name: REDIS_URL - value: {{ .Values.redis.url }} + value: {{ .Values.redis.customRedisURL }} {{- end }} command: ["sh", "-c", "until nc -z -v $(echo $REDIS_URL | cut -d'/' -f3 | cut -d':' -f1) $(echo $REDIS_URL | cut -d'/' -f3 | cut -d':' -f2); do sleep 1; done"] + {{- end }} containers: - name: {{ .Chart.Name }} securityContext: @@ -95,10 +97,10 @@ spec: {{- end }} {{- if .Values.redis.enabled }} - name: REDIS_URL - value: redis://{{.Release.Name}}-redis-headless:6379/0 - {{- else if .Values.redis.url}} + value: redis://{{ .Release.Name }}-redis-headless:6379/0 + {{- else if .Values.redis.customRedisURL }} - name: REDIS_URL - value: {{.Values.redis.url}} + value: {{ .Values.redis.customRedisURL }} {{- end }} volumeMounts: {{- if .Values.volumes }} diff --git a/charts/mageai/templates/scheduler.yaml b/charts/mageai/templates/scheduler.yaml index a469d86..bb0982b 100644 --- a/charts/mageai/templates/scheduler.yaml +++ b/charts/mageai/templates/scheduler.yaml @@ -26,18 +26,20 @@ spec: serviceAccountName: {{ include "mageai.serviceAccountName" . }} securityContext: {{- toYaml .Values.podSecurityContext | nindent 8 }} + {{- if or .Values.redis.enabled .Values.redis.customRedisURL }} initContainers: - name: wait-for-redis image: alpine env: {{- if .Values.redis.enabled }} - name: REDIS_URL - value: redis://{{.Release.Name }}-redis-headless:6379/0 - {{- else if .Values.redis.url }} + value: redis://{{ .Release.Name }}-redis-headless:6379/0 + {{- else if .Values.redis.customRedisURL }} - name: REDIS_URL - value: {{ .Values.redis.url }} + value: {{ .Values.redis.customRedisURL }} {{- end }} command: ["sh", "-c", "until nc -z -v $(echo $REDIS_URL | cut -d'/' -f3 | cut -d':' -f1) $(echo $REDIS_URL | cut -d'/' -f3 | cut -d':' -f2); do sleep 1; done"] + {{- end }} containers: - name: {{ .Chart.Name }} securityContext: @@ -67,10 +69,10 @@ spec: {{- end }} {{- if .Values.redis.enabled }} - name: REDIS_URL - value: redis://{{.Release.Name}}-redis-headless:6379/0 - {{- else if .Values.redis.url}} + value: redis://{{ .Release.Name }}-redis-headless:6379/0 + {{- else if .Values.redis.customRedisURL }} - name: REDIS_URL - value: {{ .Values.redis.url }} + value: {{ .Values.redis.customRedisURL }} {{- end }} volumeMounts: {{- if .Values.volumes }} diff --git a/charts/mageai/templates/webservice.yaml b/charts/mageai/templates/webservice.yaml index 2b05033..29d731f 100644 --- a/charts/mageai/templates/webservice.yaml +++ b/charts/mageai/templates/webservice.yaml @@ -26,18 +26,20 @@ spec: serviceAccountName: {{ include "mageai.serviceAccountName" . }} securityContext: {{- toYaml .Values.podSecurityContext | nindent 8 }} + {{- if or .Values.redis.enabled .Values.redis.customRedisURL }} initContainers: - name: wait-for-redis image: alpine env: {{- if .Values.redis.enabled }} - name: REDIS_URL - value: redis://{{.Release.Name}}-redis-headless:6379/0 - {{- else if .Values.redis.url }} + value: redis://{{ .Release.Name }}-redis-headless:6379/0 + {{- else if .Values.redis.customRedisURL }} - name: REDIS_URL - value: {{ .Values.redis.url }} + value: {{ .Values.redis.customRedisURL }} {{- end }} command: ["sh", "-c", "until nc -z -v $(echo $REDIS_URL | cut -d'/' -f3 | cut -d':' -f1) $(echo $REDIS_URL | cut -d'/' -f3 | cut -d':' -f2); do sleep 1; done"] + {{- end }} containers: - name: {{ .Chart.Name }} securityContext: @@ -96,9 +98,9 @@ spec: {{- if .Values.redis.enabled }} - name: REDIS_URL value: redis://{{.Release.Name}}-redis-headless:6379/0 - {{- else if .Values.redis.url}} + {{- else if .Values.redis.customRedisURL }} - name: REDIS_URL - value: {{ .Values.redis.url }} + value: {{ .Values.redis.customRedisURL }} {{- end }} volumeMounts: {{- if .Values.volumes }} diff --git a/charts/mageai/values.yaml b/charts/mageai/values.yaml index 9ffb924..945032b 100644 --- a/charts/mageai/values.yaml +++ b/charts/mageai/values.yaml @@ -23,7 +23,7 @@ redis: auth: enabled: false # Your custom redis url (make sure redis.enabled is set to false) - # url: redis://redis:6379/0 + customRedisURL: "" image: repository: mageai/mageai