Update Helm Charts with initial support for full config files

ohsu-comp-bio · Oct 18, 2024 · c6a84f4 · c6a84f4
1 parent a520047
commit c6a84f4
Show file tree

Hide file tree

Showing 13 changed files with 233 additions and 162 deletions.
diff --git a/.gitignore b/.gitignore
@@ -20,6 +20,5 @@ storage/_test_download
 docker/funnel
 deployments/kubernetes/**/funnel
 deployments/kubernetes/**/*.json
-funnel
 dist/
 .vscode
diff --git a/compute/kubernetes/backend.go b/compute/kubernetes/backend.go
@@ -147,11 +147,17 @@ func (b *Backend) createJob(task *tes.Task) (*v1.Job, error) {
 		"Cpus":      res.GetCpuCores(),
 		"RamGb":     res.GetRamGb(),
 		"DiskGb":    res.GetDiskGb(),
+		// TODO: How to handle multiple executors/images?
+		"Image": task.Executors[0].Image,
 	})
 	if err != nil {
 		return nil, fmt.Errorf("executing template: %v", err)
 	}
 
+	fmt.Println("DEBUG: task.Id:", task.Id)
+	fmt.Println("DEBUG: b.template:", b.template)
+	fmt.Println("DEBUG: buf.String():", buf.String())
+
 	decode := scheme.Codecs.UniversalDeserializer().Decode
 	obj, _, err := decode(buf.Bytes(), nil, nil)
 	if err != nil {
@@ -165,7 +171,7 @@ func (b *Backend) createJob(task *tes.Task) (*v1.Job, error) {
 	return job, nil
 }
 
-// Submit submits a task to the as a kubernetes v1/batch job.
+// Submit submits a task to the server as a kubernetes v1/batch job.
 func (b *Backend) Submit(ctx context.Context, task *tes.Task) error {
 	job, err := b.createJob(task)
 	if err != nil {
@@ -176,7 +182,7 @@ func (b *Backend) Submit(ctx context.Context, task *tes.Task) error {
 		FieldManager: task.Id,
 	})
 	if err != nil {
-		return fmt.Errorf("creating job: %v", err)
+		return fmt.Errorf("creating job in backend: %v", err)
 	}
 	return nil
 }

diff --git a/config/default-config.yaml b/config/default-config.yaml
@@ -3,7 +3,7 @@
 Database: boltdb
 
 # The name of the active compute backend
-# Available backends: local, htcondor, slurm, pbs, gridengine, manual, aws-batch
+# Available backends: local, htcondor, slurm, pbs, gridengine, manual, aws-batch, kubernetes
 Compute: local
 
 # The name of the active event writer backend(s).

diff --git a/config/internal/bundle.go b/config/internal/bundle.go
diff --git a/config/kubernetes-executor-template.yaml b/config/kubernetes-executor-template.yaml
@@ -11,7 +11,7 @@ spec:
   template:
     spec:
       restartPolicy: Never
-      serviceAccountName: {{.ServiceAccount}}
+      serviceAccountName: funnel-sa
       containers: 
       - name: funnel-worker-{{.TaskId}}
         image: {{.Image}}
@@ -36,4 +36,4 @@ spec:
       volumes: 
       - name: storage
         persistentVolumeClaim:
-          claimName: storage-pvc 
+          claimName: funnel-pvc 
diff --git a/config/kubernetes-template.yaml b/config/kubernetes-template.yaml
@@ -9,17 +9,17 @@ spec:
   completions: 1
   template:
     spec:
-      serviceAccountName: {{.ServiceAccount}}
+      serviceAccountName: funnel-sa
       restartPolicy: Never
       containers: 
         - name: funnel-worker-{{.TaskId}}
-          image: quay.io/ohsu-comp-bio/funnel:latest
+          image: quay.io/ohsu-comp-bio/funnel:{{.Image}}
           imagePullPolicy: Always
           args:
             - "worker"
             - "run"
             - "--config"
-            - "/etc/config/funnel-worker-config.yml"
+            - "/etc/config/funnel-worker.yaml"
             - "--taskID"
             - {{.TaskId}}
           resources:
@@ -40,4 +40,4 @@ spec:
             name: funnel-config
         - name: funnel-storage-{{.TaskId}}
           persistentVolumeClaim:
-            claimName: storage-pvc 
+            claimName: funnel-pvc 
diff --git a/deployments/kubernetes/README.md b/deployments/kubernetes/README.md
@@ -22,10 +22,18 @@ Kuberenetes Resources:
 kubectl apply -f funnel-service.yml
 ```
 
+## 2. Create Funnel config files
+
+> *[funnel-server.yaml](https://github.com/ohsu-comp-bio/funnel/blob/develop/deployments/kubernetes/funnel-server.yaml)*
+
+> *[funnel-worker.yaml](https://github.com/ohsu-comp-bio/funnel/blob/develop/deployments/kubernetes/funnel-worker.yaml)*
+
 Get the clusterIP:
 
 ```sh
-kubectl get services funnel --output=yaml | grep "clusterIP:"
+export HOSTNAME=$(kubectl get services funnel --output=jsonpath='{.spec.clusterIP}')
+
+sed -i "s|\${HOSTNAME}|${HOSTNAME}|g" funnel-worker.yaml
 ```
 
 Use this value to configure the server hostname of the worker config.
@@ -43,7 +51,7 @@ We recommend setting `DisableJobCleanup` to `true` for debugging - otherwise fai
 ## 3. Create a ConfigMap
 
 ```sh
-kubectl create configmap funnel-config --from-file=funnel-server-config.yml --from-file=funnel-worker-config.yml
+kubectl create configmap funnel-config --from-file=funnel-server.yaml --from-file=funnel-worker.yaml
 ```
 
 ## 4. Create a Service Account for Funnel

diff --git a/deployments/kubernetes/funnel-deployment.yml b/deployments/kubernetes/funnel-deployment.yml
@@ -24,7 +24,7 @@ spec:
             - 'server'
             - 'run'
             - '--config'
-            - '/etc/config/funnel-server-config.yml'
+            - '/etc/config/funnel-server.yaml'
           resources: 
             requests: 
               cpu: 500m

diff --git a/deployments/kubernetes/funnel-server-config.yml b/deployments/kubernetes/funnel-server-config.yml
@@ -12,48 +12,3 @@ Kubernetes:
   DisableReconciler: false
   ReconcileRate: 5m
   Namespace: default
-  Template: |
-    apiVersion: batch/v1
-    kind: Job
-    metadata:
-      ## DO NOT CHANGE NAME
-      name: {{.TaskId}}
-      namespace: {{.Namespace}}
-    spec:
-      backoffLimit: 0
-      completions: 1
-      template:
-        spec:
-          restartPolicy: Never
-          serviceAccountName: funnel-sa
-          containers:
-            - name: {{printf "funnel-worker-%s" .TaskId}}
-              image: quay.io/ohsu-comp-bio/funnel:latest
-              imagePullPolicy: IfNotPresent
-              args:
-                - "worker"
-                - "run"
-                - "--config"
-                - "/etc/config/funnel-worker-config.yml"
-                - "--taskID"
-                - {{.TaskId}}
-              resources:
-                  requests:
-                    cpu: {{if ne .Cpus 0 -}}{{.Cpus}}{{ else }}{{"100m"}}{{end}}
-                    memory: {{if ne .RamGb 0.0 -}}{{printf "%.0fG" .RamGb}}{{else}}{{"16M"}}{{end}}
-                    ephemeral-storage: {{if ne .DiskGb 0.0 -}}{{printf "%.0fG" .DiskGb}}{{else}}{{"100M"}}{{end}}
-              volumeMounts:
-                - name: {{printf "funnel-storage-%s" .TaskId}}
-                  mountPath: {{printf "/opt/funnel/funnel-work-dir/%s" .TaskId}}
-                - name: config-volume
-                  mountPath: /etc/config
-
-              securityContext:
-                privileged: true
-
-          volumes:
-            - name: {{printf "funnel-storage-%s" .TaskId}}
-              emptyDir: {}
-            - name: config-volume
-              configMap:
-                name: funnel-config