From 87fbe0b9271c94abe36e381cbc74adc1c6c07913 Mon Sep 17 00:00:00 2001 From: Liam Beckman Date: Mon, 9 Sep 2024 18:21:30 -0700 Subject: [PATCH] Update K8s support and documentation - Update docker images to use quay.io/ohsu-comp-bio/funnel --- Dockerfile | 2 +- Dockerfile.dind | 4 +- Dockerfile.dind-rootless | 4 +- compute/kubernetes/backend.go | 33 +++++++--- compute/kubernetes/backend_test.go | 61 ++++++++++++++++++- deployments/kubernetes/README.md | 52 +++++++++------- deployments/kubernetes/funnel-deployment.yml | 14 +++-- .../kubernetes/funnel-server-config.yml | 7 +-- 8 files changed, 134 insertions(+), 43 deletions(-) diff --git a/Dockerfile b/Dockerfile index 39c481ab5..695461b78 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,5 +1,5 @@ # build stage -FROM golang:1.21-alpine AS build-env +FROM golang:1.23-alpine AS build-env RUN apk add make git bash build-base ENV GOPATH=/go ENV PATH="/go/bin:${PATH}" diff --git a/Dockerfile.dind b/Dockerfile.dind index 29390cf7e..fc57ab1a2 100644 --- a/Dockerfile.dind +++ b/Dockerfile.dind @@ -1,5 +1,5 @@ # build stage -FROM golang:1.20-alpine AS build-env +FROM golang:1.23-alpine AS build-env RUN apk add make git bash build-base ENV GOPATH=/go ENV PATH="/go/bin:${PATH}" @@ -7,7 +7,7 @@ ADD ./ /go/src/github.com/ohsu-comp-bio/funnel RUN cd /go/src/github.com/ohsu-comp-bio/funnel && make build # final stage -FROM docker:stable-dind +FROM docker:dind WORKDIR /opt/funnel VOLUME /opt/funnel/funnel-work-dir EXPOSE 8000 9090 diff --git a/Dockerfile.dind-rootless b/Dockerfile.dind-rootless index 3daa08593..a7c4c83c5 100644 --- a/Dockerfile.dind-rootless +++ b/Dockerfile.dind-rootless @@ -1,5 +1,5 @@ # build stage -FROM golang:1.20-alpine AS build-env +FROM golang:1.23-alpine AS build-env RUN apk add make git bash build-base ENV GOPATH=/go ENV PATH="/go/bin:${PATH}" @@ -7,7 +7,7 @@ ADD ./ /go/src/github.com/ohsu-comp-bio/funnel RUN cd /go/src/github.com/ohsu-comp-bio/funnel && make build # final stage -FROM docker:stable-dind-rootless +FROM docker:dind-rootless WORKDIR /opt/funnel VOLUME /opt/funnel/funnel-work-dir EXPOSE 8000 9090 diff --git a/compute/kubernetes/backend.go b/compute/kubernetes/backend.go index 9a29c263e..d2cd3159d 100644 --- a/compute/kubernetes/backend.go +++ b/compute/kubernetes/backend.go @@ -5,6 +5,7 @@ import ( "bytes" "context" "encoding/json" + "errors" "fmt" "io/ioutil" "text/template" @@ -82,15 +83,32 @@ func NewBackend(ctx context.Context, conf config.Kubernetes, reader tes.ReadOnly // Backend represents the local backend. type Backend struct { - client batchv1.JobInterface - namespace string - template string - event events.Writer - database tes.ReadOnlyServer - log *logger.Logger + client batchv1.JobInterface + namespace string + template string + event events.Writer + database tes.ReadOnlyServer + log *logger.Logger + backendParameters map[string]string events.Computer } +func (b Backend) CheckBackendParameterSupport(task *tes.Task) error { + if !task.Resources.GetBackendParametersStrict() { + return nil + } + + taskBackendParameters := task.Resources.GetBackendParameters() + for k := range taskBackendParameters { + _, ok := b.backendParameters[k] + if !ok { + return errors.New("backend parameters not supported") + } + } + + return nil +} + // WriteEvent writes an event to the compute backend. // Currently, only TASK_CREATED is handled, which calls Submit. func (b *Backend) WriteEvent(ctx context.Context, ev *events.Event) error { @@ -153,7 +171,8 @@ func (b *Backend) Submit(ctx context.Context, task *tes.Task) error { if err != nil { return fmt.Errorf("creating job spec: %v", err) } - _, err = b.client.Create(ctx, job, metav1.CreateOptions{ + ctx = context.Background() + job, err = b.client.Create(ctx, job, metav1.CreateOptions{ FieldManager: task.Id, }) if err != nil { diff --git a/compute/kubernetes/backend_test.go b/compute/kubernetes/backend_test.go index 313247ea6..d1e0768c5 100644 --- a/compute/kubernetes/backend_test.go +++ b/compute/kubernetes/backend_test.go @@ -1,15 +1,62 @@ package kubernetes import ( + "context" "fmt" "io/ioutil" "testing" + "path/filepath" + "github.com/ohsu-comp-bio/funnel/config" "github.com/ohsu-comp-bio/funnel/logger" "github.com/ohsu-comp-bio/funnel/tes" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/rest" + "k8s.io/client-go/tools/clientcmd" + "k8s.io/client-go/util/homedir" ) +// getKubernetesClient creates a Kubernetes clientset for communication with the Kubernetes API +func getKubernetesClient() (*kubernetes.Clientset, error) { + var config *rest.Config + var err error + var contextName string + + // Try to create an in-cluster client + config, err = rest.InClusterConfig() + if err != nil { + // If in-cluster config fails, fallback to kubeconfig file for out-of-cluster + kubeconfig := filepath.Join(homedir.HomeDir(), ".kube", "config") + kubeConfig, err := clientcmd.LoadFromFile(kubeconfig) + if err != nil { + return nil, fmt.Errorf("failed to load kubeconfig: %v", err) + } + + // Get the current context name + contextName = kubeConfig.CurrentContext + + // Build config from flags or kubeconfig + config, err = clientcmd.BuildConfigFromFlags("", kubeconfig) + if err != nil { + return nil, fmt.Errorf("failed to create kubernetes config: %v", err) + } + } else { + contextName = "in-cluster" + } + + // Create the clientset + clientset, err := kubernetes.NewForConfig(config) + if err != nil { + return nil, fmt.Errorf("failed to create kubernetes clientset: %v", err) + } + + // Log or print the current context being used + fmt.Printf("Using Kubernetes context: %s\n", contextName) + + return clientset, nil +} + func TestCreateJobc(t *testing.T) { conf := config.DefaultConfig().Kubernetes content, err := ioutil.ReadFile("../../config/kubernetes-template.yaml") @@ -18,8 +65,15 @@ func TestCreateJobc(t *testing.T) { } conf.Template = string(content) log := logger.NewLogger("test", logger.DefaultConfig()) + + // Create Kubernetes client + clientset, err := getKubernetesClient() + if err != nil { + t.Fatal(fmt.Errorf("creating kubernetes client: %v", err)) + } + b := &Backend{ - client: nil, + client: clientset.BatchV1().Jobs(conf.Namespace), namespace: conf.Namespace, template: conf.Template, event: nil, @@ -42,4 +96,9 @@ func TestCreateJobc(t *testing.T) { t.Fatal(err) } t.Logf("%+v", job) + + err = b.Submit(context.Background(), task) + if err != nil { + t.Fatal(err) + } } diff --git a/deployments/kubernetes/README.md b/deployments/kubernetes/README.md index 2f6a54a24..b3443a2ac 100644 --- a/deployments/kubernetes/README.md +++ b/deployments/kubernetes/README.md @@ -1,3 +1,6 @@ +> [!NOTE] +> Kubernetes support is in active development and may involve frequent updates + # Using Funnel with Kubernetes Examples can be found here: https://github.com/ohsu-comp-bio/funnel/tree/master/deployments/kubernetes @@ -6,7 +9,7 @@ Examples can be found here: https://github.com/ohsu-comp-bio/funnel/tree/master/ *funnel-service.yml* -``` +```yaml apiVersion: v1 kind: Service metadata: @@ -23,18 +26,17 @@ spec: protocol: TCP port: 9090 targetPort: 9090 - ``` Deploy it: -``` +```sh kubectl apply -f funnel-service.yml ``` Get the clusterIP: -``` +```sh kubectl get services funnel --output=yaml | grep clusterIP ``` @@ -42,13 +44,17 @@ Use this value to configure the server hostname of the worker config. #### Create Funnel config files -*Note*: The configures job template uses the image, `ohsucompbio/funnel-dind:latest`, which is built on docker's official [docker-in-docker image (dind)](https://hub.docker.com/_/docker). You can also use the experimental [rootless dind variant](https://docs.docker.com/engine/security/rootless/) by changing the image to `ohsucompbio/funnel-dind-rootless:latest`. +> [!TIP] +> The configures job template uses the image, `ohsucompbio/funnel-dind:latest`, which is built on docker's official [docker-in-docker image (dind)](https://hub.docker.com/_/docker). You can also use the experimental [rootless dind variant](https://docs.docker.com/engine/security/rootless/) by changing the image to `quay.io/ohsu-comp-bio/funnel-dind-rootless:latest` *funnel-server-config.yml* -``` +```yaml Database: boltdb +BoltDB: + Path: /opt/funnel/funnel-work-dir/funnel.bolt.db + Compute: kubernetes Logger: @@ -74,7 +80,7 @@ Kubernetes: restartPolicy: Never containers: - name: {{printf "funnel-worker-%s" .TaskId}} - image: ohsucompbio/funnel-kube-dind:latest + image: quay.io/ohsu-comp-bio/funnel-dind:latest imagePullPolicy: IfNotPresent args: - "funnel" @@ -87,8 +93,8 @@ Kubernetes: resources: requests: cpu: {{if ne .Cpus 0 -}}{{.Cpus}}{{ else }}{{"100m"}}{{end}} - memory: {{if ne .RamGb 0.0 -}}{{printf "%.0fG" .RamGb}}{{else}}{{"16M"}}{{end}} - ephemeral-storage: {{if ne .DiskGb 0.0 -}}{{printf "%.0fG" .DiskGb}}{{else}}{{"100M"}}{{end}} + memory: {{if ne .RamGb 0.0 -}}{{printf "%.0fG" .RamGb}}{{else}}{{"16Mi"}}{{end}} + ephemeral-storage: {{if ne .DiskGb 0.0 -}}{{printf "%.0fG" .DiskGb}}{{else}}{{"100Mi"}}{{end}} volumeMounts: - name: {{printf "funnel-storage-%s" .TaskId}} mountPath: {{printf "/opt/funnel/funnel-work-dir/%s" .TaskId}} @@ -112,7 +118,7 @@ I recommend setting `DisableJobCleanup` to `true` for debugging - otherwise fail ***Remember to modify the template below to have the actual server hostname.*** -``` +```yaml Database: boltdb BoltDB: @@ -138,7 +144,7 @@ Server: #### Create a ConfigMap -``` +```sh kubectl create configmap funnel-config --from-file=funnel-server-config.yml --from-file=funnel-worker-config.yml ``` @@ -148,7 +154,7 @@ Define a Role and RoleBinding: *role.yml* -``` +```yaml kind: Role apiVersion: rbac.authorization.k8s.io/v1 metadata: @@ -168,7 +174,7 @@ rules: *role_binding.yml* -``` +```yaml kind: RoleBinding apiVersion: rbac.authorization.k8s.io/v1 metadata: @@ -195,7 +201,7 @@ kubectl create -f role_binding.yml *funnel-deployment.yml* -``` +```yaml apiVersion: apps/v1 kind: Deployment metadata: @@ -215,7 +221,7 @@ spec: serviceAccountName: funnel-sa containers: - name: funnel - image: ohsucompbio/funnel:latest + image: quay.io/ohsu-comp-bio/funnel:latest imagePullPolicy: IfNotPresent command: - 'funnel' @@ -225,9 +231,13 @@ spec: - '/etc/config/funnel-server-config.yml' resources: requests: - cpu: 2 + cpu: 500m + memory: 1G + ephemeral-storage: 25G + limits: + cpu: 2000m memory: 4G - ephemeral-storage: 25G # needed since we are using boltdb + ephemeral-storage: 25G volumeMounts: - name: funnel-deployment-storage mountPath: /opt/funnel/funnel-work-dir @@ -247,25 +257,25 @@ spec: Deploy it: -``` +```sh kubectl apply -f funnel-deployment.yml ``` #### Proxy the Service for local testing -``` +```sh kubectl port-forward service/funnel 8000:8000 ``` Now you can access the funnel server locally. Verify by running: -``` +```sh funnel task list ``` Now try running a task: -``` +```sh funnel examples hello-world > hello.json funnel task create hello.json ``` diff --git a/deployments/kubernetes/funnel-deployment.yml b/deployments/kubernetes/funnel-deployment.yml index c861bc3fe..cc41ac518 100644 --- a/deployments/kubernetes/funnel-deployment.yml +++ b/deployments/kubernetes/funnel-deployment.yml @@ -17,7 +17,7 @@ spec: serviceAccountName: funnel-sa containers: - name: funnel - image: ohsucompbio/funnel:latest + image: quay.io/ohsu-comp-bio/funnel:latest imagePullPolicy: IfNotPresent command: - 'funnel' @@ -27,9 +27,13 @@ spec: - '/etc/config/funnel-server-config.yml' resources: requests: - cpu: 2 + cpu: 500m + memory: 1G + ephemeral-storage: 25G + limits: + cpu: 2000m memory: 4G - ephemeral-storage: 25G # needed since we are using boltdb + ephemeral-storage: 25G volumeMounts: - name: funnel-deployment-storage mountPath: /opt/funnel/funnel-work-dir @@ -38,10 +42,10 @@ spec: ports: - containerPort: 8000 - containerPort: 9090 - + volumes: - name: funnel-deployment-storage emptyDir: {} - name: config-volume configMap: - name: funnel-config + name: funnel-config \ No newline at end of file diff --git a/deployments/kubernetes/funnel-server-config.yml b/deployments/kubernetes/funnel-server-config.yml index ae30805ff..c9c10f6c8 100644 --- a/deployments/kubernetes/funnel-server-config.yml +++ b/deployments/kubernetes/funnel-server-config.yml @@ -28,7 +28,7 @@ Kubernetes: restartPolicy: Never containers: - name: {{printf "funnel-worker-%s" .TaskId}} - image: ohsucompbio/funnel-dind:latest + image: quay.io/ohsu-comp-bio/funnel-dind:latest imagePullPolicy: IfNotPresent args: - "funnel" @@ -41,8 +41,8 @@ Kubernetes: resources: requests: cpu: {{if ne .Cpus 0 -}}{{.Cpus}}{{ else }}{{"100m"}}{{end}} - memory: {{if ne .RamGb 0.0 -}}{{printf "%.0fG" .RamGb}}{{else}}{{"16M"}}{{end}} - ephemeral-storage: {{if ne .DiskGb 0.0 -}}{{printf "%.0fG" .DiskGb}}{{else}}{{"100M"}}{{end}} + memory: {{if ne .RamGb 0.0 -}}{{printf "%.0fG" .RamGb}}{{else}}{{"16Mi"}}{{end}} + ephemeral-storage: {{if ne .DiskGb 0.0 -}}{{printf "%.0fG" .DiskGb}}{{else}}{{"100Mi"}}{{end}} volumeMounts: - name: {{printf "funnel-storage-%s" .TaskId}} mountPath: {{printf "/opt/funnel/funnel-work-dir/%s" .TaskId}} @@ -58,4 +58,3 @@ Kubernetes: - name: config-volume configMap: name: funnel-config -