Merge pull request #74 from KWasm/kwasm-lifecycle-manager

Kwasm lifecycle manager

Makefile

@@ -98,7 +98,7 @@ build: manifests generate fmt vet golangci-build ## Build manager binary.
 
 .PHONY: run
 run: manifests generate fmt vet ## Run a controller from your host.
-	go run -ldflags "${LDFLAGS}" ./cmd/main.go
+	CONTROLLER_NAMESPACE="default" go run -ldflags "${LDFLAGS}" ./cmd/main.go
 
 # If you wish to build the manager image targeting other platforms you can use the --platform flag.
 # (i.e. docker build --platform linux/arm64). However, you must enable docker buildKit for it.
@@ -138,10 +138,6 @@ endif
 install: manifests kustomize ## Install CRDs into the K8s cluster specified in ~/.kube/config.
 	$(KUSTOMIZE) build config/crd | $(KUBECTL) apply -f -
 
-.PHONY: install-test-shims
-install-test-shims: ## Install test shims from ./hack/.
-	kubectl apply -f ./hack/*.yaml
-
 .PHONY: uninstall
 uninstall: manifests kustomize ## Uninstall CRDs from the K8s cluster specified in ~/.kube/config. Call with ignore-not-found=true to ignore resource not found errors during deletion.
 	$(KUSTOMIZE) build config/crd | $(KUBECTL) delete --ignore-not-found=$(ignore-not-found) -f -
@@ -151,6 +147,14 @@ deploy: manifests kustomize ## Deploy controller to the K8s cluster specified in
 	cd config/manager && $(KUSTOMIZE) edit set image controller=${IMG}
 	$(KUSTOMIZE) build config/default | $(KUBECTL) apply -f -
 
+.PHONY: deploy-samples
+deploy-samples: ## Install test shims from ./hack/.
+	$(KUSTOMIZE) build config/samples | $(KUBECTL) apply -f -
+
+.PHONY: undeploy-samples
+undeploy-samples: ## Install test shims from ./hack/.
+	$(KUSTOMIZE) build config/samples | $(KUBECTL) delete --ignore-not-found=$(ignore-not-found) -f -
+
 .PHONY: undeploy
 undeploy: ## Undeploy controller from the K8s cluster specified in ~/.kube/config. Call with ignore-not-found=true to ignore resource not found errors during deletion.
 	$(KUSTOMIZE) build config/default | $(KUBECTL) delete --ignore-not-found=$(ignore-not-found) -f -
@@ -191,3 +195,14 @@ $(CONTROLLER_GEN): $(LOCALBIN)
 envtest: $(ENVTEST) ## Download envtest-setup locally if necessary.
 $(ENVTEST): $(LOCALBIN)
 	test -s $(LOCALBIN)/setup-envtest || GOBIN=$(LOCALBIN) go install sigs.k8s.io/controller-runtime/tools/setup-envtest@latest
+
+.PHONY: create-test-cluster
+create-kind-cluster:
+	kind create cluster --config ./hack/kind.yaml --name kwasm
+
+.PHONY: kind-delete
+kind-delete:
+	kind delete cluster --name kwasm
+
+.PHONY: kind
+kind: create-kind-cluster install

PROJECT

@@ -22,4 +22,9 @@ resources:
   group: runtime
   kind: Node
   version: v1alpha1
+- controller: true
+  domain: kwasm.sh
+  group: runtime
+  kind: Job
+  version: v1alpha1
 version: "3"

api/v1alpha1/shim_types.go

@@ -54,11 +54,16 @@ type RollingSpec struct {
 // ShimStatus defines the observed state of Shim
 // +operator-sdk:csv:customresourcedefinitions:type=status
 type ShimStatus struct {
-	Conditions []metav1.Condition `json:"conditions,omitempty"`
+	Conditions     []metav1.Condition `json:"conditions,omitempty"`
+	NodeCount      int                `json:"nodes"`
+	NodeReadyCount int                `json:"nodesReady"`
 }
 
 // +kubebuilder:object:root=true
 // +kubebuilder:resource:path=shims,scope=Cluster
+// +kubebuilder:printcolumn:JSONPath=".spec.runtimeClass.name",name=RuntimeClass,type=string
+// +kubebuilder:printcolumn:JSONPath=".status.nodesReady",name=Ready,type=integer
+// +kubebuilder:printcolumn:JSONPath=".status.nodes",name=Nodes,type=integer
 // Shim is the Schema for the shims API
 type Shim struct {
 	metav1.TypeMeta   `json:",inline"`

cmd/main.go

@@ -18,6 +18,7 @@ package main
 
 import (
 	"flag"
+	"fmt"
 	"os"
 
 	// Import all Kubernetes client auth plugins (e.g. Azure, GCP, OIDC, etc.)
@@ -51,18 +52,18 @@ func init() {
 }
 
 // getWatchNamespace returns the Namespace the operator should be watching for changes
-// func getWatchNamespace() string {
-// 	// WatchNamespaceEnvVar is the constant for env variable WATCH_NAMESPACE
-// 	// which specifies the Namespace to watch.
-// 	// An empty value means the operator will fail to start.
-// 	watchNamespaceEnvVar := "CONTROLLER_NAMESPACE"
-
-// 	ns, found := os.LookupEnv(watchNamespaceEnvVar)
-// 	if !found {
-// 		panic(fmt.Sprintf("env var '%s' must be set", watchNamespaceEnvVar))
-// 	}
-// 	return ns
-// }
+func getWatchNamespace() string {
+	// WatchNamespaceEnvVar is the constant for env variable WATCH_NAMESPACE
+	// which specifies the Namespace to watch.
+	// An empty value means the operator will fail to start.
+	watchNamespaceEnvVar := "   "
+
+	ns, found := os.LookupEnv(watchNamespaceEnvVar)
+	if !found {
+		panic(fmt.Sprintf("env var '%s' must be set", watchNamespaceEnvVar))
+	}
+	return ns
+}
 
 func main() {
 	var metricsAddr string
@@ -118,6 +119,13 @@ func main() {
 	// 	setupLog.Error(err, "unable to create controller", "controller", "Node")
 	// 	os.Exit(1)
 	// }
+	if err = (&controller.JobReconciler{
+		Client: mgr.GetClient(),
+		Scheme: mgr.GetScheme(),
+	}).SetupWithManager(mgr); err != nil {
+		setupLog.Error(err, "unable to create controller", "controller", "Job")
+		os.Exit(1)
+	}
 	//+kubebuilder:scaffold:builder
 
 	if err := mgr.AddHealthzCheck("healthz", healthz.Ping); err != nil {

config/crd/bases/runtime.kwasm.sh_shims.yaml

@@ -14,7 +14,17 @@ spec:
     singular: shim
   scope: Cluster
   versions:
-  - name: v1alpha1
+  - additionalPrinterColumns:
+    - jsonPath: .spec.runtimeClass.name
+      name: RuntimeClass
+      type: string
+    - jsonPath: .status.nodesReady
+      name: Ready
+      type: integer
+    - jsonPath: .status.nodes
+      name: Nodes
+      type: integer
+    name: v1alpha1
     schema:
       openAPIV3Schema:
         description: Shim is the Schema for the shims API
@@ -153,7 +163,15 @@ spec:
                   - type
                   type: object
                 type: array
+              nodes:
+                type: integer
+              nodesReady:
+                type: integer
+            required:
+            - nodes
+            - nodesReady
             type: object
         type: object
     served: true
     storage: true
+    subresources: {}

config/rbac/role.yaml

@@ -4,6 +4,32 @@ kind: ClusterRole
 metadata:
   name: manager-role
 rules:
+- apiGroups:
+  - runtime.kwasm.sh
+  resources:
+  - jobs
+  verbs:
+  - create
+  - delete
+  - get
+  - list
+  - patch
+  - update
+  - watch
+- apiGroups:
+  - runtime.kwasm.sh
+  resources:
+  - jobs/finalizers
+  verbs:
+  - update
+- apiGroups:
+  - runtime.kwasm.sh
+  resources:
+  - jobs/status
+  verbs:
+  - get
+  - patch
+  - update
 - apiGroups:
   - runtime.kwasm.sh
   resources:

config/samples/kustomization.yaml

@@ -1,4 +1,7 @@
 ## Append samples of your project ##
 resources:
-- runtime_v1alpha1_shim.yaml
+- test_shim_lunatic.yaml
+- test_shim_slight.yaml
+- test_shim_spin.yaml
+- test_shim_wws.yaml
 #+kubebuilder:scaffold:manifestskustomizesamples

config/samples/test_shim_lunatic.yaml

@@ -2,6 +2,12 @@ apiVersion: runtime.kwasm.sh/v1alpha1
 kind: Shim
 metadata:
   name: lunatic-v1
+  labels:
+    app.kubernetes.io/name: lunatic-v1
+    app.kubernetes.io/instance: lunatic-v1
+    app.kubernetes.io/part-of: kwasm-operator
+    app.kubernetes.io/managed-by: kustomize
+    app.kubernetes.io/created-by: kwasm-operator
 spec:
   nodeSelector:
     lunatic: "true"

config/samples/test_shim_slight.yaml

@@ -2,6 +2,12 @@ apiVersion: runtime.kwasm.sh/v1alpha1
 kind: Shim
 metadata:
   name: slight-v1
+  labels:
+    app.kubernetes.io/name: slight-v1
+    app.kubernetes.io/instance: slight-v1
+    app.kubernetes.io/part-of: kwasm-operator
+    app.kubernetes.io/managed-by: kustomize
+    app.kubernetes.io/created-by: kwasm-operator
 spec:
   nodeSelector:
     slight: "true"

config/samples/test_shim_spin.yaml

@@ -2,6 +2,12 @@ apiVersion: runtime.kwasm.sh/v1alpha1
 kind: Shim
 metadata:
   name: wasmtime-spin-v2
+  labels:
+    app.kubernetes.io/name: wasmtime-spin-v2
+    app.kubernetes.io/instance: wasmtime-spin-v2
+    app.kubernetes.io/part-of: kwasm-operator
+    app.kubernetes.io/managed-by: kustomize
+    app.kubernetes.io/created-by: kwasm-operator
 spec:
   nodeSelector:
     spin: "true"

config/samples/test_shim_wws.yaml

@@ -2,6 +2,12 @@ apiVersion: runtime.kwasm.sh/v1alpha1
 kind: Shim
 metadata:
   name: wws-v1
+  labels:
+    app.kubernetes.io/name: wws-v1
+    app.kubernetes.io/instance: wws-v1
+    app.kubernetes.io/part-of: kwasm-operator
+    app.kubernetes.io/managed-by: kustomize
+    app.kubernetes.io/created-by: kwasm-operator
 spec:
   nodeSelector:
     wws: "true"

docs/supported_distros.md

@@ -0,0 +1,15 @@
+# Supported-Matrix
+
+We support the same Kubernetes distributions as Kwasm.
+
+| Container runtimes  | Kind | Azure Kubernetes | GCP Kubernetes | AWS Kubernetes | Digital Ocean Kubernetes | CIVO Kubernetes | Kubernetes in Docker | Minikube | Canonical MicroK8s |
+|---------------------|------|------------------|----------------|----------------|--------------------------|-----------------|----------------------|----------|--------------------|
+| WasmEdge            | ✅   | ✅                | (✅)           | (✅)           | ✅                        | ✅              | ✅                    | ✅       | ✅                 |
+| Wasmtime            | ✅   | ✅                | (✅)           | (✅)           | ✅                        | ✅              | ✅                    | ✅       | ✅                 |
+| Fermion Spin        | ✅   | ✅                | (✅)           | (✅)           | ✅                        | ✅              | ✅                    | ✅       | ✅                 |
+| Wasm Workers Server | ✅   | ✅                | (✅)           | (✅)           | ✅                        | ✅              | ✅                    | ✅       | ✅                 |
+| Lunatic             | ✅   | ✅                | (✅)           | (✅)           | ✅                        | ✅              | ✅                    | ✅       | ✅                 |
+| Slight              | ✅   | ✅                | (✅)           | (✅)           | ✅                        | ✅              | ✅                    | ✅       | ✅                 |
+
+✅   = officially supported
+(✅) = only with Ubuntu Nodes

go.mod

@@ -5,6 +5,9 @@ go 1.21
 require (
 	github.com/onsi/ginkgo/v2 v2.11.0
 	github.com/onsi/gomega v1.27.10
+	github.com/prometheus/common v0.44.0
+	github.com/rs/zerolog v1.31.0
+	k8s.io/api v0.28.3
 	k8s.io/apimachinery v0.28.3
 	k8s.io/client-go v0.28.3
 	sigs.k8s.io/controller-runtime v0.16.3
@@ -44,9 +47,7 @@ require (
 	github.com/pkg/errors v0.9.1 // indirect
 	github.com/prometheus/client_golang v1.16.0 // indirect
 	github.com/prometheus/client_model v0.4.0 // indirect
-	github.com/prometheus/common v0.44.0 // indirect
 	github.com/prometheus/procfs v0.10.1 // indirect
-	github.com/rs/zerolog v1.31.0 // indirect
 	github.com/spf13/pflag v1.0.5 // indirect
 	go.uber.org/multierr v1.11.0 // indirect
 	go.uber.org/zap v1.25.0 // indirect
@@ -64,7 +65,6 @@ require (
 	gopkg.in/inf.v0 v0.9.1 // indirect
 	gopkg.in/yaml.v2 v2.4.0 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
-	k8s.io/api v0.28.3 // indirect
 	k8s.io/apiextensions-apiserver v0.28.3 // indirect
 	k8s.io/component-base v0.28.3 // indirect
 	k8s.io/klog/v2 v2.100.1 // indirect

go.sum

@@ -1,5 +1,6 @@
 github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA=
 github.com/benbjohnson/clock v1.3.0 h1:ip6w0uFQkncKQ979AypyG0ER7mqUSBdKLOgAle/AT8A=
+github.com/benbjohnson/clock v1.3.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA=
 github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
 github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
 github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44=
@@ -15,6 +16,7 @@ github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSs
 github.com/emicklei/go-restful/v3 v3.11.0 h1:rAQeMHw1c7zTmncogyy8VvRZwtkmkZ4FxERmMY4rD+g=
 github.com/emicklei/go-restful/v3 v3.11.0/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc=
 github.com/evanphx/json-patch v5.6.0+incompatible h1:jBYDEEiFBPxA0v50tFdvOzQQTCvpL6mnFh5mB2/l16U=
+github.com/evanphx/json-patch v5.6.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk=
 github.com/evanphx/json-patch/v5 v5.6.0 h1:b91NhWfaz02IuVxO9faSllyAtNXHMPkC5J8sJCLunww=
 github.com/evanphx/json-patch/v5 v5.6.0/go.mod h1:G79N1coSVB93tBe7j6PhzjmR3/2VvlbKOFpnXhI9Bw4=
 github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4HY=
@@ -67,6 +69,7 @@ github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+o
 github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
 github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
 github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
+github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk=
 github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
 github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
 github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
@@ -105,6 +108,7 @@ github.com/prometheus/common v0.44.0/go.mod h1:ofAIvZbQ1e/nugmZGz4/qCb9Ap1VoSTIO
 github.com/prometheus/procfs v0.10.1 h1:kYK1Va/YMlutzCGazswoHKo//tZVlFpKYh+PymziUAg=
 github.com/prometheus/procfs v0.10.1/go.mod h1:nwNm2aOCAYw8uTR/9bWRREkZFxAUcWzPHWJq+XBB/FM=
 github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ=
+github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog=
 github.com/rs/xid v1.5.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg=
 github.com/rs/zerolog v1.31.0 h1:FcTR3NnLWW+NnTwwhFWiJSZr4ECLpqCm6QsEnyvbV4A=
 github.com/rs/zerolog v1.31.0/go.mod h1:/7mN4D5sKwJLZQ2b/znpjC3/GQWY/xaDXUM0kKWRHss=
@@ -120,12 +124,14 @@ github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/
 github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
 github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
 github.com/stretchr/testify v1.8.2 h1:+h33VjcLVPDHtOdpUCuF+7gSuG3yGIftsP1YvFihtJ8=
+github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
 github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k=
 go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc=
 go.uber.org/goleak v1.1.11/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ=
 go.uber.org/goleak v1.2.1 h1:NBol2c7O1ZokfZ0LEU9K6Whx/KnwvepVetCUhtKja4A=
+go.uber.org/goleak v1.2.1/go.mod h1:qlT2yGI9QafXHhZZLxlSuNsMw3FFLxBr+tBRlmO1xH4=
 go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU=
 go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0=
 go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y=
@@ -142,6 +148,7 @@ golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/mod v0.10.0 h1:lFO9qtOdlre5W1jxS3r/4szv2/6iXxScdzjoBMXNhYk=
+golang.org/x/mod v0.10.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
 golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
 golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
 golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks=