diff --git a/HACKING.md b/HACKING.md index d183b02ed..7f2497c79 100644 --- a/HACKING.md +++ b/HACKING.md @@ -96,6 +96,51 @@ A sample VSCode `launch.json` configuration is provided below: ], "host": "localhost", "port": 30052 + }, + { + "name": "Attach to Debug Job Task Runner on Kind", + "type": "go", + "debugAdapter": "dlv-dap", + "request": "attach", + "mode": "remote", + "substitutePath": [ + { + "from": "${workspaceFolder}", + "to": "/workspace" + } + ], + "host": "localhost", + "port": 30053 + }, + { + "name": "Attach to Debug Kpack Image Builder on Kind", + "type": "go", + "debugAdapter": "dlv-dap", + "request": "attach", + "mode": "remote", + "substitutePath": [ + { + "from": "${workspaceFolder}", + "to": "/workspace" + } + ], + "host": "localhost", + "port": 30054 + }, + { + "name": "Attach to Debug Statefulset Runner on Kind", + "type": "go", + "debugAdapter": "dlv-dap", + "request": "attach", + "mode": "remote", + "substitutePath": [ + { + "from": "${workspaceFolder}", + "to": "/workspace" + } + ], + "host": "localhost", + "port": 30055 } ] } diff --git a/controllers/Dockerfile b/controllers/Dockerfile index bd0cde1a5..d28872886 100644 --- a/controllers/Dockerfile +++ b/controllers/Dockerfile @@ -12,9 +12,6 @@ RUN --mount=type=cache,target=/go/pkg/mod \ COPY model model COPY controllers controllers -COPY kpack-image-builder kpack-image-builder -COPY job-task-runner job-task-runner -COPY statefulset-runner statefulset-runner COPY tools tools COPY version version diff --git a/controllers/cleanup/build_cleaner.go b/controllers/cleanup/build_cleaner.go index eb020beff..1313c731c 100644 --- a/controllers/cleanup/build_cleaner.go +++ b/controllers/cleanup/build_cleaner.go @@ -6,7 +6,6 @@ import ( "sort" korifiv1alpha1 "code.cloudfoundry.org/korifi/controllers/api/v1alpha1" - "code.cloudfoundry.org/korifi/statefulset-runner/controllers" "github.com/go-logr/logr" "k8s.io/apimachinery/pkg/api/meta" "k8s.io/apimachinery/pkg/types" @@ -35,7 +34,7 @@ func (c BuildCleaner) Clean(ctx context.Context, app types.NamespacedName) error err = c.k8sClient.List(ctx, &cfBuilds, client.InNamespace(app.Namespace), client.MatchingLabels{ - controllers.LabelAppGUID: app.Name, + korifiv1alpha1.CFAppGUIDLabelKey: app.Name, }, ) if err != nil { diff --git a/controllers/cleanup/build_cleaner_test.go b/controllers/cleanup/build_cleaner_test.go index e7d3f5a09..30b710d40 100644 --- a/controllers/cleanup/build_cleaner_test.go +++ b/controllers/cleanup/build_cleaner_test.go @@ -5,7 +5,6 @@ import ( korifiv1alpha1 "code.cloudfoundry.org/korifi/controllers/api/v1alpha1" "code.cloudfoundry.org/korifi/controllers/cleanup" - "code.cloudfoundry.org/korifi/statefulset-runner/controllers" "github.com/google/uuid" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" @@ -108,7 +107,7 @@ func createBuild(namespace, appGUID, name string) *korifiv1alpha1.CFBuild { Name: name, Namespace: namespace, Labels: map[string]string{ - controllers.LabelAppGUID: appGUID, + korifiv1alpha1.CFAppGUIDLabelKey: appGUID, }, }, Spec: korifiv1alpha1.CFBuildSpec{Lifecycle: korifiv1alpha1.Lifecycle{Type: "buildpack"}}, diff --git a/controllers/cleanup/package_cleaner.go b/controllers/cleanup/package_cleaner.go index e0976ae73..6ede73c7f 100644 --- a/controllers/cleanup/package_cleaner.go +++ b/controllers/cleanup/package_cleaner.go @@ -5,7 +5,6 @@ import ( "sort" korifiv1alpha1 "code.cloudfoundry.org/korifi/controllers/api/v1alpha1" - "code.cloudfoundry.org/korifi/statefulset-runner/controllers" "k8s.io/apimachinery/pkg/api/meta" "k8s.io/apimachinery/pkg/types" @@ -45,7 +44,7 @@ func (c PackageCleaner) Clean(ctx context.Context, app types.NamespacedName) err err = c.k8sClient.List(ctx, &cfPackages, client.InNamespace(app.Namespace), client.MatchingLabels{ - controllers.LabelAppGUID: app.Name, + korifiv1alpha1.CFAppGUIDLabelKey: app.Name, }, ) if err != nil { diff --git a/controllers/cleanup/package_cleaner_test.go b/controllers/cleanup/package_cleaner_test.go index 8a349f167..82fb86ac6 100644 --- a/controllers/cleanup/package_cleaner_test.go +++ b/controllers/cleanup/package_cleaner_test.go @@ -5,7 +5,6 @@ import ( korifiv1alpha1 "code.cloudfoundry.org/korifi/controllers/api/v1alpha1" "code.cloudfoundry.org/korifi/controllers/cleanup" - "code.cloudfoundry.org/korifi/statefulset-runner/controllers" "github.com/google/uuid" . "github.com/onsi/ginkgo/v2" @@ -119,7 +118,7 @@ func createPackage(namespace, appGUID, name string) *korifiv1alpha1.CFPackage { Name: name, Namespace: namespace, Labels: map[string]string{ - controllers.LabelAppGUID: appGUID, + korifiv1alpha1.CFAppGUIDLabelKey: appGUID, }, }, Spec: korifiv1alpha1.CFPackageSpec{Type: "bits"}, diff --git a/controllers/config/config.go b/controllers/config/config.go index 348d650ca..f932454af 100644 --- a/controllers/config/config.go +++ b/controllers/config/config.go @@ -9,11 +9,6 @@ import ( ) type ControllerConfig struct { - // components - IncludeKpackImageBuilder bool `yaml:"includeKpackImageBuilder"` - IncludeJobTaskRunner bool `yaml:"includeJobTaskRunner"` - IncludeStatefulsetRunner bool `yaml:"includeStatefulsetRunner"` - // core controllers CFProcessDefaults CFProcessDefaults `yaml:"cfProcessDefaults"` CFStagingResources CFStagingResources `yaml:"cfStagingResources"` @@ -29,16 +24,7 @@ type ControllerConfig struct { LogLevel zapcore.Level `yaml:"logLevel"` SpaceFinalizerAppDeletionTimeout *int32 `yaml:"spaceFinalizerAppDeletionTimeout"` - // job-task-runner - JobTTL string `yaml:"jobTTL"` - - // kpack-image-builder - ClusterBuilderName string `yaml:"clusterBuilderName"` - BuilderServiceAccount string `yaml:"builderServiceAccount"` - BuilderReadinessTimeout string `yaml:"builderReadinessTimeout"` - ContainerRepositoryPrefix string `yaml:"containerRepositoryPrefix"` - ContainerRegistryType string `yaml:"containerRegistryType"` - Networking Networking `yaml:"networking"` + Networking Networking `yaml:"networking"` ExperimentalManagedServicesEnabled bool `yaml:"experimentalManagedServicesEnabled"` TrustInsecureServiceBrokers bool `yaml:"trustInsecureServiceBrokers"` @@ -106,15 +92,3 @@ func (c ControllerConfig) ParseTaskTTL() (time.Duration, error) { return tools.ParseDuration(c.TaskTTL) } - -func (c ControllerConfig) ParseBuilderReadinessTimeout() (time.Duration, error) { - return tools.ParseDuration(c.BuilderReadinessTimeout) -} - -func (c ControllerConfig) ParseJobTTL() (time.Duration, error) { - if c.JobTTL == "" { - return defaultJobTTL, nil - } - - return tools.ParseDuration(c.JobTTL) -} diff --git a/controllers/config/config_test.go b/controllers/config/config_test.go index 5a9c2c139..f90b7e9ff 100644 --- a/controllers/config/config_test.go +++ b/controllers/config/config_test.go @@ -45,7 +45,6 @@ var _ = Describe("LoadFromPath", func() { TaskTTL: "taskTTL", BuilderName: "buildReconciler", RunnerName: "statefulset-runner", - JobTTL: "jobTTL", LogLevel: zapcore.DebugLevel, SpaceFinalizerAppDeletionTimeout: tools.PtrTo(int32(42)), Networking: config.Networking{ @@ -89,7 +88,6 @@ var _ = Describe("LoadFromPath", func() { RunnerName: "statefulset-runner", NamespaceLabels: map[string]string{}, ExtraVCAPApplicationValues: map[string]any{}, - JobTTL: "jobTTL", LogLevel: zapcore.DebugLevel, SpaceFinalizerAppDeletionTimeout: tools.PtrTo(int32(42)), Networking: config.Networking{ @@ -187,48 +185,3 @@ var _ = Describe("ParseTaskTTL", func() { }) }) }) - -var _ = Describe("ParseJobTTL", func() { - var ( - jobTTL time.Duration - parseErr error - jobTTLStr string - ) - - BeforeEach(func() { - jobTTLStr = "" - }) - - JustBeforeEach(func() { - cfg := config.ControllerConfig{ - JobTTL: jobTTLStr, - } - jobTTL, parseErr = cfg.ParseJobTTL() - }) - - It("return 30 days by default", func() { - Expect(parseErr).NotTo(HaveOccurred()) - Expect(jobTTL).To(Equal(24 * time.Hour)) - }) - - When("jobTTL is something parseable by tools.ParseDuration", func() { - BeforeEach(func() { - jobTTLStr = "5d12h" - }) - - It("parses ok", func() { - Expect(parseErr).NotTo(HaveOccurred()) - Expect(jobTTL).To(Equal(5*24*time.Hour + 12*time.Hour)) - }) - }) - - When("entering something that cannot be parsed", func() { - BeforeEach(func() { - jobTTLStr = "foreva" - }) - - It("returns an error", func() { - Expect(parseErr).To(HaveOccurred()) - }) - }) -}) diff --git a/controllers/controllers/workloads/orgs/controller.go b/controllers/controllers/workloads/orgs/controller.go index 9f0e74530..dd2450904 100644 --- a/controllers/controllers/workloads/orgs/controller.go +++ b/controllers/controllers/workloads/orgs/controller.go @@ -118,6 +118,8 @@ func (r *Reconciler) enqueueCFOrgRequests(ctx context.Context, object client.Obj //+kubebuilder:rbac:groups="metrics.k8s.io",resources=pods,verbs=get;list;watch //+kubebuilder:rbac:groups="policy",resources=poddisruptionbudgets,verbs=create;deletecollection //+kubebuilder:rbac:groups="policy",resources=podsecuritypolicies,verbs=use +//+kubebuilder:rbac:groups=korifi.cloudfoundry.org,resources=runnerinfos,verbs=get;list;watch;create;patch;delete +//+kubebuilder:rbac:groups=korifi.cloudfoundry.org,resources=runnerinfos/status,verbs=get;patch func (r *Reconciler) ReconcileResource(ctx context.Context, cfOrg *korifiv1alpha1.CFOrg) (ctrl.Result, error) { nsReconcileResult, err := r.namespaceReconciler.ReconcileResource(ctx, cfOrg) diff --git a/controllers/main.go b/controllers/main.go index af124ef50..f6544aed1 100644 --- a/controllers/main.go +++ b/controllers/main.go @@ -62,13 +62,8 @@ import ( packageswebhook "code.cloudfoundry.org/korifi/controllers/webhooks/workloads/packages" spaceswebhook "code.cloudfoundry.org/korifi/controllers/webhooks/workloads/spaces" taskswebhook "code.cloudfoundry.org/korifi/controllers/webhooks/workloads/tasks" - jobtaskrunnercontrollers "code.cloudfoundry.org/korifi/job-task-runner/controllers" - "code.cloudfoundry.org/korifi/kpack-image-builder/controllers" - kpackimagebuilderfinalizer "code.cloudfoundry.org/korifi/kpack-image-builder/controllers/webhooks/finalizer" - statefulsetcontrollers "code.cloudfoundry.org/korifi/statefulset-runner/controllers" "code.cloudfoundry.org/korifi/tools" "code.cloudfoundry.org/korifi/tools/image" - "code.cloudfoundry.org/korifi/tools/registry" "code.cloudfoundry.org/korifi/version" buildv1alpha2 "github.com/pivotal/kpack/pkg/apis/build/v1alpha2" @@ -345,91 +340,6 @@ func main() { os.Exit(1) } - if controllerConfig.IncludeKpackImageBuilder { - var builderReadinessTimeout time.Duration - builderReadinessTimeout, err = controllerConfig.ParseBuilderReadinessTimeout() - if err != nil { - setupLog.Error(err, "error parsing builderReadinessTimeout") - os.Exit(1) - } - if err = controllers.NewBuildWorkloadReconciler( - mgr.GetClient(), - mgr.GetScheme(), - controllersLog, - controllerConfig, - imageClient, - controllerConfig.ContainerRepositoryPrefix, - registry.NewRepositoryCreator(controllerConfig.ContainerRegistryType), - builderReadinessTimeout, - ).SetupWithManager(mgr); err != nil { - setupLog.Error(err, "unable to create controller", "controller", "BuildWorkload") - os.Exit(1) - } - - if err = controllers.NewBuilderInfoReconciler( - mgr.GetClient(), - mgr.GetScheme(), - controllersLog, - controllerConfig.ClusterBuilderName, - controllerConfig.CFRootNamespace, - ).SetupWithManager(mgr); err != nil { - setupLog.Error(err, "unable to create controller", "controller", "BuilderInfo") - os.Exit(1) - } - - if err = controllers.NewKpackBuildController( - mgr.GetClient(), - controllersLog, - imageClient, - controllerConfig.BuilderServiceAccount, - ).SetupWithManager(mgr); err != nil { - setupLog.Error(err, "unable to create controller", "controller", "KpackBuild") - os.Exit(1) - } - } - - if controllerConfig.IncludeJobTaskRunner { - var jobTTL time.Duration - jobTTL, err = controllerConfig.ParseJobTTL() - if err != nil { - panic(err) - } - - taskWorkloadReconciler := jobtaskrunnercontrollers.NewTaskWorkloadReconciler( - controllersLog, - mgr.GetClient(), - mgr.GetScheme(), - jobtaskrunnercontrollers.NewStatusGetter(mgr.GetClient()), - jobTTL, - ) - if err = taskWorkloadReconciler.SetupWithManager(mgr); err != nil { - setupLog.Error(err, "unable to create controller", "controller", "TaskWorkload") - os.Exit(1) - } - } - - if controllerConfig.IncludeStatefulsetRunner { - if err = statefulsetcontrollers.NewAppWorkloadReconciler( - mgr.GetClient(), - mgr.GetScheme(), - statefulsetcontrollers.NewAppWorkloadToStatefulsetConverter(mgr.GetScheme()), - statefulsetcontrollers.NewPDBUpdater(mgr.GetClient()), - controllersLog, - ).SetupWithManager(mgr); err != nil { - setupLog.Error(err, "unable to create controller", "controller", "AppWorkload") - os.Exit(1) - } - - if err = statefulsetcontrollers.NewRunnerInfoReconciler( - mgr.GetClient(), - mgr.GetScheme(), - controllersLog, - ).SetupWithManager(mgr); err != nil { - setupLog.Error(err, "unable to create controller", "controller", "RunnerInfo") - os.Exit(1) - } - } - if err = routes.NewReconciler( mgr.GetClient(), mgr.GetScheme(), @@ -563,10 +473,6 @@ func main() { relationships.NewSpaceGUIDWebhook().SetupWebhookWithManager(mgr) - if controllerConfig.IncludeKpackImageBuilder { - kpackimagebuilderfinalizer.NewKpackImageBuilderFinalizerWebhook().SetupWebhookWithManager(mgr) - } - if err = mgr.AddReadyzCheck("readyz", mgr.GetWebhookServer().StartedChecker()); err != nil { setupLog.Error(err, "unable to set up ready check") os.Exit(1) diff --git a/controllers/remote-debug/Dockerfile b/controllers/remote-debug/Dockerfile index f21d2884a..791823ef5 100644 --- a/controllers/remote-debug/Dockerfile +++ b/controllers/remote-debug/Dockerfile @@ -12,9 +12,6 @@ RUN --mount=type=cache,target=/go/pkg/mod \ COPY model model COPY controllers controllers -COPY kpack-image-builder kpack-image-builder -COPY job-task-runner job-task-runner -COPY statefulset-runner statefulset-runner COPY tools tools COPY version version diff --git a/helm/korifi/controllers/configmap.yaml b/helm/korifi/controllers/configmap.yaml index 59b200354..c2e40abdf 100644 --- a/helm/korifi/controllers/configmap.yaml +++ b/helm/korifi/controllers/configmap.yaml @@ -5,9 +5,6 @@ metadata: namespace: {{ .Release.Namespace }} data: config.yaml: |- - includeKpackImageBuilder: {{ .Values.kpackImageBuilder.include }} - includeJobTaskRunner: {{ .Values.jobTaskRunner.include }} - includeStatefulsetRunner: {{ .Values.statefulsetRunner.include }} builderName: {{ .Values.reconcilers.build }} runnerName: {{ .Values.reconcilers.run }} cfProcessDefaults: @@ -38,22 +35,6 @@ data: maxRetainedPackagesPerApp: {{ .Values.controllers.maxRetainedPackagesPerApp }} maxRetainedBuildsPerApp: {{ .Values.controllers.maxRetainedBuildsPerApp }} logLevel: {{ .Values.logLevel }} - {{- if .Values.kpackImageBuilder.include }} - clusterBuilderName: {{ .Values.kpackImageBuilder.clusterBuilderName | default "cf-kpack-cluster-builder" }} - builderReadinessTimeout: {{ required "builderReadinessTimeout is required" .Values.kpackImageBuilder.builderReadinessTimeout }} - containerRepositoryPrefix: {{ .Values.containerRepositoryPrefix | quote }} - builderServiceAccount: kpack-service-account - cfStagingResources: - buildCacheMB: {{ .Values.stagingRequirements.buildCacheMB }} - diskMB: {{ .Values.stagingRequirements.diskMB }} - memoryMB: {{ .Values.stagingRequirements.memoryMB }} - {{- if .Values.eksContainerRegistryRoleARN }} - containerRegistryType: "ECR" - {{- end }} - {{- end }} - {{- if .Values.jobTaskRunner.include }} - jobTTL: {{ required "jobTTL is required" .Values.jobTaskRunner.jobTTL }} - {{- end }} networking: gatewayNamespace: {{ .Release.Namespace }}-gateway gatewayName: korifi diff --git a/helm/korifi/controllers/rbac.yaml b/helm/korifi/controllers/rbac.yaml index 75500ef3a..75bf0a4f1 100644 --- a/helm/korifi/controllers/rbac.yaml +++ b/helm/korifi/controllers/rbac.yaml @@ -79,51 +79,3 @@ subjects: - kind: ServiceAccount name: korifi-controllers-controller-manager namespace: {{ .Release.Namespace }} - -{{- if .Values.jobTaskRunner.include }} ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - name: korifi-job-task-runner-manager-rolebinding -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: korifi-job-task-runner-taskworkload-manager-role -subjects: -- kind: ServiceAccount - name: korifi-controllers-controller-manager - namespace: {{ .Release.Namespace }} -{{- end }} - -{{- if .Values.kpackImageBuilder.include }} ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - name: korifi-kpack-build-manager-rolebinding -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: korifi-kpack-build-manager-role -subjects: -- kind: ServiceAccount - name: korifi-controllers-controller-manager - namespace: {{ .Release.Namespace }} -{{- end }} - -{{- if .Values.statefulsetRunner.include }} ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - name: korifi-statefulset-runner-manager-rolebinding -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: korifi-statefulset-runner-appworkload-manager-role -subjects: -- kind: ServiceAccount - name: korifi-controllers-controller-manager - namespace: {{ .Release.Namespace }} -{{- end }} diff --git a/helm/korifi/controllers/role.yaml b/helm/korifi/controllers/role.yaml index ad805b568..98edc0031 100644 --- a/helm/korifi/controllers/role.yaml +++ b/helm/korifi/controllers/role.yaml @@ -190,6 +190,7 @@ rules: - korifi.cloudfoundry.org resources: - cfdomains + - runnerinfos - taskworkloads verbs: - create @@ -198,6 +199,13 @@ rules: - list - patch - watch +- apiGroups: + - korifi.cloudfoundry.org + resources: + - runnerinfos/status + verbs: + - get + - patch - apiGroups: - kpack.io resources: diff --git a/helm/korifi/job-task-runner/deployment.yaml b/helm/korifi/job-task-runner/deployment.yaml new file mode 100644 index 000000000..e2f809c1d --- /dev/null +++ b/helm/korifi/job-task-runner/deployment.yaml @@ -0,0 +1,70 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + labels: + app: job-task-runner + name: job-task-runner-controller-manager + namespace: {{ .Release.Namespace }} +spec: + replicas: {{ .Values.jobTaskRunner.replicas }} + selector: + matchLabels: + app: job-task-runner + template: + metadata: + annotations: + kubectl.kubernetes.io/default-container: manager + prometheus.io/path: /metrics + prometheus.io/port: "8080" + prometheus.io/scrape: "true" + labels: + app: job-task-runner + spec: + containers: + - name: manager + image: {{ .Values.jobTaskRunner.image }} +{{- if .Values.debug }} + command: + - "/dlv" + args: + - "--listen=:40000" + - "--headless=true" + - "--api-version=2" + - "exec" + - "/manager" + - "--continue" + - "--accept-multiclient" + - "--" + - "--health-probe-bind-address=:8081" + - "--leader-elect" + - "--ttl={{ required "jobTTL is required" .Values.jobTaskRunner.jobTTL }}" +{{- else }} + args: + - --health-probe-bind-address=:8081 + - --leader-elect + - --ttl={{ required "jobTTL is required" .Values.jobTaskRunner.jobTTL }} +{{- end }} + livenessProbe: + httpGet: + path: /healthz + port: 8081 + initialDelaySeconds: 15 + periodSeconds: 20 + ports: + - containerPort: 8080 + name: metrics + protocol: TCP + resources: + {{- .Values.jobTaskRunner.resources | toYaml | nindent 10 }} + {{- include "korifi.securityContext" . | indent 8 }} + {{- include "korifi.podSecurityContext" . | indent 6 }} + serviceAccountName: job-task-runner-controller-manager +{{- if .Values.jobTaskRunner.nodeSelector }} + nodeSelector: + {{ toYaml .Values.jobTaskRunner.nodeSelector | indent 8 }} +{{- end }} +{{- if .Values.jobTaskRunner.tolerations }} + tolerations: + {{- toYaml .Values.jobTaskRunner.tolerations | nindent 8 }} +{{- end }} + terminationGracePeriodSeconds: 10 diff --git a/helm/korifi/job-task-runner/rbac.yaml b/helm/korifi/job-task-runner/rbac.yaml new file mode 100644 index 000000000..b860e0e5d --- /dev/null +++ b/helm/korifi/job-task-runner/rbac.yaml @@ -0,0 +1,37 @@ +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: job-task-runner-controller-manager + namespace: {{ .Release.Namespace }} +imagePullSecrets: +{{- range .Values.systemImagePullSecrets }} +- name: {{ . | quote }} +{{- end }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: job-task-runner-leader-election-rolebinding + namespace: {{ .Release.Namespace }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: korifi-controllers-leader-election-role +subjects: +- kind: ServiceAccount + name: job-task-runner-controller-manager + namespace: {{ .Release.Namespace }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: job-task-runner-manager-rolebinding +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: korifi-job-task-runner-taskworkload-manager-role +subjects: +- kind: ServiceAccount + name: job-task-runner-controller-manager + namespace: {{ .Release.Namespace }} diff --git a/helm/korifi/job-task-runner/service.yaml b/helm/korifi/job-task-runner/service.yaml new file mode 100644 index 000000000..217a4b389 --- /dev/null +++ b/helm/korifi/job-task-runner/service.yaml @@ -0,0 +1,18 @@ +--- +{{- if .Values.debug }} +apiVersion: v1 +kind: Service +metadata: + name: job-task-runner-debug-port + namespace: {{ .Release.Namespace }} +spec: + ports: + - name: debug-30053 + nodePort: 30053 + port: 30053 + protocol: TCP + targetPort: 40000 + selector: + app: job-task-runner + type: NodePort +{{- end }} diff --git a/helm/korifi/kpack-image-builder/cert.yaml b/helm/korifi/kpack-image-builder/cert.yaml new file mode 100644 index 000000000..2dba93cfd --- /dev/null +++ b/helm/korifi/kpack-image-builder/cert.yaml @@ -0,0 +1,13 @@ +apiVersion: cert-manager.io/v1 +kind: Certificate +metadata: + name: kpack-image-builder-serving-cert + namespace: {{ .Release.Namespace }} +spec: + dnsNames: + - kpack-image-builder-webhook-service.{{ .Release.Namespace }}.svc + - kpack-image-builder-webhook-service.{{ .Release.Namespace }}.svc.cluster.local + issuerRef: + kind: Issuer + name: selfsigned-issuer + secretName: kpack-image-builder-webhook-cert diff --git a/helm/korifi/kpack-image-builder/configmap.yaml b/helm/korifi/kpack-image-builder/configmap.yaml new file mode 100644 index 000000000..3592153b5 --- /dev/null +++ b/helm/korifi/kpack-image-builder/configmap.yaml @@ -0,0 +1,19 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: kpack-image-builder-config + namespace: {{ .Release.Namespace }} +data: + config.yaml: |- + cfRootNamespace: {{ .Values.rootNamespace }} + clusterBuilderName: {{ .Values.kpackImageBuilder.clusterBuilderName | default "cf-kpack-cluster-builder" }} + builderReadinessTimeout: {{ required "builderReadinessTimeout is required" .Values.kpackImageBuilder.builderReadinessTimeout }} + containerRepositoryPrefix: {{ .Values.containerRepositoryPrefix | quote }} + builderServiceAccount: kpack-service-account + cfStagingResources: + buildCacheMB: {{ .Values.stagingRequirements.buildCacheMB }} + diskMB: {{ .Values.stagingRequirements.diskMB }} + memoryMB: {{ .Values.stagingRequirements.memoryMB }} + {{- if .Values.eksContainerRegistryRoleARN }} + containerRegistryType: "ECR" + {{- end }} diff --git a/helm/korifi/kpack-image-builder/deployment.yaml b/helm/korifi/kpack-image-builder/deployment.yaml new file mode 100644 index 000000000..42f34960b --- /dev/null +++ b/helm/korifi/kpack-image-builder/deployment.yaml @@ -0,0 +1,95 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + labels: + app: kpack-image-builder + name: kpack-image-builder-controller-manager + namespace: {{ .Release.Namespace }} +spec: + replicas: {{ .Values.kpackImageBuilder.replicas | default 1}} + selector: + matchLabels: + app: kpack-image-builder + template: + metadata: + annotations: + kubectl.kubernetes.io/default-container: manager + prometheus.io/path: /metrics + prometheus.io/port: "8080" + prometheus.io/scrape: "true" + checksum/config: {{ tpl ($.Files.Get "controllers/configmap.yaml") $ | sha256sum }} + labels: + app: kpack-image-builder + spec: + containers: + - name: manager + image: {{ .Values.kpackImageBuilder.image }} +{{- if .Values.debug }} + command: + - "/dlv" + args: + - "--listen=:40000" + - "--headless=true" + - "--api-version=2" + - "exec" + - "/manager" + - "--continue" + - "--accept-multiclient" + - "--" + - "--health-probe-bind-address=:8081" + - "--leader-elect" + - "--config=/etc/kpack-image-builder-config" +{{- else }} + args: + - --health-probe-bind-address=:8081 + - --leader-elect + - --config=/etc/kpack-image-builder-config +{{- end }} + livenessProbe: + httpGet: + path: /healthz + port: 8081 + initialDelaySeconds: 15 + periodSeconds: 20 + ports: + - containerPort: 9443 + name: webhook-server + protocol: TCP + - containerPort: 8080 + name: metrics + protocol: TCP + readinessProbe: + httpGet: + path: /readyz + port: 8081 + initialDelaySeconds: 5 + periodSeconds: 10 + resources: + {{- .Values.jobTaskRunner.resources | toYaml | nindent 10 }} + {{- include "korifi.securityContext" . | indent 8 }} + volumeMounts: + - mountPath: /tmp/k8s-webhook-server/serving-certs + name: cert + readOnly: true + - mountPath: /etc/kpack-image-builder-config + name: kpack-image-builder-config + readOnly: true + {{- include "korifi.podSecurityContext" . | indent 6 }} + serviceAccountName: kpack-image-builder-controller-manager +{{- if .Values.kpackImageBuilder.nodeSelector }} + nodeSelector: + {{ toYaml .Values.kpackImageBuilder.nodeSelector | indent 8 }} +{{- end }} +{{- if .Values.kpackImageBuilder.tolerations }} + tolerations: + {{- toYaml .Values.kpackImageBuilder.tolerations | nindent 8 }} +{{- end }} + terminationGracePeriodSeconds: 10 + volumes: + - name: cert + secret: + defaultMode: 420 + secretName: kpack-image-builder-webhook-cert + - configMap: + name: kpack-image-builder-config + name: kpack-image-builder-config diff --git a/helm/korifi/kpack-image-builder/manifests.yaml b/helm/korifi/kpack-image-builder/manifests.yaml index 9261c5af1..2b17cbb09 100644 --- a/helm/korifi/kpack-image-builder/manifests.yaml +++ b/helm/korifi/kpack-image-builder/manifests.yaml @@ -4,14 +4,14 @@ kind: MutatingWebhookConfiguration metadata: name: korifi-kpack-image-builder-mutating-webhook-configuration annotations: - cert-manager.io/inject-ca-from: '{{ .Release.Namespace }}/korifi-controllers-serving-cert' + cert-manager.io/inject-ca-from: '{{ .Release.Namespace }}/kpack-image-builder-serving-cert' webhooks: - admissionReviewVersions: - v1 - v1beta1 clientConfig: service: - name: korifi-controllers-webhook-service + name: kpack-image-builder-webhook-service namespace: '{{ .Release.Namespace }}' path: /mutate-korifi-cloudfoundry-org-v1alpha1-kpack-image-builder-finalizer failurePolicy: Fail diff --git a/helm/korifi/kpack-image-builder/rbac.yaml b/helm/korifi/kpack-image-builder/rbac.yaml new file mode 100644 index 000000000..ebcddc839 --- /dev/null +++ b/helm/korifi/kpack-image-builder/rbac.yaml @@ -0,0 +1,41 @@ +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: kpack-image-builder-controller-manager + namespace: {{ .Release.Namespace }} + {{- if .Values.eksContainerRegistryRoleARN }} + annotations: + eks.amazonaws.com/role-arn: {{ .Values.eksContainerRegistryRoleARN }} + {{- end }} +imagePullSecrets: +{{- range .Values.systemImagePullSecrets }} +- name: {{ . | quote }} +{{- end }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: kpack-image-builder-leader-election-rolebinding + namespace: {{ .Release.Namespace }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: korifi-controllers-leader-election-role +subjects: +- kind: ServiceAccount + name: kpack-image-builder-controller-manager + namespace: {{ .Release.Namespace }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: kpack-image-builder-manager-rolebinding +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: korifi-kpack-build-manager-role +subjects: +- kind: ServiceAccount + name: kpack-image-builder-controller-manager + namespace: {{ .Release.Namespace }} diff --git a/helm/korifi/kpack-image-builder/service.yaml b/helm/korifi/kpack-image-builder/service.yaml new file mode 100644 index 000000000..ac41475ba --- /dev/null +++ b/helm/korifi/kpack-image-builder/service.yaml @@ -0,0 +1,29 @@ +apiVersion: v1 +kind: Service +metadata: + name: kpack-image-builder-webhook-service + namespace: {{ .Release.Namespace }} +spec: + ports: + - port: 443 + targetPort: 9443 + selector: + app: kpack-image-builder +--- +{{- if .Values.debug }} +apiVersion: v1 +kind: Service +metadata: + name: kpack-image-builder-debug-port + namespace: {{ .Release.Namespace }} +spec: + ports: + - name: debug-30054 + nodePort: 30054 + port: 30054 + protocol: TCP + targetPort: 40000 + selector: + app: kpack-image-builder + type: NodePort +{{- end }} diff --git a/helm/korifi/statefulset-runner/deployment.yaml b/helm/korifi/statefulset-runner/deployment.yaml new file mode 100644 index 000000000..665ae98c4 --- /dev/null +++ b/helm/korifi/statefulset-runner/deployment.yaml @@ -0,0 +1,68 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + labels: + app: statefulset-runner + name: statefulset-runner-controller-manager + namespace: {{ .Release.Namespace }} +spec: + replicas: {{ .Values.statefulsetRunner.replicas }} + selector: + matchLabels: + app: statefulset-runner + template: + metadata: + annotations: + kubectl.kubernetes.io/default-container: manager + prometheus.io/path: /metrics + prometheus.io/port: "8080" + prometheus.io/scrape: "true" + labels: + app: statefulset-runner + spec: + containers: + - name: manager + image: {{ .Values.statefulsetRunner.image }} +{{- if .Values.debug }} + command: + - "/dlv" + args: + - "--listen=:40000" + - "--headless=true" + - "--api-version=2" + - "exec" + - "/manager" + - "--continue" + - "--accept-multiclient" + - "--" + - "--health-probe-bind-address=:8081" + - "--leader-elect" +{{- else }} + args: + - --health-probe-bind-address=:8081 + - --leader-elect +{{- end }} + livenessProbe: + httpGet: + path: /healthz + port: 8081 + initialDelaySeconds: 15 + periodSeconds: 20 + ports: + - containerPort: 8080 + name: metrics + protocol: TCP + resources: + {{- .Values.statefulsetRunner.resources | toYaml | nindent 10 }} + {{- include "korifi.securityContext" . | indent 8 }} + {{- include "korifi.podSecurityContext" . | indent 6 }} + serviceAccountName: statefulset-runner-controller-manager +{{- if .Values.statefulsetRunner.nodeSelector }} + nodeSelector: + {{ toYaml .Values.statefulsetRunner.nodeSelector | indent 8 }} +{{- end }} +{{- if .Values.statefulsetRunner.tolerations }} + tolerations: + {{- toYaml .Values.statefulsetRunner.tolerations | nindent 8 }} +{{- end }} + terminationGracePeriodSeconds: 10 diff --git a/helm/korifi/statefulset-runner/post-install-runnerinfo.yaml b/helm/korifi/statefulset-runner/post-install-runnerinfo.yaml index 1179f55e3..fb845ba64 100644 --- a/helm/korifi/statefulset-runner/post-install-runnerinfo.yaml +++ b/helm/korifi/statefulset-runner/post-install-runnerinfo.yaml @@ -23,7 +23,7 @@ spec: app.kubernetes.io/instance: {{ .Release.Name | quote }} helm.sh/chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" spec: - serviceAccountName: korifi-controllers-controller-manager + serviceAccountName: statefulset-runner-controller-manager restartPolicy: Never {{- include "korifi.podSecurityContext" . | indent 6 }} containers: diff --git a/helm/korifi/statefulset-runner/rbac.yaml b/helm/korifi/statefulset-runner/rbac.yaml new file mode 100644 index 000000000..e5ebde3ec --- /dev/null +++ b/helm/korifi/statefulset-runner/rbac.yaml @@ -0,0 +1,36 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: statefulset-runner-controller-manager + namespace: {{ .Release.Namespace }} +imagePullSecrets: +{{- range .Values.systemImagePullSecrets }} +- name: {{ . | quote }} +{{- end }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: statefulset-runner-leader-election-rolebinding + namespace: {{ .Release.Namespace }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: korifi-controllers-leader-election-role +subjects: +- kind: ServiceAccount + name: statefulset-runner-controller-manager + namespace: {{ .Release.Namespace }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: statefulset-runner-manager-rolebinding +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: korifi-statefulset-runner-appworkload-manager-role +subjects: +- kind: ServiceAccount + name: statefulset-runner-controller-manager + namespace: {{ .Release.Namespace }} diff --git a/helm/korifi/statefulset-runner/service.yaml b/helm/korifi/statefulset-runner/service.yaml new file mode 100644 index 000000000..b8e449281 --- /dev/null +++ b/helm/korifi/statefulset-runner/service.yaml @@ -0,0 +1,18 @@ +--- +{{- if .Values.debug }} +apiVersion: v1 +kind: Service +metadata: + name: statefulset-runner-debug-port + namespace: {{ .Release.Namespace }} +spec: + ports: + - name: debug-30055 + nodePort: 30055 + port: 30055 + protocol: TCP + targetPort: 40000 + selector: + app: statefulset-runner + type: NodePort +{{- end }} diff --git a/helm/korifi/values.yaml b/helm/korifi/values.yaml index d2fa46530..85afeba37 100644 --- a/helm/korifi/values.yaml +++ b/helm/korifi/values.yaml @@ -94,6 +94,7 @@ controllers: kpackImageBuilder: include: true + image: cloudfoundry/korifi-kpack-image-builder:latest replicas: 1 resources: limits: @@ -112,25 +113,27 @@ kpackImageBuilder: statefulsetRunner: include: true + image: cloudfoundry/korifi-statefulset-runner:latest replicas: 1 resources: limits: - cpu: 500m - memory: 128Mi + cpu: 1000m + memory: 1Gi requests: - cpu: 10m - memory: 64Mi + cpu: 50m + memory: 100Mi jobTaskRunner: include: true + image: cloudfoundry/korifi-job-task-runner:latest replicas: 1 resources: limits: - cpu: 500m - memory: 128Mi + cpu: 1000m + memory: 1Gi requests: - cpu: 10m - memory: 64Mi + cpu: 50m + memory: 100Mi jobTTL: 24h diff --git a/job-task-runner/Dockerfile b/job-task-runner/Dockerfile new file mode 100644 index 000000000..145f3598e --- /dev/null +++ b/job-task-runner/Dockerfile @@ -0,0 +1,32 @@ +# syntax = docker/dockerfile:experimental +FROM golang:1.23 as builder + +ARG version=dev + +WORKDIR /workspace + +COPY go.mod go.sum ./ + +RUN --mount=type=cache,target=/go/pkg/mod \ + go mod download + +COPY api api +COPY controllers controllers +COPY job-task-runner job-task-runner +COPY model model +COPY tools tools +COPY version version + +RUN --mount=type=cache,target=/root/.cache/go-build \ + --mount=type=cache,target=/go/pkg/mod \ + CGO_ENABLED=0 GOOS=linux go build -ldflags "-X code.cloudfoundry.org/korifi/version.Version=${version}" -o manager job-task-runner/main.go + +# Use distroless as minimal base image to package the manager binary +# Refer to https://github.com/GoogleContainerTools/distroless for more details +FROM gcr.io/distroless/static:nonroot + +WORKDIR / +COPY --from=builder /workspace/manager . +USER 65532:65532 + +ENTRYPOINT ["/manager"] diff --git a/job-task-runner/main.go b/job-task-runner/main.go new file mode 100644 index 000000000..3fe01dfa5 --- /dev/null +++ b/job-task-runner/main.go @@ -0,0 +1,98 @@ +package main + +import ( + "flag" + "fmt" + "os" + + korifiv1alpha1 "code.cloudfoundry.org/korifi/controllers/api/v1alpha1" + jobtaskcontrollers "code.cloudfoundry.org/korifi/job-task-runner/controllers" + "code.cloudfoundry.org/korifi/tools" + "go.uber.org/zap/zapcore" + utilruntime "k8s.io/apimachinery/pkg/util/runtime" + clientgoscheme "k8s.io/client-go/kubernetes/scheme" + "k8s.io/klog/v2" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/healthz" + metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server" + + "k8s.io/apimachinery/pkg/runtime" +) + +var ( + scheme = runtime.NewScheme() + setupLog = ctrl.Log.WithName("setup") +) + +func init() { + utilruntime.Must(clientgoscheme.AddToScheme(scheme)) + utilruntime.Must(korifiv1alpha1.AddToScheme(scheme)) +} + +func main() { + var ( + metricsAddr string + enableLeaderElection bool + probeAddr string + ttl string + ) + + flag.StringVar(&metricsAddr, "metrics-bind-address", ":8080", "The address the metric endpoint binds to.") + flag.StringVar(&probeAddr, "health-probe-bind-address", ":8081", "The address the probe endpoint binds to.") + flag.StringVar(&ttl, "ttl", "24h", "The time to live for a task job.") + flag.BoolVar(&enableLeaderElection, "leader-elect", false, + "Enable leader election for controller manager. "+ + "Enabling this will ensure there is only one active controller manager.") + flag.Parse() + + logger, _, err := tools.NewZapLogger(zapcore.InfoLevel) + if err != nil { + panic(fmt.Sprintf("error creating new zap logger: %v", err)) + } + + ctrl.SetLogger(logger) + klog.SetLogger(ctrl.Log) + + mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), ctrl.Options{ + Scheme: scheme, + Metrics: metricsserver.Options{ + BindAddress: metricsAddr, + }, + HealthProbeBindAddress: probeAddr, + LeaderElection: enableLeaderElection, + LeaderElectionID: "13c300bs.cloudfoundry.org", + }) + if err != nil { + setupLog.Error(err, "unable to initialize manager") + os.Exit(1) + } + + jobTTL, err := tools.ParseDuration(ttl) + if err != nil { + setupLog.Error(err, "unable to parse job TTL", "ttl", ttl) + os.Exit(1) + } + + controllersLog := ctrl.Log.WithName("controllers") + if err = jobtaskcontrollers.NewTaskWorkloadReconciler( + controllersLog, + mgr.GetClient(), + mgr.GetScheme(), + jobtaskcontrollers.NewStatusGetter(mgr.GetClient()), + jobTTL, + ).SetupWithManager(mgr); err != nil { + setupLog.Error(err, "unable to create controller", "controller", "TaskWorkload") + os.Exit(1) + } + + if err := mgr.AddHealthzCheck("healthz", healthz.Ping); err != nil { + setupLog.Error(err, "unable to set up health check") + os.Exit(1) + } + + setupLog.Info("starting manager") + if err := mgr.Start(ctrl.SetupSignalHandler()); err != nil { + setupLog.Error(err, "problem running manager") + os.Exit(1) + } +} diff --git a/job-task-runner/remote-debug/Dockerfile b/job-task-runner/remote-debug/Dockerfile new file mode 100644 index 000000000..e66fe63aa --- /dev/null +++ b/job-task-runner/remote-debug/Dockerfile @@ -0,0 +1,35 @@ +# syntax = docker/dockerfile:experimental +FROM golang:1.23 as builder + +ARG version=dev + +WORKDIR /workspace + +COPY go.mod go.sum ./ + +RUN --mount=type=cache,target=/go/pkg/mod \ + go mod download + +COPY api api +COPY controllers controllers +COPY job-task-runner job-task-runner +COPY model model +COPY tools tools +COPY version version + +RUN --mount=type=cache,target=/root/.cache/go-build \ + --mount=type=cache,target=/go/pkg/mod \ + CGO_ENABLED=0 GOOS=linux go build -ldflags "-X code.cloudfoundry.org/korifi/version.Version=${version}" -gcflags=all="-N -l" -o manager job-task-runner/main.go + +# Get Delve from a GOPATH not from a Go Modules project +WORKDIR /go/src/ +RUN go install github.com/go-delve/delve/cmd/dlv@latest + +FROM ubuntu + +WORKDIR / +COPY --from=builder /workspace/manager . +COPY --from=builder /go/bin/dlv . +EXPOSE 8080 8081 9443 40000 + +CMD ["/dlv", "--listen=:40000", "--headless=true", "--api-version=2", "exec", "/manager", "--continue", "--accept-multiclient"] diff --git a/kpack-image-builder/Dockerfile b/kpack-image-builder/Dockerfile new file mode 100644 index 000000000..6a2942f33 --- /dev/null +++ b/kpack-image-builder/Dockerfile @@ -0,0 +1,32 @@ +# syntax = docker/dockerfile:experimental +FROM golang:1.23 as builder + +ARG version=dev + +WORKDIR /workspace + +COPY go.mod go.sum ./ + +RUN --mount=type=cache,target=/go/pkg/mod \ + go mod download + +COPY api api +COPY controllers controllers +COPY kpack-image-builder kpack-image-builder +COPY model model +COPY tools tools +COPY version version + +RUN --mount=type=cache,target=/root/.cache/go-build \ + --mount=type=cache,target=/go/pkg/mod \ + CGO_ENABLED=0 GOOS=linux go build -ldflags "-X code.cloudfoundry.org/korifi/version.Version=${version}" -o manager kpack-image-builder/main.go + +# Use distroless as minimal base image to package the manager binary +# Refer to https://github.com/GoogleContainerTools/distroless for more details +FROM gcr.io/distroless/static:nonroot + +WORKDIR / +COPY --from=builder /workspace/manager . +USER 65532:65532 + +ENTRYPOINT ["/manager"] diff --git a/kpack-image-builder/Makefile b/kpack-image-builder/Makefile index 3cb601339..fc0043cc7 100644 --- a/kpack-image-builder/Makefile +++ b/kpack-image-builder/Makefile @@ -42,10 +42,10 @@ manifests: bin/controller-gen output:rbac:artifacts:config=../helm/korifi/kpack-image-builder \ output:webhook:artifacts:config=../helm/korifi/kpack-image-builder - yq -i 'with(.metadata; .annotations["cert-manager.io/inject-ca-from"]="{{ .Release.Namespace }}/korifi-controllers-serving-cert")' $(webhooks-file) + yq -i 'with(.metadata; .annotations["cert-manager.io/inject-ca-from"]="{{ .Release.Namespace }}/kpack-image-builder-serving-cert")' $(webhooks-file) yq -i 'with(.metadata; .name="korifi-kpack-image-builder-" + .name)' $(webhooks-file) yq -i 'with(.webhooks[]; .clientConfig.service.namespace="{{ .Release.Namespace }}")' $(webhooks-file) - yq -i 'with(.webhooks[]; .clientConfig.service.name="korifi-controllers-" + .clientConfig.service.name)' $(webhooks-file) + yq -i 'with(.webhooks[]; .clientConfig.service.name="kpack-image-builder-" + .clientConfig.service.name)' $(webhooks-file) diff --git a/kpack-image-builder/controllers/buildworkload_controller.go b/kpack-image-builder/controllers/buildworkload_controller.go index 475b23896..20f0d2d33 100644 --- a/kpack-image-builder/controllers/buildworkload_controller.go +++ b/kpack-image-builder/controllers/buildworkload_controller.go @@ -28,7 +28,7 @@ import ( "time" korifiv1alpha1 "code.cloudfoundry.org/korifi/controllers/api/v1alpha1" - "code.cloudfoundry.org/korifi/controllers/config" + "code.cloudfoundry.org/korifi/kpack-image-builder/controllers/config" "code.cloudfoundry.org/korifi/tools/image" "code.cloudfoundry.org/korifi/tools/k8s" @@ -78,35 +78,29 @@ func NewBuildWorkloadReconciler( c client.Client, scheme *runtime.Scheme, log logr.Logger, - config *config.ControllerConfig, + config *config.Config, imageConfigGetter ImageConfigGetter, - imageRepoPrefix string, imageRepoCreator RepositoryCreator, - builderReadinessTimeout time.Duration, ) *k8s.PatchingReconciler[korifiv1alpha1.BuildWorkload, *korifiv1alpha1.BuildWorkload] { buildWorkloadReconciler := BuildWorkloadReconciler{ - k8sClient: c, - scheme: scheme, - log: log, - controllerConfig: config, - imageConfigGetter: imageConfigGetter, - imageRepoPrefix: imageRepoPrefix, - imageRepoCreator: imageRepoCreator, - builderReadinessTimeout: builderReadinessTimeout, + k8sClient: c, + scheme: scheme, + log: log, + controllerConfig: config, + imageConfigGetter: imageConfigGetter, + imageRepoCreator: imageRepoCreator, } return k8s.NewPatchingReconciler[korifiv1alpha1.BuildWorkload, *korifiv1alpha1.BuildWorkload](log, c, &buildWorkloadReconciler) } // BuildWorkloadReconciler reconciles a BuildWorkload object type BuildWorkloadReconciler struct { - k8sClient client.Client - scheme *runtime.Scheme - log logr.Logger - controllerConfig *config.ControllerConfig - imageConfigGetter ImageConfigGetter - imageRepoPrefix string - imageRepoCreator RepositoryCreator - builderReadinessTimeout time.Duration + k8sClient client.Client + scheme *runtime.Scheme + log logr.Logger + controllerConfig *config.Config + imageConfigGetter ImageConfigGetter + imageRepoCreator RepositoryCreator } func (r *BuildWorkloadReconciler) SetupWithManager(mgr ctrl.Manager) *builder.Builder { @@ -210,7 +204,7 @@ func (r *BuildWorkloadReconciler) ReconcileResource(ctx context.Context, buildWo } if !builderReadyCondition.IsTrue() { - if time.Since(buildWorkload.CreationTimestamp.Time) < r.builderReadinessTimeout { + if time.Since(buildWorkload.CreationTimestamp.Time) < r.controllerConfig.BuilderReadinessTimeout { log.Info("waiting for builder to be ready") return ctrl.Result{RequeueAfter: time.Second}, nil } @@ -440,7 +434,7 @@ func (r *BuildWorkloadReconciler) ensureKpackBuilderForBuildpacks(ctx context.Co } builderName := ComputeBuilderName(buildWorkload.Spec.Buildpacks) - builderRepo := fmt.Sprintf("%sbuilders-%s", r.imageRepoPrefix, builderName) + builderRepo := fmt.Sprintf("%sbuilders-%s", r.controllerConfig.ContainerRepositoryPrefix, builderName) err = r.imageRepoCreator.CreateRepository(ctx, builderRepo) if err != nil { log.Info("failed creating builder repo", "reason", err) @@ -930,5 +924,5 @@ func (r *BuildWorkloadReconciler) hasRemainingBuilds(ctx context.Context, buildW } func (r *BuildWorkloadReconciler) repositoryRef(appGUID string) string { - return r.imageRepoPrefix + appGUID + "-droplets" + return r.controllerConfig.ContainerRepositoryPrefix + appGUID + "-droplets" } diff --git a/kpack-image-builder/controllers/config/config.go b/kpack-image-builder/controllers/config/config.go new file mode 100644 index 000000000..d220b822d --- /dev/null +++ b/kpack-image-builder/controllers/config/config.go @@ -0,0 +1,17 @@ +package config + +import ( + "time" + + controllersconfig "code.cloudfoundry.org/korifi/controllers/config" +) + +type Config struct { + CFRootNamespace string `yaml:"cfRootNamespace"` + CFStagingResources controllersconfig.CFStagingResources `yaml:"cfStagingResources"` + ClusterBuilderName string `yaml:"clusterBuilderName"` + BuilderServiceAccount string `yaml:"builderServiceAccount"` + BuilderReadinessTimeout time.Duration `yaml:"builderReadinessTimeout"` + ContainerRepositoryPrefix string `yaml:"containerRepositoryPrefix"` + ContainerRegistryType string `yaml:"containerRegistryType"` +} diff --git a/kpack-image-builder/controllers/suite_test.go b/kpack-image-builder/controllers/suite_test.go index e42c7ddc5..7c04cb8ce 100644 --- a/kpack-image-builder/controllers/suite_test.go +++ b/kpack-image-builder/controllers/suite_test.go @@ -23,14 +23,15 @@ import ( "time" korifiv1alpha1 "code.cloudfoundry.org/korifi/controllers/api/v1alpha1" - "code.cloudfoundry.org/korifi/controllers/config" "code.cloudfoundry.org/korifi/kpack-image-builder/controllers" + "code.cloudfoundry.org/korifi/kpack-image-builder/controllers/config" "code.cloudfoundry.org/korifi/kpack-image-builder/controllers/fake" "code.cloudfoundry.org/korifi/kpack-image-builder/controllers/webhooks/finalizer" "code.cloudfoundry.org/korifi/tests/helpers" "code.cloudfoundry.org/korifi/tools" "code.cloudfoundry.org/korifi/tools/k8s" + controllersconfig "code.cloudfoundry.org/korifi/controllers/config" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" buildv1alpha2 "github.com/pivotal/kpack/pkg/apis/build/v1alpha2" @@ -105,12 +106,13 @@ var _ = BeforeSuite(func() { finalizer.NewKpackImageBuilderFinalizerWebhook().SetupWebhookWithManager(k8sManager) - controllerConfig := &config.ControllerConfig{ + controllerConfig := &config.Config{ CFRootNamespace: PrefixedGUID("cf"), ClusterBuilderName: "cf-kpack-builder", - ContainerRepositoryPrefix: "image/registry/tag", BuilderServiceAccount: "builder-service-account", - CFStagingResources: config.CFStagingResources{ + BuilderReadinessTimeout: 4 * time.Second, + ContainerRepositoryPrefix: "my.repository/my-prefix/", + CFStagingResources: controllersconfig.CFStagingResources{ BuildCacheMB: 1024, DiskMB: 2048, MemoryMB: 1234, @@ -125,9 +127,7 @@ var _ = BeforeSuite(func() { ctrl.Log.WithName("kpack-image-builder").WithName("BuildWorkload"), controllerConfig, fakeImageConfigGetter, - "my.repository/my-prefix/", imageRepoCreator, - 4*time.Second, ) err = buildWorkloadReconciler.SetupWithManager(k8sManager) Expect(err).NotTo(HaveOccurred()) diff --git a/kpack-image-builder/main.go b/kpack-image-builder/main.go new file mode 100644 index 000000000..605e373dd --- /dev/null +++ b/kpack-image-builder/main.go @@ -0,0 +1,161 @@ +package main + +import ( + "flag" + "fmt" + "os" + + korifiv1alpha1 "code.cloudfoundry.org/korifi/controllers/api/v1alpha1" + "code.cloudfoundry.org/korifi/kpack-image-builder/controllers" + "code.cloudfoundry.org/korifi/kpack-image-builder/controllers/config" + kpackimagebuilderfinalizer "code.cloudfoundry.org/korifi/kpack-image-builder/controllers/webhooks/finalizer" + "code.cloudfoundry.org/korifi/tools" + "code.cloudfoundry.org/korifi/tools/image" + "code.cloudfoundry.org/korifi/tools/registry" + buildv1alpha2 "github.com/pivotal/kpack/pkg/apis/build/v1alpha2" + "go.uber.org/zap/zapcore" + utilruntime "k8s.io/apimachinery/pkg/util/runtime" + k8sclient "k8s.io/client-go/kubernetes" + clientgoscheme "k8s.io/client-go/kubernetes/scheme" + "k8s.io/client-go/rest" + "k8s.io/klog/v2" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/healthz" + "sigs.k8s.io/controller-runtime/pkg/manager" + metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server" + "sigs.k8s.io/controller-runtime/pkg/webhook" + + "k8s.io/apimachinery/pkg/runtime" +) + +var ( + scheme = runtime.NewScheme() + setupLog = ctrl.Log.WithName("setup") +) + +func init() { + utilruntime.Must(clientgoscheme.AddToScheme(scheme)) + utilruntime.Must(korifiv1alpha1.AddToScheme(scheme)) + utilruntime.Must(buildv1alpha2.AddToScheme(scheme)) +} + +func main() { + var ( + metricsAddr string + enableLeaderElection bool + probeAddr string + configPath string + ) + + flag.StringVar(&metricsAddr, "metrics-bind-address", ":8080", "The address the metric endpoint binds to.") + flag.StringVar(&probeAddr, "health-probe-bind-address", ":8081", "The address the probe endpoint binds to.") + flag.StringVar(&configPath, "config", "", "") + flag.BoolVar(&enableLeaderElection, "leader-elect", false, + "Enable leader election for controller manager. "+ + "Enabling this will ensure there is only one active controller manager.") + flag.Parse() + + logger, _, err := tools.NewZapLogger(zapcore.InfoLevel) + if err != nil { + setupLog.Error(err, "unable to set up zap logger") + os.Exit(1) + } + + ctrl.SetLogger(logger) + klog.SetLogger(ctrl.Log) + + conf := ctrl.GetConfigOrDie() + mgr, err := ctrl.NewManager(conf, ctrl.Options{ + Scheme: scheme, + WebhookServer: webhook.NewServer(webhook.Options{ + Port: 9443, + }), + Metrics: metricsserver.Options{ + BindAddress: metricsAddr, + }, + HealthProbeBindAddress: probeAddr, + LeaderElection: enableLeaderElection, + LeaderElectionID: "13w500bs.cloudfoundry.org", + }) + if err != nil { + setupLog.Error(err, "unable to initialize manager") + os.Exit(1) + } + + if err = setupControllers(mgr, conf, configPath); err != nil { + setupLog.Error(err, "unable to set up controllers") + os.Exit(1) + } + + if err = setupWebhooks(mgr); err != nil { + setupLog.Error(err, "unable to set up webhooks") + os.Exit(1) + } + + setupLog.Info("starting manager") + if err := mgr.Start(ctrl.SetupSignalHandler()); err != nil { + setupLog.Error(err, "problem running manager") + os.Exit(1) + } +} + +func setupControllers(mgr manager.Manager, restConf *rest.Config, configPath string) error { + controllersLog := ctrl.Log.WithName("controllers") + imageClientSet, err := k8sclient.NewForConfig(restConf) + if err != nil { + return fmt.Errorf("could not create k8s client: %v", err) + } + + controllerConfig := &config.Config{} + err = tools.LoadConfigInto(controllerConfig, configPath) + if err != nil { + return fmt.Errorf("config could not be read: %v", err) + } + + imageClient := image.NewClient(imageClientSet) + if err = controllers.NewBuildWorkloadReconciler( + mgr.GetClient(), + mgr.GetScheme(), + controllersLog, + controllerConfig, + imageClient, + registry.NewRepositoryCreator(controllerConfig.ContainerRegistryType), + ).SetupWithManager(mgr); err != nil { + return fmt.Errorf("unable to create BuildWorkload controller: %v", err) + } + + if err = controllers.NewBuilderInfoReconciler( + mgr.GetClient(), + mgr.GetScheme(), + controllersLog, + controllerConfig.ClusterBuilderName, + controllerConfig.CFRootNamespace, + ).SetupWithManager(mgr); err != nil { + return fmt.Errorf("unable to create BuilderInfo controller: %v", err) + } + + if err = controllers.NewKpackBuildController( + mgr.GetClient(), + controllersLog, + imageClient, + controllerConfig.BuilderServiceAccount, + ).SetupWithManager(mgr); err != nil { + return fmt.Errorf("unable to create KpackBuild controller: %v", err) + } + + return nil +} + +func setupWebhooks(mgr manager.Manager) error { + kpackimagebuilderfinalizer.NewKpackImageBuilderFinalizerWebhook().SetupWebhookWithManager(mgr) + + if err := mgr.AddHealthzCheck("healthz", healthz.Ping); err != nil { + return fmt.Errorf("unable to set up health check: %v", err) + } + + if err := mgr.AddReadyzCheck("readyz", mgr.GetWebhookServer().StartedChecker()); err != nil { + return fmt.Errorf("unable to set up ready check: %v", err) + } + + return nil +} diff --git a/kpack-image-builder/remote-debug/Dockerfile b/kpack-image-builder/remote-debug/Dockerfile new file mode 100644 index 000000000..2e2e153a8 --- /dev/null +++ b/kpack-image-builder/remote-debug/Dockerfile @@ -0,0 +1,35 @@ +# syntax = docker/dockerfile:experimental +FROM golang:1.23 as builder + +ARG version=dev + +WORKDIR /workspace + +COPY go.mod go.sum ./ + +RUN --mount=type=cache,target=/go/pkg/mod \ + go mod download + +COPY api api +COPY controllers controllers +COPY kpack-image-builder kpack-image-builder +COPY model model +COPY tools tools +COPY version version + +RUN --mount=type=cache,target=/root/.cache/go-build \ + --mount=type=cache,target=/go/pkg/mod \ + CGO_ENABLED=0 GOOS=linux go build -ldflags "-X code.cloudfoundry.org/korifi/version.Version=${version}" -gcflags=all="-N -l" -o manager kpack-image-builder/main.go + +# Get Delve from a GOPATH not from a Go Modules project +WORKDIR /go/src/ +RUN go install github.com/go-delve/delve/cmd/dlv@latest + +FROM ubuntu + +WORKDIR / +COPY --from=builder /workspace/manager . +COPY --from=builder /go/bin/dlv . +EXPOSE 8080 8081 9443 40000 + +CMD ["/dlv", "--listen=:40000", "--headless=true", "--api-version=2", "exec", "/manager", "--continue", "--accept-multiclient"] diff --git a/scripts/assets/kind-config.yaml b/scripts/assets/kind-config.yaml index 74915116e..d495c045d 100644 --- a/scripts/assets/kind-config.yaml +++ b/scripts/assets/kind-config.yaml @@ -27,3 +27,12 @@ nodes: - containerPort: 30052 hostPort: 30052 protocol: TCP + - containerPort: 30053 + hostPort: 30053 + protocol: TCP + - containerPort: 30054 + hostPort: 30054 + protocol: TCP + - containerPort: 30055 + hostPort: 30055 + protocol: TCP diff --git a/scripts/assets/korifi-debug-kbld.yml b/scripts/assets/korifi-debug-kbld.yml index 588202340..04e3cedcd 100644 --- a/scripts/assets/korifi-debug-kbld.yml +++ b/scripts/assets/korifi-debug-kbld.yml @@ -13,3 +13,21 @@ sources: docker: buildx: file: controllers/remote-debug/Dockerfile + +- image: cloudfoundry/korifi-kpack-image-builder:latest + path: . + docker: + buildx: + file: kpack-image-builder/remote-debug/Dockerfile + +- image: cloudfoundry/korifi-statefulset-runner:latest + path: . + docker: + buildx: + file: statefulset-runner/remote-debug/Dockerfile + +- image: cloudfoundry/korifi-job-task-runner:latest + path: . + docker: + buildx: + file: job-task-runner/remote-debug/Dockerfile diff --git a/scripts/assets/korifi-kbld.yml b/scripts/assets/korifi-kbld.yml index 132aa5f73..dc5e6130a 100644 --- a/scripts/assets/korifi-kbld.yml +++ b/scripts/assets/korifi-kbld.yml @@ -13,3 +13,21 @@ sources: docker: buildx: file: controllers/Dockerfile + +- image: cloudfoundry/korifi-kpack-image-builder:latest + path: . + docker: + buildx: + file: kpack-image-builder/Dockerfile + +- image: cloudfoundry/korifi-statefulset-runner:latest + path: . + docker: + buildx: + file: statefulset-runner/Dockerfile + +- image: cloudfoundry/korifi-job-task-runner:latest + path: . + docker: + buildx: + file: job-task-runner/Dockerfile diff --git a/scripts/deploy-on-kind.sh b/scripts/deploy-on-kind.sh index 2ee3be97f..c76669ce4 100755 --- a/scripts/deploy-on-kind.sh +++ b/scripts/deploy-on-kind.sh @@ -216,6 +216,11 @@ function create_namespaces() { local security_policy security_policy="restricted" + + if [[ "$DEBUG" == "true" ]]; then + security_policy="privileged" + fi + for ns in cf korifi; do cat <