statefulset: Use stateful sets for postgres deployments (PROJQUAY-6672)

.github/workflows/ci.yaml

@@ -43,12 +43,5 @@ jobs:
         uses: actions/checkout@v3
       - name: OS Dependencies
         run: apt-get update && apt-get install -y tar make gcc
-      - name: Install Kubebuilder
-        run: |
-          os=$(go env GOOS)
-          arch=$(go env GOARCH)
-          curl -L https://github.com/kubernetes-sigs/kubebuilder/releases/download/v2.3.1/kubebuilder_2.3.1_${os}_${arch}.tar.gz | tar -xz -C /tmp/
-          mv /tmp/kubebuilder_2.3.1_${os}_${arch} /usr/local/kubebuilder
-          export PATH=$PATH:/usr/local/kubebuilder/bin
       - name: Tests
-        run: go test -v ./...
+        run: make test

Makefile

@@ -17,7 +17,7 @@ all: manager
 
 # Run tests
 test: manifests generate fmt vet envtest ## Run tests.
-	KUBEBUILDER_ASSETS="$(shell $(ENVTEST) use $(ENVTEST_K8S_VERSION) --bin-dir $(LOCALBIN) -p path)" go test ./... -coverprofile cover.out
+	KUBEBUILDER_ASSETS="$(shell $(ENVTEST) use $(ENVTEST_K8S_VERSION) --bin-dir $(LOCALBIN) -p path)" go test -v ./... -coverprofile cover.out
 
 test-e2e:
 	mkdir -p ./bin
@@ -107,7 +107,7 @@ $(CONTROLLER_GEN): $(LOCALBIN)
 .PHONY: envtest
 envtest: $(ENVTEST) ## Download envtest-setup locally if necessary.
 $(ENVTEST): $(LOCALBIN)
-	test -s $(LOCALBIN)/setup-envtest || GOBIN=$(LOCALBIN) go install sigs.k8s.io/controller-runtime/tools/setup-envtest@latest
+	test -s $(LOCALBIN)/setup-envtest || GOBIN=$(LOCALBIN) go install sigs.k8s.io/controller-runtime/tools/setup-envtest@v0.0.0-20230216140739-c98506dc3b8e
 
 # This target called from the prepare-release github action.
 # CHANNEL         - operator channel (eg. stable-3.6, candidate-3.9)

controllers/quay/features.go

@@ -410,8 +410,8 @@ func (r *QuayRegistryReconciler) checkNeedsPostgresUpgradeForComponent(
 	ctx context.Context, qctx *quaycontext.QuayRegistryContext, quay *v1.QuayRegistry, component v1.ComponentKind,
 ) error {
 	componentInfo := map[v1.ComponentKind]struct {
-		deploymentSuffix string
-		upgradeField     *bool
+		resourceSuffix string
+		upgradeField   *bool
 	}{
 		v1.ComponentClairPostgres: {"clair-postgres", &qctx.NeedsClairPgUpgrade},
 		v1.ComponentPostgres:      {"quay-database", &qctx.NeedsPgUpgrade},
@@ -422,23 +422,63 @@ func (r *QuayRegistryReconciler) checkNeedsPostgresUpgradeForComponent(
 		return fmt.Errorf("invalid component kind: %s", component)
 	}
 
-	deploymentName := fmt.Sprintf("%s-%s", quay.GetName(), info.deploymentSuffix)
+	resourceName := fmt.Sprintf("%s-%s", quay.GetName(), info.resourceSuffix)
 	r.Log.Info(fmt.Sprintf("getting %s version", component))
 
-	postgresDeployment := &appsv1.Deployment{}
-	if err := r.Client.Get(
-		ctx,
-		types.NamespacedName{
-			Name:      deploymentName,
-			Namespace: quay.GetNamespace(),
-		},
-		postgresDeployment,
-	); err != nil {
-		r.Log.Info(fmt.Sprintf("%s deployment not found, skipping", component))
-		return nil
+	var deployedImageName string
+
+	if component == v1.ComponentClairPostgres {
+		statefulSet := &appsv1.StatefulSet{}
+		err := r.Client.Get(
+			ctx,
+			types.NamespacedName{
+				Name:      resourceName,
+				Namespace: quay.GetNamespace(),
+			},
+			statefulSet,
+		)
+		if err != nil {
+			if !errors.IsNotFound(err) {
+				return err
+			}
+			// NOTE: Check for Deployment to support migration from Deployment to StatefulSet.
+			// This ensures compatibility with both old and new setups during the upgrade process.
+			deployment := &appsv1.Deployment{}
+			err = r.Client.Get(
+				ctx,
+				types.NamespacedName{
+					Name:      resourceName,
+					Namespace: quay.GetNamespace(),
+				},
+				deployment,
+			)
+			if err != nil {
+				if errors.IsNotFound(err) {
+					r.Log.Info(fmt.Sprintf("%s statefulset and deployment not found, skipping", component))
+					return nil
+				}
+				return err
+			}
+			deployedImageName = deployment.Spec.Template.Spec.Containers[0].Image
+		} else {
+			deployedImageName = statefulSet.Spec.Template.Spec.Containers[0].Image
+		}
+	} else {
+		deployment := &appsv1.Deployment{}
+		if err := r.Client.Get(
+			ctx,
+			types.NamespacedName{
+				Name:      resourceName,
+				Namespace: quay.GetNamespace(),
+			},
+			deployment,
+		); err != nil {
+			r.Log.Info(fmt.Sprintf("%s deployment not found, skipping", component))
+			return nil
+		}
+		deployedImageName = deployment.Spec.Template.Spec.Containers[0].Image
 	}
 
-	deployedImageName := postgresDeployment.Spec.Template.Spec.Containers[0].Image
 	r.Log.Info(fmt.Sprintf("%s deployment found", component), "image", deployedImageName)
 
 	expectedImage, err := kustomize.ComponentImageFor(component)

controllers/quay/quayregistry_controller.go

@@ -246,12 +246,16 @@ func (r *QuayRegistryReconciler) checkPostgresUpgradeStatus(
 		if job.Status.Succeeded == 1 {
 			log.Info(fmt.Sprintf("%s upgrade complete", jobName))
 			var oldPostgresDeploymentName string
+			var postgresDeploymentName string
 			if jobName == clairPostgresUpgradeJobName {
 				oldPostgresDeploymentName = fmt.Sprintf("%s-%s", quay.GetName(), "clair-postgres-old")
+				postgresDeploymentName = fmt.Sprintf("%s-%s", quay.GetName(), "clair-postgres")
 			} else {
 				oldPostgresDeploymentName = fmt.Sprintf("%s-%s", quay.GetName(), "quay-database-old")
+				postgresDeploymentName = fmt.Sprintf("%s-%s", quay.GetName(), "quay-database")
 			}
 			oldPostgresDeployment := &appsv1.Deployment{}
+			postgresDeployment := &appsv1.Deployment{}
 			if err := r.Client.Get(
 				ctx,
 				types.NamespacedName{
@@ -263,7 +267,20 @@ func (r *QuayRegistryReconciler) checkPostgresUpgradeStatus(
 				r.Log.Info(fmt.Sprintf("%s deployment not found, skipping", oldPostgresDeploymentName))
 				continue
 			}
-
+			if err := r.Client.Get(
+				ctx,
+				types.NamespacedName{
+					Name:      postgresDeploymentName,
+					Namespace: quay.GetNamespace(),
+				},
+				postgresDeployment,
+			); err != nil {
+				r.Log.Info(fmt.Sprintf("%s deployment not found, skipping", postgresDeploymentName))
+				continue
+			}
+			if err := r.Client.Delete(ctx, postgresDeployment); err != nil {
+				r.Log.Error(err, fmt.Sprintf("%s deployment could not be deleted", postgresDeploymentName))
+			}
 			// Remove owner reference
 			obj, err := v1.RemoveOwnerReference(quay, oldPostgresDeployment)
 			if err != nil {
@@ -548,7 +565,7 @@ func (r *QuayRegistryReconciler) Reconcile(ctx context.Context, req ctrl.Request
 				v1.ConditionTypeRolloutBlocked,
 				metav1.ConditionTrue,
 				v1.ConditionReasonPostgresUpgradeFailed,
-				fmt.Sprintf("error checking for pg upgrade: %s", err),
+				fmt.Sprintf("error checking for clair pg upgrade: %s", err),
 			)
 		}
 	}

kustomize/components/clairpgupgrade/base/clair-pg.deployment.patch.yaml → kustomize/components/clairpgupgrade/base/clair-pg.statefulset.patch.yaml

@@ -1,5 +1,5 @@
 apiVersion: apps/v1
-kind: Deployment
+kind: StatefulSet
 metadata:
   name: clair-postgres
 spec:

kustomize/components/clairpgupgrade/base/kustomization.yaml

@@ -5,4 +5,4 @@ resources:
   - ./clair-pg-old.persistentvolumeclaim.yaml
   - ./clair-pg-old.deployment.yaml
 patchesStrategicMerge:
-  - ./clair-pg.deployment.patch.yaml
+  - ./clair-pg.statefulset.patch.yaml

kustomize/components/clairpostgres/kustomization.yaml

@@ -1,9 +1,9 @@
 # Clair component adds Clair v4 security scanner and its database.
 apiVersion: kustomize.config.k8s.io/v1alpha1
 kind: Component
-resources: 
+resources:
   - ./postgres.serviceaccount.yaml
   - ./postgres.persistentvolumeclaim.yaml
-  - ./postgres.deployment.yaml
+  - ./postgres.statefulset.yaml
   - ./postgres.service.yaml
   - ./clair-postgres-conf-sample.configmap.yaml

kustomize/components/clairpostgres/postgres.deployment.yaml → kustomize/components/clairpostgres/postgres.statefulset.yaml

@@ -1,15 +1,14 @@
 apiVersion: apps/v1
-kind: Deployment
+kind: StatefulSet
 metadata:
   name: clair-postgres
   labels:
     quay-component: clair-postgres
   annotations:
     quay-component: clair-postgres
 spec:
+  serviceName: clair-postgres
   replicas: 1
-  strategy:
-    type: Recreate
   selector:
     matchLabels:
       quay-component: clair-postgres
@@ -57,3 +56,4 @@ spec:
             requests:
               cpu: 500m
               memory: 2Gi
+  volumeClaimTemplates: []

pkg/kustomize/kustomize.go

@@ -221,6 +221,8 @@ func ModelFor(gvk schema.GroupVersionKind) client.Object {
 		return &prometheusv1.ServiceMonitor{}
 	case schema.GroupVersionKind{Group: "monitoring.coreos.com", Version: "v1", Kind: "PrometheusRule"}.String():
 		return &prometheusv1.PrometheusRule{}
+	case schema.GroupVersionKind{Group: "apps", Version: "v1", Kind: "StatefulSet"}.String():
+		return &apps.StatefulSet{}
 	default:
 		panic(fmt.Sprintf("Missing model for GVK %s", gvk.String()))
 	}