Add kuttle test for cell deletion

controllers/nova_controller.go

@@ -601,14 +601,31 @@ func (r *NovaReconciler) Reconcile(ctx context.Context, req ctrl.Request) (resul
 	}
 
 	for _, cr := range novaCellList.Items {
+		cellDbs := [][]string{}
 		_, ok := instance.Spec.CellTemplates[cr.Spec.CellName]
 		if !ok {
-			err := r.ensureCellDeleted(ctx, h, instance, cr.Spec.CellName, apiTransportURL, secret, apiDB)
+			err := r.ensureCellDeleted(ctx, h, instance,
+				cr.Spec.CellName, apiTransportURL,
+				secret, apiDB, cellDBs[novav1.Cell0Name].Database.GetDatabaseHostname(), cells[novav1.Cell0Name])
 			if err != nil {
 				return ctrl.Result{}, err
 			}
+			cellDbs = append(cellDbs, []string{novaapi.ServiceName + "-" + cr.Spec.CellName, instance.Spec.CellTemplates[cr.Spec.CellName].CellDatabaseAccount})
 			delete(instance.Status.RegisteredCells, cr.Name)
 		}
+		// iterate over novaDbs and remove finalizers
+		for _, novaDb := range cellDbs {
+			dbName, accountName := novaDb[0], novaDb[1]
+			db, err := mariadbv1.GetDatabaseByNameAndAccount(ctx, h, dbName, accountName, instance.ObjectMeta.Namespace)
+			if err != nil && !k8s_errors.IsNotFound(err) {
+				return ctrl.Result{}, err
+			}
+			if !k8s_errors.IsNotFound(err) {
+				if err := db.DeleteFinalizer(ctx, h); err != nil {
+					return ctrl.Result{}, err
+				}
+			}
+		}
 	}
 
 	Log.Info("Successfully reconciled")
@@ -621,8 +638,10 @@ func (r *NovaReconciler) ensureCellDeleted(
 	instance *novav1.Nova,
 	cellName string,
 	apiTransportURL string,
-	secret corev1.Secret,
+	topLevelSecret corev1.Secret,
 	apiDB *mariadbv1.Database,
+	APIDatabaseHostname string,
+	cell0 *novav1.NovaCell,
 ) error {
 	Log := r.GetLogger(ctx)
 	cell := &novav1.NovaCell{}
@@ -652,7 +671,7 @@ func (r *NovaReconciler) ensureCellDeleted(
 	}
 
 	configHash, scriptName, configName, err := r.ensureNovaManageJobSecret(ctx, h, instance,
-		cell, secret, cell.Spec.APIDatabaseHostname, apiTransportURL, apiDB)
+		cell0, topLevelSecret, APIDatabaseHostname, apiTransportURL, apiDB)
 	if err != nil {
 		return err
 	}
@@ -675,6 +694,29 @@ func (r *NovaReconciler) ensureCellDeleted(
 		return err
 	}
 
+	// Delete secrets
+	dbSecret := fmt.Sprintf("%s-db-secret", cell.Name)
+	err = secret.DeleteSecretsWithName(ctx, h, dbSecret, instance.Namespace)
+	if err != nil {
+		return err
+	}
+	secretName := getNovaCellCRName(instance.Name, cellName)
+	err = secret.DeleteSecretsWithName(ctx, h, secretName, instance.Namespace)
+	if err != nil {
+		return err
+	}
+
+	// Delete transportURL cr
+	transportURL := &rabbitmqv1.TransportURL{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      instance.Name + "-" + cellName + "-transport",
+			Namespace: instance.Namespace,
+		},
+	}
+	err = r.Client.Delete(ctx, transportURL)
+	if err != nil {
+		return err
+	}
 	Log.Info("Cell isn't defined in the nova, so deleted cell", "cell", cell)
 	return nil
 }

controllers/novaconductor_controller.go

@@ -724,6 +724,23 @@ func (r *NovaConductorReconciler) reconcileDelete(
 		Log.Info("Removed finalizer from ourselves")
 	}
 
+	_, result, secret, err := ensureSecret(
+		ctx,
+		types.NamespacedName{Namespace: instance.Namespace, Name: instance.Spec.Secret},
+		nil,
+		h.GetClient(),
+		&instance.Status.Conditions,
+		r.RequeueTimeout,
+	)
+	if (err != nil || result != ctrl.Result{}) {
+		return err
+	}
+
+	err = r.cleanServiceFromNovaDb(ctx, h, instance, secret, Log)
+	if err != nil {
+		return err
+	}
+
 	Log.Info("Reconciled delete successfully")
 	return nil
 }

templates/nova-manage/bin/delete_cell.sh

@@ -24,6 +24,4 @@ export CELL_NAME=${CELL_NAME:?"Please specify a CELL_NAME variable."}
 # is part of the line, e.g. as the user name of the DB URL
 cell_uuid=$(nova-manage cell_v2 list_cells | tr ' ' '|' | tr --squeeze-repeats '|' | grep -e "^|$CELL_NAME|" | cut -d '|' -f 3)
 
-if [ -z "${cell_uuid}" ]; then
-    nova-manage cell_v2 delete_cell --cell_uuid "${cell_uuid}"
-fi
+nova-manage cell_v2 delete_cell --cell_uuid "${cell_uuid}"

templates/nova-manage/config/cell-delete-config.json

@@ -1,5 +1,5 @@
 {
-    "command": "/bin/ensure_cell_mapping.sh",
+    "command": "/bin/delete_cell.sh",
     "config_files": [
         {
             "source": "/var/lib/openstack/config/nova-blank.conf",

test/functional/nova_reconfiguration_test.go

@@ -22,11 +22,13 @@ import (
 	"github.com/google/go-cmp/cmp"
 	. "github.com/onsi/ginkgo/v2" //revive:disable:dot-imports
 	. "github.com/onsi/gomega"    //revive:disable:dot-imports
+	k8s_errors "k8s.io/apimachinery/pkg/api/errors"
 
 	//revive:disable-next-line:dot-imports
 	. "github.com/openstack-k8s-operators/lib-common/modules/common/test/helpers"
 
 	memcachedv1 "github.com/openstack-k8s-operators/infra-operator/apis/memcached/v1beta1"
+	rabbitmqv1 "github.com/openstack-k8s-operators/infra-operator/apis/rabbitmq/v1beta1"
 	condition "github.com/openstack-k8s-operators/lib-common/modules/common/condition"
 	mariadbv1 "github.com/openstack-k8s-operators/mariadb-operator/api/v1beta1"
 
@@ -190,7 +192,13 @@ var _ = Describe("Nova reconfiguration", func() {
 					mappingJob.Spec.Template.Spec.Containers[0].Env, "INPUT_HASH", "")
 				g.Expect(newJobInputHash).NotTo(BeNil())
 			}, timeout, interval).Should(Succeed())
+			th.AssertSecretDoesNotExist(cell1.InternalCellSecretName)
 
+			Eventually(func(g Gomega) {
+				instance := &rabbitmqv1.TransportURL{}
+				err := k8sClient.Get(ctx, cell1.TransportURLName, instance)
+				g.Expect(k8s_errors.IsNotFound(err)).To(BeTrue())
+			}, timeout, interval).Should(Succeed())
 		})
 	})
 	When("cell0 conductor replicas is set to 0", func() {

test/kuttl/test-suites/default/cell-delete-tests/00-cleanup-nova.yaml

@@ -0,0 +1 @@
+../common/cleanup-nova.yaml

test/kuttl/test-suites/default/cell-delete-tests/01-assert.yaml

@@ -0,0 +1,140 @@
+apiVersion: nova.openstack.org/v1beta1
+kind: Nova
+metadata:
+  finalizers:
+  - openstack.org/nova
+  name: nova-kuttl
+  namespace: nova-kuttl-default
+status:
+  apiServiceReadyCount: 1
+  metadataServiceReadyCount: 1
+  schedulerServiceReadyCount: 1
+  conditions:
+  - message: Setup complete
+    reason: Ready
+    status: "True"
+    type: Ready
+  - message: Input data complete
+    reason: Ready
+    status: "True"
+    type: InputReady
+  - message: Setup complete
+    reason: Ready
+    status: "True"
+    type: KeystoneServiceReady
+  - message: MariaDBAccount creation complete
+    reason: Ready
+    status: "True"
+    type: MariaDBAccountReady
+  - message: ' Memcached instance has been provisioned'
+    reason: Ready
+    status: "True"
+    type: MemcachedReady
+  - message: DB create completed
+    reason: Ready
+    status: "True"
+    type: NovaAPIDBReady
+  - message: API message bus creation successfully
+    reason: Ready
+    status: "True"
+    type: NovaAPIMQReady
+  - message: Setup complete
+    reason: Ready
+    status: "True"
+    type: NovaAPIReady
+  - message: All DBs created successfully
+    reason: Ready
+    status: "True"
+    type: NovaAllCellDBReady
+  - message: All NovaCells are ready
+    reason: Ready
+    status: "True"
+    type: NovaAllCellReady
+  - message: All message busses created successfully
+    reason: Ready
+    status: "True"
+    type: NovaAllCellsMQReady
+  - message: Setup complete
+    reason: Ready
+    status: "True"
+    type: NovaMetadataReady
+  - message: Setup complete
+    reason: Ready
+    status: "True"
+    type: NovaSchedulerReady
+  - message: RoleBinding created
+    reason: Ready
+    status: "True"
+    type: RoleBindingReady
+  - message: Role created
+    reason: Ready
+    status: "True"
+    type: RoleReady
+  - message: ServiceAccount created
+    reason: Ready
+    status: "True"
+    type: ServiceAccountReady
+---
+apiVersion: v1
+kind: Pod
+metadata:
+  annotations:
+    openshift.io/scc: anyuid
+  labels:
+    service: nova-api
+  name: nova-kuttl-api-0
+status:
+  containerStatuses:
+  - name: nova-kuttl-api-api
+    ready: true
+    started: true
+  - name: nova-kuttl-api-log
+    ready: true
+    started: true
+---
+apiVersion: v1
+kind: Pod
+metadata:
+  annotations:
+    openshift.io/scc: anyuid
+  labels:
+    service: nova-metadata
+    statefulset.kubernetes.io/pod-name: nova-kuttl-metadata-0
+  name: nova-kuttl-metadata-0
+  ownerReferences:
+  - apiVersion: apps/v1
+    blockOwnerDeletion: true
+    controller: true
+    kind: StatefulSet
+    name: nova-kuttl-metadata
+status:
+  containerStatuses:
+  - name: nova-kuttl-metadata-log
+    ready: true
+    started: true
+  - name: nova-kuttl-metadata-metadata
+    ready: true
+    started: true
+---
+apiVersion: v1
+kind: Pod
+metadata:
+  name: nova-kuttl-cell1-compute-fake1-compute-0
+status:
+  containerStatuses:
+  - name: nova-kuttl-cell1-compute-fake1-compute-compute
+    ready: true
+    started: true
+---
+apiVersion: kuttl.dev/v1beta1
+kind: TestAssert
+namespaced: true
+commands:
+  - script: |
+      set -euxo pipefail
+      MESSAGE=$(oc exec -n $NAMESPACE openstackclient -- openstack flavor create my-flavor 2>&1) || true
+      echo $MESSAGE | grep "Policy doesn't allow os_compute_api:os-flavor-manage:create to be performed"
+  - script: |
+      set -euxo pipefail
+      RP_UUID=$(oc exec -n $NAMESPACE openstackclient -- openstack resource provider list --name nova-kuttl-cell1-compute-fake1-compute-0 -f value -c uuid)
+      oc exec -n $NAMESPACE openstackclient -- openstack resource provider trait list $RP_UUID | grep CUSTOM_FOO

test/kuttl/test-suites/default/cell-delete-tests/01-deploy-with-default-config-overwrite.yaml

@@ -0,0 +1,31 @@
+apiVersion: nova.openstack.org/v1beta1
+kind: Nova
+metadata:
+  name: nova-kuttl
+spec:
+  secret: osp-secret
+  cellTemplates:
+    cell0:
+      cellDatabaseInstance: openstack
+      cellDatabaseAccount: nova-cell0
+      cellMessageBusInstance: rabbitmq
+      hasAPIAccess: true
+      memcachedInstance: memcached
+    cell1:
+      cellDatabaseInstance: openstack-cell1
+      cellDatabaseAccount: nova-cell1
+      cellMessageBusInstance: rabbitmq-cell1
+      memcachedInstance: memcached
+      novaComputeTemplates:
+        compute-fake1:
+          computeDriver: fake.FakeDriver
+          defaultConfigOverwrite:
+            provider.yaml: |
+              meta:
+                schema_version: '1.0'
+              providers:
+                - identification:
+                    uuid: '$COMPUTE_NODE'
+                  traits:
+                    additional:
+                      - 'CUSTOM_FOO'