From b316362c6dd252a36eb6bf5a012997e31eeb808a Mon Sep 17 00:00:00 2001 From: Kamil Sambor Date: Mon, 2 Sep 2024 14:38:28 +0200 Subject: [PATCH] Add kuttle test for cell deletion --- controllers/nova_controller.go | 48 +++++- controllers/novaconductor_controller.go | 17 +++ templates/nova-manage/bin/delete_cell.sh | 4 +- .../config/cell-delete-config.json | 2 +- test/functional/nova_reconfiguration_test.go | 8 + .../cell-delete-tests/00-cleanup-nova.yaml | 1 + .../default/cell-delete-tests/01-assert.yaml | 140 ++++++++++++++++++ ...-deploy-with-default-config-overwrite.yaml | 31 ++++ .../default/cell-delete-tests/02-assert.yaml | 130 ++++++++++++++++ .../02-delete-cell-nova.yaml | 5 + .../cell-delete-tests/03-cleanup-nova.yaml | 1 + 11 files changed, 380 insertions(+), 7 deletions(-) create mode 120000 test/kuttl/test-suites/default/cell-delete-tests/00-cleanup-nova.yaml create mode 100644 test/kuttl/test-suites/default/cell-delete-tests/01-assert.yaml create mode 100644 test/kuttl/test-suites/default/cell-delete-tests/01-deploy-with-default-config-overwrite.yaml create mode 100644 test/kuttl/test-suites/default/cell-delete-tests/02-assert.yaml create mode 100644 test/kuttl/test-suites/default/cell-delete-tests/02-delete-cell-nova.yaml create mode 120000 test/kuttl/test-suites/default/cell-delete-tests/03-cleanup-nova.yaml diff --git a/controllers/nova_controller.go b/controllers/nova_controller.go index 687bc50af..f58fab6b1 100644 --- a/controllers/nova_controller.go +++ b/controllers/nova_controller.go @@ -601,14 +601,31 @@ func (r *NovaReconciler) Reconcile(ctx context.Context, req ctrl.Request) (resul } for _, cr := range novaCellList.Items { + cellDbs := [][]string{} _, ok := instance.Spec.CellTemplates[cr.Spec.CellName] if !ok { - err := r.ensureCellDeleted(ctx, h, instance, cr.Spec.CellName, apiTransportURL, secret, apiDB) + err := r.ensureCellDeleted(ctx, h, instance, + cr.Spec.CellName, apiTransportURL, + secret, apiDB, cellDBs[novav1.Cell0Name].Database.GetDatabaseHostname(), cells[novav1.Cell0Name]) if err != nil { return ctrl.Result{}, err } + cellDbs = append(cellDbs, []string{novaapi.ServiceName + "-" + cr.Spec.CellName, instance.Spec.CellTemplates[cr.Spec.CellName].CellDatabaseAccount}) delete(instance.Status.RegisteredCells, cr.Name) } + // iterate over novaDbs and remove finalizers + for _, novaDb := range cellDbs { + dbName, accountName := novaDb[0], novaDb[1] + db, err := mariadbv1.GetDatabaseByNameAndAccount(ctx, h, dbName, accountName, instance.ObjectMeta.Namespace) + if err != nil && !k8s_errors.IsNotFound(err) { + return ctrl.Result{}, err + } + if !k8s_errors.IsNotFound(err) { + if err := db.DeleteFinalizer(ctx, h); err != nil { + return ctrl.Result{}, err + } + } + } } Log.Info("Successfully reconciled") @@ -621,8 +638,10 @@ func (r *NovaReconciler) ensureCellDeleted( instance *novav1.Nova, cellName string, apiTransportURL string, - secret corev1.Secret, + topLevelSecret corev1.Secret, apiDB *mariadbv1.Database, + APIDatabaseHostname string, + cell0 *novav1.NovaCell, ) error { Log := r.GetLogger(ctx) cell := &novav1.NovaCell{} @@ -652,7 +671,7 @@ func (r *NovaReconciler) ensureCellDeleted( } configHash, scriptName, configName, err := r.ensureNovaManageJobSecret(ctx, h, instance, - cell, secret, cell.Spec.APIDatabaseHostname, apiTransportURL, apiDB) + cell0, topLevelSecret, APIDatabaseHostname, apiTransportURL, apiDB) if err != nil { return err } @@ -675,6 +694,29 @@ func (r *NovaReconciler) ensureCellDeleted( return err } + // Delete secrets + dbSecret := fmt.Sprintf("%s-db-secret", cell.Name) + err = secret.DeleteSecretsWithName(ctx, h, dbSecret, instance.Namespace) + if err != nil { + return err + } + secretName := getNovaCellCRName(instance.Name, cellName) + err = secret.DeleteSecretsWithName(ctx, h, secretName, instance.Namespace) + if err != nil { + return err + } + + // Delete transportURL cr + transportURL := &rabbitmqv1.TransportURL{ + ObjectMeta: metav1.ObjectMeta{ + Name: instance.Name + "-" + cellName + "-transport", + Namespace: instance.Namespace, + }, + } + err = r.Client.Delete(ctx, transportURL) + if err != nil { + return err + } Log.Info("Cell isn't defined in the nova, so deleted cell", "cell", cell) return nil } diff --git a/controllers/novaconductor_controller.go b/controllers/novaconductor_controller.go index cda327e6a..b531bcbde 100644 --- a/controllers/novaconductor_controller.go +++ b/controllers/novaconductor_controller.go @@ -724,6 +724,23 @@ func (r *NovaConductorReconciler) reconcileDelete( Log.Info("Removed finalizer from ourselves") } + _, result, secret, err := ensureSecret( + ctx, + types.NamespacedName{Namespace: instance.Namespace, Name: instance.Spec.Secret}, + nil, + h.GetClient(), + &instance.Status.Conditions, + r.RequeueTimeout, + ) + if (err != nil || result != ctrl.Result{}) { + return err + } + + err = r.cleanServiceFromNovaDb(ctx, h, instance, secret, Log) + if err != nil { + return err + } + Log.Info("Reconciled delete successfully") return nil } diff --git a/templates/nova-manage/bin/delete_cell.sh b/templates/nova-manage/bin/delete_cell.sh index 0356324de..21ed94591 100755 --- a/templates/nova-manage/bin/delete_cell.sh +++ b/templates/nova-manage/bin/delete_cell.sh @@ -24,6 +24,4 @@ export CELL_NAME=${CELL_NAME:?"Please specify a CELL_NAME variable."} # is part of the line, e.g. as the user name of the DB URL cell_uuid=$(nova-manage cell_v2 list_cells | tr ' ' '|' | tr --squeeze-repeats '|' | grep -e "^|$CELL_NAME|" | cut -d '|' -f 3) -if [ -z "${cell_uuid}" ]; then - nova-manage cell_v2 delete_cell --cell_uuid "${cell_uuid}" -fi +nova-manage cell_v2 delete_cell --cell_uuid "${cell_uuid}" diff --git a/templates/nova-manage/config/cell-delete-config.json b/templates/nova-manage/config/cell-delete-config.json index 8866cf24e..955ea4ca2 100644 --- a/templates/nova-manage/config/cell-delete-config.json +++ b/templates/nova-manage/config/cell-delete-config.json @@ -1,5 +1,5 @@ { - "command": "/bin/ensure_cell_mapping.sh", + "command": "/bin/delete_cell.sh", "config_files": [ { "source": "/var/lib/openstack/config/nova-blank.conf", diff --git a/test/functional/nova_reconfiguration_test.go b/test/functional/nova_reconfiguration_test.go index 41a9c9295..11c5b44bf 100644 --- a/test/functional/nova_reconfiguration_test.go +++ b/test/functional/nova_reconfiguration_test.go @@ -22,11 +22,13 @@ import ( "github.com/google/go-cmp/cmp" . "github.com/onsi/ginkgo/v2" //revive:disable:dot-imports . "github.com/onsi/gomega" //revive:disable:dot-imports + k8s_errors "k8s.io/apimachinery/pkg/api/errors" //revive:disable-next-line:dot-imports . "github.com/openstack-k8s-operators/lib-common/modules/common/test/helpers" memcachedv1 "github.com/openstack-k8s-operators/infra-operator/apis/memcached/v1beta1" + rabbitmqv1 "github.com/openstack-k8s-operators/infra-operator/apis/rabbitmq/v1beta1" condition "github.com/openstack-k8s-operators/lib-common/modules/common/condition" mariadbv1 "github.com/openstack-k8s-operators/mariadb-operator/api/v1beta1" @@ -190,7 +192,13 @@ var _ = Describe("Nova reconfiguration", func() { mappingJob.Spec.Template.Spec.Containers[0].Env, "INPUT_HASH", "") g.Expect(newJobInputHash).NotTo(BeNil()) }, timeout, interval).Should(Succeed()) + th.AssertSecretDoesNotExist(cell1.InternalCellSecretName) + Eventually(func(g Gomega) { + instance := &rabbitmqv1.TransportURL{} + err := k8sClient.Get(ctx, cell1.TransportURLName, instance) + g.Expect(k8s_errors.IsNotFound(err)).To(BeTrue()) + }, timeout, interval).Should(Succeed()) }) }) When("cell0 conductor replicas is set to 0", func() { diff --git a/test/kuttl/test-suites/default/cell-delete-tests/00-cleanup-nova.yaml b/test/kuttl/test-suites/default/cell-delete-tests/00-cleanup-nova.yaml new file mode 120000 index 000000000..23d51de43 --- /dev/null +++ b/test/kuttl/test-suites/default/cell-delete-tests/00-cleanup-nova.yaml @@ -0,0 +1 @@ +../common/cleanup-nova.yaml \ No newline at end of file diff --git a/test/kuttl/test-suites/default/cell-delete-tests/01-assert.yaml b/test/kuttl/test-suites/default/cell-delete-tests/01-assert.yaml new file mode 100644 index 000000000..1179c685b --- /dev/null +++ b/test/kuttl/test-suites/default/cell-delete-tests/01-assert.yaml @@ -0,0 +1,140 @@ +apiVersion: nova.openstack.org/v1beta1 +kind: Nova +metadata: + finalizers: + - openstack.org/nova + name: nova-kuttl + namespace: nova-kuttl-default +status: + apiServiceReadyCount: 1 + metadataServiceReadyCount: 1 + schedulerServiceReadyCount: 1 + conditions: + - message: Setup complete + reason: Ready + status: "True" + type: Ready + - message: Input data complete + reason: Ready + status: "True" + type: InputReady + - message: Setup complete + reason: Ready + status: "True" + type: KeystoneServiceReady + - message: MariaDBAccount creation complete + reason: Ready + status: "True" + type: MariaDBAccountReady + - message: ' Memcached instance has been provisioned' + reason: Ready + status: "True" + type: MemcachedReady + - message: DB create completed + reason: Ready + status: "True" + type: NovaAPIDBReady + - message: API message bus creation successfully + reason: Ready + status: "True" + type: NovaAPIMQReady + - message: Setup complete + reason: Ready + status: "True" + type: NovaAPIReady + - message: All DBs created successfully + reason: Ready + status: "True" + type: NovaAllCellDBReady + - message: All NovaCells are ready + reason: Ready + status: "True" + type: NovaAllCellReady + - message: All message busses created successfully + reason: Ready + status: "True" + type: NovaAllCellsMQReady + - message: Setup complete + reason: Ready + status: "True" + type: NovaMetadataReady + - message: Setup complete + reason: Ready + status: "True" + type: NovaSchedulerReady + - message: RoleBinding created + reason: Ready + status: "True" + type: RoleBindingReady + - message: Role created + reason: Ready + status: "True" + type: RoleReady + - message: ServiceAccount created + reason: Ready + status: "True" + type: ServiceAccountReady +--- +apiVersion: v1 +kind: Pod +metadata: + annotations: + openshift.io/scc: anyuid + labels: + service: nova-api + name: nova-kuttl-api-0 +status: + containerStatuses: + - name: nova-kuttl-api-api + ready: true + started: true + - name: nova-kuttl-api-log + ready: true + started: true +--- +apiVersion: v1 +kind: Pod +metadata: + annotations: + openshift.io/scc: anyuid + labels: + service: nova-metadata + statefulset.kubernetes.io/pod-name: nova-kuttl-metadata-0 + name: nova-kuttl-metadata-0 + ownerReferences: + - apiVersion: apps/v1 + blockOwnerDeletion: true + controller: true + kind: StatefulSet + name: nova-kuttl-metadata +status: + containerStatuses: + - name: nova-kuttl-metadata-log + ready: true + started: true + - name: nova-kuttl-metadata-metadata + ready: true + started: true +--- +apiVersion: v1 +kind: Pod +metadata: + name: nova-kuttl-cell1-compute-fake1-compute-0 +status: + containerStatuses: + - name: nova-kuttl-cell1-compute-fake1-compute-compute + ready: true + started: true +--- +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +namespaced: true +commands: + - script: | + set -euxo pipefail + MESSAGE=$(oc exec -n $NAMESPACE openstackclient -- openstack flavor create my-flavor 2>&1) || true + echo $MESSAGE | grep "Policy doesn't allow os_compute_api:os-flavor-manage:create to be performed" + - script: | + set -euxo pipefail + RP_UUID=$(oc exec -n $NAMESPACE openstackclient -- openstack resource provider list --name nova-kuttl-cell1-compute-fake1-compute-0 -f value -c uuid) + oc exec -n $NAMESPACE openstackclient -- openstack resource provider trait list $RP_UUID | grep CUSTOM_FOO diff --git a/test/kuttl/test-suites/default/cell-delete-tests/01-deploy-with-default-config-overwrite.yaml b/test/kuttl/test-suites/default/cell-delete-tests/01-deploy-with-default-config-overwrite.yaml new file mode 100644 index 000000000..4cc6c0bf0 --- /dev/null +++ b/test/kuttl/test-suites/default/cell-delete-tests/01-deploy-with-default-config-overwrite.yaml @@ -0,0 +1,31 @@ +apiVersion: nova.openstack.org/v1beta1 +kind: Nova +metadata: + name: nova-kuttl +spec: + secret: osp-secret + cellTemplates: + cell0: + cellDatabaseInstance: openstack + cellDatabaseAccount: nova-cell0 + cellMessageBusInstance: rabbitmq + hasAPIAccess: true + memcachedInstance: memcached + cell1: + cellDatabaseInstance: openstack-cell1 + cellDatabaseAccount: nova-cell1 + cellMessageBusInstance: rabbitmq-cell1 + memcachedInstance: memcached + novaComputeTemplates: + compute-fake1: + computeDriver: fake.FakeDriver + defaultConfigOverwrite: + provider.yaml: | + meta: + schema_version: '1.0' + providers: + - identification: + uuid: '$COMPUTE_NODE' + traits: + additional: + - 'CUSTOM_FOO' diff --git a/test/kuttl/test-suites/default/cell-delete-tests/02-assert.yaml b/test/kuttl/test-suites/default/cell-delete-tests/02-assert.yaml new file mode 100644 index 000000000..ec06251de --- /dev/null +++ b/test/kuttl/test-suites/default/cell-delete-tests/02-assert.yaml @@ -0,0 +1,130 @@ +apiVersion: nova.openstack.org/v1beta1 +kind: Nova +metadata: + finalizers: + - openstack.org/nova + name: nova-kuttl + namespace: nova-kuttl-default +status: + apiServiceReadyCount: 1 + metadataServiceReadyCount: 1 + schedulerServiceReadyCount: 1 + conditions: + - message: Setup complete + reason: Ready + status: "True" + type: Ready + - message: Input data complete + reason: Ready + status: "True" + type: InputReady + - message: Setup complete + reason: Ready + status: "True" + type: KeystoneServiceReady + - message: MariaDBAccount creation complete + reason: Ready + status: "True" + type: MariaDBAccountReady + - message: ' Memcached instance has been provisioned' + reason: Ready + status: "True" + type: MemcachedReady + - message: DB create completed + reason: Ready + status: "True" + type: NovaAPIDBReady + - message: API message bus creation successfully + reason: Ready + status: "True" + type: NovaAPIMQReady + - message: Setup complete + reason: Ready + status: "True" + type: NovaAPIReady + - message: All DBs created successfully + reason: Ready + status: "True" + type: NovaAllCellDBReady + - message: All NovaCells are ready + reason: Ready + status: "True" + type: NovaAllCellReady + - message: All message busses created successfully + reason: Ready + status: "True" + type: NovaAllCellsMQReady + - message: Setup complete + reason: Ready + status: "True" + type: NovaMetadataReady + - message: Setup complete + reason: Ready + status: "True" + type: NovaSchedulerReady + - message: RoleBinding created + reason: Ready + status: "True" + type: RoleBindingReady + - message: Role created + reason: Ready + status: "True" + type: RoleReady + - message: ServiceAccount created + reason: Ready + status: "True" + type: ServiceAccountReady +--- +apiVersion: v1 +kind: Pod +metadata: + annotations: + openshift.io/scc: anyuid + labels: + service: nova-api + name: nova-kuttl-api-0 +status: + containerStatuses: + - name: nova-kuttl-api-api + ready: true + started: true + - name: nova-kuttl-api-log + ready: true + started: true +--- +apiVersion: v1 +kind: Pod +metadata: + annotations: + openshift.io/scc: anyuid + labels: + service: nova-metadata + statefulset.kubernetes.io/pod-name: nova-kuttl-metadata-0 + name: nova-kuttl-metadata-0 + ownerReferences: + - apiVersion: apps/v1 + blockOwnerDeletion: true + controller: true + kind: StatefulSet + name: nova-kuttl-metadata +status: + containerStatuses: + - name: nova-kuttl-metadata-log + ready: true + started: true + - name: nova-kuttl-metadata-metadata + ready: true + started: true +--- +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +namespaced: true +commands: + - script: | + set -euxo pipefail + MESSAGE=$(oc exec -n $NAMESPACE openstackclient -- openstack flavor create my-flavor 2>&1) || true + echo $MESSAGE | grep "Policy doesn't allow os_compute_api:os-flavor-manage:create to be performed" + - script: | + set -euxo pipefail + RP_UUID=$(oc exec -n $NAMESPACE openstackclient -- openstack resource provider list --name nova-kuttl-cell1-compute-fake1-compute-0 -f value -c uuid) + oc exec -n $NAMESPACE openstackclient -- openstack resource provider trait list $RP_UUID | grep CUSTOM_FOO diff --git a/test/kuttl/test-suites/default/cell-delete-tests/02-delete-cell-nova.yaml b/test/kuttl/test-suites/default/cell-delete-tests/02-delete-cell-nova.yaml new file mode 100644 index 000000000..df6625a64 --- /dev/null +++ b/test/kuttl/test-suites/default/cell-delete-tests/02-delete-cell-nova.yaml @@ -0,0 +1,5 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +commands: + - script: | + oc patch -n nova-kuttl-default nova/nova-kuttl --type='json' -p='[{"op": "remove", "path": "/spec/cellTemplates/cell1"}]' diff --git a/test/kuttl/test-suites/default/cell-delete-tests/03-cleanup-nova.yaml b/test/kuttl/test-suites/default/cell-delete-tests/03-cleanup-nova.yaml new file mode 120000 index 000000000..49b58eee9 --- /dev/null +++ b/test/kuttl/test-suites/default/cell-delete-tests/03-cleanup-nova.yaml @@ -0,0 +1 @@ +00-cleanup-nova.yaml \ No newline at end of file