Add deleting cells automation

controllers/nova_controller.go

@@ -591,10 +591,198 @@ func (r *NovaReconciler) Reconcile(ctx context.Context, req ctrl.Request) (resul
 		}
 	}
 
+	// We need to check and delete cells
+	novaCellList := &novav1.NovaCellList{}
+	listOpts := []client.ListOption{
+		client.InNamespace(instance.Namespace),
+	}
+	if err := r.Client.List(ctx, novaCellList, listOpts...); err != nil {
+		return ctrl.Result{}, err
+	}
+
+	for _, cr := range novaCellList.Items {
+		_, ok := instance.Spec.CellTemplates[cr.Spec.CellName]
+		if !ok {
+			err := r.ensureCellDeleted(ctx, h, instance,
+				cr.Spec.CellName, apiTransportURL,
+				secret, apiDB, cellDBs[novav1.Cell0Name].Database.GetDatabaseHostname(), cells[novav1.Cell0Name])
+			if err != nil {
+				return ctrl.Result{}, err
+			}
+			Log.Info("Cell deleted", "cell", cr.Spec.CellName)
+			delete(instance.Status.RegisteredCells, cr.Name)
+		}
+
+	}
+
 	Log.Info("Successfully reconciled")
 	return ctrl.Result{}, nil
 }
 
+func (r *NovaReconciler) ensureAccountDeletedIfOwned(
+	ctx context.Context,
+	h *helper.Helper,
+	instance *novav1.Nova,
+	accountName string,
+) error {
+	Log := r.GetLogger(ctx)
+
+	account, err := mariadbv1.GetAccount(ctx, h, accountName, instance.Namespace)
+	if err != nil && !k8s_errors.IsNotFound(err) {
+		return err
+	}
+	if k8s_errors.IsNotFound(err) {
+		// Nothing to delete
+		return nil
+	}
+
+	// If it is not created by us, we don't clean it up
+	if !OwnedBy(account, instance) {
+		Log.Info("MariaDBAccount in not owned by Nova, not deleting", "account", account)
+		return nil
+	}
+
+	// NOTE(gibi): We need to delete the Secret first and then the Account
+	// otherwise we cannot retry the Secret deletion when the Account is
+	// gone as we will not know the name of the Secret. This logic should
+	// be moved to the mariadb-operator.
+	err = secret.DeleteSecretsWithName(ctx, h, account.Spec.Secret, instance.Namespace)
+	if err != nil && !k8s_errors.IsNotFound(err) {
+		return err
+	}
+	err = r.Client.Delete(ctx, account)
+	if err != nil && !k8s_errors.IsNotFound(err) {
+		return err
+	}
+
+	Log.Info("Deleted MariaDBAccount", "account", account)
+	return nil
+}
+
+func (r *NovaReconciler) ensureCellDeleted(
+	ctx context.Context,
+	h *helper.Helper,
+	instance *novav1.Nova,
+	cellName string,
+	apiTransportURL string,
+	topLevelSecret corev1.Secret,
+	apiDB *mariadbv1.Database,
+	APIDatabaseHostname string,
+	cell0 *novav1.NovaCell,
+) error {
+	Log := r.GetLogger(ctx)
+	cell := &novav1.NovaCell{}
+	fullCellName := types.NamespacedName{
+		Name:      getNovaCellCRName(instance.Name, cellName),
+		Namespace: instance.GetNamespace(),
+	}
+
+	err := r.Client.Get(ctx, fullCellName, cell)
+	if k8s_errors.IsNotFound(err) {
+		// We cannot do further cleanup of the MariaDBDatabase and
+		// MariaDBAccount as their name is only available in the NovaCell CR
+		// since the cell definition is removed from the Nova CR already.
+		return nil
+	}
+	if err != nil {
+		return err
+	}
+	// If it is not created by us, we don't touch it
+	if !OwnedBy(cell, instance) {
+		Log.Info("Cell isn't defined in the Nova, but there is a  "+
+			"Cell CR not owned by us. Not deleting it.",
+			"cell", cell)
+		return nil
+	}
+
+	dbName, accountName := novaapi.ServiceName+"-"+cell.Spec.CellName, cell.Spec.CellDatabaseAccount
+
+	configHash, scriptName, configName, err := r.ensureNovaManageJobSecret(ctx, h, instance,
+		cell0, topLevelSecret, APIDatabaseHostname, apiTransportURL, apiDB)
+	if err != nil {
+		return err
+	}
+	inputHash, err := util.HashOfInputHashes(configHash)
+	if err != nil {
+		return err
+	}
+
+	labels := map[string]string{
+		common.AppSelector: NovaLabelPrefix,
+	}
+	jobDef := nova.CellDeleteJob(instance, cell, configName, scriptName, inputHash, labels)
+	job := job.NewJob(
+		jobDef, cell.Name+"-cell-delete",
+		instance.Spec.PreserveJobs, r.RequeueTimeout,
+		inputHash)
+
+	_, err = job.DoJob(ctx, h)
+	if err != nil {
+		return err
+	}
+
+	secretName := getNovaCellCRName(instance.Name, cellName)
+	err = secret.DeleteSecretsWithName(ctx, h, secretName, instance.Namespace)
+	if err != nil && !k8s_errors.IsNotFound(err) {
+		return err
+	}
+	configSecret, scriptSecret := r.getNovaManageJobSecretNames(cell)
+	err = secret.DeleteSecretsWithName(ctx, h, configSecret, instance.Namespace)
+	if err != nil && !k8s_errors.IsNotFound(err) {
+		return err
+	}
+	err = secret.DeleteSecretsWithName(ctx, h, scriptSecret, instance.Namespace)
+	if err != nil && !k8s_errors.IsNotFound(err) {
+		return err
+	}
+
+	// Delete transportURL cr
+	transportURL := &rabbitmqv1.TransportURL{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      instance.Name + "-" + cellName + "-transport",
+			Namespace: instance.Namespace,
+		},
+	}
+	err = r.Client.Delete(ctx, transportURL)
+	if err != nil && !k8s_errors.IsNotFound(err) {
+		return err
+	}
+
+	err = mariadbv1.DeleteDatabaseAndAccountFinalizers(
+		ctx, h, dbName, accountName, instance.Namespace)
+	if err != nil {
+		return err
+	}
+
+	err = r.ensureAccountDeletedIfOwned(ctx, h, instance, accountName)
+	if err != nil {
+		return err
+	}
+
+	database := &mariadbv1.MariaDBDatabase{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      dbName,
+			Namespace: instance.Namespace,
+		},
+	}
+	err = r.Client.Delete(ctx, database)
+	if err != nil && !k8s_errors.IsNotFound(err) {
+		return err
+	}
+	Log.Info("Deleted MariaDBDatabase", "database", database)
+
+	// Finally we delete the NovaCell CR. We need to do it as the last step
+	// otherwise we cannot retry the above cleanup as we won't have the data
+	// what to clean up.
+	err = r.Client.Delete(ctx, cell)
+	if err != nil && k8s_errors.IsNotFound(err) {
+		return err
+	}
+
+	Log.Info("Cell isn't defined in the Nova CR, so it is deleted", "cell", cell)
+	return nil
+}
+
 func (r *NovaReconciler) initStatus(
 	instance *novav1.Nova,
 ) error {
@@ -703,6 +891,14 @@ func (r *NovaReconciler) initConditions(
 	return nil
 }
 
+func (r *NovaReconciler) getNovaManageJobSecretNames(
+	cell *novav1.NovaCell,
+) (configName string, scriptName string) {
+	configName = fmt.Sprintf("%s-config-data", cell.Name+"-manage")
+	scriptName = fmt.Sprintf("%s-scripts", cell.Name+"-manage")
+	return
+}
+
 func (r *NovaReconciler) ensureNovaManageJobSecret(
 	ctx context.Context, h *helper.Helper, instance *novav1.Nova,
 	cell *novav1.NovaCell,
@@ -711,8 +907,7 @@ func (r *NovaReconciler) ensureNovaManageJobSecret(
 	cellTransportURL string,
 	cellDB *mariadbv1.Database,
 ) (map[string]env.Setter, string, string, error) {
-	configName := fmt.Sprintf("%s-config-data", cell.Name+"-manage")
-	scriptName := fmt.Sprintf("%s-scripts", cell.Name+"-manage")
+	configName, scriptName := r.getNovaManageJobSecretNames(cell)
 
 	cmLabels := labels.GetLabels(
 		instance, labels.GetGroupLabel(NovaLabelPrefix), map[string]string{},

go.mod

@@ -15,7 +15,7 @@ require (
 	github.com/openstack-k8s-operators/lib-common/modules/common v0.4.1-0.20240926101719-8fc1c3da53f7
 	github.com/openstack-k8s-operators/lib-common/modules/openstack v0.4.1-0.20240926101719-8fc1c3da53f7
 	github.com/openstack-k8s-operators/lib-common/modules/test v0.4.1-0.20240926101719-8fc1c3da53f7
-	github.com/openstack-k8s-operators/mariadb-operator/api v0.4.1-0.20240927143624-61d230f582d6
+	github.com/openstack-k8s-operators/mariadb-operator/api v0.4.1-0.20241008102252-6f4b7a67029f
 	github.com/openstack-k8s-operators/nova-operator/api v0.0.0-20221209164002-f9e6b9363961
 	go.uber.org/zap v1.27.0
 	gopkg.in/yaml.v3 v3.0.1

go.sum

@@ -87,8 +87,8 @@ github.com/openstack-k8s-operators/lib-common/modules/openstack v0.4.1-0.2024092
 github.com/openstack-k8s-operators/lib-common/modules/openstack v0.4.1-0.20240926101719-8fc1c3da53f7/go.mod h1:oCopeVBDHbCoPFXOMNAPae/XvPQ3H9Sdaag16HlLZC0=
 github.com/openstack-k8s-operators/lib-common/modules/test v0.4.1-0.20240926101719-8fc1c3da53f7 h1:dNf8GQ+6Dv2twGwYlyTfiYttpSESfcQ89p0py2XAbdo=
 github.com/openstack-k8s-operators/lib-common/modules/test v0.4.1-0.20240926101719-8fc1c3da53f7/go.mod h1:LV0jo5etIsGyINpmB37i4oWR8zU6ApIuh7fsqGGA41o=
-github.com/openstack-k8s-operators/mariadb-operator/api v0.4.1-0.20240927143624-61d230f582d6 h1:t0TJCqVvIR+2ab77er24bjUTS8uR2xJuuXt3cBlom90=
-github.com/openstack-k8s-operators/mariadb-operator/api v0.4.1-0.20240927143624-61d230f582d6/go.mod h1:13K91HQShjM0y1zVTupCybaTpWJYzOhMPd+rJUpxIg8=
+github.com/openstack-k8s-operators/mariadb-operator/api v0.4.1-0.20241008102252-6f4b7a67029f h1:Tkg3SwrWfLhlc00MQYYLli5TdUQCxC5KFDq5BToSVaw=
+github.com/openstack-k8s-operators/mariadb-operator/api v0.4.1-0.20241008102252-6f4b7a67029f/go.mod h1:13K91HQShjM0y1zVTupCybaTpWJYzOhMPd+rJUpxIg8=
 github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
 github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=

pkg/nova/celldelete.go

@@ -0,0 +1,86 @@
+package nova
+
+import (
+	batchv1 "k8s.io/api/batch/v1"
+	corev1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/utils/ptr"
+
+	"github.com/openstack-k8s-operators/lib-common/modules/common/env"
+	novav1 "github.com/openstack-k8s-operators/nova-operator/api/v1beta1"
+)
+
+func CellDeleteJob(
+	instance *novav1.Nova,
+	cell *novav1.NovaCell,
+	configName string,
+	scriptName string,
+	inputHash string,
+	labels map[string]string,
+) *batchv1.Job {
+	args := []string{"-c", KollaServiceCommand}
+
+	envVars := map[string]env.Setter{}
+	envVars["KOLLA_CONFIG_STRATEGY"] = env.SetValue("COPY_ALWAYS")
+	envVars["KOLLA_BOOTSTRAP"] = env.SetValue("true")
+	envVars["CELL_NAME"] = env.SetValue(cell.Spec.CellName)
+
+	// This is stored in the Job so that if the input of the job changes
+	// then it results in a new job hash and therefore lib-common will re-run
+	// the job
+	envVars["INPUT_HASH"] = env.SetValue(inputHash)
+
+	env := env.MergeEnvs([]corev1.EnvVar{}, envVars)
+
+	jobName := instance.Name + "-" + cell.Spec.CellName + "-cell-delete"
+
+	volumes := []corev1.Volume{
+		GetConfigVolume(configName),
+		GetScriptVolume(scriptName),
+	}
+	volumeMounts := []corev1.VolumeMount{
+		GetConfigVolumeMount(),
+		GetScriptVolumeMount(),
+		GetKollaConfigVolumeMount("cell-delete"),
+	}
+
+	// add CA cert if defined
+	if instance.Spec.APIServiceTemplate.TLS.CaBundleSecretName != "" {
+		volumes = append(volumes, instance.Spec.APIServiceTemplate.TLS.CreateVolume())
+		volumeMounts = append(volumeMounts, instance.Spec.APIServiceTemplate.TLS.CreateVolumeMounts(nil)...)
+	}
+
+	job := &batchv1.Job{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      jobName,
+			Namespace: instance.Namespace,
+			Labels:    labels,
+		},
+		Spec: batchv1.JobSpec{
+			Template: corev1.PodTemplateSpec{
+				Spec: corev1.PodSpec{
+					RestartPolicy:      corev1.RestartPolicyOnFailure,
+					ServiceAccountName: instance.RbacResourceName(),
+					Volumes:            volumes,
+					Containers: []corev1.Container{
+						{
+							Name: "nova-manage",
+							Command: []string{
+								"/bin/bash",
+							},
+							Args:  args,
+							Image: cell.Spec.ConductorContainerImageURL,
+							SecurityContext: &corev1.SecurityContext{
+								RunAsUser: ptr.To(NovaUserID),
+							},
+							Env:          env,
+							VolumeMounts: volumeMounts,
+						},
+					},
+				},
+			},
+		},
+	}
+
+	return job
+}

templates/nova-manage/bin/delete_cell.sh

@@ -0,0 +1,27 @@
+#!/bin/bash
+# Copyright 2023.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -xe
+
+export CELL_NAME=${CELL_NAME:?"Please specify a CELL_NAME variable."}
+
+# NOTE(gibi): nova-manage should be enhanced upstream to get rid of this
+# uglyness
+# Note the "|" around the CELL_NAME, that is needed as a single line from
+# nova-manage cell_v2 cell_list can match to multiple cells if the cell name
+# is part of the line, e.g. as the user name of the DB URL
+cell_uuid=$(nova-manage cell_v2 list_cells | tr ' ' '|' | tr --squeeze-repeats '|' | grep -e "^|$CELL_NAME|" | cut -d '|' -f 3)
+
+nova-manage cell_v2 delete_cell --cell_uuid "${cell_uuid}"

templates/nova-manage/config/cell-delete-config.json

@@ -0,0 +1,36 @@
+{
+    "command": "/bin/delete_cell.sh",
+    "config_files": [
+        {
+            "source": "/var/lib/openstack/config/nova-blank.conf",
+            "dest": "/etc/nova/nova.conf",
+            "owner": "nova",
+            "perm": "0600"
+        },
+        {
+            "source": "/var/lib/openstack/config/01-nova.conf",
+            "dest": "/etc/nova/nova.conf.d/01-nova.conf",
+            "owner": "nova",
+            "perm": "0600"
+        },
+        {
+            "source": "/var/lib/openstack/config/02-nova-override.conf",
+            "dest": "/etc/nova/nova.conf.d/02-nova-override.conf",
+            "owner": "nova",
+            "perm": "0600",
+            "optional": true
+        },
+        {
+            "source": "/var/lib/openstack/bin/delete_cell.sh",
+            "dest": "/bin/",
+            "owner": "nova",
+            "perm": "0700"
+        },
+        {
+            "source": "/var/lib/openstack/config/my.cnf",
+            "dest": "/etc/my.cnf",
+            "owner": "nova",
+            "perm": "0644"
+        }
+    ]
+}