Merge pull request #9 from cschwede/ring-rebalance

Ring rebalance

controllers/swift_controller.go

@@ -125,7 +125,7 @@ func (r *SwiftReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl
 		{
 			APIGroups: []string{""},
 			Resources: []string{"configmaps"},
-			Verbs:     []string{"create", "update", "delete"},
+			Verbs:     []string{"create", "get", "update", "delete"},
 		},
 		{
 			APIGroups: []string{""},
@@ -159,44 +159,44 @@ func (r *SwiftReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl
 		}
 	}
 
-	// create or update Swift rings
-	swiftRing, op, err := r.ringCreateOrUpdate(ctx, instance)
+	// create or update Swift storage
+	swiftStorage, op, err := r.storageCreateOrUpdate(ctx, instance)
 	if err != nil {
 		instance.Status.Conditions.Set(condition.FalseCondition(
-			swiftv1beta1.SwiftRingReadyCondition,
+			swiftv1beta1.SwiftStorageReadyCondition,
 			condition.ErrorReason,
 			condition.SeverityWarning,
-			swiftv1beta1.SwiftRingReadyErrorMessage,
+			swiftv1beta1.SwiftStorageReadyErrorMessage,
 			err.Error()))
 		return ctrl.Result{}, err
 	}
 	if op != controllerutil.OperationResultNone {
 		r.Log.Info(fmt.Sprintf("Deployment %s successfully reconciled - operation: %s", instance.Name, string(op)))
 	}
 
-	// Mirror SwiftRing's condition status
-	c := swiftRing.Status.Conditions.Mirror(swiftv1beta1.SwiftRingReadyCondition)
+	// Mirror SwiftStorage's condition status
+	c := swiftStorage.Status.Conditions.Mirror(swiftv1beta1.SwiftStorageReadyCondition)
 	if c != nil {
 		instance.Status.Conditions.Set(c)
 	}
 
-	// create or update Swift storage
-	swiftStorage, op, err := r.storageCreateOrUpdate(ctx, instance)
+	// create or update Swift rings
+	swiftRing, op, err := r.ringCreateOrUpdate(ctx, instance)
 	if err != nil {
 		instance.Status.Conditions.Set(condition.FalseCondition(
-			swiftv1beta1.SwiftStorageReadyCondition,
+			swiftv1beta1.SwiftRingReadyCondition,
 			condition.ErrorReason,
 			condition.SeverityWarning,
-			swiftv1beta1.SwiftStorageReadyErrorMessage,
+			swiftv1beta1.SwiftRingReadyErrorMessage,
 			err.Error()))
 		return ctrl.Result{}, err
 	}
 	if op != controllerutil.OperationResultNone {
 		r.Log.Info(fmt.Sprintf("Deployment %s successfully reconciled - operation: %s", instance.Name, string(op)))
 	}
 
-	// Mirror SwiftStorage's condition status
-	c = swiftStorage.Status.Conditions.Mirror(swiftv1beta1.SwiftStorageReadyCondition)
+	// Mirror SwiftRing's condition status
+	c = swiftRing.Status.Conditions.Mirror(swiftv1beta1.SwiftRingReadyCondition)
 	if c != nil {
 		instance.Status.Conditions.Set(c)
 	}

controllers/swiftproxy_controller.go

@@ -34,7 +34,6 @@ import (
 	ctrl "sigs.k8s.io/controller-runtime"
 
 	"github.com/openstack-k8s-operators/lib-common/modules/common/condition"
-	"github.com/openstack-k8s-operators/lib-common/modules/common/configmap"
 	"github.com/openstack-k8s-operators/lib-common/modules/common/deployment"
 	"github.com/openstack-k8s-operators/lib-common/modules/common/endpoint"
 	"github.com/openstack-k8s-operators/lib-common/modules/common/env"
@@ -120,22 +119,6 @@ func (r *SwiftProxyReconciler) Reconcile(ctx context.Context, req ctrl.Request)
 		return r.reconcileDelete(ctx, instance, helper)
 	}
 
-	// Check if there is a ConfigMap for the Swift rings
-	cm, ctrlResult, err := configmap.GetConfigMap(
-		ctx, helper, instance, swiftv1beta1.RingConfigMapName, 5*time.Second)
-	if err != nil {
-		return ctrlResult, err
-	} else if (ctrlResult != ctrl.Result{}) {
-		return ctrlResult, nil
-	}
-
-	// This will only exist after SwiftStorage instances are up and rings
-	// are set up properly
-	_, ok := cm.BinaryData["swiftrings.tar.gz"]
-	if !ok {
-		return ctrl.Result{RequeueAfter: time.Duration(10) * time.Second}, nil
-	}
-
 	serviceLabels := swiftproxy.Labels()
 
 	// Create a Service and endpoints for the proxy

controllers/swiftring_controller.go

@@ -126,13 +126,6 @@ func (r *SwiftRingReconciler) Reconcile(ctx context.Context, req ctrl.Request) (
 		return ctrl.Result{}, err
 	}
 
-	// Create a ConfigMap for the Swift rings
-	tpl = swiftring.ConfigMapTemplates(instance, serviceLabels)
-	err = configmap.EnsureConfigMaps(ctx, helper, instance, tpl, &envVars)
-	if err != nil {
-		return ctrl.Result{}, err
-	}
-
 	// Swift ring init job - start
 	if instance.Status.Hash == nil {
 		instance.Status.Hash = map[string]string{}

controllers/swiftstorage_controller.go

@@ -115,23 +115,31 @@ func (r *SwiftStorageReconciler) Reconcile(ctx context.Context, req ctrl.Request
 	}
 
 	serviceLabels := swiftstorage.Labels()
+	envVars := make(map[string]env.Setter)
+
+	// Check if there is already an existing ConfigMap and device list. If
+	// not, create an initial device list to bootstrap the cluster with The
+	// weights are simply set to the requested size, this will be changed
+	// once all StatefulSets are running
+	_, ctrlResult, err := configmap.GetConfigMap(ctx, helper, instance, swiftv1beta1.DeviceConfigMapName, 5*time.Second)
+	if err != nil {
+		return ctrlResult, err
+	} else if (ctrlResult != ctrl.Result{}) {
+		devices := swiftstorage.DeviceList(ctx, helper, instance)
+		tpl := swiftstorage.DeviceConfigMapTemplates(instance, devices)
+		err = configmap.EnsureConfigMaps(ctx, helper, instance, tpl, &envVars)
+		if err != nil {
+			return ctrl.Result{}, err
+		}
+	}
 
 	// Create a ConfigMap populated with content from templates/
-	envVars := make(map[string]env.Setter)
 	tpl := swiftstorage.ConfigMapTemplates(instance, serviceLabels)
 	err = configmap.EnsureConfigMaps(ctx, helper, instance, tpl, &envVars)
 	if err != nil {
 		return ctrl.Result{}, err
 	}
 
-	// Check if there is a ConfigMap for the Swift rings
-	_, ctrlResult, err := configmap.GetConfigMap(ctx, helper, instance, swiftv1beta1.RingConfigMapName, 5*time.Second)
-	if err != nil {
-		return ctrlResult, err
-	} else if (ctrlResult != ctrl.Result{}) {
-		return ctrlResult, nil
-	}
-
 	// Headless Service
 	svc := service.NewService(swiftstorage.Service(instance), serviceLabels, 5*time.Second)
 	ctrlResult, err = svc.CreateOrPatch(ctx, helper)
@@ -177,10 +185,7 @@ func (r *SwiftStorageReconciler) Reconcile(ctx context.Context, req ctrl.Request
 
 	if sset.GetStatefulSet().Status.ReadyReplicas == instance.Spec.Replicas {
 		envVars := make(map[string]env.Setter)
-		devices, err := swiftstorage.DeviceList(ctx, helper, instance)
-		if err != nil {
-			return ctrl.Result{}, err
-		}
+		devices := swiftstorage.DeviceList(ctx, helper, instance)
 		tpl = swiftstorage.DeviceConfigMapTemplates(instance, devices)
 		err = configmap.EnsureConfigMaps(ctx, helper, instance, tpl, &envVars)
 		if err != nil {

pkg/swiftproxy/deployment.go

@@ -27,20 +27,6 @@ import (
 	swift "github.com/openstack-k8s-operators/swift-operator/pkg/swift"
 )
 
-func getInitContainers(swiftproxy *swiftv1beta1.SwiftProxy) []corev1.Container {
-	securityContext := swift.GetSecurityContext()
-	return []corev1.Container{
-		{
-			Name:            "swift-init",
-			Image:           swiftproxy.Spec.ContainerImageProxy,
-			ImagePullPolicy: corev1.PullIfNotPresent,
-			SecurityContext: &securityContext,
-			VolumeMounts:    getProxyVolumeMounts(),
-			Command:         []string{"/usr/local/bin/container-scripts/swift-init.sh"},
-		},
-	}
-}
-
 func Deployment(
 	instance *swiftv1beta1.SwiftProxy, labels map[string]string) *appsv1.Deployment {
 
@@ -89,9 +75,18 @@ func Deployment(
 							Type: corev1.SeccompProfileTypeRuntimeDefault,
 						},
 					},
-					Volumes:        getProxyVolumes(instance),
-					InitContainers: getInitContainers(instance),
+					Volumes: getProxyVolumes(instance),
 					Containers: []corev1.Container{
+						{
+							Name:            "ring-sync",
+							Image:           instance.Spec.ContainerImageProxy,
+							ImagePullPolicy: corev1.PullIfNotPresent,
+							SecurityContext: &securityContext,
+							ReadinessProbe:  readinessProbe,
+							LivenessProbe:   livenessProbe,
+							VolumeMounts:    getProxyVolumeMounts(),
+							Command:         []string{"/usr/local/bin/container-scripts/ring-sync.sh"},
+						},
 						{
 							Image:           instance.Spec.ContainerImageProxy,
 							Name:            "proxy-server",
@@ -118,16 +113,6 @@ func Deployment(
 							VolumeMounts: getProxyVolumeMounts(),
 							Command:      []string{"/usr/bin/memcached", "-p", "11211", "-u", "memcached"},
 						},
-						{
-							Name:            "ring-sync",
-							Image:           instance.Spec.ContainerImageProxy,
-							ImagePullPolicy: corev1.PullIfNotPresent,
-							SecurityContext: &securityContext,
-							ReadinessProbe:  readinessProbe,
-							LivenessProbe:   livenessProbe,
-							VolumeMounts:    getProxyVolumeMounts(),
-							Command:         []string{"/usr/local/bin/container-scripts/ring-sync.sh"},
-						},
 					},
 				},
 			},

pkg/swiftproxy/templates.go

@@ -41,7 +41,7 @@ func SecretTemplates(instance *swiftv1beta1.SwiftProxy, labels map[string]string
 			Name:               fmt.Sprintf("%s-scripts", instance.Name),
 			Namespace:          instance.Namespace,
 			Type:               util.TemplateTypeScripts,
-			AdditionalTemplate: map[string]string{"swift-init.sh": "/common/swift-init.sh", "ring-sync.sh": "/common/ring-sync.sh"},
+			AdditionalTemplate: map[string]string{"ring-sync.sh": "/common/ring-sync.sh"},
 			InstanceType:       instance.Kind,
 			Labels:             labels,
 		},

pkg/swiftring/volumes.go

@@ -57,16 +57,6 @@ func getRingVolumes(instance *swiftv1beta1.SwiftRing) []corev1.Volume {
 				},
 			},
 		},
-		{
-			Name: "ring-data",
-			VolumeSource: corev1.VolumeSource{
-				ConfigMap: &corev1.ConfigMapVolumeSource{
-					LocalObjectReference: corev1.LocalObjectReference{
-						Name: swiftv1beta1.RingConfigMapName,
-					},
-				},
-			},
-		},
 	}
 }
 
@@ -92,10 +82,5 @@ func getRingVolumeMounts() []corev1.VolumeMount {
 			MountPath: "/var/lib/config-data/ring-devices",
 			ReadOnly:  true,
 		},
-		{
-			Name:      "ring-data",
-			MountPath: "/var/lib/config-data/rings",
-			ReadOnly:  true,
-		},
 	}
 }

pkg/swiftstorage/funcs.go

@@ -22,6 +22,7 @@ import (
 	"strings"
 
 	corev1 "k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/api/resource"
 	"k8s.io/apimachinery/pkg/types"
 
 	"github.com/openstack-k8s-operators/lib-common/modules/common/helper"
@@ -32,24 +33,29 @@ import (
 
 //+kubebuilder:rbac:groups=core,resources=persistentvolumeclaims,verbs=get;list;watch
 
-func DeviceList(ctx context.Context, h *helper.Helper, instance *swiftv1beta1.SwiftStorage) (string, error) {
+func DeviceList(ctx context.Context, h *helper.Helper, instance *swiftv1beta1.SwiftStorage) string {
+	// Creates a CSV list of devices. If PVCs do not exist yet (because not
+	// all StatefulSets are up yet), it will just use the request capacity
+	// as value.
 	var devices strings.Builder
 
 	foundClaim := &corev1.PersistentVolumeClaim{}
 	for replica := 0; replica < int(instance.Spec.Replicas); replica++ {
 		cn := fmt.Sprintf("%s-%s-%d", swift.ClaimName, instance.Name, replica)
 		err := h.GetClient().Get(ctx, types.NamespacedName{Name: cn, Namespace: instance.Namespace}, foundClaim)
+		capacity := resource.MustParse(instance.Spec.StorageRequest)
+		weight, _ := capacity.AsInt64()
 		if err == nil {
-			fsc := foundClaim.Status.Capacity["storage"]
-			c, _ := (&fsc).AsInt64()
-			c = c / (1000 * 1000 * 1000)
-			host := fmt.Sprintf("%s-%d.%s", instance.Name, replica, instance.Name)
-			devices.WriteString(fmt.Sprintf("%s,%s,%d\n", host, "d1", c))
+			capacity := foundClaim.Status.Capacity["storage"]
+			weight, _ = capacity.AsInt64()
 		} else {
-			return "", err
+			h.GetLogger().Info(fmt.Sprintf("Did not find PVC %s, assuming %s as capacity", cn, instance.Spec.StorageRequest))
 		}
+		weight = weight / (1000 * 1000 * 1000) // 10GiB gets a weight of 10 etc.
+		// CSV: region,zone,hostname,devicename,weight
+		devices.WriteString(fmt.Sprintf("1,1,%s-%d.%s,%s,%d\n", instance.Name, replica, instance.Name, "d1", weight))
 	}
-	return devices.String(), nil
+	return devices.String()
 }
 
 func Labels() map[string]string {

pkg/swiftstorage/statefulset.go

@@ -35,25 +35,18 @@ func getPorts(port int32, name string) []corev1.ContainerPort {
 	}
 }
 
-func getStorageInitContainers(swiftstorage *swiftv1beta1.SwiftStorage) []corev1.Container {
+func getStorageContainers(swiftstorage *swiftv1beta1.SwiftStorage) []corev1.Container {
 	securityContext := swift.GetSecurityContext()
 
 	return []corev1.Container{
 		{
-			Name:            "swift-init",
-			Image:           swiftstorage.Spec.ContainerImageAccount,
+			Name:            "ring-sync",
+			Image:           swiftstorage.Spec.ContainerImageProxy,
 			ImagePullPolicy: corev1.PullIfNotPresent,
 			SecurityContext: &securityContext,
 			VolumeMounts:    getStorageVolumeMounts(),
-			Command:         []string{"/usr/local/bin/container-scripts/swift-init.sh"},
+			Command:         []string{"/usr/local/bin/container-scripts/ring-sync.sh"},
 		},
-	}
-}
-
-func getStorageContainers(swiftstorage *swiftv1beta1.SwiftStorage) []corev1.Container {
-	securityContext := swift.GetSecurityContext()
-
-	return []corev1.Container{
 		{
 			Name:            "account-server",
 			Image:           swiftstorage.Spec.ContainerImageAccount,
@@ -178,14 +171,6 @@ func getStorageContainers(swiftstorage *swiftv1beta1.SwiftStorage) []corev1.Cont
 			Ports:           getPorts(swift.MemcachedPort, "memcached"),
 			Command:         []string{"/usr/bin/memcached", "-p", "11211", "-u", "memcached"},
 		},
-		{
-			Name:            "ring-sync",
-			Image:           swiftstorage.Spec.ContainerImageProxy,
-			ImagePullPolicy: corev1.PullIfNotPresent,
-			SecurityContext: &securityContext,
-			VolumeMounts:    getStorageVolumeMounts(),
-			Command:         []string{"/usr/local/bin/container-scripts/ring-sync.sh"},
-		},
 	}
 }
 
@@ -226,9 +211,8 @@ func StatefulSet(
 							Type: corev1.SeccompProfileTypeRuntimeDefault,
 						},
 					},
-					Volumes:        getStorageVolumes(swiftstorage),
-					InitContainers: getStorageInitContainers(swiftstorage),
-					Containers:     getStorageContainers(swiftstorage),
+					Volumes:    getStorageVolumes(swiftstorage),
+					Containers: getStorageContainers(swiftstorage),
 				},
 			},
 			VolumeClaimTemplates: []corev1.PersistentVolumeClaim{{

pkg/swiftstorage/templates.go

@@ -40,7 +40,7 @@ func ConfigMapTemplates(instance *swiftv1beta1.SwiftStorage, labels map[string]s
 			Type:               util.TemplateTypeScripts,
 			InstanceType:       instance.Kind,
 			Labels:             labels,
-			AdditionalTemplate: map[string]string{"swift-init.sh": "/common/swift-init.sh", "ring-sync.sh": "/common/ring-sync.sh"},
+			AdditionalTemplate: map[string]string{"ring-sync.sh": "/common/ring-sync.sh"},
 		},
 	}
 }

templates/common/ring-sync.sh

@@ -2,6 +2,9 @@
 TARFILE="/var/lib/config-data/rings/swiftrings.tar.gz"
 MTIME="0"
 
+# This is done only initially, should be done by Kolla probably
+cp -t /etc/swift/ /var/lib/config-data/default/* /var/lib/config-data/swiftconf/*
+
 while true; do
     if [ -e $TARFILE ] ; then
         _MTIME=$(stat -L --printf "%Y" $TARFILE)