diff --git a/Documentation/ceph-object-store-crd.md b/Documentation/ceph-object-store-crd.md index 2e5c9f6e3fe5..a3549c07bd56 100644 --- a/Documentation/ceph-object-store-crd.md +++ b/Documentation/ceph-object-store-crd.md @@ -36,7 +36,6 @@ spec: port: 80 securePort: instances: 1 - allNodes: false # A key/value list of annotations annotations: # key: value @@ -85,8 +84,7 @@ The gateway settings correspond to the RGW daemon settings. - `sslCertificateRef`: If the certificate is not specified, SSL will not be configured. If specified, this is the name of the Kubernetes secret that contains the SSL certificate to be used for secure connections to the object store. Rook will look in the secret provided at the `cert` key name. The value of the `cert` key must be in the format expected by the [RGW service](http://docs.ceph.com/docs/master/install/install-ceph-gateway/#using-ssl-with-civetweb): "The server key, server certificate, and any other CA or intermediate certificates be supplied in one file. Each of these items must be in pem form." - `port`: The port on which the RGW pods and the RGW service will be listening (not encrypted). - `securePort`: The secure port on which RGW pods will be listening. An SSL certificate must be specified. -- `instances`: The number of pods that will be started to load balance this object store. Ignored if `allNodes` is true. -- `allNodes`: Whether RGW pods should be started on all nodes. If true, a daemonset is created. If false, `instances` must be set. +- `instances`: The number of pods that will be started to load balance this object store. - `annotations`: Key value pair list of annotations to add. - `placement`: The Kubernetes placement settings to determine where the RGW pods should be started in the cluster. - `resources`: Set resource requests/limits for the Gateway Pod(s), see [Resource Requirements/Limits](ceph-cluster-crd.md#resource-requirementslimits). diff --git a/Documentation/ceph-object.md b/Documentation/ceph-object.md index 56f424870b51..3ec411da6920 100644 --- a/Documentation/ceph-object.md +++ b/Documentation/ceph-object.md @@ -42,7 +42,6 @@ spec: port: 80 securePort: instances: 1 - allNodes: false ``` When the object store is created the Rook operator will create all the pools and other resources necessary to start the service. This may take a minute to complete. diff --git a/PendingReleaseNotes.md b/PendingReleaseNotes.md index 2a236ada0891..8b6ddeb771a6 100644 --- a/PendingReleaseNotes.md +++ b/PendingReleaseNotes.md @@ -13,6 +13,7 @@ an example usage - Rgw pods have liveness probe enabled - Rgw is now configured with the Beast backend as of the Nautilus release - OSD: newly updated cluster from 0.9 to 1.0.3 and thus Ceph Nautilus will have their OSDs allowing new features for Nautilus +- Rgw instances have their own key and thus are properly reflected in the Ceph status ## Breaking Changes @@ -24,4 +25,11 @@ an example usage ## Deprecations +### Ceph + +- For rgw, when deploying an object store with `object.yaml`, using `allNodes` is not supported anymore, a transition path has been implemented in the code though. +So if you were using `allNodes: true`, Rook will replace each daemonset with a deployment (one for one replacement) gradually. +This operation will be triggered on an update or when a new version of the operator is deployed. +Once complete, it is expected that you edit your object CR with `kubectl -n rook-ceph edit cephobjectstore.ceph.rook.io/my-store` and set `allNodes: false` and `instances` with the current number of rgw instances. + ### diff --git a/cluster/examples/kubernetes/ceph/object-ec.yaml b/cluster/examples/kubernetes/ceph/object-ec.yaml index 94022033c001..4da1ee74d3bd 100644 --- a/cluster/examples/kubernetes/ceph/object-ec.yaml +++ b/cluster/examples/kubernetes/ceph/object-ec.yaml @@ -31,10 +31,8 @@ spec: port: 80 # The port that RGW pods will listen on (https). An ssl certificate is required. securePort: - # The number of pods in the rgw deployment (ignored if allNodes=true) + # The number of pods in the rgw deployment instances: 1 - # Whether the rgw pods should be deployed on all nodes as a daemonset - allNodes: false # The affinity rules to apply to the rgw deployment or daemonset. placement: # nodeAffinity: diff --git a/cluster/examples/kubernetes/ceph/object-openshift.yaml b/cluster/examples/kubernetes/ceph/object-openshift.yaml index c607d25e3e83..8460a9f3e2b6 100644 --- a/cluster/examples/kubernetes/ceph/object-openshift.yaml +++ b/cluster/examples/kubernetes/ceph/object-openshift.yaml @@ -30,10 +30,8 @@ spec: port: 8080 # The port that RGW pods will listen on (https). An ssl certificate is required. securePort: - # The number of pods in the rgw deployment (ignored if allNodes=true) + # The number of pods in the rgw deployment instances: 1 - # Whether the rgw pods should be deployed on all nodes as a daemonset - allNodes: false # The affinity rules to apply to the rgw deployment or daemonset. placement: # nodeAffinity: diff --git a/cluster/examples/kubernetes/ceph/object-test.yaml b/cluster/examples/kubernetes/ceph/object-test.yaml index 65c073f9e52f..c98f6a5d60a1 100644 --- a/cluster/examples/kubernetes/ceph/object-test.yaml +++ b/cluster/examples/kubernetes/ceph/object-test.yaml @@ -20,4 +20,3 @@ spec: port: 80 securePort: instances: 1 - allNodes: false diff --git a/cluster/examples/kubernetes/ceph/object.yaml b/cluster/examples/kubernetes/ceph/object.yaml index 309e73eebfc2..e92e9f98137f 100644 --- a/cluster/examples/kubernetes/ceph/object.yaml +++ b/cluster/examples/kubernetes/ceph/object.yaml @@ -30,10 +30,8 @@ spec: port: 80 # The port that RGW pods will listen on (https). An ssl certificate is required. securePort: - # The number of pods in the rgw deployment (ignored if allNodes=true) + # The number of pods in the rgw deployment instances: 1 - # Whether the rgw pods should be deployed on all nodes as a daemonset - allNodes: false # The affinity rules to apply to the rgw deployment or daemonset. placement: # nodeAffinity: diff --git a/cluster/olm/ceph/assemble/metadata-common.yaml b/cluster/olm/ceph/assemble/metadata-common.yaml index 8df89e1a1e2a..8ccf4e724be8 100644 --- a/cluster/olm/ceph/assemble/metadata-common.yaml +++ b/cluster/olm/ceph/assemble/metadata-common.yaml @@ -151,7 +151,6 @@ metadata: "port": 8080, "securePort": null, "instances": 1, - "allNodes": false, "placement": null, "annotations": null, "resources": null diff --git a/design/object-store.md b/design/object-store.md index 6a8f8aea2813..c0ac0fc506a1 100644 --- a/design/object-store.md +++ b/design/object-store.md @@ -34,7 +34,6 @@ spec: port: 80 securePort: 443 instances: 3 - allNodes: false ``` Now create the object store. @@ -82,8 +81,7 @@ The gateway settings correspond to the RGW service. - `sslCertificateRef`: If specified, this is the name of the Kubernetes secret that contains the SSL certificate to be used for secure connections to the object store. The secret must be in the same namespace as the Rook cluster. Rook will look in the secret provided at the `cert` key name. The value of the `cert` key must be in the format expected by the [RGW service](http://docs.ceph.com/docs/master/install/install-ceph-gateway/#using-ssl-with-civetweb): "The server key, server certificate, and any other CA or intermediate certificates be supplied in one file. Each of these items must be in pem form." If the certificate is not specified, SSL will not be configured. - `port`: The service port where the RGW service will be listening (http) - `securePort`: The service port where the RGW service will be listening (https) -- `instances`: The number of RGW pods that will be started for this object store (ignored if allNodes=true) -- `allNodes`: Whether all nodes in the cluster should run RGW as a daemonset +- `instances`: The number of RGW pods that will be started for this object store - `placement`: The rgw pods can be given standard Kubernetes placement restrictions with `nodeAffinity`, `tolerations`, `podAffinity`, and `podAntiAffinity` similar to placement defined for daemons configured by the [cluster CRD](/cluster/examples/kubernetes/ceph/cluster.yaml). The RGW service can be configured to listen on both http and https by specifying both `port` and `securePort`. @@ -95,7 +93,6 @@ The RGW service can be configured to listen on both http and https by specifying port: 80 securePort: 443 instances: 1 - allNodes: false ``` ### Realms, Zone Groups, and Zones diff --git a/pkg/operator/ceph/cluster/mgr/config.go b/pkg/operator/ceph/cluster/mgr/config.go index 0910fb49a907..260869e18a88 100644 --- a/pkg/operator/ceph/cluster/mgr/config.go +++ b/pkg/operator/ceph/cluster/mgr/config.go @@ -62,7 +62,7 @@ func (c *Cluster) generateKeyring(m *mgrConfig) error { /* TODO: can we change this ownerref to be the deployment or service? */ s := keyring.GetSecretStore(c.context, c.Namespace, &c.ownerRef) - key, err := s.GenerateKey(m.ResourceName, user, access) + key, err := s.GenerateKey(user, access) if err != nil { return err } diff --git a/pkg/operator/ceph/cluster/rbd/config.go b/pkg/operator/ceph/cluster/rbd/config.go index 4a904a19a661..75c12341a613 100644 --- a/pkg/operator/ceph/cluster/rbd/config.go +++ b/pkg/operator/ceph/cluster/rbd/config.go @@ -46,7 +46,7 @@ func (m *Mirroring) generateKeyring(daemonConfig *daemonConfig) error { access := []string{"mon", "profile rbd-mirror", "osd", "profile rbd"} s := keyring.GetSecretStore(m.context, m.Namespace, &m.ownerRef) - key, err := s.GenerateKey(daemonConfig.ResourceName, user, access) + key, err := s.GenerateKey(user, access) if err != nil { return err } diff --git a/pkg/operator/ceph/config/keyring/store.go b/pkg/operator/ceph/config/keyring/store.go index 5faca88b2c3a..403bb1eb3dda 100644 --- a/pkg/operator/ceph/config/keyring/store.go +++ b/pkg/operator/ceph/config/keyring/store.go @@ -26,7 +26,7 @@ import ( "github.com/rook/rook/pkg/clusterd" "github.com/rook/rook/pkg/daemon/ceph/client" "github.com/rook/rook/pkg/operator/k8sutil" - "k8s.io/api/core/v1" + v1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) @@ -65,7 +65,7 @@ func keyringSecretName(resourceName string) string { // GenerateKey generates a key for a Ceph user with the given access permissions. It returns the key // generated on success. Ceph will always return the most up-to-date key for a daemon, and the key // usually does not change. -func (k *SecretStore) GenerateKey(resourceName, user string, access []string) (string, error) { +func (k *SecretStore) GenerateKey(user string, access []string) (string, error) { // get-or-create-key for the user account key, err := client.AuthGetOrCreateKey(k.context, k.namespace, user, access) if err != nil { diff --git a/pkg/operator/ceph/config/keyring/store_test.go b/pkg/operator/ceph/config/keyring/store_test.go index bc8b114a5da5..a03e27dfcd8d 100644 --- a/pkg/operator/ceph/config/keyring/store_test.go +++ b/pkg/operator/ceph/config/keyring/store_test.go @@ -52,20 +52,20 @@ func TestGenerateKey(t *testing.T) { generateKey = "generatedsecretkey" failGenerateKey = false - k, e := s.GenerateKey("testresource", "testuser", []string{"test", "access"}) + k, e := s.GenerateKey("testuser", []string{"test", "access"}) assert.NoError(t, e) assert.Equal(t, "generatedsecretkey", k) generateKey = "differentsecretkey" failGenerateKey = false - k, e = s.GenerateKey("testresource", "testuser", []string{"test", "access"}) + k, e = s.GenerateKey("testuser", []string{"test", "access"}) assert.NoError(t, e) assert.Equal(t, "differentsecretkey", k) // make sure error on fail generateKey = "failgeneratekey" failGenerateKey = true - _, e = s.GenerateKey("newresource", "newuser", []string{"new", "access"}) + _, e = s.GenerateKey("newuser", []string{"new", "access"}) assert.Error(t, e) } diff --git a/pkg/operator/ceph/file/mds/config.go b/pkg/operator/ceph/file/mds/config.go index 56bb1a2f7410..cf90c826cc1c 100644 --- a/pkg/operator/ceph/file/mds/config.go +++ b/pkg/operator/ceph/file/mds/config.go @@ -46,7 +46,7 @@ func (c *Cluster) generateKeyring(m *mdsConfig, deploymentUID types.UID) error { } s := keyring.GetSecretStore(c.context, c.fs.Namespace, ownerRef) - key, err := s.GenerateKey(m.ResourceName, user, access) + key, err := s.GenerateKey(user, access) if err != nil { return err } diff --git a/pkg/operator/ceph/object/config.go b/pkg/operator/ceph/object/config.go index 0319a265eb05..c684685b9195 100644 --- a/pkg/operator/ceph/object/config.go +++ b/pkg/operator/ceph/object/config.go @@ -20,17 +20,17 @@ import ( "fmt" "path" "strconv" + "strings" cephconfig "github.com/rook/rook/pkg/operator/ceph/config" "github.com/rook/rook/pkg/operator/ceph/config/keyring" cephver "github.com/rook/rook/pkg/operator/ceph/version" - "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) const ( keyringTemplate = ` -[client.radosgw.gateway] +[%s] key = %s caps mon = "allow rw" caps osd = "allow rwx" @@ -87,27 +87,22 @@ func (c *clusterConfig) portString() string { return portString } +func generateCephXUser(name string) string { + user := strings.TrimPrefix(name, AppName) + return "client" + strings.Replace(user, "-", ".", -1) +} + func (c *clusterConfig) generateKeyring(replicationControllerOwnerRef *metav1.OwnerReference) error { - user := "client.radosgw.gateway" + user := generateCephXUser(replicationControllerOwnerRef.Name) /* TODO: this says `osd allow rwx` while template says `osd allow *`; which is correct? */ access := []string{"osd", "allow rwx", "mon", "allow rw"} s := keyring.GetSecretStore(c.context, c.store.Namespace, replicationControllerOwnerRef) - key, err := s.GenerateKey(c.instanceName(), user, access) + key, err := s.GenerateKey(user, access) if err != nil { return err } - // Delete legacy key store for upgrade from Rook v0.9.x to v1.0.x - err = c.context.Clientset.CoreV1().Secrets(c.store.Namespace).Delete(c.instanceName(), &metav1.DeleteOptions{}) - if err != nil { - if errors.IsNotFound(err) { - logger.Debugf("legacy rgw key %s is already removed", c.instanceName()) - } else { - logger.Warningf("legacy rgw key %s could not be removed. %+v", c.instanceName(), err) - } - } - - keyring := fmt.Sprintf(keyringTemplate, key) - return s.CreateOrUpdate(c.instanceName(), keyring) + keyring := fmt.Sprintf(keyringTemplate, user, key) + return s.CreateOrUpdate(replicationControllerOwnerRef.Name, keyring) } diff --git a/pkg/operator/ceph/object/config_test.go b/pkg/operator/ceph/object/config_test.go index c9a5d6e4c0e3..b3a94dc2b52e 100644 --- a/pkg/operator/ceph/object/config_test.go +++ b/pkg/operator/ceph/object/config_test.go @@ -86,3 +86,8 @@ func TestFrontend(t *testing.T) { result = rgwFrontend(cfg.clusterInfo.CephVersion) assert.Equal(t, "beast", result) } + +func TestGenerateCephXUser(t *testing.T) { + fakeUser := generateCephXUser("rook-ceph-rgw-fake-store-fake-user") + assert.Equal(t, "client.fake.store.fake.user", fakeUser) +} diff --git a/pkg/operator/ceph/object/rgw.go b/pkg/operator/ceph/object/rgw.go index 44792592bbcb..18c3bd0a9a4c 100644 --- a/pkg/operator/ceph/object/rgw.go +++ b/pkg/operator/ceph/object/rgw.go @@ -19,16 +19,17 @@ package object import ( "fmt" + "strings" cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1" "github.com/rook/rook/pkg/clusterd" + "github.com/rook/rook/pkg/daemon/ceph/client" cephconfig "github.com/rook/rook/pkg/daemon/ceph/config" "github.com/rook/rook/pkg/operator/ceph/config" "github.com/rook/rook/pkg/operator/ceph/pool" "github.com/rook/rook/pkg/operator/k8sutil" "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/types" ) type clusterConfig struct { @@ -42,6 +43,15 @@ type clusterConfig struct { DataPathMap *config.DataPathMap } +type rgwConfig struct { + ResourceName string + DaemonID string +} + +const ( + oldRgwKeyName = "client.radosgw.gateway" +) + // Start the rgw manager func (c *clusterConfig) createStore() error { return c.createOrUpdate(false) @@ -91,64 +101,145 @@ func (c *clusterConfig) createOrUpdate(update bool) error { } func (c *clusterConfig) startRGWPods(update bool) error { + // backward compatibility, triggered during updates + if c.store.Spec.Gateway.AllNodes { + // log we don't support that anymore + logger.Warningf( + "setting 'AllNodes' to %t is not supported anymore, please use 'instances' instead, removing old DaemonSets if any and replace them with Deployments in object store %s", + c.store.Spec.Gateway.AllNodes, c.store.Name) + } + if c.store.Spec.Gateway.Instances < 1 { + // Set the minimum of at least one instance + logger.Warningf("spec.gateway.instances must be set to at least 1") + c.store.Spec.Gateway.Instances = 1 + } - // if intended to update, remove the old pods so they can be created with the new spec settings - if update { - err := k8sutil.DeleteDeployment(c.context.Clientset, c.store.Namespace, c.instanceName()) - if err != nil { - logger.Warning(err.Error()) + // start a new deployment and scale up + desiredRgwInstances := int(c.store.Spec.Gateway.Instances) + for i := 0; i < desiredRgwInstances; i++ { + daemonLetterID := k8sutil.IndexToName(i) + // Each rgw is id'ed by - + daemonName := fmt.Sprintf("%s-%s", c.store.Name, daemonLetterID) + // resource name is rook-ceph-rgw-- + resourceName := fmt.Sprintf("%s-%s-%s", AppName, c.store.Name, daemonLetterID) + + rgwConfig := &rgwConfig{ + ResourceName: resourceName, + DaemonID: daemonName, } - err = k8sutil.DeleteDaemonset(c.context.Clientset, c.store.Namespace, c.instanceName()) - if err != nil { - logger.Warning(err.Error()) - } - } - // start the deployment or daemonset - var uid types.UID - var controllerType string - if c.store.Spec.Gateway.AllNodes { - daemonSet, err := c.startDaemonset() + deployment, err := c.startDeployment(rgwConfig) if err != nil { return err } - uid = daemonSet.UID - controllerType = "daemonset" - } else { - deployment, err := c.startDeployment() + + resourceControllerOwnerRef := &metav1.OwnerReference{ + UID: deployment.UID, + APIVersion: "v1", + Kind: "deployment", + Name: rgwConfig.ResourceName, + } + + // Generate the keyring after starting the replication controller so that the keyring may use + // the controller as its owner reference; the keyring is deleted with the controller + err = c.generateKeyring(resourceControllerOwnerRef) if err != nil { - return err + return fmt.Errorf("failed to create rgw keyring. %+v", err) } - uid = deployment.UID - controllerType = "deployment" - } - resourceControllerOwnerRef := &metav1.OwnerReference{ - UID: uid, - APIVersion: "v1", - Kind: controllerType, - Name: c.instanceName(), + // Generate the mime.types file after the rep. controller as well for the same reason as keyring + if err := c.generateMimeTypes(resourceControllerOwnerRef); err != nil { + return fmt.Errorf("failed to generate the rgw mime.types config. %+v", err) + } } - // Generate the keyring after starting the replication controller so that the keyring may use - // the controller as its owner reference; the keyring is deleted with the controller - err := c.generateKeyring(resourceControllerOwnerRef) + // scale down scenario + deps, err := k8sutil.GetDeployments(c.context.Clientset, c.store.Namespace, c.storeLabelSelector()) if err != nil { - return fmt.Errorf("failed to create rgw keyring. %+v", err) + logger.Warning("could not get deployments for object store %s (matching label selector '%s'): %+v", c.store.Name, c.storeLabelSelector(), err) } - // Generate the mime.types file after the rep. controller as well for the same reason as keyring - if err := c.generateMimeTypes(resourceControllerOwnerRef); err != nil { - return fmt.Errorf("failed to generate the rgw mime.types config. %+v", err) + currentRgwInstances := int(len(deps.Items)) + if currentRgwInstances > desiredRgwInstances { + logger.Infof("found more rgw deployments %d than desired %d in object store %s, scaling down", currentRgwInstances, c.store.Spec.Gateway.Instances, c.store.Name) + diffCount := currentRgwInstances - desiredRgwInstances + for i := 0; i < diffCount; { + depIDToRemove := currentRgwInstances - 1 + depNameToRemove := fmt.Sprintf("%s-%s-%s", AppName, c.store.Name, k8sutil.IndexToName(depIDToRemove)) + if err := k8sutil.DeleteDeployment(c.context.Clientset, c.store.Namespace, depNameToRemove); err != nil { + logger.Warning("error during deletion of deployment %s resource: %+v", depNameToRemove, err) + } + currentRgwInstances = currentRgwInstances - 1 + i++ + + // Delete the auth key + err = client.AuthDelete(c.context, c.store.Namespace, generateCephXUser(depNameToRemove)) + if err != nil { + logger.Infof("failed to delete rgw key %s. %+v", depNameToRemove, err) + } + } + // verify scale down was successful + deps, err = k8sutil.GetDeployments(c.context.Clientset, c.store.Namespace, c.storeLabelSelector()) + if err != nil { + logger.Warning("could not get deployments for object store %s (matching label selector '%s'): %+v", c.store.Name, c.storeLabelSelector(), err) + } + currentRgwInstances = len(deps.Items) + if currentRgwInstances == desiredRgwInstances { + logger.Infof("successfully scaled down rgw deployments to %d in object store %s", desiredRgwInstances, c.store.Name) + } } + c.deleteLegacyDaemons() return nil } +// deleteLegacyDaemons removes legacy rgw components that might have existed in Rook v1.0 +func (c *clusterConfig) deleteLegacyDaemons() { + // Make a best effort to delete the rgw pods daemonsets + daemons, err := k8sutil.GetDaemonsets(c.context.Clientset, c.store.Namespace, c.storeLabelSelector()) + if err != nil { + logger.Warningf("could not get deployments for object store %s (matching label selector '%s'): %+v", c.store.Name, c.storeLabelSelector(), err) + } + daemonsetNum := len(daemons.Items) + if daemonsetNum > 0 { + for _, d := range daemons.Items { + // Delete any existing daemonset + if err := k8sutil.DeleteDaemonset(c.context.Clientset, c.store.Namespace, d.Name); err != nil { + logger.Errorf("error during deletion of daemonset %s resource: %+v", d.Name, err) + } + } + // Delete legacy rgw key + err = client.AuthDelete(c.context, c.store.Namespace, oldRgwKeyName) + if err != nil { + logger.Infof("failed to delete legacy rgw key %s. %+v", oldRgwKeyName, err) + } + } + + // legacy deployment detection + logger.Debugf("looking for legacy deployment in object store %s", c.store.Name) + deps, err := k8sutil.GetDeployments(c.context.Clientset, c.store.Namespace, c.storeLabelSelector()) + if err != nil { + logger.Warning("could not get deployments for object store %s (matching label selector '%s'): %+v", c.store.Name, c.storeLabelSelector(), err) + } + for _, d := range deps.Items { + if d.Name == c.instanceName() { + logger.Infof("legacy deployment in object store %s found %s", c.store.Name, d.Name) + if err := k8sutil.DeleteDeployment(c.context.Clientset, c.store.Namespace, d.Name); err != nil { + logger.Warning("error during deletion of deployment %s resource: %+v", d.Name, err) + } + // Delete legacy rgw key + err = client.AuthDelete(c.context, c.store.Namespace, oldRgwKeyName) + if err != nil { + logger.Infof("failed to delete legacy rgw key %s. %+v", oldRgwKeyName, err) + } + } + } +} + // Delete the object store. // WARNING: This is a very destructive action that deletes all metadata and data pools. func (c *clusterConfig) deleteStore() error { - // check if the object store exists + // check if the object store exists exists, err := c.storeExists() if err != nil { return fmt.Errorf("failed to detect if there is an object store to delete. %+v", err) @@ -170,22 +261,33 @@ func (c *clusterConfig) deleteStore() error { logger.Warningf("failed to delete rgw service. %+v", err) } - // Make a best effort to delete the rgw pods - err = k8sutil.DeleteDeployment(c.context.Clientset, c.store.Namespace, c.instanceName()) + // Make a best effort to delete the rgw pods deployments + deps, err := k8sutil.GetDeployments(c.context.Clientset, c.store.Namespace, c.storeLabelSelector()) if err != nil { - logger.Warning(err.Error()) + logger.Warning("could not get deployments for object store %s (matching label selector '%s'): %+v", c.store.Namespace, c.storeLabelSelector(), err) } - err = k8sutil.DeleteDaemonset(c.context.Clientset, c.store.Namespace, c.instanceName()) - if err != nil { - logger.Warning(err.Error()) + for _, d := range deps.Items { + if err := k8sutil.DeleteDeployment(c.context.Clientset, c.store.Namespace, d.Name); err != nil { + logger.Warning("error during deletion of deployment %s resource: %+v", d.Name, err) + } } - // Delete the rgw keyring + // Delete the rgw config map keyrings err = c.context.Clientset.CoreV1().Secrets(c.store.Namespace).Delete(c.instanceName(), options) if err != nil && !errors.IsNotFound(err) { logger.Warningf("failed to delete rgw secret. %+v", err) } + // Delete rgw CephX keys + for i := 0; i < int(c.store.Spec.Gateway.Instances); i++ { + daemonLetterID := k8sutil.IndexToName(i) + keyName := fmt.Sprintf("client.%s.%s", strings.Replace(c.store.Name, "-", ".", -1), daemonLetterID) + err = client.AuthDelete(c.context, c.store.Namespace, keyName) + if err != nil { + return err + } + } + // Delete the realm and pools objContext := NewContext(c.context, c.store.Name, c.store.Namespace) err = deleteRealmAndPools(objContext) @@ -199,8 +301,9 @@ func (c *clusterConfig) deleteStore() error { // Check if the object store exists depending on either the deployment or the daemonset func (c *clusterConfig) storeExists() (bool, error) { - _, err := c.context.Clientset.AppsV1().Deployments(c.store.Namespace).Get(c.instanceName(), metav1.GetOptions{}) - if err == nil { + d, err := k8sutil.GetDeployments(c.context.Clientset, c.store.Namespace, c.storeLabelSelector()) + // k8sutil.GetDeployments can return an empty list! + if err == nil && len(d.Items) != 0 { // the deployment was found return true, nil } @@ -208,8 +311,9 @@ func (c *clusterConfig) storeExists() (bool, error) { return false, err } - _, err = c.context.Clientset.AppsV1().DaemonSets(c.store.Namespace).Get(c.instanceName(), metav1.GetOptions{}) - if err == nil { + dd, err := k8sutil.GetDaemonsets(c.context.Clientset, c.store.Namespace, c.storeLabelSelector()) + // k8sutil.GetDaemonsets can return an empty list! + if err == nil && len(dd.Items) != 0 { // the daemonset was found return true, nil } @@ -225,6 +329,10 @@ func (c *clusterConfig) instanceName() string { return fmt.Sprintf("%s-%s", AppName, c.store.Name) } +func (c *clusterConfig) storeLabelSelector() string { + return fmt.Sprintf("rook_object_store=%s", c.store.Name) +} + // Validate the object store arguments func validateStore(context *clusterd.Context, s cephv1.CephObjectStore) error { if s.Name == "" { diff --git a/pkg/operator/ceph/object/rgw_test.go b/pkg/operator/ceph/object/rgw_test.go index 4acc2f03bb1e..c47abb140aaf 100644 --- a/pkg/operator/ceph/object/rgw_test.go +++ b/pkg/operator/ceph/object/rgw_test.go @@ -27,7 +27,6 @@ import ( testop "github.com/rook/rook/pkg/operator/test" exectest "github.com/rook/rook/pkg/util/exec/test" "github.com/stretchr/testify/assert" - "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/kubernetes/fake" ) @@ -48,6 +47,7 @@ func TestStartRGW(t *testing.T) { info := testop.CreateConfigDir(1) context := &clusterd.Context{Clientset: clientset, Executor: executor, ConfigDir: configDir} store := simpleStore() + store.Spec.Gateway.Instances = 1 version := "v1.1.0" data := cephconfig.NewStatelessDaemonDataPathMap(cephconfig.RgwType, "my-fs", "rook-ceph", "/var/lib/rook/") @@ -56,32 +56,18 @@ func TestStartRGW(t *testing.T) { err := c.createStore() assert.Nil(t, err) - validateStart(t, c, clientset, false) + validateStart(t, c, clientset) // starting again should update the pods with the new settings - c.store.Spec.Gateway.AllNodes = true err = c.updateStore() assert.Nil(t, err) - - validateStart(t, c, clientset, true) } -func validateStart(t *testing.T, c *clusterConfig, clientset *fake.Clientset, allNodes bool) { - if !allNodes { - r, err := clientset.AppsV1().Deployments(c.store.Namespace).Get(c.instanceName(), metav1.GetOptions{}) - assert.Nil(t, err) - assert.Equal(t, c.instanceName(), r.Name) - - _, err = clientset.AppsV1().DaemonSets(c.store.Namespace).Get(c.instanceName(), metav1.GetOptions{}) - assert.True(t, errors.IsNotFound(err)) - } else { - r, err := clientset.AppsV1().DaemonSets(c.store.Namespace).Get(c.instanceName(), metav1.GetOptions{}) - assert.Nil(t, err) - assert.Equal(t, c.instanceName(), r.Name) - - _, err = clientset.AppsV1().Deployments(c.store.Namespace).Get(c.instanceName(), metav1.GetOptions{}) - assert.True(t, errors.IsNotFound(err)) - } +func validateStart(t *testing.T, c *clusterConfig, clientset *fake.Clientset) { + rgwName := c.instanceName() + "-a" + r, err := clientset.AppsV1().Deployments(c.store.Namespace).Get(rgwName, metav1.GetOptions{}) + assert.Nil(t, err) + assert.Equal(t, rgwName, r.Name) s, err := clientset.CoreV1().Services(c.store.Namespace).Get(c.instanceName(), metav1.GetOptions{}) assert.Nil(t, err) diff --git a/pkg/operator/ceph/object/spec.go b/pkg/operator/ceph/object/spec.go index 84b26c188d25..0404382f44d9 100644 --- a/pkg/operator/ceph/object/spec.go +++ b/pkg/operator/ceph/object/spec.go @@ -29,10 +29,11 @@ import ( "k8s.io/apimachinery/pkg/util/intstr" ) -func (c *clusterConfig) startDeployment() (*apps.Deployment, error) { +func (c *clusterConfig) startDeployment(rgwConfig *rgwConfig) (*apps.Deployment, error) { + replicas := int32(1) d := &apps.Deployment{ ObjectMeta: metav1.ObjectMeta{ - Name: c.instanceName(), + Name: rgwConfig.ResourceName, Namespace: c.store.Namespace, Labels: c.getLabels(), }, @@ -40,8 +41,8 @@ func (c *clusterConfig) startDeployment() (*apps.Deployment, error) { Selector: &metav1.LabelSelector{ MatchLabels: c.getLabels(), }, - Template: c.makeRGWPodSpec(), - Replicas: &c.store.Spec.Gateway.Instances, + Template: c.makeRGWPodSpec(rgwConfig), + Replicas: &replicas, Strategy: apps.DeploymentStrategy{ Type: apps.RecreateDeploymentStrategyType, }, @@ -79,55 +80,15 @@ func (c *clusterConfig) startDeployment() (*apps.Deployment, error) { return deployment, err } -func (c *clusterConfig) startDaemonset() (*apps.DaemonSet, error) { - d := &apps.DaemonSet{ - ObjectMeta: metav1.ObjectMeta{ - Name: c.instanceName(), - Namespace: c.store.Namespace, - Labels: c.getLabels(), - }, - Spec: apps.DaemonSetSpec{ - Selector: &metav1.LabelSelector{ - MatchLabels: c.getLabels(), - }, - UpdateStrategy: apps.DaemonSetUpdateStrategy{ - Type: apps.RollingUpdateDaemonSetStrategyType, - }, - Template: c.makeRGWPodSpec(), - }, - } - k8sutil.AddRookVersionLabelToDaemonSet(d) - opspec.AddCephVersionLabelToDaemonSet(c.clusterInfo.CephVersion, d) - k8sutil.SetOwnerRefs(c.context.Clientset, c.store.Namespace, &d.ObjectMeta, c.ownerRefs) - - logger.Debugf("starting rgw daemonset: %+v", d) - daemonSet, err := c.context.Clientset.AppsV1().DaemonSets(c.store.Namespace).Create(d) - if err != nil { - if !errors.IsAlreadyExists(err) { - return nil, fmt.Errorf("failed to create rgw daemonset %s: %+v", c.instanceName(), err) - } - logger.Infof("daemonset for rgw %s already exists. updating if needed", c.instanceName()) - // There may be a *lot* of rgws, and they are stateless, so don't bother waiting until the - // entire daemonset is updated to move on. - // TODO: is the above statement safe to assume? - // TODO: Are there any steps for RGW that need to happen before the daemons upgrade? - daemonSet, err = c.context.Clientset.AppsV1().DaemonSets(c.store.Namespace).Update(d) - if err != nil { - return nil, fmt.Errorf("failed to update rgw daemonset %s. %+v", c.instanceName(), err) - } - } - return daemonSet, nil -} - -func (c *clusterConfig) makeRGWPodSpec() v1.PodTemplateSpec { +func (c *clusterConfig) makeRGWPodSpec(rgwConfig *rgwConfig) v1.PodTemplateSpec { podSpec := v1.PodSpec{ InitContainers: []v1.Container{}, Containers: []v1.Container{ - c.makeDaemonContainer(), + c.makeDaemonContainer(rgwConfig), }, RestartPolicy: v1.RestartPolicyAlways, Volumes: append( - opspec.DaemonVolumes(c.DataPathMap, c.instanceName()), + opspec.DaemonVolumes(c.DataPathMap, rgwConfig.ResourceName), c.mimeTypesVolume(), ), HostNetwork: c.hostNetwork, @@ -154,7 +115,7 @@ func (c *clusterConfig) makeRGWPodSpec() v1.PodTemplateSpec { podTemplateSpec := v1.PodTemplateSpec{ ObjectMeta: metav1.ObjectMeta{ - Name: c.instanceName(), + Name: rgwConfig.ResourceName, Labels: c.getLabels(), }, Spec: podSpec, @@ -164,7 +125,7 @@ func (c *clusterConfig) makeRGWPodSpec() v1.PodTemplateSpec { return podTemplateSpec } -func (c *clusterConfig) makeDaemonContainer() v1.Container { +func (c *clusterConfig) makeDaemonContainer(rgwConfig *rgwConfig) v1.Container { // start the rgw daemon in the foreground container := v1.Container{ @@ -177,13 +138,13 @@ func (c *clusterConfig) makeDaemonContainer() v1.Container { append( opspec.DaemonFlags(c.clusterInfo, c.store.Name), "--foreground", - "--name=client.radosgw.gateway", + cephconfig.NewFlag("name", generateCephXUser(rgwConfig.ResourceName)), cephconfig.NewFlag("host", opspec.ContainerEnvVarReference("POD_NAME")), cephconfig.NewFlag("rgw-mime-types-file", mimeTypesMountPath()), ), c.defaultSettings().GlobalFlags()..., // use default settings as flags until mon kv store supported ), VolumeMounts: append( - opspec.DaemonVolumeMounts(c.DataPathMap, c.instanceName()), + opspec.DaemonVolumeMounts(c.DataPathMap, rgwConfig.ResourceName), c.mimeTypesVolumeMount(), ), Env: opspec.DaemonEnvVars(c.cephVersion.Image), diff --git a/pkg/operator/ceph/object/spec_test.go b/pkg/operator/ceph/object/spec_test.go index 85c26f3507b5..4e3cc9045742 100644 --- a/pkg/operator/ceph/object/spec_test.go +++ b/pkg/operator/ceph/object/spec_test.go @@ -17,6 +17,7 @@ limitations under the License. package object import ( + "fmt" "testing" cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1" @@ -56,7 +57,12 @@ func TestPodSpecs(t *testing.T) { DataPathMap: data, } - s := c.makeRGWPodSpec() + resourceName := fmt.Sprintf("%s-%s", AppName, c.store.Name) + rgwConfig := &rgwConfig{ + ResourceName: resourceName, + } + + s := c.makeRGWPodSpec(rgwConfig) podTemplate := cephtest.NewPodTemplateSpecTester(t, &s) podTemplate.RunFullSuite(cephconfig.RgwType, "default", "rook-ceph-rgw", "mycluster", "ceph/ceph:myversion", @@ -91,7 +97,11 @@ func TestSSLPodSpec(t *testing.T) { } c.hostNetwork = true - s := c.makeRGWPodSpec() + resourceName := fmt.Sprintf("%s-%s", AppName, c.store.Name) + rgwConfig := &rgwConfig{ + ResourceName: resourceName, + } + s := c.makeRGWPodSpec(rgwConfig) podTemplate := cephtest.NewPodTemplateSpecTester(t, &s) podTemplate.RunFullSuite(cephconfig.RgwType, "default", "rook-ceph-rgw", "mycluster", "ceph/ceph:myversion", diff --git a/pkg/operator/k8sutil/daemonset.go b/pkg/operator/k8sutil/daemonset.go index 8f5fa9e0f099..18f6981676ad 100644 --- a/pkg/operator/k8sutil/daemonset.go +++ b/pkg/operator/k8sutil/daemonset.go @@ -19,8 +19,8 @@ package k8sutil import ( "fmt" - "k8s.io/api/apps/v1" apps "k8s.io/api/apps/v1" + v1 "k8s.io/api/apps/v1" k8serrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/kubernetes" @@ -63,3 +63,15 @@ func AddRookVersionLabelToDaemonSet(d *v1.DaemonSet) { } addRookVersionLabel(d.Labels) } + +// GetDaemonsets returns a list of daemonsets names labels matching a given selector +// example of a label selector might be "app=rook-ceph-mon, mon!=b" +// more: https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/ +func GetDaemonsets(clientset kubernetes.Interface, namespace, labelSelector string) (*apps.DaemonSetList, error) { + listOptions := metav1.ListOptions{LabelSelector: labelSelector} + daemonsets, err := clientset.AppsV1().DaemonSets(namespace).List(listOptions) + if err != nil { + return nil, fmt.Errorf("failed to list deployments with labelSelector %s: %v", labelSelector, err) + } + return daemonsets, nil +}