ceph: refactor rgw bootstrap

This commit does multiple things:

* remove support for AllNodes where we would deploy one rgw per node on
all the nodes.
* a transition path is implemented in the code so that if someone has an
existing deployment, daemonsets will be removed and replaced by an
deployments.
* when using "instances", each rgw deployed has its own key which makes
Ceph reporting the exact number of rgw running, see:

```
[root@rook-ceph-operator-775cf575c5-bh4sr /]# ceph -s
  cluster:
    id:     611fcf39-0669-4864-9a12-debb35c0397a
    health: HEALTH_OK

  services:
    mon: 3 daemons, quorum a,b,c (age 12h)
    mgr: a(active, since 12h)
    osd: 3 osds: 3 up (since 12h), 3 in (since 12h)
    rgw: 3 daemons active (my.store.a, my.store.b, my.store.c)

  data:
    pools:   6 pools, 600 pgs
    objects: 235 objects, 3.8 KiB
    usage:   3.0 GiB used, 84 GiB / 87 GiB avail
    pgs:     600 active+clean
```

Closes: rook#2474, rook#2957 and rook#3245
Signed-off-by: Sébastien Han <[email protected]>

Documentation/ceph-object-store-crd.md

@@ -36,7 +36,6 @@ spec:
     port: 80
     securePort:
     instances: 1
-    allNodes: false
     # A key/value list of annotations
     annotations:
     #  key: value
@@ -85,8 +84,7 @@ The gateway settings correspond to the RGW daemon settings.
 - `sslCertificateRef`: If the certificate is not specified, SSL will not be configured. If specified, this is the name of the Kubernetes secret that contains the SSL certificate to be used for secure connections to the object store. Rook will look in the secret provided at the `cert` key name. The value of the `cert` key must be in the format expected by the [RGW service](http://docs.ceph.com/docs/master/install/install-ceph-gateway/#using-ssl-with-civetweb): "The server key, server certificate, and any other CA or intermediate certificates be supplied in one file. Each of these items must be in pem form."
 - `port`: The port on which the RGW pods and the RGW service will be listening (not encrypted).
 - `securePort`: The secure port on which RGW pods will be listening. An SSL certificate must be specified.
-- `instances`: The number of pods that will be started to load balance this object store. Ignored if `allNodes` is true.
-- `allNodes`: Whether RGW pods should be started on all nodes. If true, a daemonset is created. If false, `instances` must be set.
+- `instances`: The number of pods that will be started to load balance this object store.
 - `annotations`: Key value pair list of annotations to add.
 - `placement`: The Kubernetes placement settings to determine where the RGW pods should be started in the cluster.
 - `resources`: Set resource requests/limits for the Gateway Pod(s), see [Resource Requirements/Limits](ceph-cluster-crd.md#resource-requirementslimits).

Documentation/ceph-object.md

@@ -42,7 +42,6 @@ spec:
     port: 80
     securePort:
     instances: 1
-    allNodes: false
 ```
 
 When the object store is created the Rook operator will create all the pools and other resources necessary to start the service. This may take a minute to complete.

PendingReleaseNotes.md

@@ -13,6 +13,7 @@ an example usage
 - Rgw pods have liveness probe enabled
 - Rgw is now configured with the Beast backend as of the Nautilus release
 - OSD: newly updated cluster from 0.9 to 1.0.3 and thus Ceph Nautilus will have their OSDs allowing new features for Nautilus
+- Rgw instances have their own key and thus are properly reflected in the Ceph status
 
 ## Breaking Changes
 
@@ -24,4 +25,11 @@ an example usage
 
 ## Deprecations
 
+### Ceph
+
+- For rgw, when deploying an object store with `object.yaml`, using `allNodes` is not supported anymore, a transition path has been implemented in the code though.
+So if you were using `allNodes: true`, Rook will replace each daemonset with a deployment (one for one replacement) gradually.
+This operation will be triggered on an update or when a new version of the operator is deployed.
+Once complete, it is expected that you edit your object CR with `kubectl -n rook-ceph edit cephobjectstore.ceph.rook.io/my-store` and set `allNodes: false` and `instances` with the current number of rgw instances.
+
 ### <Storage Provider>

cluster/examples/kubernetes/ceph/object-ec.yaml

@@ -31,10 +31,8 @@ spec:
     port: 80
     # The port that RGW pods will listen on (https). An ssl certificate is required.
     securePort:
-    # The number of pods in the rgw deployment (ignored if allNodes=true)
+    # The number of pods in the rgw deployment
     instances: 1
-    # Whether the rgw pods should be deployed on all nodes as a daemonset
-    allNodes: false
     # The affinity rules to apply to the rgw deployment or daemonset.
     placement:
     #  nodeAffinity:

cluster/examples/kubernetes/ceph/object-openshift.yaml

@@ -30,10 +30,8 @@ spec:
     port: 8080
     # The port that RGW pods will listen on (https). An ssl certificate is required.
     securePort:
-    # The number of pods in the rgw deployment (ignored if allNodes=true)
+    # The number of pods in the rgw deployment
     instances: 1
-    # Whether the rgw pods should be deployed on all nodes as a daemonset
-    allNodes: false
     # The affinity rules to apply to the rgw deployment or daemonset.
     placement:
     #  nodeAffinity:

cluster/examples/kubernetes/ceph/object-test.yaml

@@ -20,4 +20,3 @@ spec:
     port: 80
     securePort:
     instances: 1
-    allNodes: false

cluster/examples/kubernetes/ceph/object.yaml

@@ -30,10 +30,8 @@ spec:
     port: 80
     # The port that RGW pods will listen on (https). An ssl certificate is required.
     securePort:
-    # The number of pods in the rgw deployment (ignored if allNodes=true)
+    # The number of pods in the rgw deployment
     instances: 1
-    # Whether the rgw pods should be deployed on all nodes as a daemonset
-    allNodes: false
     # The affinity rules to apply to the rgw deployment or daemonset.
     placement:
     #  nodeAffinity:

cluster/olm/ceph/assemble/metadata-common.yaml

@@ -151,7 +151,6 @@ metadata:
               "port": 8080,
               "securePort": null,
               "instances": 1,
-              "allNodes": false,
               "placement": null,
               "annotations": null,
               "resources": null

design/object-store.md

@@ -34,7 +34,6 @@ spec:
     port: 80
     securePort: 443
     instances: 3
-    allNodes: false
 ```
 
 Now create the object store.
@@ -82,8 +81,7 @@ The gateway settings correspond to the RGW service.
 - `sslCertificateRef`: If specified, this is the name of the Kubernetes secret that contains the SSL certificate to be used for secure connections to the object store. The secret must be in the same namespace as the Rook cluster. Rook will look in the secret provided at the `cert` key name. The value of the `cert` key must be in the format expected by the [RGW service](http://docs.ceph.com/docs/master/install/install-ceph-gateway/#using-ssl-with-civetweb): "The server key, server certificate, and any other CA or intermediate certificates be supplied in one file. Each of these items must be in pem form." If the certificate is not specified, SSL will not be configured.
 - `port`: The service port where the RGW service will be listening (http)
 - `securePort`: The service port where the RGW service will be listening (https)
-- `instances`: The number of RGW pods that will be started for this object store (ignored if allNodes=true)
-- `allNodes`: Whether all nodes in the cluster should run RGW as a daemonset
+- `instances`: The number of RGW pods that will be started for this object store
 - `placement`: The rgw pods can be given standard Kubernetes placement restrictions with `nodeAffinity`, `tolerations`, `podAffinity`, and `podAntiAffinity` similar to placement defined for daemons configured by the [cluster CRD](/cluster/examples/kubernetes/ceph/cluster.yaml).
 
 The RGW service can be configured to listen on both http and https by specifying both `port` and `securePort`.
@@ -95,7 +93,6 @@ The RGW service can be configured to listen on both http and https by specifying
     port: 80
     securePort: 443
     instances: 1
-    allNodes: false
 ```
 
 ### Realms, Zone Groups, and Zones

pkg/operator/ceph/cluster/mgr/config.go

@@ -62,7 +62,7 @@ func (c *Cluster) generateKeyring(m *mgrConfig) error {
 	/* TODO: can we change this ownerref to be the deployment or service? */
 	s := keyring.GetSecretStore(c.context, c.Namespace, &c.ownerRef)
 
-	key, err := s.GenerateKey(m.ResourceName, user, access)
+	key, err := s.GenerateKey(user, access)
 	if err != nil {
 		return err
 	}

pkg/operator/ceph/cluster/rbd/config.go

@@ -46,7 +46,7 @@ func (m *Mirroring) generateKeyring(daemonConfig *daemonConfig) error {
 	access := []string{"mon", "profile rbd-mirror", "osd", "profile rbd"}
 	s := keyring.GetSecretStore(m.context, m.Namespace, &m.ownerRef)
 
-	key, err := s.GenerateKey(daemonConfig.ResourceName, user, access)
+	key, err := s.GenerateKey(user, access)
 	if err != nil {
 		return err
 	}

pkg/operator/ceph/config/keyring/store.go

@@ -26,7 +26,7 @@ import (
 	"github.com/rook/rook/pkg/clusterd"
 	"github.com/rook/rook/pkg/daemon/ceph/client"
 	"github.com/rook/rook/pkg/operator/k8sutil"
-	"k8s.io/api/core/v1"
+	v1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 )
@@ -65,7 +65,7 @@ func keyringSecretName(resourceName string) string {
 // GenerateKey generates a key for a Ceph user with the given access permissions. It returns the key
 // generated on success. Ceph will always return the most up-to-date key for a daemon, and the key
 // usually does not change.
-func (k *SecretStore) GenerateKey(resourceName, user string, access []string) (string, error) {
+func (k *SecretStore) GenerateKey(user string, access []string) (string, error) {
 	// get-or-create-key for the user account
 	key, err := client.AuthGetOrCreateKey(k.context, k.namespace, user, access)
 	if err != nil {

pkg/operator/ceph/config/keyring/store_test.go

@@ -52,20 +52,20 @@ func TestGenerateKey(t *testing.T) {
 
 	generateKey = "generatedsecretkey"
 	failGenerateKey = false
-	k, e := s.GenerateKey("testresource", "testuser", []string{"test", "access"})
+	k, e := s.GenerateKey("testuser", []string{"test", "access"})
 	assert.NoError(t, e)
 	assert.Equal(t, "generatedsecretkey", k)
 
 	generateKey = "differentsecretkey"
 	failGenerateKey = false
-	k, e = s.GenerateKey("testresource", "testuser", []string{"test", "access"})
+	k, e = s.GenerateKey("testuser", []string{"test", "access"})
 	assert.NoError(t, e)
 	assert.Equal(t, "differentsecretkey", k)
 
 	// make sure error on fail
 	generateKey = "failgeneratekey"
 	failGenerateKey = true
-	_, e = s.GenerateKey("newresource", "newuser", []string{"new", "access"})
+	_, e = s.GenerateKey("newuser", []string{"new", "access"})
 	assert.Error(t, e)
 }
 

pkg/operator/ceph/file/mds/config.go

@@ -46,7 +46,7 @@ func (c *Cluster) generateKeyring(m *mdsConfig, deploymentUID types.UID) error {
 	}
 	s := keyring.GetSecretStore(c.context, c.fs.Namespace, ownerRef)
 
-	key, err := s.GenerateKey(m.ResourceName, user, access)
+	key, err := s.GenerateKey(user, access)
 	if err != nil {
 		return err
 	}

pkg/operator/ceph/object/config.go

@@ -20,6 +20,7 @@ import (
 	"fmt"
 	"path"
 	"strconv"
+	"strings"
 
 	cephconfig "github.com/rook/rook/pkg/operator/ceph/config"
 	"github.com/rook/rook/pkg/operator/ceph/config/keyring"
@@ -29,7 +30,7 @@ import (
 
 const (
 	keyringTemplate = `
-[client.radosgw.gateway]
+[%s]
 key = %s
 caps mon = "allow rw"
 caps osd = "allow rwx"
@@ -86,17 +87,22 @@ func (c *clusterConfig) portString() string {
 	return portString
 }
 
+func generateCephXUser(name string) string {
+	user := strings.TrimPrefix(name, AppName)
+	return "client" + strings.Replace(user, "-", ".", -1)
+}
+
 func (c *clusterConfig) generateKeyring(replicationControllerOwnerRef *metav1.OwnerReference) error {
-	user := "client.radosgw.gateway"
+	user := generateCephXUser(replicationControllerOwnerRef.Name)
 	/* TODO: this says `osd allow rwx` while template says `osd allow *`; which is correct? */
 	access := []string{"osd", "allow rwx", "mon", "allow rw"}
 	s := keyring.GetSecretStore(c.context, c.store.Namespace, replicationControllerOwnerRef)
 
-	key, err := s.GenerateKey(c.instanceName(), user, access)
+	key, err := s.GenerateKey(user, access)
 	if err != nil {
 		return err
 	}
 
-	keyring := fmt.Sprintf(keyringTemplate, key)
-	return s.CreateOrUpdate(c.instanceName(), keyring)
+	keyring := fmt.Sprintf(keyringTemplate, user, key)
+	return s.CreateOrUpdate(replicationControllerOwnerRef.Name, keyring)
 }

pkg/operator/ceph/object/config_test.go

@@ -86,3 +86,8 @@ func TestFrontend(t *testing.T) {
 	result = rgwFrontend(cfg.clusterInfo.CephVersion)
 	assert.Equal(t, "beast", result)
 }
+
+func TestGenerateCephXUser(t *testing.T) {
+	fakeUser := generateCephXUser("rook-ceph-rgw-fake-store-fake-user")
+	assert.Equal(t, "client.fake.store.fake.user", fakeUser)
+}