Add replication kuttl test

This is basically a full deployment test, ensuring all parts work
together as expected:

- Create a 3x replicated deployment
- Upload some data and ensure it is accessible
- Forcefully delete one replica on the disk
- Ensure replication fixes this and gets back to 3 replicas

It uses the swift-dispersion tools and user, which creates objects
distributed across the nodes and fetches them later again to ensure
they are replicated properly.

This kuttl uses all important parts of the Swift deployment:
- Swift proxy using Keystone for authentication
- network connectivity between proxy and storage services
- replication between storage services

The test takes approx 2.5 minutes of time to finish.

tests/kuttl/common/00-deps.yaml

@@ -0,0 +1,7 @@
+apiVersion: kuttl.dev/v1beta1
+kind: TestStep
+commands:
+  - script: |
+      if ! [[ `oc get crd openstackdataplanenodesets.dataplane.openstack.org` ]]; then
+        oc apply -f ../../deps/dataplane.openstack.org_openstackdataplanenodesets.yaml
+      fi

tests/kuttl/common/scripts/check_ring_rebalance_output.sh

@@ -5,5 +5,5 @@ LOGS=$(oc logs -n $NAMESPACE job/swift-ring-rebalance)
 PODS=$(oc get pods -n $NAMESPACE -l component=swift-storage -o name | cut -f 2 -d "/")
 
 for pod in $PODS; do
-	echo $LOGS | grep -q $pod || exit 1
+    echo $LOGS | grep -q $pod || exit 1
 done

tests/kuttl/tests/basic-deploy/00-deps.yaml

@@ -0,0 +1 @@
+../../common/00-deps.yaml

tests/kuttl/tests/basic-deploy_tls/00-deps.yaml

@@ -0,0 +1 @@
+../../common/00-deps.yaml

tests/kuttl/tests/replication/00-deps.yaml

@@ -0,0 +1 @@
+../../common/00-deps.yaml

tests/kuttl/tests/replication/01-assert-deploy-swift.yaml

@@ -0,0 +1,53 @@
+---
+apiVersion: swift.openstack.org/v1beta1
+kind: Swift
+metadata:
+  name: swift
+spec:
+  swiftRing:
+    ringReplicas: 3
+  swiftStorage:
+    storageClass: local-storage
+    replicas: 3
+  swiftProxy:
+    replicas: 1
+    passwordSelectors:
+      service: SwiftPassword
+status:
+  conditions:
+  - message: Setup complete
+    reason: Ready
+    status: "True"
+    type: Ready
+  - message: " Memcached instance has been provisioned"
+    reason: Ready
+    status: "True"
+    type: MemcachedReady
+  - message: RoleBinding created
+    reason: Ready
+    status: "True"
+    type: RoleBindingReady
+  - message: Role created
+    reason: Ready
+    status: "True"
+    type: RoleReady
+  - message: ServiceAccount created
+    reason: Ready
+    status: "True"
+    type: ServiceAccountReady
+  - message: Service config create completed
+    reason: Ready
+    status: "True"
+    type: ServiceConfigReady
+  - message: Setup complete
+    reason: Ready
+    status: "True"
+    type: SwiftProxyReady
+  - message: Setup complete
+    reason: Ready
+    status: "True"
+    type: SwiftRingReady
+  - message: Setup complete
+    reason: Ready
+    status: "True"
+    type: SwiftStorageReady

tests/kuttl/tests/replication/01-deploy-swift.yaml

@@ -0,0 +1,6 @@
+apiVersion: kuttl.dev/v1beta1
+kind: TestStep
+commands:
+  - script: |
+      cp ../../../../config/samples/swift_v1beta1_swift.yaml deploy
+      oc kustomize deploy | oc apply -n $NAMESPACE -f -

tests/kuttl/tests/replication/02-assert-store-data.yaml

@@ -0,0 +1,6 @@
+---
+apiVersion: kuttl.dev/v1beta1
+kind: TestAssert
+commands:
+  - script: |
+      oc debug -n $NAMESPACE --keep-labels=true job/swift-ring-rebalance -- /bin/sh -c 'swift-ring-tool get && swift-dispersion-report --object-only' | grep '100.00% of object copies found'

tests/kuttl/tests/replication/02-store-data.yaml

@@ -0,0 +1,6 @@
+---
+apiVersion: kuttl.dev/v1beta1
+kind: TestStep
+commands:
+  - script: |
+      oc debug -n $NAMESPACE --keep-labels=true job/swift-ring-rebalance -- /bin/sh -c 'swift-ring-tool get && swift-dispersion-populate --object-only'

tests/kuttl/tests/replication/03-assert-force-replication.yaml

@@ -0,0 +1,6 @@
+---
+apiVersion: kuttl.dev/v1beta1
+kind: TestAssert
+commands:
+  - script: |
+      oc debug -n $NAMESPACE --keep-labels=true job/swift-ring-rebalance -- /bin/sh -c 'swift-ring-tool get && swift-dispersion-report --object-only' | grep '100.00% of object copies found'

tests/kuttl/tests/replication/03-force-replication.yaml

@@ -0,0 +1,6 @@
+---
+apiVersion: kuttl.dev/v1beta1
+kind: TestStep
+commands:
+  - script: |
+      oc rsh -n $NAMESPACE pod/swift-storage-0 /bin/sh -c 'rm -rf /srv/node/d1/* || true'

tests/kuttl/tests/replication/04-cleanup.yaml

@@ -0,0 +1,11 @@
+apiVersion: kuttl.dev/v1beta1
+kind: TestStep
+delete:
+- apiVersion: swift.openstack.org/v1beta1
+  kind: Swift
+  name: swift
+commands:
+- script: |
+    oc delete --ignore-not-found=true -n $NAMESPACE pvc srv-swift-storage-0
+    oc delete --ignore-not-found=true -n $NAMESPACE pvc srv-swift-storage-1
+    oc delete --ignore-not-found=true -n $NAMESPACE pvc srv-swift-storage-2

tests/kuttl/tests/replication/04-errors-cleanup.yaml

@@ -0,0 +1,15 @@
+apiVersion: swift.openstack.org/v1beta1
+kind: Swift
+metadata:
+  name: swift
+---
+apiVersion: v1
+kind: Pod
+metadata:
+  labels:
+    app.kubernetes.io/name: SwiftProxy
+---
+apiVersion: v1
+kind: Pod
+metadata:
+  name: swift-storage-0

tests/kuttl/tests/replication/deploy/kustomization.yaml

@@ -0,0 +1,14 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+resources:
+- ./swift_v1beta1_swift.yaml
+patches:
+- patch: |-
+    - op: replace
+      path: /spec/swiftRing/ringReplicas
+      value: 3
+    - op: replace
+      path: /spec/swiftStorage/replicas
+      value: 3
+  target:
+    kind: Swift