Merge branch 'improvement/ZENKO-4856' into w/2.7/improvement/ZENKO-4856

.devcontainer/README.md

@@ -6,7 +6,7 @@
 To run the CTST tests in the codespace, simply head to `.github/script/end2end/` and run `run-e2e-ctst.sh` script.
 
 ```bash
-    cd .github/script/end2end/
+    cd .github/scripts/end2end/
     bash run-e2e-ctst.sh
 ```
 

.devcontainer/devcontainer.json

@@ -74,13 +74,17 @@
                     "permissions": {
                         "contents": "read"
                     }
+                },
+                "scality/zenko-drctl": {
+                    "permissions": {
+                        "contents": "read"
+                    }
                 }
             }
         }
     },
     "containerEnv": {
         "ZENKO_MONGODB_DATABASE": "zenko-database",
-        "ZENKO_MONGODB_SHARDED": "true" 
         },
     // Use 'postCreateCommand' to run commands after the container is created.
     "postCreateCommand": "bash .devcontainer/setup.sh"

.devcontainer/setup.sh

@@ -1,22 +1,22 @@
 #!/bin/bash
 
+set -e
+
 env_variables=$(yq eval '.env | to_entries | .[] | .key + "=" + .value' .github/workflows/end2end.yaml | sed 's/\${{[^}]*}}//g') && export $env_variables
 export GIT_ACCESS_TOKEN=${GITHUB_TOKEN}
 export E2E_IMAGE_TAG=latest
 
-array_length=`yq ".runs.steps | length - 1" .github/actions/deploy/action.yaml`
+array_length=$(yq ".runs.steps | length - 1" .github/actions/deploy/action.yaml)
 for i in $(seq 0 $array_length); do
-    step=`yq ".runs.steps[$i]" .github/actions/deploy/action.yaml`
-    working_dir=`yq ".runs.steps[$i].working-directory" .github/actions/deploy/action.yaml`
-    run_command=`yq ".runs.steps[$i].run" .github/actions/deploy/action.yaml`
+    #step=$(yq ".runs.steps[$i]" .github/actions/deploy/action.yaml)
+    working_dir=$(yq ".runs.steps[$i].working-directory" .github/actions/deploy/action.yaml)
+    run_command=$(yq ".runs.steps[$i].run" .github/actions/deploy/action.yaml)
 
     # We don't want to run `run-e2e-test.sh` because it is used for linting here, user will run it manually if needed after deployment
     # We can't run `configure-e2e.sh` here because it needs an image that is not yet built and sent to kind, will be run after
     (
-        if [[ "$run_command" != "null" && "$run_command" != *"configure-e2e.sh"* && "$run_command" != *"run-e2e-test.sh"* ]]
-        then
-            if [ "$working_dir" != "null" ]
-            then
+        if [[ "$run_command" != "null" && "$run_command" != *"configure-e2e.sh"* && "$run_command" != *"run-e2e-test.sh"* ]]; then
+            if [ "$working_dir" != "null" ]; then
                 echo "Changing working dir: $working_dir"
                 cd $working_dir
             fi
@@ -33,7 +33,7 @@ done
 
     envsubst < 'e2e-config.yaml.template' > 'e2e-config.yaml'
     if [[ "${ENABLE_RING_TESTS}" == "false" ]]; then
-    yq -i 'del(.locations[] | select(.locationType == "location-scality-ring-s3-v1"))' e2e-config.yaml
+        yq -i 'del(.locations[] | select(.locationType == "location-scality-ring-s3-v1"))' e2e-config.yaml
     fi
     docker build -t $E2E_IMAGE_NAME:$E2E_IMAGE_TAG .
     kind load docker-image  ${E2E_IMAGE_NAME}:${E2E_IMAGE_TAG}
@@ -50,5 +50,6 @@ docker image prune -af
 
 CTST_TAG=$(sed 's/.*"cli-testing": ".*#\(.*\)".*/\1/;t;d' ./tests/ctst/package.json)
 SORBET_TAG=$(yq eval '.sorbet.tag' solution/deps.yaml)
-docker build --build-arg CTST_TAG=$CTST_TAG --build-arg SORBET_TAG=$SORBET_TAG -t $E2E_CTST_IMAGE_NAME:$E2E_IMAGE_TAG ./tests/ctst
+DRCTL_TAG=$(yq eval '.drctl.tag' solution/deps.yaml)
+docker build --build-arg CTST_TAG=$CTST_TAG --build-arg SORBET_TAG=$SORBET_TAG --build-arg DRCTL_TAG=$DRCTL_TAG -t $E2E_CTST_IMAGE_NAME:$E2E_IMAGE_TAG ./tests/ctst
 kind load docker-image  ${E2E_CTST_IMAGE_NAME}:${E2E_IMAGE_TAG}

.github/actions/archive-artifacts/action.yaml

@@ -16,6 +16,7 @@ runs:
         tar zcvf /tmp/artifacts/${{ github.sha }}-${STAGE}-logs-volumes.tgz /tmp/artifacts/data/${STAGE}/kind-logs;
     - name: Dump kafka
       shell: bash
+      continue-on-error: true
       run: |-
         set -exu
 

.github/actions/debug-wait/action.yaml

@@ -13,4 +13,3 @@ runs:
         tmate-server-port: ${{ env.TMATE_SERVER_PORT }}
         tmate-server-rsa-fingerprint: ${{ env.TMATE_SERVER_RSA_FINGERPRINT }}
         tmate-server-ed25519-fingerprint: ${{ env.TMATE_SERVER_ED25519_FINGERPRINT }}
-      if: failure() && runner.debug == '1'

.github/actions/deploy/action.yaml

@@ -7,10 +7,6 @@ inputs:
     description: "The tag of the Zenko Operator image to use"
     required: false
     default: ""
-  extra_components:
-    description: "Extra components to add to zenkoversion"
-    required: false
-    default: ""
 runs:
   using: composite
   steps:
@@ -78,8 +74,6 @@ runs:
       shell: bash
       run: bash deploy-zenko.sh end2end default
       working-directory: ./.github/scripts/end2end
-      env:
-        EXTRA_COMPONENTS: ${{ inputs.extra_components }}
     - name: Add Keycloak user and assign StorageManager role
       shell: bash
       run: bash keycloak-helper.sh add-user default

.github/scripts/end2end/configs/prometheus.yaml

@@ -0,0 +1,48 @@
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  name: ${PROMETHEUS_NAME}
+rules:
+- apiGroups:
+  - ""
+  resources:
+  - pods
+  - services
+  - endpoints
+  verbs:
+  - '*'
+---
+kind: RoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: ${PROMETHEUS_NAME}
+subjects:
+- kind: ServiceAccount
+  name: default
+  namespace: default
+roleRef:
+  kind: Role
+  name: ${PROMETHEUS_NAME}
+  apiGroup: rbac.authorization.k8s.io
+---
+apiVersion: monitoring.coreos.com/v1
+kind: Prometheus
+metadata:
+  name: ${PROMETHEUS_NAME}
+  labels:
+    prometheus: ${PROMETHEUS_NAME}
+spec:
+  version: v2.35.0
+  evaluationInterval: 30s
+  logFormat: logfmt
+  logLevel: info
+  podMonitorNamespaceSelector: {}
+  podMonitorSelector:
+    matchLabels:
+      metalk8s.scality.com/monitor: ""
+  probeSelector:
+    matchLabels:
+      metalk8s.scality.com/monitor: ""
+  ruleSelector:
+    matchLabels:
+      metalk8s.scality.com/monitor: ""

.github/scripts/end2end/configs/zenko.yaml

@@ -4,6 +4,8 @@ kind: Zenko
 metadata:
   name: ${ZENKO_NAME}
   ${ZENKO_ANNOTATIONS}
+    zenko.io/x-backbeat-oneshard-replicaset: data-db-mongodb-sharded-shard-0
+    zenko.io/x-backbeat-oneshard-replicaset-hosts: data-db-mongodb-sharded-shard0-data-0.data-db-mongodb-sharded-headless.default.svc.cluster.local:27017
 spec:
   version: ${ZENKO_VERSION_NAME}
   replicas: 1

.github/scripts/end2end/configs/zenkoversion.yaml

@@ -66,6 +66,9 @@ spec:
     backbeat:
       image: '${BACKBEAT_IMAGE}' 
       tag: '${BACKBEAT_TAG}'
+    drctl:
+      image: '${DRCTL_IMAGE}'
+      tag: '${DRCTL_TAG}'
     utapi:
       image: '${UTAPI_IMAGE}' 
       tag: '${UTAPI_TAG}'
@@ -106,9 +109,9 @@ spec:
       monitoring:
         image: '${JMX_JAVAAGENT_IMAGE}'
         tag: '${JMX_JAVAAGENT_TAG}'
-      cleaner:
-        image: '${KAFKA_CLEANER_IMAGE}'
-        tag: '${KAFKA_CLEANER_TAG}'
+      listener:
+        image: haproxy
+        tag: '${HAPROXY_TAG}'
     vault:
       image: '${VAULT_IMAGE}'
       tag: '${VAULT_TAG}'
@@ -127,7 +130,6 @@ spec:
         image: '${REDIS_EXPORTER_IMAGE}'
         tag: '${REDIS_EXPORTER_TAG}'
       kubedb: '${REDIS_KUBEDB_TAG}'
-    ${EXTRA_COMPONENTS}
   defaults:
     backbeatConcurrency:
       lifecycleBucketProcessor: 30

.github/scripts/end2end/deploy-zenko.sh

@@ -38,21 +38,14 @@ else
 fi
 
 # TODO: use kustomize
-ZENKO_MONGODB_SHARDED=${ZENKO_MONGODB_SHARDED:-'false'}
-if [ "${ZENKO_MONGODB_SHARDED}" = 'true' ]; then
-    export ZENKO_ANNOTATIONS="annotations:
-    zenko.io/x-backbeat-oneshard-replicaset: data-db-mongodb-sharded-shard-0
-    zenko.io/x-backbeat-oneshard-replicaset-hosts: data-db-mongodb-sharded-shard0-data-0.data-db-mongodb-sharded-headless.default.svc.cluster.local:27017"
-    export ZENKO_MONGODB_ENDPOINT="data-db-mongodb-sharded.default.svc.cluster.local:27017"
-    export ZENKO_MONGODB_CONFIG="writeConcern: 'majority'
+export ZENKO_ANNOTATIONS="annotations:"
+export ZENKO_MONGODB_ENDPOINT="data-db-mongodb-sharded.default.svc.cluster.local:27017"
+export ZENKO_MONGODB_CONFIG="writeConcern: 'majority'
     enableSharding: true"
-else 
-    export ZENKO_MONGODB_ENDPOINT="dev-db-mongodb-primary-0.dev-db-mongodb-headless.default.svc.cluster.local:27017"
-fi
 export ZENKO_MONGODB_DATABASE="${ZENKO_MONGODB_DATABASE:-'datadb'}"
 
 if [ "${TIME_PROGRESSION_FACTOR}" -gt 1 ]; then
-    export ZENKO_ANNOTATIONS="${ZENKO_ANNOTATIONS:-annotations:}
+    export ZENKO_ANNOTATIONS="$ZENKO_ANNOTATIONS
     zenko.io/time-progression-factor: \"${TIME_PROGRESSION_FACTOR}\""
 fi
 

.github/scripts/end2end/install-kind-dependencies.sh

@@ -16,23 +16,15 @@ KAFKA_OPERATOR_VERSION=0.23.0
 INGRESS_NGINX_VERSION=controller-v1.1.0
 PROMETHEUS_VERSION=v0.52.1
 KEYCLOAK_VERSION=18.4.4
-BITNAMI_MONGODB_VER=7.8.0
 
 MONGODB_ROOT_USERNAME=root
 MONGODB_ROOT_PASSWORD=rootpass
 MONGODB_APP_USERNAME=data
 MONGODB_APP_PASSWORD=datapass
 MONGODB_APP_DATABASE="${ZENKO_MONGODB_DATABASE:-'datadb'}"
 MONGODB_RS_KEY=0123456789abcdef
-# force a 4.0 image as that's what artesca uses
-DEPS_FILE="$DIR/../../../solution-base/deps.yaml"
-MONGODB_IMAGE_TAG=$(yq eval ".mongodb.tag" $DEPS_FILE)
-MONGODB_INIT_IMAGE_NAME=$(yq eval ".mongodb-shell.image" $DEPS_FILE)
-MONGODB_INIT_IMAGE_TAG=$(yq eval ".mongodb-shell.tag" $DEPS_FILE)
-MONGODB_EXPORTER_IMAGE_TAG=$(yq eval ".mongodb-exporter.tag" $DEPS_FILE)
 
 ENABLE_KEYCLOAK_HTTPS=${ENABLE_KEYCLOAK_HTTPS:-'false'}
-ZENKO_MONGODB_SHARDED=${ZENKO_MONGODB_SHARDED:-'false'}
 
 KAFKA_CHART=banzaicloud-stable/kafka-operator
 
@@ -65,13 +57,22 @@ kubectl apply -f https://raw.githubusercontent.com/kubernetes/ingress-nginx/${IN
 kubectl rollout status -n ingress-nginx deployment/ingress-nginx-controller --timeout=10m
 
 # cert-manager
-kubectl apply --validate=false -f https://github.com/jetstack/cert-manager/releases/download/${CERT_MANAGER_VERSION}/cert-manager.yaml
+kubectl apply --validate=false -f https://github.com/jetstack/cert-manager/releases/download/${CERT_MANAGER_VERSION}/cert-manager.yaml --wait
+# kubectl apply --validate=false -f - <<EOF
+# apiVersion: cert-manager.io/v1
+# kind: ClusterIssuer
+# metadata:
+#   name: artesca-root-ca-issuer
+# spec:
+#   selfSigned: {}
+# EOF
 
 # prometheus
 # last-applied-configuration can end up larger than 256kB  which is too large for an annotation
 # so if apply fails, replace can work
 prom_url=https://raw.githubusercontent.com/coreos/prometheus-operator/${PROMETHEUS_VERSION}/bundle.yaml
-kubectl create -f $prom_url || kubectl replace -f $prom_url
+kubectl create -f $prom_url || kubectl replace -f $prom_url --wait
+envsubst < configs/prometheus.yaml | kubectl apply -f -
 
 # zookeeper
 helm upgrade --install --version ${ZK_OPERATOR_VERSION} -n default zk-operator pravega/zookeeper-operator --set "watchNamespace=default"
@@ -132,39 +133,6 @@ get_image_from_deps() {
     yq eval ".$dep_name | (.sourceRegistry // \"docker.io\") + \"/\" + .image + \":\" + .tag" $SOLUTION_BASE_DIR/deps.yaml
 }
 
-mongodb_replicaset() {
-    ### TODO:  update to use chart in project
-    kubectl create configmap \
-        --from-file=${DIR}/../../../solution-base/mongodb/scripts mongodb-init-scripts \
-        --dry-run=client -o yaml | kubectl apply -f -
-
-    helm upgrade --install dev-db bitnami/mongodb \
-        --version $BITNAMI_MONGODB_VER \
-        -f "${DIR}/configs/mongodb_options.yaml"  \
-        --set "volumePermissions.enabled=true" \
-        --set "volumePermissions.image.repository=${MONGODB_INIT_IMAGE_NAME}" \
-        --set "volumePermissions.image.tag=${MONGODB_INIT_IMAGE_TAG}" \
-        --set "persistence.storageClass=standard" \
-        --set "usePassword=true" \
-        --set "mongodbRootPassword=$MONGODB_ROOT_PASSWORD" \
-        --set "replicaSet.key=$MONGODB_RS_KEY" \
-        --set "extraEnvVars[0].name=MONGODB_APP_USERNAME" \
-        --set "extraEnvVars[0].value=$MONGODB_APP_USERNAME" \
-        --set "extraEnvVars[1].name=MONGODB_APP_PASSWORD" \
-        --set "extraEnvVars[1].value=$MONGODB_APP_PASSWORD" \
-        --set "extraEnvVars[2].name=MONGODB_APP_DATABASE" \
-        --set "extraEnvVars[2].value=$MONGODB_APP_DATABASE" \
-        --set "extraEnvVars[3].name=MONGODB_NAMESPACE" \
-        --set "extraEnvVars[3].value=default" \
-        --set "replicaSet.pdb.minAvailable.secondary=1" \
-        --set "replicaSet.pdb.minAvailable.arbiter=0" \
-        --set "replicaSet.replicas.secondary=0" \
-        --set "replicaSet.replicas.arbiter=0"
-
-    kubectl rollout status statefulset dev-db-mongodb-primary
-    kubectl rollout status statefulset dev-db-mongodb-secondary
-}
-
 retry() {
     local count=0
     local errMsg=${1:-'reached max retry attempts'}
@@ -216,9 +184,5 @@ mongodb_sharded() {
 }
 
 build_solution_base_manifests
-if [ $ZENKO_MONGODB_SHARDED = 'true' ]; then
-    mongodb_sharded
-else
-    mongodb_replicaset
-fi
+mongodb_sharded
 

.github/scripts/end2end/prepare-pra.sh

@@ -6,14 +6,14 @@ export MONGODB_PRA_DATABASE="${MONGODB_PRA_DATABASE:-'pradb'}"
 export ZENKO_MONGODB_DATABASE="${MONGODB_PRA_DATABASE}"
 export ZENKO_MONGODB_SECRET_NAME="mongodb-db-creds-pra"
 
-echo 'ZENKO_MONGODB_DATABASE="pradb"' >> $GITHUB_ENV
-echo 'ZENKO_MONGODB_SECRET_NAME="mongodb-db-creds-pra"' >> $GITHUB_ENV
+echo 'ZENKO_MONGODB_DATABASE="pradb"' >> "$GITHUB_ENV"
+echo 'ZENKO_MONGODB_SECRET_NAME="mongodb-db-creds-pra"' >> "$GITHUB_ENV"
 
-echo 'ZENKO_IAM_INGRESS="iam.dr.zenko.local"' >> $GITHUB_ENV
-echo 'ZENKO_STS_INGRESS="sts.dr.zenko.local"' >> $GITHUB_ENV
-echo 'ZENKO_MANAGEMENT_INGRESS="management.dr.zenko.local"' >> $GITHUB_ENV
-echo 'ZENKO_S3_INGRESS="s3.dr.zenko.local"' >> $GITHUB_ENV
-echo 'ZENKO_UI_INGRESS="ui.dr.zenko.local"' >> $GITHUB_ENV
+echo 'ZENKO_IAM_INGRESS="iam.dr.zenko.local"' >> "$GITHUB_ENV"
+echo 'ZENKO_STS_INGRESS="sts.dr.zenko.local"' >> "$GITHUB_ENV"
+echo 'ZENKO_MANAGEMENT_INGRESS="management.dr.zenko.local"' >> "$GITHUB_ENV"
+echo 'ZENKO_S3_INGRESS="s3.dr.zenko.local"' >> "$GITHUB_ENV"
+echo 'ZENKO_UI_INGRESS="ui.dr.zenko.local"' >> "$GITHUB_ENV"
 
 MONGODB_ROOT_USERNAME="${MONGODB_ROOT_USERNAME:-'root'}"
 MONGODB_ROOT_PASSWORD="${MONGODB_ROOT_PASSWORD:-'rootpass'}"
@@ -36,3 +36,36 @@ stringData:
   mongodb-password: $MONGODB_PRA_PASSWORD 
   mongodb-database: $MONGODB_PRA_DATABASE
 EOF
+
+# Pre-create volume, to ensure it ends up on first node (dev-worker)
+KAFKA_NODE="${CLUSTER_NAME:-kind}-worker"
+kubectl -n ${PRA_NAMESPACE} apply -f - <<EOF
+apiVersion: v1
+kind: PersistentVolume
+metadata:
+  name: zenko-dr-kafka-broker0
+  labels:
+    brokerId: '0'
+    app: kafka-dr-sink
+spec:
+  persistentVolumeReclaimPolicy: Recycle
+  capacity:
+    storage: 1Gi
+  accessModes:
+  - ReadWriteOnce
+  hostPath:
+    path: /data/kafka-dr-broker0
+    type: DirectoryOrCreate
+  storageClassName: ""
+  nodeAffinity:
+    required:
+      nodeSelectorTerms:
+      - matchExpressions:
+        - key: kubernetes.io/hostname
+          operator: In
+          values:
+          - "${KAFKA_NODE}"
+EOF
+
+KAFKA_EXTERNAL_IP=$(kubectl get node "${KAFKA_NODE}" -o yaml | yq '.status.addresses.[] | select(.type == "InternalIP") | .address')
+echo "KAFKA_EXTERNAL_IP=${KAFKA_EXTERNAL_IP}" >> "$GITHUB_ENV"

.github/scripts/end2end/run-e2e-ctst.sh

@@ -81,6 +81,8 @@ WORLD_PARAMETERS="$(jq -c <<EOF
   "NotificationDestinationTopic":"${NOTIF_DEST_TOPIC}",
   "NotificationDestinationAlt":"${NOTIF_ALT_DEST_NAME}",
   "NotificationDestinationTopicAlt":"${NOTIF_ALT_DEST_TOPIC}",
+  "KafkaExternalIps": "${KAFKA_EXTERNAL_IP:-}",
+  "PrometheusService":"${PROMETHEUS_NAME}-operated.default.svc.cluster.local",
   "KafkaHosts":"${KAFKA_HOST_PORT}",
   "KeycloakPassword":"${KEYCLOAK_TEST_PASSWORD}",
   "KeycloakHost":"${KEYCLOAK_TEST_HOST}",