From e5445ae5f41c29a5a62503438d86dcddf2c59617 Mon Sep 17 00:00:00 2001 From: Kavindu Dodanduwa Date: Thu, 17 Aug 2023 11:23:38 -0700 Subject: [PATCH] chore: improve e2e tests (#521) Signed-off-by: Kavindu Dodanduwa Signed-off-by: Kavindu Dodanduwa Co-authored-by: Florian Bacher --- Makefile | 25 ++-- docs/development_notes.md | 2 +- kuttl-test-local.yaml | 2 +- kuttl-test.yaml | 4 +- test/e2e/DEVELOPER.md | 21 --- test/e2e/README.md | 41 ++++++ test/e2e/e2e.yml | 137 ------------------ test/e2e/flag-evaluation.sh | 37 ----- test/e2e/kuttl/flagd-disabled/00-assert.yaml | 2 + .../kuttl/fsconfig-file-sync/00-assert.yaml | 2 + .../fsconfig-flagd-proxy-sync/00-assert.yaml | 2 + .../kuttl/fsconfig-k8s-sync/00-assert.yaml | 2 + test/e2e/kuttl/inject-flagd/00-assert.yaml | 2 + test/e2e/run.sh | 23 --- test/e2e/tests/001.flagd-response.sh | 20 --- .../002.crd-configuration-reconciliation.sh | 29 ---- ....configmap-configuration-reconciliation.sh | 27 ---- test/e2e/tests/004.disabled.sh | 61 -------- 18 files changed, 69 insertions(+), 370 deletions(-) delete mode 100644 test/e2e/DEVELOPER.md create mode 100644 test/e2e/README.md delete mode 100644 test/e2e/e2e.yml delete mode 100755 test/e2e/flag-evaluation.sh delete mode 100755 test/e2e/run.sh delete mode 100755 test/e2e/tests/001.flagd-response.sh delete mode 100755 test/e2e/tests/002.crd-configuration-reconciliation.sh delete mode 100755 test/e2e/tests/003.configmap-configuration-reconciliation.sh delete mode 100755 test/e2e/tests/004.disabled.sh diff --git a/Makefile b/Makefile index e812f9cee..d709801c1 100644 --- a/Makefile +++ b/Makefile @@ -65,20 +65,23 @@ vet: ## Run go vet against code. unit-test: manifests fmt vet generate envtest ## Run tests. go test ./... -v -short -coverprofile cover.out -## Requires the operator to be deployed -.PHONY: e2e-test -e2e-test: manifests generate fmt vet - kubectl -n open-feature-operator-system apply -f ./test/e2e/e2e.yml - kubectl wait --for=condition=Available=True deploy --all -n 'open-feature-operator-system' - ./test/e2e/run.sh - -.PHONY: e2e-test-kuttl #these tests should run on a real cluster! +## e2e tests require the operator to be deployed in a real cluster +.PHONY: e2e-test-kuttl e2e-test-kuttl: - kubectl kuttl test --start-kind=false ./test/e2e/kuttl --config=./kuttl-test.yaml + kubectl kuttl test --start-kind=false --config=./kuttl-test.yaml -.PHONY: e2e-test-kuttl-local #these tests should run on a real cluster! +.PHONY: e2e-test-kuttl-local e2e-test-kuttl-local: - kubectl kuttl test --start-kind=false ./test/e2e/kuttl/scenarios --config=./kuttl-test-local.yaml + kubectl kuttl test --start-kind=false --config=./kuttl-test-local.yaml + +.PHONY: e2e-test-validate-local +e2e-test-validate-local: + docker build . -t open-feature-operator-local:validate + kind create cluster --config ./test/e2e/kind-cluster.yml --name e2e-tests + kind load docker-image open-feature-operator-local:validate --name e2e-tests + IMG=open-feature-operator-local:validate make deploy-operator + IMG=open-feature-operator-local:validate make e2e-test-kuttl + kind delete cluster --name e2e-tests .PHONY: lint lint: diff --git a/docs/development_notes.md b/docs/development_notes.md index 01974aa83..77f2957bc 100644 --- a/docs/development_notes.md +++ b/docs/development_notes.md @@ -13,7 +13,7 @@ Run `make test` to run the test suite. The controller integration tests use [env This provides means of asserting that the Kubernetes components reach the desired state without the overhead of using an actual cluster, keeping test runtime and resource consumption down. -An e2e test suite can also be found in the [`/test/e2e`](../test/e2e/DEVELOPER.md) directory. These tests are run as part of the `pr-lint` github action, they work by deploying an nginx reverse proxy and asserting that curls to the proxy elicit expected behaviour from the flagd sidecar created by open-feature-operator. +An e2e test suite can also be found in the [`/test/e2e`](../test/e2e/README.md) directory. These tests are run as part of the `pr-checks` github action, they work by deploying an nginx reverse proxy and asserting that curls to the proxy elicit expected behaviour from the flagd sidecar created by open-feature-operator. ## Releases diff --git a/kuttl-test-local.yaml b/kuttl-test-local.yaml index 5769b5f65..2d91fa49c 100644 --- a/kuttl-test-local.yaml +++ b/kuttl-test-local.yaml @@ -2,5 +2,5 @@ apiVersion: kuttl.dev/v1 kind: TestSuite crdDir: ./config/crd/bases testDirs: - - ./test/e2e/kuttl/scenarios/ + - ./test/e2e/kuttl timeout: 300 \ No newline at end of file diff --git a/kuttl-test.yaml b/kuttl-test.yaml index c76198704..17cbf9146 100644 --- a/kuttl-test.yaml +++ b/kuttl-test.yaml @@ -2,6 +2,6 @@ apiVersion: kuttl.dev/v1 kind: TestSuite crdDir: ./config/crd/bases testDirs: - - ./test/e2e/kuttl/scenarios/ + - ./test/e2e/kuttl timeout: 300 -skipDelete: true +skipDelete: true # skip removing test related resources (ex:- namespaces) diff --git a/test/e2e/DEVELOPER.md b/test/e2e/DEVELOPER.md deleted file mode 100644 index 0cbc835bc..000000000 --- a/test/e2e/DEVELOPER.md +++ /dev/null @@ -1,21 +0,0 @@ -# E2E Testing - -This suite tests the end-to-end deployment of open-feature-operator by deploying an nginx reverse proxy and asserting that curls to the proxy elicit expected behaviour from the flagd sidecar created by open-feature-operator. - -## Running on a Kind cluster - -```shell -kind create cluster --config ./test/e2e/kind-cluster.yml -IMG=ghcr.io/open-feature/open-feature-operator:main make deploy-operator -IMG=ghcr.io/open-feature/open-feature-operator:main make e2e-test -``` - -## Running on a Kind cluster using a locally built image - -```shell -kind create cluster --config ./test/e2e/kind-cluster.yml -kind load docker-image local-image-tag:latest -IMG=local-image-tag:latest make deploy-operator -IMG=local-image-tag:latest make e2e-test -``` - diff --git a/test/e2e/README.md b/test/e2e/README.md new file mode 100644 index 000000000..545826141 --- /dev/null +++ b/test/e2e/README.md @@ -0,0 +1,41 @@ +# E2E Testing + +This suite tests the end-to-end operation of the open-feature-operator. + +Tests are written with [kuttl](https://kuttl.dev/) and assertions are executed from a curl enabled Job. +Ngnix reverse proxy is used as the workload where flagd get injected using OFO annotations. + +## Running and validating locally + +It is recommended to run and validate e2e test locally before opening a pull request. + +To run locally (commands are executed from the project root level), + +1. Build the operator locally - `docker build . -t open-feature-operator-local:validate` +2. Create a kind cluster - `kind create cluster --config ./test/e2e/kind-cluster.yml --name e2e-tests` +3. Load locally build operator image - `kind load docker-image open-feature-operator-local:validate --name e2e-tests` +4. Deploy Operator to kind cluster - `IMG=open-feature-operator-local:validate make deploy-operator` +5. Execute kuttl tests - `IMG=open-feature-operator-local:validate make e2e-test-kuttl` + +Alternatively, you can use `e2e-test-validate-local` Makefile rule to execute all above and cleanup the kind cluster, + +> make e2e-test-validate-local + +After the test run, make sure test status by validating kuttl output, + +```text +--- PASS: kuttl (48.71s) + --- PASS: kuttl/harness (0.00s) + --- PASS: kuttl/harness/assets (0.01s) + --- PASS: kuttl/harness/flagd-disabled (12.58s) + --- PASS: kuttl/harness/inject-flagd (26.41s) + --- PASS: kuttl/harness/fsconfig-file-sync (31.73s) + --- PASS: kuttl/harness/fsconfig-k8s-sync (31.74s) + --- PASS: kuttl/harness/fsconfig-flagd-proxy-sync (48.49s) +``` + +### Running individual tests + +You can use kuttl command options to execute individual tests. Consider the example command below, + +>$ kubectl kuttl test --start-kind=false ./test/e2e/kuttl --config=kuttl-test.yaml --test=flagd-disabled \ No newline at end of file diff --git a/test/e2e/e2e.yml b/test/e2e/e2e.yml deleted file mode 100644 index cc844d61b..000000000 --- a/test/e2e/e2e.yml +++ /dev/null @@ -1,137 +0,0 @@ -# This configuration deploys the means to test the mutating injection webhook by proxying through an nginx server -# to assert that flagd is reachable ---- -apiVersion: core.openfeature.dev/v1alpha1 -kind: FeatureFlagConfiguration -metadata: - name: end-to-end-test-default -spec: - featureFlagSpec: | - { - "flags": { - "simple-flag": { - "state": "ENABLED", - "variants": { - "on": true, - "off": false - }, - "defaultVariant": "on" - } - } - } ---- -apiVersion: core.openfeature.dev/v1alpha2 -kind: FeatureFlagConfiguration -metadata: - name: end-to-end-test-filepath -spec: - syncProvider: - name: filepath - featureFlagSpec: - flags: - simple-flag-filepath: - state: ENABLED - variants: - "on": true - "off": false - defaultVariant: "on" ---- -apiVersion: core.openfeature.dev/v1alpha2 -kind: FeatureFlagConfiguration -metadata: - name: end-to-end-test-filepath2 -spec: - syncProvider: - name: filepath - featureFlagSpec: - flags: - simple-flag-filepath2: - state: ENABLED - variants: - "on": true - "off": false - defaultVariant: "on" ---- -apiVersion: core.openfeature.dev/v1alpha3 -kind: FlagSourceConfiguration -metadata: - name: end-to-end-test-fs-config -spec: - sources: - - source: end-to-end-test-filepath2 - provider: kubernetes ---- -apiVersion: v1 -kind: ServiceAccount -metadata: - name: open-feature-e2e-test-sa -automountServiceAccountToken: true ---- -apiVersion: v1 -kind: ConfigMap -metadata: - name: open-feature-e2e-nginx-conf -data: - nginx.conf: | - events {} - http { - server { - location / { - proxy_pass http://127.0.0.1:8013; - } - } - } ---- -# Deployment of nginx using our custom resource -apiVersion: apps/v1 -kind: Deployment -metadata: - name: open-feature-e2e-test-deployment - labels: - app: open-feature-e2e-test -spec: - replicas: 1 - selector: - matchLabels: - app: open-feature-e2e-test - template: - metadata: - labels: - app: open-feature-e2e-test - annotations: - openfeature.dev/enabled: "true" - openfeature.dev/featureflagconfiguration: "end-to-end-test-default,end-to-end-test-filepath" - openfeature.dev/flagsourceconfiguration: "end-to-end-test-fs-config" - spec: - serviceAccountName: open-feature-e2e-test-sa - volumes: - - name: open-feature-e2e-nginx-conf - configMap: - name: open-feature-e2e-nginx-conf - items: - - key: nginx.conf - path: nginx.conf - containers: - - name: open-feature-e2e-test - image: nginx:stable-alpine - ports: - - containerPort: 80 - volumeMounts: - - name: open-feature-e2e-nginx-conf - mountPath: /etc/nginx - readOnly: true ---- -# Service exposed using NodePort -apiVersion: v1 -kind: Service -metadata: - name: open-feature-e2e-test-service -spec: - type: NodePort - selector: - app: open-feature-e2e-test - ports: - - protocol: TCP - port: 30000 - targetPort: 80 - nodePort: 30000 \ No newline at end of file diff --git a/test/e2e/flag-evaluation.sh b/test/e2e/flag-evaluation.sh deleted file mode 100755 index a58f7f484..000000000 --- a/test/e2e/flag-evaluation.sh +++ /dev/null @@ -1,37 +0,0 @@ -#!/bin/bash - -FLAG_KEY="$1" -EXPECTED_RESPONSE_CONTAIN="$2" - -# attempt up to 5 times -MAX_ATTEMPTS=5 -# retry every x seconds -RETRY_INTERVAL=1 -if [[ "$3" =~ ^[0-9]+$ ]] - then - MAX_ATTEMPTS=$3 -fi -if [[ "$4" =~ ^[0-9]+$ ]] - then - RETRY_INTERVAL=$4 -fi - - -for (( ATTEMPT_COUNTER=0; ATTEMPT_COUNTER<${MAX_ATTEMPTS}; ATTEMPT_COUNTER++ )) -do - - RESPONSE=$(curl -s -X POST "localhost:30000/schema.v1.Service/ResolveBoolean" -d "{\"flagKey\":\"$FLAG_KEY\",\"context\":{}}" -H "Content-Type: application/json") - RESPONSE="${RESPONSE//[[:space:]]/}" # strip whitespace from response - - if [[ "$RESPONSE" == *"$EXPECTED_RESPONSE_CONTAIN"* ]] - then - exit 0 - fi - - echo "Expected response for flag $FLAG_KEY to contain: EXPECTED_RESPONSE_CONTAIN" - echo "Got response for flag $FLAG_KEY: $RESPONSE" - echo "Retrying in ${RETRY_INTERVAL} seconds" - sleep "${RETRY_INTERVAL}" -done -echo "Max attempts reached" -exit 1 \ No newline at end of file diff --git a/test/e2e/kuttl/flagd-disabled/00-assert.yaml b/test/e2e/kuttl/flagd-disabled/00-assert.yaml index 0ce96ba41..7e70f3316 100644 --- a/test/e2e/kuttl/flagd-disabled/00-assert.yaml +++ b/test/e2e/kuttl/flagd-disabled/00-assert.yaml @@ -2,3 +2,5 @@ apiVersion: kuttl.dev/v1beta1 kind: TestAssert commands: - command: kubectl wait --for=condition=complete job flagd-query-test -n $NAMESPACE +collectors: + - command: kubectl logs -l job-name=flagd-query-test -n $NAMESPACE diff --git a/test/e2e/kuttl/fsconfig-file-sync/00-assert.yaml b/test/e2e/kuttl/fsconfig-file-sync/00-assert.yaml index 0ce96ba41..7e70f3316 100644 --- a/test/e2e/kuttl/fsconfig-file-sync/00-assert.yaml +++ b/test/e2e/kuttl/fsconfig-file-sync/00-assert.yaml @@ -2,3 +2,5 @@ apiVersion: kuttl.dev/v1beta1 kind: TestAssert commands: - command: kubectl wait --for=condition=complete job flagd-query-test -n $NAMESPACE +collectors: + - command: kubectl logs -l job-name=flagd-query-test -n $NAMESPACE diff --git a/test/e2e/kuttl/fsconfig-flagd-proxy-sync/00-assert.yaml b/test/e2e/kuttl/fsconfig-flagd-proxy-sync/00-assert.yaml index 0ce96ba41..e916130f0 100644 --- a/test/e2e/kuttl/fsconfig-flagd-proxy-sync/00-assert.yaml +++ b/test/e2e/kuttl/fsconfig-flagd-proxy-sync/00-assert.yaml @@ -2,3 +2,5 @@ apiVersion: kuttl.dev/v1beta1 kind: TestAssert commands: - command: kubectl wait --for=condition=complete job flagd-query-test -n $NAMESPACE +collectors: + - command: kubectl logs -l job-name=flagd-query-test -n $NAMESPACE \ No newline at end of file diff --git a/test/e2e/kuttl/fsconfig-k8s-sync/00-assert.yaml b/test/e2e/kuttl/fsconfig-k8s-sync/00-assert.yaml index 0ce96ba41..7e70f3316 100644 --- a/test/e2e/kuttl/fsconfig-k8s-sync/00-assert.yaml +++ b/test/e2e/kuttl/fsconfig-k8s-sync/00-assert.yaml @@ -2,3 +2,5 @@ apiVersion: kuttl.dev/v1beta1 kind: TestAssert commands: - command: kubectl wait --for=condition=complete job flagd-query-test -n $NAMESPACE +collectors: + - command: kubectl logs -l job-name=flagd-query-test -n $NAMESPACE diff --git a/test/e2e/kuttl/inject-flagd/00-assert.yaml b/test/e2e/kuttl/inject-flagd/00-assert.yaml index 0ce96ba41..7e70f3316 100644 --- a/test/e2e/kuttl/inject-flagd/00-assert.yaml +++ b/test/e2e/kuttl/inject-flagd/00-assert.yaml @@ -2,3 +2,5 @@ apiVersion: kuttl.dev/v1beta1 kind: TestAssert commands: - command: kubectl wait --for=condition=complete job flagd-query-test -n $NAMESPACE +collectors: + - command: kubectl logs -l job-name=flagd-query-test -n $NAMESPACE diff --git a/test/e2e/run.sh b/test/e2e/run.sh deleted file mode 100755 index dfdc4300a..000000000 --- a/test/e2e/run.sh +++ /dev/null @@ -1,23 +0,0 @@ -#!/bin/bash - -FAILURE=0 - -for FILE in "$(dirname "${BASH_SOURCE[0]}")"/tests/*; -do - echo "Running ${FILE##*/}"; - ./"${FILE}" - EXIT_CODE=$? - if [ $EXIT_CODE -ne 0 ]; - then - FAILURE=1 - echo "${FILE##*/} failed." - else - echo "${FILE##*/} succeeded." - fi -done - -if [ $FAILURE -eq 1 ]; -then - exit 1 -fi - diff --git a/test/e2e/tests/001.flagd-response.sh b/test/e2e/tests/001.flagd-response.sh deleted file mode 100755 index 996cfe4d5..000000000 --- a/test/e2e/tests/001.flagd-response.sh +++ /dev/null @@ -1,20 +0,0 @@ -#!/bin/bash - -FAILURE=0 - -flagKeys=('simple-flag' 'simple-flag-filepath' 'simple-flag-filepath2') -expectedResponseContain=('value":true,"reason":"STATIC","variant":"on"' '"value":true,"reason":"STATIC","variant":"on"' '"value":true,"reason":"STATIC","variant":"on"') - -for i in "${!flagKeys[@]}"; do - ./"$(dirname "${BASH_SOURCE[0]}")"/../flag-evaluation.sh "${flagKeys[$i]}" "${expectedResponseContain[$i]}" - EXIT_CODE=$? - if [ $EXIT_CODE -ne 0 ]; - then - FAILURE=1 - fi -done - -if [ $FAILURE -eq 1 ]; -then - exit 1 -fi diff --git a/test/e2e/tests/002.crd-configuration-reconciliation.sh b/test/e2e/tests/002.crd-configuration-reconciliation.sh deleted file mode 100755 index 151b0053d..000000000 --- a/test/e2e/tests/002.crd-configuration-reconciliation.sh +++ /dev/null @@ -1,29 +0,0 @@ -#!/bin/bash - -cat < /dev/null # reset state quietly - -exit $EXIT_CODE diff --git a/test/e2e/tests/003.configmap-configuration-reconciliation.sh b/test/e2e/tests/003.configmap-configuration-reconciliation.sh deleted file mode 100755 index 47d8f36c2..000000000 --- a/test/e2e/tests/003.configmap-configuration-reconciliation.sh +++ /dev/null @@ -1,27 +0,0 @@ -#!/bin/bash - -cat < /dev/null # reset state quietly - -exit $EXIT_CODE diff --git a/test/e2e/tests/004.disabled.sh b/test/e2e/tests/004.disabled.sh deleted file mode 100755 index 8655c0eeb..000000000 --- a/test/e2e/tests/004.disabled.sh +++ /dev/null @@ -1,61 +0,0 @@ -#!/bin/bash - -# delete existing deployment -kubectl -n open-feature-operator-system delete deployment open-feature-e2e-test-deployment - -# set openfeature.dev/enabled annotation to false and redeploy -cat < /dev/null -kubectl wait --for=condition=Available=True deploy --all -n 'open-feature-operator-system' - -if [ "$STATUS_CODE" -eq 200 ]; then - echo "Expected curl to nginx reverse proxy to return non 200 status code when openfeature.dev/enabled annotation is false." - exit 1 -else - exit 0 -fi