chore: improve e2e tests (#521)

Signed-off-by: Kavindu Dodanduwa <[email protected]> Signed-off-by: Kavindu Dodanduwa <[email protected]> Co-authored-by: Florian Bacher <[email protected]>
open-feature · Aug 17, 2023 · e5445ae · e5445ae
1 parent 4d5dfdd
commit e5445ae
Show file tree

Hide file tree

Showing 18 changed files with 69 additions and 370 deletions.
diff --git a/Makefile b/Makefile
@@ -65,20 +65,23 @@ vet: ## Run go vet against code.
 unit-test: manifests fmt vet generate envtest ## Run tests.
 	go test ./... -v -short -coverprofile cover.out
 
-## Requires the operator to be deployed
-.PHONY: e2e-test
-e2e-test: manifests generate fmt vet
-	kubectl -n open-feature-operator-system apply -f ./test/e2e/e2e.yml
-	kubectl wait --for=condition=Available=True deploy --all -n 'open-feature-operator-system'
-	./test/e2e/run.sh
-
-.PHONY: e2e-test-kuttl #these tests should run on a real cluster!
+## e2e tests require the operator to be deployed in a real cluster
+.PHONY: e2e-test-kuttl
 e2e-test-kuttl:
-	kubectl kuttl test --start-kind=false ./test/e2e/kuttl --config=./kuttl-test.yaml
+	kubectl kuttl test --start-kind=false --config=./kuttl-test.yaml
 
-.PHONY: e2e-test-kuttl-local #these tests should run on a real cluster!
+.PHONY: e2e-test-kuttl-local
 e2e-test-kuttl-local:
-	kubectl kuttl test --start-kind=false ./test/e2e/kuttl/scenarios --config=./kuttl-test-local.yaml
+	kubectl kuttl test --start-kind=false --config=./kuttl-test-local.yaml
+
+.PHONY: e2e-test-validate-local
+e2e-test-validate-local:
+	docker build . -t open-feature-operator-local:validate
+	kind create cluster --config ./test/e2e/kind-cluster.yml --name e2e-tests
+	kind load docker-image open-feature-operator-local:validate --name e2e-tests
+	IMG=open-feature-operator-local:validate make deploy-operator
+	IMG=open-feature-operator-local:validate make e2e-test-kuttl
+	kind delete cluster --name e2e-tests
 
 .PHONY: lint
 lint:

diff --git a/docs/development_notes.md b/docs/development_notes.md
@@ -13,7 +13,7 @@ Run `make test` to run the test suite. The controller integration tests use [env
 This provides means of asserting that the Kubernetes components reach the desired state without the overhead of using an actual cluster, keeping
 test runtime and resource consumption down.
 
-An e2e test suite can also be found in the [`/test/e2e`](../test/e2e/DEVELOPER.md) directory. These tests are run as part of the `pr-lint` github action, they work by deploying an nginx reverse proxy and asserting that curls to the proxy elicit expected behaviour from the flagd sidecar created by open-feature-operator.
+An e2e test suite can also be found in the [`/test/e2e`](../test/e2e/README.md) directory. These tests are run as part of the `pr-checks` github action, they work by deploying an nginx reverse proxy and asserting that curls to the proxy elicit expected behaviour from the flagd sidecar created by open-feature-operator.
 
 ## Releases
 

diff --git a/kuttl-test-local.yaml b/kuttl-test-local.yaml
@@ -2,5 +2,5 @@ apiVersion: kuttl.dev/v1
 kind: TestSuite
 crdDir: ./config/crd/bases
 testDirs:
-  - ./test/e2e/kuttl/scenarios/
+  - ./test/e2e/kuttl
 timeout: 300
diff --git a/kuttl-test.yaml b/kuttl-test.yaml
@@ -2,6 +2,6 @@ apiVersion: kuttl.dev/v1
 kind: TestSuite
 crdDir: ./config/crd/bases
 testDirs:
-  - ./test/e2e/kuttl/scenarios/
+  - ./test/e2e/kuttl
 timeout: 300
-skipDelete: true
+skipDelete: true # skip removing test related resources (ex:- namespaces)
diff --git a/test/e2e/DEVELOPER.md b/test/e2e/DEVELOPER.md
diff --git a/test/e2e/README.md b/test/e2e/README.md
@@ -0,0 +1,41 @@
+# E2E Testing
+
+This suite tests the end-to-end operation of the open-feature-operator.
+
+Tests are written with [kuttl](https://kuttl.dev/) and assertions are executed from a curl enabled Job.
+Ngnix reverse proxy is used as the workload where flagd get injected using OFO annotations.
+
+## Running and validating locally
+
+It is recommended to run and validate e2e test locally before opening a pull request.
+
+To run locally (commands are executed from the project root level),
+
+1. Build the operator locally - `docker build . -t open-feature-operator-local:validate`
+2. Create a kind cluster - `kind create cluster --config ./test/e2e/kind-cluster.yml --name e2e-tests`
+3. Load locally build operator image - `kind load docker-image open-feature-operator-local:validate --name e2e-tests`
+4. Deploy Operator to kind cluster - `IMG=open-feature-operator-local:validate make deploy-operator`
+5. Execute kuttl tests - `IMG=open-feature-operator-local:validate make e2e-test-kuttl`
+
+Alternatively, you can use `e2e-test-validate-local` Makefile rule to execute all above and cleanup the kind cluster,
+
+> make e2e-test-validate-local
+
+After the test run, make sure test status by validating kuttl output,
+
+```text
+--- PASS: kuttl (48.71s)
+    --- PASS: kuttl/harness (0.00s)
+        --- PASS: kuttl/harness/assets (0.01s)
+        --- PASS: kuttl/harness/flagd-disabled (12.58s)
+        --- PASS: kuttl/harness/inject-flagd (26.41s)
+        --- PASS: kuttl/harness/fsconfig-file-sync (31.73s)
+        --- PASS: kuttl/harness/fsconfig-k8s-sync (31.74s)
+        --- PASS: kuttl/harness/fsconfig-flagd-proxy-sync (48.49s)
+```
+
+### Running individual tests
+
+You can use kuttl command options to execute individual tests. Consider the example command below,
+
+>$ kubectl kuttl test --start-kind=false ./test/e2e/kuttl --config=kuttl-test.yaml --test=flagd-disabled
diff --git a/test/e2e/e2e.yml b/test/e2e/e2e.yml
diff --git a/test/e2e/flag-evaluation.sh b/test/e2e/flag-evaluation.sh
diff --git a/test/e2e/kuttl/flagd-disabled/00-assert.yaml b/test/e2e/kuttl/flagd-disabled/00-assert.yaml
@@ -2,3 +2,5 @@ apiVersion: kuttl.dev/v1beta1
 kind: TestAssert
 commands:
   - command: kubectl wait --for=condition=complete job flagd-query-test -n $NAMESPACE
+collectors:
+  - command: kubectl logs -l job-name=flagd-query-test -n $NAMESPACE
diff --git a/test/e2e/kuttl/fsconfig-file-sync/00-assert.yaml b/test/e2e/kuttl/fsconfig-file-sync/00-assert.yaml
@@ -2,3 +2,5 @@ apiVersion: kuttl.dev/v1beta1
 kind: TestAssert
 commands:
   - command: kubectl wait --for=condition=complete job flagd-query-test -n $NAMESPACE
+collectors:
+  - command: kubectl logs -l job-name=flagd-query-test -n $NAMESPACE
diff --git a/test/e2e/kuttl/fsconfig-flagd-proxy-sync/00-assert.yaml b/test/e2e/kuttl/fsconfig-flagd-proxy-sync/00-assert.yaml
@@ -2,3 +2,5 @@ apiVersion: kuttl.dev/v1beta1
 kind: TestAssert
 commands:
   - command: kubectl wait --for=condition=complete job flagd-query-test -n $NAMESPACE
+collectors:
+  - command: kubectl logs -l job-name=flagd-query-test -n $NAMESPACE
diff --git a/test/e2e/kuttl/fsconfig-k8s-sync/00-assert.yaml b/test/e2e/kuttl/fsconfig-k8s-sync/00-assert.yaml
@@ -2,3 +2,5 @@ apiVersion: kuttl.dev/v1beta1
 kind: TestAssert
 commands:
   - command: kubectl wait --for=condition=complete job flagd-query-test -n $NAMESPACE
+collectors:
+  - command: kubectl logs -l job-name=flagd-query-test -n $NAMESPACE
diff --git a/test/e2e/kuttl/inject-flagd/00-assert.yaml b/test/e2e/kuttl/inject-flagd/00-assert.yaml
@@ -2,3 +2,5 @@ apiVersion: kuttl.dev/v1beta1
 kind: TestAssert
 commands:
   - command: kubectl wait --for=condition=complete job flagd-query-test -n $NAMESPACE
+collectors:
+  - command: kubectl logs -l job-name=flagd-query-test -n $NAMESPACE
diff --git a/test/e2e/run.sh b/test/e2e/run.sh
diff --git a/test/e2e/tests/001.flagd-response.sh b/test/e2e/tests/001.flagd-response.sh