From 70134129dfb83943485671ca9a0903a8c1faeba9 Mon Sep 17 00:00:00 2001 From: Marcin Owsiany Date: Mon, 25 Nov 2024 13:43:01 +0100 Subject: [PATCH] e2e test suite Signed-off-by: Marcin Owsiany --- .github/workflows/e2e-test.yml | 22 +++ Makefile | 7 +- test/junit/.gitignore | 6 + test/junit/Makefile | 24 +++ test/junit/kuttl-report-step.json.golden | 204 +++++++++++++++++++++++ test/junit/kuttl-report-step.xml.golden | 44 +++++ test/junit/suite1/test0/00-assert.yaml | 7 + test/junit/suite1/test0/00-run.yaml | 7 + test/junit/suite1/test1/00-assert.yaml | 7 + test/junit/suite1/test1/00-run.yaml | 7 + test/junit/suite1/test1/01-assert.yaml | 7 + test/junit/suite1/test1/01-run.yaml | 7 + test/junit/suite1/test1/02-run.yaml | 7 + test/junit/suite1/test2/00-assert.yaml | 7 + test/junit/suite1/test2/00-run.yaml | 7 + test/junit/suite1/test2/01-assert.yaml | 7 + test/junit/suite1/test2/01-run.yaml | 7 + test/junit/suite1/test2/02-run.yaml | 7 + test/junit/suite2/test0/00-assert.yaml | 7 + test/junit/suite2/test0/00-run.yaml | 7 + test/junit/suite2/test1/00-assert.yaml | 7 + test/junit/suite2/test1/00-run.yaml | 7 + test/junit/suite2/test1/01-assert.yaml | 7 + test/junit/suite2/test1/01-run.yaml | 7 + test/junit/suite2/test1/02-run.yaml | 7 + test/junit/suite2/test2/00-assert.yaml | 7 + test/junit/suite2/test2/00-run.yaml | 7 + test/junit/suite2/test2/01-assert.yaml | 7 + test/junit/suite2/test2/01-run.yaml | 7 + test/junit/suite2/test2/02-run.yaml | 7 + 30 files changed, 474 insertions(+), 1 deletion(-) create mode 100644 .github/workflows/e2e-test.yml create mode 100644 test/junit/.gitignore create mode 100644 test/junit/Makefile create mode 100644 test/junit/kuttl-report-step.json.golden create mode 100644 test/junit/kuttl-report-step.xml.golden create mode 100644 test/junit/suite1/test0/00-assert.yaml create mode 100644 test/junit/suite1/test0/00-run.yaml create mode 100644 test/junit/suite1/test1/00-assert.yaml create mode 100644 test/junit/suite1/test1/00-run.yaml create mode 100644 test/junit/suite1/test1/01-assert.yaml create mode 100644 test/junit/suite1/test1/01-run.yaml create mode 100644 test/junit/suite1/test1/02-run.yaml create mode 100644 test/junit/suite1/test2/00-assert.yaml create mode 100644 test/junit/suite1/test2/00-run.yaml create mode 100644 test/junit/suite1/test2/01-assert.yaml create mode 100644 test/junit/suite1/test2/01-run.yaml create mode 100644 test/junit/suite1/test2/02-run.yaml create mode 100644 test/junit/suite2/test0/00-assert.yaml create mode 100644 test/junit/suite2/test0/00-run.yaml create mode 100644 test/junit/suite2/test1/00-assert.yaml create mode 100644 test/junit/suite2/test1/00-run.yaml create mode 100644 test/junit/suite2/test1/01-assert.yaml create mode 100644 test/junit/suite2/test1/01-run.yaml create mode 100644 test/junit/suite2/test1/02-run.yaml create mode 100644 test/junit/suite2/test2/00-assert.yaml create mode 100644 test/junit/suite2/test2/00-run.yaml create mode 100644 test/junit/suite2/test2/01-assert.yaml create mode 100644 test/junit/suite2/test2/01-run.yaml create mode 100644 test/junit/suite2/test2/02-run.yaml diff --git a/.github/workflows/e2e-test.yml b/.github/workflows/e2e-test.yml new file mode 100644 index 00000000..52cfea56 --- /dev/null +++ b/.github/workflows/e2e-test.yml @@ -0,0 +1,22 @@ +name: "End-to-end tests" + +on: + push: + branches: + - main + - releases/* + pull_request: + branches: + - main + - releases/* + +jobs: + e2e-tests: + runs-on: ubuntu-20.04 + steps: + - uses: actions/setup-go@v2.1.4 + with: + go-version: 1.21 + - uses: actions/checkout@v2.4.0 + - name: "Run end-to-end tests" + run: make e2e-test diff --git a/Makefile b/Makefile index 3c14b167..97481340 100644 --- a/Makefile +++ b/Makefile @@ -122,7 +122,7 @@ todo: ## Shows todos from code ##@ Tests .PHONY: all -all: lint test integration-test ## Runs lint, unit and integration tests +all: lint test integration-test e2e-test ## Runs lint, unit, integration and e2e tests # Run unit tests .PHONY: test @@ -139,6 +139,11 @@ endif integration-test: envtest ## Runs integration tests KUBEBUILDER_ASSETS="$(shell $(ENVTEST) use $(ENVTEST_K8S_VERSION) --bin-dir $(LOCALBIN) -p path)" ./hack/run-integration-tests.sh +.PHONY: e2e-test +# Run e2e tests +e2e-test: envtest ## Runs end-to-end tests + KUBEBUILDER_ASSETS="$(shell $(ENVTEST) use $(ENVTEST_K8S_VERSION) --bin-dir $(LOCALBIN) -p path)" $(MAKE) -C ./test/junit + ##@ Build Dependencies ## Location to install dependencies to diff --git a/test/junit/.gitignore b/test/junit/.gitignore new file mode 100644 index 00000000..fa0c69dd --- /dev/null +++ b/test/junit/.gitignore @@ -0,0 +1,6 @@ +/kuttl-ouput-step-json.txt +/kuttl-ouput-step-xml.txt +/kuttl-report-step.json +/kuttl-report-step.json.normalized +/kuttl-report-step.xml +/kuttl-report-step.xml.normalized diff --git a/test/junit/Makefile b/test/junit/Makefile new file mode 100644 index 00000000..1c8f6f6d --- /dev/null +++ b/test/junit/Makefile @@ -0,0 +1,24 @@ +.PHONY: test +test: + $(MAKE) -C ../../ cli + rm -f kuttl-report-step.xml.normalized kuttl-report-step.json.normalized + ../../bin/kubectl-kuttl test --config /dev/null --start-control-plane --report-name kuttl-report-step --timeout 10 --report xml suite1 suite2 > kuttl-ouput-step-xml.txt 2>&1 || true # this is meant to fail + if [ ! -e kuttl-report-step.xml ]; then cat kuttl-output-step-xml.txt; exit 1; fi + ../../bin/kubectl-kuttl test --config /dev/null --start-control-plane --report-name kuttl-report-step --timeout 10 --report json suite1 suite2 > kuttl-ouput-step-json.txt 2>&1 || true # this is meant to fail + if [ ! -e kuttl-report-step.json ]; then cat kuttl-output-step-json.txt; exit 1; fi + $(MAKE) kuttl-report-step.xml.normalized kuttl-report-step.json.normalized + diff -u kuttl-report-step.xml.golden kuttl-report-step.xml.normalized + diff -u kuttl-report-step.json.golden kuttl-report-step.json.normalized + +.PHONY: update-golden +update-golden: + cp kuttl-report-step.json.normalized kuttl-report-step.json.golden + cp kuttl-report-step.xml.normalized kuttl-report-step.xml.golden + +# The following targets replace all timestamps and durations with dummy values to make comparisons easy. + +%.xml.normalized: %.xml + sed -E -e 's/time="[^"]+"/time="1.0"/g; s/[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}[.][0-9]{6,10}(Z|[-+][0-9]{2}:[0-9]{2})/2000-01-01T00:00:00.00000000+00:00/g' < $< > $@ + +%.json.normalized: %.json + sed -E -e 's/"time": *"[^"]+"/"time": "1.0"/g; s/[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}[.][0-9]{6,10}(Z|[-+][0-9]{2}:[0-9]{2})/2000-01-01T00:00:00.00000000+00:00/g' < $< > $@ diff --git a/test/junit/kuttl-report-step.json.golden b/test/junit/kuttl-report-step.json.golden new file mode 100644 index 00000000..ba555cc9 --- /dev/null +++ b/test/junit/kuttl-report-step.json.golden @@ -0,0 +1,204 @@ +{ + "name": "", + "tests": 18, + "failures": 4, + "time": "1.0", + "testsuite": [ + { + "tests": 9, + "failures": 2, + "timestamp": "2000-01-01T00:00:00.00000000+00:00", + "time": "1.0", + "name": "suite1", + "testsuite": [ + { + "tests": 2, + "failures": 0, + "timestamp": "2000-01-01T00:00:00.00000000+00:00", + "time": "1.0", + "name": "test0", + "testcase": [ + { + "classname": "test0", + "name": "setup", + "timestamp": "2000-01-01T00:00:00.00000000+00:00", + "time": "1.0" + }, + { + "classname": "test0", + "name": "step 0-run", + "timestamp": "2000-01-01T00:00:00.00000000+00:00", + "time": "1.0" + } + ] + }, + { + "tests": 4, + "failures": 1, + "timestamp": "2000-01-01T00:00:00.00000000+00:00", + "time": "1.0", + "name": "test1", + "testcase": [ + { + "classname": "test1", + "name": "setup", + "timestamp": "2000-01-01T00:00:00.00000000+00:00", + "time": "1.0" + }, + { + "classname": "test1", + "name": "step 0-run", + "timestamp": "2000-01-01T00:00:00.00000000+00:00", + "time": "1.0" + }, + { + "classname": "test1", + "name": "step 1-run", + "timestamp": "2000-01-01T00:00:00.00000000+00:00", + "time": "1.0" + }, + { + "classname": "test1", + "name": "step 2-run", + "timestamp": "2000-01-01T00:00:00.00000000+00:00", + "time": "1.0", + "failure": { + "text": "command \"echo step stdout\\\\n echo \u003e\u00262 step stderr\\\\n false\" failed, exit status 1", + "message": "failed in step 2-run" + } + } + ] + }, + { + "tests": 3, + "failures": 1, + "timestamp": "2000-01-01T00:00:00.00000000+00:00", + "time": "1.0", + "name": "test2", + "testcase": [ + { + "classname": "test2", + "name": "setup", + "timestamp": "2000-01-01T00:00:00.00000000+00:00", + "time": "1.0" + }, + { + "classname": "test2", + "name": "step 0-run", + "timestamp": "2000-01-01T00:00:00.00000000+00:00", + "time": "1.0" + }, + { + "classname": "test2", + "name": "step 1-run", + "timestamp": "2000-01-01T00:00:00.00000000+00:00", + "time": "1.0", + "failure": { + "text": "command \"echo assert stdout\\\\n echo \u003e\u00262 assert stderr\\\\n false\" failed, exit status 1", + "message": "failed in step 1-run" + } + } + ] + } + ] + }, + { + "tests": 9, + "failures": 2, + "timestamp": "2000-01-01T00:00:00.00000000+00:00", + "time": "1.0", + "name": "suite2", + "testsuite": [ + { + "tests": 2, + "failures": 0, + "timestamp": "2000-01-01T00:00:00.00000000+00:00", + "time": "1.0", + "name": "test0", + "testcase": [ + { + "classname": "test0", + "name": "setup", + "timestamp": "2000-01-01T00:00:00.00000000+00:00", + "time": "1.0" + }, + { + "classname": "test0", + "name": "step 0-run", + "timestamp": "2000-01-01T00:00:00.00000000+00:00", + "time": "1.0" + } + ] + }, + { + "tests": 4, + "failures": 1, + "timestamp": "2000-01-01T00:00:00.00000000+00:00", + "time": "1.0", + "name": "test1", + "testcase": [ + { + "classname": "test1", + "name": "setup", + "timestamp": "2000-01-01T00:00:00.00000000+00:00", + "time": "1.0" + }, + { + "classname": "test1", + "name": "step 0-run", + "timestamp": "2000-01-01T00:00:00.00000000+00:00", + "time": "1.0" + }, + { + "classname": "test1", + "name": "step 1-run", + "timestamp": "2000-01-01T00:00:00.00000000+00:00", + "time": "1.0" + }, + { + "classname": "test1", + "name": "step 2-run", + "timestamp": "2000-01-01T00:00:00.00000000+00:00", + "time": "1.0", + "failure": { + "text": "command \"echo step stdout\\\\n echo \u003e\u00262 step stderr\\\\n false\" failed, exit status 1", + "message": "failed in step 2-run" + } + } + ] + }, + { + "tests": 3, + "failures": 1, + "timestamp": "2000-01-01T00:00:00.00000000+00:00", + "time": "1.0", + "name": "test2", + "testcase": [ + { + "classname": "test2", + "name": "setup", + "timestamp": "2000-01-01T00:00:00.00000000+00:00", + "time": "1.0" + }, + { + "classname": "test2", + "name": "step 0-run", + "timestamp": "2000-01-01T00:00:00.00000000+00:00", + "time": "1.0" + }, + { + "classname": "test2", + "name": "step 1-run", + "timestamp": "2000-01-01T00:00:00.00000000+00:00", + "time": "1.0", + "failure": { + "text": "command \"echo assert stdout\\\\n echo \u003e\u00262 assert stderr\\\\n false\" failed, exit status 1", + "message": "failed in step 1-run" + } + } + ] + } + ] + } + ] + } \ No newline at end of file diff --git a/test/junit/kuttl-report-step.xml.golden b/test/junit/kuttl-report-step.xml.golden new file mode 100644 index 00000000..80e02e88 --- /dev/null +++ b/test/junit/kuttl-report-step.xml.golden @@ -0,0 +1,44 @@ + + + + + + + + + + + + command "echo step stdout\\n echo >&2 step stderr\\n false" failed, exit status 1 + + + + + + + command "echo assert stdout\\n echo >&2 assert stderr\\n false" failed, exit status 1 + + + + + + + + + + + + + + command "echo step stdout\\n echo >&2 step stderr\\n false" failed, exit status 1 + + + + + + + command "echo assert stdout\\n echo >&2 assert stderr\\n false" failed, exit status 1 + + + + \ No newline at end of file diff --git a/test/junit/suite1/test0/00-assert.yaml b/test/junit/suite1/test0/00-assert.yaml new file mode 100644 index 00000000..397a7fc9 --- /dev/null +++ b/test/junit/suite1/test0/00-assert.yaml @@ -0,0 +1,7 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +commands: +- script: | + echo assert stdout + echo >&2 assert stderr + true diff --git a/test/junit/suite1/test0/00-run.yaml b/test/junit/suite1/test0/00-run.yaml new file mode 100644 index 00000000..2e07fd0a --- /dev/null +++ b/test/junit/suite1/test0/00-run.yaml @@ -0,0 +1,7 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +commands: +- script: | + echo step stdout + echo >&2 step stderr + true diff --git a/test/junit/suite1/test1/00-assert.yaml b/test/junit/suite1/test1/00-assert.yaml new file mode 100644 index 00000000..397a7fc9 --- /dev/null +++ b/test/junit/suite1/test1/00-assert.yaml @@ -0,0 +1,7 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +commands: +- script: | + echo assert stdout + echo >&2 assert stderr + true diff --git a/test/junit/suite1/test1/00-run.yaml b/test/junit/suite1/test1/00-run.yaml new file mode 100644 index 00000000..2e07fd0a --- /dev/null +++ b/test/junit/suite1/test1/00-run.yaml @@ -0,0 +1,7 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +commands: +- script: | + echo step stdout + echo >&2 step stderr + true diff --git a/test/junit/suite1/test1/01-assert.yaml b/test/junit/suite1/test1/01-assert.yaml new file mode 100644 index 00000000..397a7fc9 --- /dev/null +++ b/test/junit/suite1/test1/01-assert.yaml @@ -0,0 +1,7 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +commands: +- script: | + echo assert stdout + echo >&2 assert stderr + true diff --git a/test/junit/suite1/test1/01-run.yaml b/test/junit/suite1/test1/01-run.yaml new file mode 100644 index 00000000..2e07fd0a --- /dev/null +++ b/test/junit/suite1/test1/01-run.yaml @@ -0,0 +1,7 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +commands: +- script: | + echo step stdout + echo >&2 step stderr + true diff --git a/test/junit/suite1/test1/02-run.yaml b/test/junit/suite1/test1/02-run.yaml new file mode 100644 index 00000000..2e87aa1d --- /dev/null +++ b/test/junit/suite1/test1/02-run.yaml @@ -0,0 +1,7 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +commands: +- script: | + echo step stdout + echo >&2 step stderr + false diff --git a/test/junit/suite1/test2/00-assert.yaml b/test/junit/suite1/test2/00-assert.yaml new file mode 100644 index 00000000..397a7fc9 --- /dev/null +++ b/test/junit/suite1/test2/00-assert.yaml @@ -0,0 +1,7 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +commands: +- script: | + echo assert stdout + echo >&2 assert stderr + true diff --git a/test/junit/suite1/test2/00-run.yaml b/test/junit/suite1/test2/00-run.yaml new file mode 100644 index 00000000..2e07fd0a --- /dev/null +++ b/test/junit/suite1/test2/00-run.yaml @@ -0,0 +1,7 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +commands: +- script: | + echo step stdout + echo >&2 step stderr + true diff --git a/test/junit/suite1/test2/01-assert.yaml b/test/junit/suite1/test2/01-assert.yaml new file mode 100644 index 00000000..5ca2cdb4 --- /dev/null +++ b/test/junit/suite1/test2/01-assert.yaml @@ -0,0 +1,7 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +commands: +- script: | + echo assert stdout + echo >&2 assert stderr + false diff --git a/test/junit/suite1/test2/01-run.yaml b/test/junit/suite1/test2/01-run.yaml new file mode 100644 index 00000000..2e07fd0a --- /dev/null +++ b/test/junit/suite1/test2/01-run.yaml @@ -0,0 +1,7 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +commands: +- script: | + echo step stdout + echo >&2 step stderr + true diff --git a/test/junit/suite1/test2/02-run.yaml b/test/junit/suite1/test2/02-run.yaml new file mode 100644 index 00000000..2e87aa1d --- /dev/null +++ b/test/junit/suite1/test2/02-run.yaml @@ -0,0 +1,7 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +commands: +- script: | + echo step stdout + echo >&2 step stderr + false diff --git a/test/junit/suite2/test0/00-assert.yaml b/test/junit/suite2/test0/00-assert.yaml new file mode 100644 index 00000000..397a7fc9 --- /dev/null +++ b/test/junit/suite2/test0/00-assert.yaml @@ -0,0 +1,7 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +commands: +- script: | + echo assert stdout + echo >&2 assert stderr + true diff --git a/test/junit/suite2/test0/00-run.yaml b/test/junit/suite2/test0/00-run.yaml new file mode 100644 index 00000000..2e07fd0a --- /dev/null +++ b/test/junit/suite2/test0/00-run.yaml @@ -0,0 +1,7 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +commands: +- script: | + echo step stdout + echo >&2 step stderr + true diff --git a/test/junit/suite2/test1/00-assert.yaml b/test/junit/suite2/test1/00-assert.yaml new file mode 100644 index 00000000..397a7fc9 --- /dev/null +++ b/test/junit/suite2/test1/00-assert.yaml @@ -0,0 +1,7 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +commands: +- script: | + echo assert stdout + echo >&2 assert stderr + true diff --git a/test/junit/suite2/test1/00-run.yaml b/test/junit/suite2/test1/00-run.yaml new file mode 100644 index 00000000..2e07fd0a --- /dev/null +++ b/test/junit/suite2/test1/00-run.yaml @@ -0,0 +1,7 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +commands: +- script: | + echo step stdout + echo >&2 step stderr + true diff --git a/test/junit/suite2/test1/01-assert.yaml b/test/junit/suite2/test1/01-assert.yaml new file mode 100644 index 00000000..397a7fc9 --- /dev/null +++ b/test/junit/suite2/test1/01-assert.yaml @@ -0,0 +1,7 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +commands: +- script: | + echo assert stdout + echo >&2 assert stderr + true diff --git a/test/junit/suite2/test1/01-run.yaml b/test/junit/suite2/test1/01-run.yaml new file mode 100644 index 00000000..2e07fd0a --- /dev/null +++ b/test/junit/suite2/test1/01-run.yaml @@ -0,0 +1,7 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +commands: +- script: | + echo step stdout + echo >&2 step stderr + true diff --git a/test/junit/suite2/test1/02-run.yaml b/test/junit/suite2/test1/02-run.yaml new file mode 100644 index 00000000..2e87aa1d --- /dev/null +++ b/test/junit/suite2/test1/02-run.yaml @@ -0,0 +1,7 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +commands: +- script: | + echo step stdout + echo >&2 step stderr + false diff --git a/test/junit/suite2/test2/00-assert.yaml b/test/junit/suite2/test2/00-assert.yaml new file mode 100644 index 00000000..397a7fc9 --- /dev/null +++ b/test/junit/suite2/test2/00-assert.yaml @@ -0,0 +1,7 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +commands: +- script: | + echo assert stdout + echo >&2 assert stderr + true diff --git a/test/junit/suite2/test2/00-run.yaml b/test/junit/suite2/test2/00-run.yaml new file mode 100644 index 00000000..2e07fd0a --- /dev/null +++ b/test/junit/suite2/test2/00-run.yaml @@ -0,0 +1,7 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +commands: +- script: | + echo step stdout + echo >&2 step stderr + true diff --git a/test/junit/suite2/test2/01-assert.yaml b/test/junit/suite2/test2/01-assert.yaml new file mode 100644 index 00000000..5ca2cdb4 --- /dev/null +++ b/test/junit/suite2/test2/01-assert.yaml @@ -0,0 +1,7 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +commands: +- script: | + echo assert stdout + echo >&2 assert stderr + false diff --git a/test/junit/suite2/test2/01-run.yaml b/test/junit/suite2/test2/01-run.yaml new file mode 100644 index 00000000..2e07fd0a --- /dev/null +++ b/test/junit/suite2/test2/01-run.yaml @@ -0,0 +1,7 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +commands: +- script: | + echo step stdout + echo >&2 step stderr + true diff --git a/test/junit/suite2/test2/02-run.yaml b/test/junit/suite2/test2/02-run.yaml new file mode 100644 index 00000000..2e87aa1d --- /dev/null +++ b/test/junit/suite2/test2/02-run.yaml @@ -0,0 +1,7 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +commands: +- script: | + echo step stdout + echo >&2 step stderr + false