e2e test suite

Signed-off-by: Marcin Owsiany <[email protected]>
kudobuilder · Nov 28, 2024 · 7013412 · 7013412
1 parent 60fc134
commit 7013412
Show file tree

Hide file tree

Showing 30 changed files with 474 additions and 1 deletion.
diff --git a/.github/workflows/e2e-test.yml b/.github/workflows/e2e-test.yml
@@ -0,0 +1,22 @@
+name: "End-to-end tests"
+
+on:
+  push:
+    branches:
+      - main
+      - releases/*
+  pull_request:
+    branches:
+      - main
+      - releases/*
+
+jobs:
+  e2e-tests:
+    runs-on: ubuntu-20.04
+    steps:
+    - uses: actions/[email protected]
+      with:
+        go-version: 1.21
+    - uses: actions/[email protected]
+    - name: "Run end-to-end tests"
+      run: make e2e-test
diff --git a/Makefile b/Makefile
@@ -122,7 +122,7 @@ todo: ## Shows todos from code
 ##@ Tests
 
 .PHONY: all
-all: lint test integration-test  ## Runs lint, unit and integration tests
+all: lint test integration-test e2e-test  ## Runs lint, unit, integration and e2e tests
 
 # Run unit tests
 .PHONY: test
@@ -139,6 +139,11 @@ endif
 integration-test: envtest  ## Runs integration tests
 	KUBEBUILDER_ASSETS="$(shell $(ENVTEST) use $(ENVTEST_K8S_VERSION) --bin-dir $(LOCALBIN) -p path)" ./hack/run-integration-tests.sh
 
+.PHONY: e2e-test
+# Run e2e tests
+e2e-test: envtest  ## Runs end-to-end tests
+	KUBEBUILDER_ASSETS="$(shell $(ENVTEST) use $(ENVTEST_K8S_VERSION) --bin-dir $(LOCALBIN) -p path)" $(MAKE) -C ./test/junit
+
 ##@ Build Dependencies
 
 ## Location to install dependencies to

diff --git a/test/junit/.gitignore b/test/junit/.gitignore
@@ -0,0 +1,6 @@
+/kuttl-ouput-step-json.txt
+/kuttl-ouput-step-xml.txt
+/kuttl-report-step.json
+/kuttl-report-step.json.normalized
+/kuttl-report-step.xml
+/kuttl-report-step.xml.normalized
diff --git a/test/junit/Makefile b/test/junit/Makefile
@@ -0,0 +1,24 @@
+.PHONY: test
+test:
+	$(MAKE) -C ../../ cli
+	rm -f kuttl-report-step.xml.normalized kuttl-report-step.json.normalized
+	../../bin/kubectl-kuttl test --config /dev/null --start-control-plane --report-name kuttl-report-step --timeout 10 --report xml suite1 suite2 > kuttl-ouput-step-xml.txt 2>&1 || true # this is meant to fail
+	if [ ! -e kuttl-report-step.xml ]; then cat kuttl-output-step-xml.txt; exit 1; fi
+	../../bin/kubectl-kuttl test --config /dev/null --start-control-plane --report-name kuttl-report-step --timeout 10 --report json suite1 suite2 > kuttl-ouput-step-json.txt 2>&1 || true # this is meant to fail
+	if [ ! -e kuttl-report-step.json ]; then cat kuttl-output-step-json.txt; exit 1; fi
+	$(MAKE) kuttl-report-step.xml.normalized kuttl-report-step.json.normalized
+	diff -u kuttl-report-step.xml.golden kuttl-report-step.xml.normalized
+	diff -u kuttl-report-step.json.golden kuttl-report-step.json.normalized
+
+.PHONY: update-golden
+update-golden:
+	cp kuttl-report-step.json.normalized kuttl-report-step.json.golden
+	cp kuttl-report-step.xml.normalized kuttl-report-step.xml.golden
+
+# The following targets replace all timestamps and durations with dummy values to make comparisons easy.
+
+%.xml.normalized: %.xml
+	sed -E -e 's/time="[^"]+"/time="1.0"/g; s/[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}[.][0-9]{6,10}(Z|[-+][0-9]{2}:[0-9]{2})/2000-01-01T00:00:00.00000000+00:00/g' < $< > $@
+
+%.json.normalized: %.json
+	sed -E -e 's/"time": *"[^"]+"/"time": "1.0"/g; s/[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}[.][0-9]{6,10}(Z|[-+][0-9]{2}:[0-9]{2})/2000-01-01T00:00:00.00000000+00:00/g' < $< > $@
diff --git a/test/junit/kuttl-report-step.json.golden b/test/junit/kuttl-report-step.json.golden
@@ -0,0 +1,204 @@
+{
+   "name": "",
+   "tests": 18,
+   "failures": 4,
+   "time": "1.0",
+   "testsuite": [
+     {
+       "tests": 9,
+       "failures": 2,
+       "timestamp": "2000-01-01T00:00:00.00000000+00:00",
+       "time": "1.0",
+       "name": "suite1",
+       "testsuite": [
+         {
+           "tests": 2,
+           "failures": 0,
+           "timestamp": "2000-01-01T00:00:00.00000000+00:00",
+           "time": "1.0",
+           "name": "test0",
+           "testcase": [
+             {
+               "classname": "test0",
+               "name": "setup",
+               "timestamp": "2000-01-01T00:00:00.00000000+00:00",
+               "time": "1.0"
+             },
+             {
+               "classname": "test0",
+               "name": "step 0-run",
+               "timestamp": "2000-01-01T00:00:00.00000000+00:00",
+               "time": "1.0"
+             }
+           ]
+         },
+         {
+           "tests": 4,
+           "failures": 1,
+           "timestamp": "2000-01-01T00:00:00.00000000+00:00",
+           "time": "1.0",
+           "name": "test1",
+           "testcase": [
+             {
+               "classname": "test1",
+               "name": "setup",
+               "timestamp": "2000-01-01T00:00:00.00000000+00:00",
+               "time": "1.0"
+             },
+             {
+               "classname": "test1",
+               "name": "step 0-run",
+               "timestamp": "2000-01-01T00:00:00.00000000+00:00",
+               "time": "1.0"
+             },
+             {
+               "classname": "test1",
+               "name": "step 1-run",
+               "timestamp": "2000-01-01T00:00:00.00000000+00:00",
+               "time": "1.0"
+             },
+             {
+               "classname": "test1",
+               "name": "step 2-run",
+               "timestamp": "2000-01-01T00:00:00.00000000+00:00",
+               "time": "1.0",
+               "failure": {
+                 "text": "command \"echo step stdout\\\\n echo \u003e\u00262 step stderr\\\\n false\" failed, exit status 1",
+                 "message": "failed in step 2-run"
+               }
+             }
+           ]
+         },
+         {
+           "tests": 3,
+           "failures": 1,
+           "timestamp": "2000-01-01T00:00:00.00000000+00:00",
+           "time": "1.0",
+           "name": "test2",
+           "testcase": [
+             {
+               "classname": "test2",
+               "name": "setup",
+               "timestamp": "2000-01-01T00:00:00.00000000+00:00",
+               "time": "1.0"
+             },
+             {
+               "classname": "test2",
+               "name": "step 0-run",
+               "timestamp": "2000-01-01T00:00:00.00000000+00:00",
+               "time": "1.0"
+             },
+             {
+               "classname": "test2",
+               "name": "step 1-run",
+               "timestamp": "2000-01-01T00:00:00.00000000+00:00",
+               "time": "1.0",
+               "failure": {
+                 "text": "command \"echo assert stdout\\\\n echo \u003e\u00262 assert stderr\\\\n false\" failed, exit status 1",
+                 "message": "failed in step 1-run"
+               }
+             }
+           ]
+         }
+       ]
+     },
+     {
+       "tests": 9,
+       "failures": 2,
+       "timestamp": "2000-01-01T00:00:00.00000000+00:00",
+       "time": "1.0",
+       "name": "suite2",
+       "testsuite": [
+         {
+           "tests": 2,
+           "failures": 0,
+           "timestamp": "2000-01-01T00:00:00.00000000+00:00",
+           "time": "1.0",
+           "name": "test0",
+           "testcase": [
+             {
+               "classname": "test0",
+               "name": "setup",
+               "timestamp": "2000-01-01T00:00:00.00000000+00:00",
+               "time": "1.0"
+             },
+             {
+               "classname": "test0",
+               "name": "step 0-run",
+               "timestamp": "2000-01-01T00:00:00.00000000+00:00",
+               "time": "1.0"
+             }
+           ]
+         },
+         {
+           "tests": 4,
+           "failures": 1,
+           "timestamp": "2000-01-01T00:00:00.00000000+00:00",
+           "time": "1.0",
+           "name": "test1",
+           "testcase": [
+             {
+               "classname": "test1",
+               "name": "setup",
+               "timestamp": "2000-01-01T00:00:00.00000000+00:00",
+               "time": "1.0"
+             },
+             {
+               "classname": "test1",
+               "name": "step 0-run",
+               "timestamp": "2000-01-01T00:00:00.00000000+00:00",
+               "time": "1.0"
+             },
+             {
+               "classname": "test1",
+               "name": "step 1-run",
+               "timestamp": "2000-01-01T00:00:00.00000000+00:00",
+               "time": "1.0"
+             },
+             {
+               "classname": "test1",
+               "name": "step 2-run",
+               "timestamp": "2000-01-01T00:00:00.00000000+00:00",
+               "time": "1.0",
+               "failure": {
+                 "text": "command \"echo step stdout\\\\n echo \u003e\u00262 step stderr\\\\n false\" failed, exit status 1",
+                 "message": "failed in step 2-run"
+               }
+             }
+           ]
+         },
+         {
+           "tests": 3,
+           "failures": 1,
+           "timestamp": "2000-01-01T00:00:00.00000000+00:00",
+           "time": "1.0",
+           "name": "test2",
+           "testcase": [
+             {
+               "classname": "test2",
+               "name": "setup",
+               "timestamp": "2000-01-01T00:00:00.00000000+00:00",
+               "time": "1.0"
+             },
+             {
+               "classname": "test2",
+               "name": "step 0-run",
+               "timestamp": "2000-01-01T00:00:00.00000000+00:00",
+               "time": "1.0"
+             },
+             {
+               "classname": "test2",
+               "name": "step 1-run",
+               "timestamp": "2000-01-01T00:00:00.00000000+00:00",
+               "time": "1.0",
+               "failure": {
+                 "text": "command \"echo assert stdout\\\\n echo \u003e\u00262 assert stderr\\\\n false\" failed, exit status 1",
+                 "message": "failed in step 1-run"
+               }
+             }
+           ]
+         }
+       ]
+     }
+   ]
+ }
diff --git a/test/junit/kuttl-report-step.xml.golden b/test/junit/kuttl-report-step.xml.golden
@@ -0,0 +1,44 @@
+ <testsuites name="" tests="18" failures="4" time="1.0">
+   <testsuite tests="9" failures="2" timestamp="2000-01-01T00:00:00.00000000+00:00" time="1.0" name="suite1">
+     <testsuite tests="2" failures="0" timestamp="2000-01-01T00:00:00.00000000+00:00" time="1.0" name="test0">
+       <testcase classname="test0" name="setup" timestamp="2000-01-01T00:00:00.00000000+00:00" time="1.0" assertions="0"></testcase>
+       <testcase classname="test0" name="step 0-run" timestamp="2000-01-01T00:00:00.00000000+00:00" time="1.0" assertions="0"></testcase>
+     </testsuite>
+     <testsuite tests="4" failures="1" timestamp="2000-01-01T00:00:00.00000000+00:00" time="1.0" name="test1">
+       <testcase classname="test1" name="setup" timestamp="2000-01-01T00:00:00.00000000+00:00" time="1.0" assertions="0"></testcase>
+       <testcase classname="test1" name="step 0-run" timestamp="2000-01-01T00:00:00.00000000+00:00" time="1.0" assertions="0"></testcase>
+       <testcase classname="test1" name="step 1-run" timestamp="2000-01-01T00:00:00.00000000+00:00" time="1.0" assertions="0"></testcase>
+       <testcase classname="test1" name="step 2-run" timestamp="2000-01-01T00:00:00.00000000+00:00" time="1.0" assertions="0">
+         <failure message="failed in step 2-run" type="">command &#34;echo step stdout\\n echo &gt;&amp;2 step stderr\\n false&#34; failed, exit status 1</failure>
+       </testcase>
+     </testsuite>
+     <testsuite tests="3" failures="1" timestamp="2000-01-01T00:00:00.00000000+00:00" time="1.0" name="test2">
+       <testcase classname="test2" name="setup" timestamp="2000-01-01T00:00:00.00000000+00:00" time="1.0" assertions="0"></testcase>
+       <testcase classname="test2" name="step 0-run" timestamp="2000-01-01T00:00:00.00000000+00:00" time="1.0" assertions="0"></testcase>
+       <testcase classname="test2" name="step 1-run" timestamp="2000-01-01T00:00:00.00000000+00:00" time="1.0" assertions="0">
+         <failure message="failed in step 1-run" type="">command &#34;echo assert stdout\\n echo &gt;&amp;2 assert stderr\\n false&#34; failed, exit status 1</failure>
+       </testcase>
+     </testsuite>
+   </testsuite>
+   <testsuite tests="9" failures="2" timestamp="2000-01-01T00:00:00.00000000+00:00" time="1.0" name="suite2">
+     <testsuite tests="2" failures="0" timestamp="2000-01-01T00:00:00.00000000+00:00" time="1.0" name="test0">
+       <testcase classname="test0" name="setup" timestamp="2000-01-01T00:00:00.00000000+00:00" time="1.0" assertions="0"></testcase>
+       <testcase classname="test0" name="step 0-run" timestamp="2000-01-01T00:00:00.00000000+00:00" time="1.0" assertions="0"></testcase>
+     </testsuite>
+     <testsuite tests="4" failures="1" timestamp="2000-01-01T00:00:00.00000000+00:00" time="1.0" name="test1">
+       <testcase classname="test1" name="setup" timestamp="2000-01-01T00:00:00.00000000+00:00" time="1.0" assertions="0"></testcase>
+       <testcase classname="test1" name="step 0-run" timestamp="2000-01-01T00:00:00.00000000+00:00" time="1.0" assertions="0"></testcase>
+       <testcase classname="test1" name="step 1-run" timestamp="2000-01-01T00:00:00.00000000+00:00" time="1.0" assertions="0"></testcase>
+       <testcase classname="test1" name="step 2-run" timestamp="2000-01-01T00:00:00.00000000+00:00" time="1.0" assertions="0">
+         <failure message="failed in step 2-run" type="">command &#34;echo step stdout\\n echo &gt;&amp;2 step stderr\\n false&#34; failed, exit status 1</failure>
+       </testcase>
+     </testsuite>
+     <testsuite tests="3" failures="1" timestamp="2000-01-01T00:00:00.00000000+00:00" time="1.0" name="test2">
+       <testcase classname="test2" name="setup" timestamp="2000-01-01T00:00:00.00000000+00:00" time="1.0" assertions="0"></testcase>
+       <testcase classname="test2" name="step 0-run" timestamp="2000-01-01T00:00:00.00000000+00:00" time="1.0" assertions="0"></testcase>
+       <testcase classname="test2" name="step 1-run" timestamp="2000-01-01T00:00:00.00000000+00:00" time="1.0" assertions="0">
+         <failure message="failed in step 1-run" type="">command &#34;echo assert stdout\\n echo &gt;&amp;2 assert stderr\\n false&#34; failed, exit status 1</failure>
+       </testcase>
+     </testsuite>
+   </testsuite>
+ </testsuites>
diff --git a/test/junit/suite1/test0/00-assert.yaml b/test/junit/suite1/test0/00-assert.yaml
@@ -0,0 +1,7 @@
+apiVersion: kuttl.dev/v1beta1
+kind: TestAssert
+commands:
+- script: |
+    echo assert stdout
+    echo >&2 assert stderr
+    true
diff --git a/test/junit/suite1/test0/00-run.yaml b/test/junit/suite1/test0/00-run.yaml
@@ -0,0 +1,7 @@
+apiVersion: kuttl.dev/v1beta1
+kind: TestStep
+commands:
+- script: |
+    echo step stdout
+    echo >&2 step stderr
+    true
diff --git a/test/junit/suite1/test1/00-assert.yaml b/test/junit/suite1/test1/00-assert.yaml
@@ -0,0 +1,7 @@
+apiVersion: kuttl.dev/v1beta1
+kind: TestAssert
+commands:
+- script: |
+    echo assert stdout
+    echo >&2 assert stderr
+    true
diff --git a/test/junit/suite1/test1/00-run.yaml b/test/junit/suite1/test1/00-run.yaml
@@ -0,0 +1,7 @@
+apiVersion: kuttl.dev/v1beta1
+kind: TestStep
+commands:
+- script: |
+    echo step stdout
+    echo >&2 step stderr
+    true
diff --git a/test/junit/suite1/test1/01-assert.yaml b/test/junit/suite1/test1/01-assert.yaml
@@ -0,0 +1,7 @@
+apiVersion: kuttl.dev/v1beta1
+kind: TestAssert
+commands:
+- script: |
+    echo assert stdout
+    echo >&2 assert stderr
+    true
diff --git a/test/junit/suite1/test1/01-run.yaml b/test/junit/suite1/test1/01-run.yaml
@@ -0,0 +1,7 @@
+apiVersion: kuttl.dev/v1beta1
+kind: TestStep
+commands:
+- script: |
+    echo step stdout
+    echo >&2 step stderr
+    true
diff --git a/test/junit/suite1/test1/02-run.yaml b/test/junit/suite1/test1/02-run.yaml
@@ -0,0 +1,7 @@
+apiVersion: kuttl.dev/v1beta1
+kind: TestStep
+commands:
+- script: |
+    echo step stdout
+    echo >&2 step stderr
+    false
diff --git a/test/junit/suite1/test2/00-assert.yaml b/test/junit/suite1/test2/00-assert.yaml
@@ -0,0 +1,7 @@
+apiVersion: kuttl.dev/v1beta1
+kind: TestAssert
+commands:
+- script: |
+    echo assert stdout
+    echo >&2 assert stderr
+    true
diff --git a/test/junit/suite1/test2/00-run.yaml b/test/junit/suite1/test2/00-run.yaml
@@ -0,0 +1,7 @@
+apiVersion: kuttl.dev/v1beta1
+kind: TestStep
+commands:
+- script: |
+    echo step stdout
+    echo >&2 step stderr
+    true