From 6ad3d324d434f9ee4d05bbb03c8fe73d0a8f0a09 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20A=2E=20Bellone?= Date: Fri, 28 Oct 2022 12:30:55 -0300 Subject: [PATCH 1/7] Added option for minimum test amount in plugin spec --- plugin.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/plugin.yml b/plugin.yml index cc03bd7..1dcaf76 100644 --- a/plugin.yml +++ b/plugin.yml @@ -20,6 +20,8 @@ configuration: type: boolean job-uuid-file-pattern: type: string + min-tests: + type: integer report-slowest: type: integer required: From 17335432822952f0979b90ec3d9fe7ece5cbb743 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20A=2E=20Bellone?= Date: Fri, 28 Oct 2022 12:31:11 -0300 Subject: [PATCH 2/7] Added documentation on new option (and tweaked the rest) --- README.md | 23 ++++++++++++++--------- 1 file changed, 14 insertions(+), 9 deletions(-) diff --git a/README.md b/README.md index a0bf946..9808f02 100644 --- a/README.md +++ b/README.md @@ -30,37 +30,42 @@ Example: `tmp/junit-*.xml` Forces the creation of the annotation even when no failures or errors are found +### `context` (optional) + +Default: `junit` + +The buildkite annotation context to use. Useful to differentiate multiple runs of this plugin in a single pipeline. + ### `job-uuid-file-pattern` (optional) + Default: `-(.*).xml` -The regular expression (with capture group) that matches the job UUID in the junit file names. This is used to create the job links in the annotation. +The regular expression (with capture group) that matches the job UUID in the junit file names. This is used to create the job links in the annotation. To use this, configure your test reporter to embed the `$BUILDKITE_JOB_ID` environment variable into your junit file names. For example `"junit-buildkite-job-$BUILDKITE_JOB_ID.xml"`. ### `failure-format` (optional) -Default: `classname` This setting controls the format of your failed test in the main annotation summary. There are two options for this: -* `classname` +* `classname` (the default) * displays: `MyClass::UnderTest text of the failed expectation in path.to.my_class.under_test` * `file` * displays: `MyClass::UnderTest text of the failed expectation in path/to/my_class/under_test.file_ext` ### `fail-build-on-error` (optional) + Default: `false` -If this setting is true and any errors are found in the JUnit XML files during -parsing, the annotation step will exit with a non-zero value, which should cause -the build to fail. +If this setting is true and any errors are found in the JUnit XML files during parsing, the annotation step will exit with a non-zero value, which should cause the build to fail. -### `context` (optional) -Default: `junit` +### `min-tests` (optional, integer) -The buildkite annotation context to use. Useful to differentiate multiple runs of this plugin in a single pipeline. +Minimum amount of run tests that need to be analyzed or a failure will be reported. It is useful to ensure that tests are actually run and report files to analyze do contain information. ### `report-slowest` (optional) + Default: `0` Include the specified number of slowest tests in the annotation. The annotation will always be shown. From 434c978fa7d5a9751fcbff28ff9daa8f716e799e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20A=2E=20Bellone?= Date: Fri, 28 Oct 2022 12:48:22 -0300 Subject: [PATCH 3/7] Implemented option to fail if less than X tests are parsed --- hooks/command | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/hooks/command b/hooks/command index 7db1f15..f548c92 100755 --- a/hooks/command +++ b/hooks/command @@ -76,6 +76,18 @@ if [ $has_errors -eq 0 ]; then echo "Will create annotation anyways" create_annotation=1 fi + + if [[ -e "${annotation_path}" ]]; then + TOTAL_TESTS=$(head -4 "${annotation_path}" | grep 'Total tests' | cut -d\ -f3) + else + TOTAL_TESTS=0 + fi + + if [[ "${BUILDKITE_PLUGIN_JUNIT_ANNOTATE_MIN_TESTS:-0}" -gt "${TOTAL_TESTS}" ]]; then + create_annotation=1 + fail_build=1 + echo ":warning: Less than ${BUILDKITE_PLUGIN_JUNIT_ANNOTATE_MIN_TESTS} tests run" + fi elif ! check_size; then echo "--- :warning: Failures too large to annotate" From 908821c694f184cbe76cced3cd17dc10c79597c3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20A=2E=20Bellone?= Date: Fri, 28 Oct 2022 12:48:48 -0300 Subject: [PATCH 4/7] Corrected tests now that output is important --- tests/command.bats | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/tests/command.bats b/tests/command.bats index 0300156..e7f1366 100644 --- a/tests/command.bats +++ b/tests/command.bats @@ -53,7 +53,7 @@ export annotation_input="tests/tmp/annotation.input" "annotate --context \* --style \* : cat >'${annotation_input}'; echo Annotation added with context \$3 and style \$5, content saved" stub docker \ - "--log-level error run --rm --volume \* --volume \* --env \* --env \* --env \* ruby:2.7-alpine ruby /src/bin/annotate /junits : echo '
Failure
' && exit 64" + "--log-level error run --rm --volume \* --volume \* --env \* --env \* --env \* ruby:2.7-alpine ruby /src/bin/annotate /junits : cat tests/2-tests-1-failure.output && exit 64" run "$PWD/hooks/command" @@ -80,7 +80,7 @@ export annotation_input="tests/tmp/annotation.input" "annotate --context \* --style \* : cat >'${annotation_input}'; echo Annotation added with context \$3 and style \$5, content saved" stub docker \ - "--log-level error run --rm --volume \* --volume \* --env BUILDKITE_PLUGIN_JUNIT_ANNOTATE_JOB_UUID_FILE_PATTERN='custom_(*)_pattern.xml' --env \* --env \* ruby:2.7-alpine ruby /src/bin/annotate /junits : echo '
Failure
' && exit 64" + "--log-level error run --rm --volume \* --volume \* --env BUILDKITE_PLUGIN_JUNIT_ANNOTATE_JOB_UUID_FILE_PATTERN='custom_(*)_pattern.xml' --env \* --env \* ruby:2.7-alpine ruby /src/bin/annotate /junits : cat tests/2-tests-1-failure.output && exit 64" run "$PWD/hooks/command" @@ -107,7 +107,7 @@ export annotation_input="tests/tmp/annotation.input" "annotate --context \* --style \* : cat >'${annotation_input}'; echo Annotation added with context \$3 and style \$5, content saved" stub docker \ - "--log-level error run --rm --volume \* --volume \* --env \* --env BUILDKITE_PLUGIN_JUNIT_ANNOTATE_FAILURE_FORMAT='file' --env \* ruby:2.7-alpine ruby /src/bin/annotate /junits : echo '
Failure
' && exit 64" + "--log-level error run --rm --volume \* --volume \* --env \* --env BUILDKITE_PLUGIN_JUNIT_ANNOTATE_FAILURE_FORMAT='file' --env \* ruby:2.7-alpine ruby /src/bin/annotate /junits : cat tests/2-tests-1-failure.output && exit 64" run "$PWD/hooks/command" @@ -132,7 +132,7 @@ export annotation_input="tests/tmp/annotation.input" "artifact download \* \* : echo Downloaded artifact \$3 to \$4" stub docker \ - "--log-level error run --rm --volume \* --volume \* --env \* --env \* --env \* ruby:2.7-alpine ruby /src/bin/annotate /junits : echo No test errors" + "--log-level error run --rm --volume \* --volume \* --env \* --env \* --env \* ruby:2.7-alpine ruby /src/bin/annotate /junits : echo 'Total tests: 0'" run "$PWD/hooks/command" @@ -156,13 +156,14 @@ export annotation_input="tests/tmp/annotation.input" "annotate --context \* --style \* : cat >'${annotation_input}'; echo Annotation added with context \$3 and style \$5, content saved" stub docker \ - "--log-level error run --rm --volume \* --volume \* --env \* --env \* --env \* ruby:2.7-alpine ruby /src/bin/annotate /junits : echo No test errors" + "--log-level error run --rm --volume \* --volume \* --env \* --env \* --env \* ruby:2.7-alpine ruby /src/bin/annotate /junits : echo 'Total tests: 0'" run "$PWD/hooks/command" assert_success - assert_output --partial "No test errors" + assert_output --partial "Total tests: 0" assert_output --partial "Will create annotation anyways" + assert_equal "$(cat "${annotation_input}")" 'Total tests: 0' unstub mktemp unstub buildkite-agent @@ -194,7 +195,7 @@ export annotation_input="tests/tmp/annotation.input" "artifact download \* \* : echo Downloaded artifact \$3 to \$4" stub docker \ - "--log-level error run --rm --volume \* --volume \* --env \* --env \* --env \* ruby:2.7-alpine ruby /src/bin/annotate /junits : echo '
Failure
' && exit 64" + "--log-level error run --rm --volume \* --volume \* --env \* --env \* --env \* ruby:2.7-alpine ruby /src/bin/annotate /junits : cat tests/2-tests-1-failure.output && exit 64" run "$PWD/hooks/command" @@ -256,7 +257,7 @@ export annotation_input="tests/tmp/annotation.input" "annotate --context \* --style \* : cat >'${annotation_input}'; echo Annotation added with context \$3 and style \$5, content saved" stub docker \ - "--log-level error run --rm --volume \* --volume \* --env \* --env \* --env \* ruby:2.7-alpine ruby /src/bin/annotate /junits : echo '
Failure
' && exit 64" + "--log-level error run --rm --volume \* --volume \* --env \* --env \* --env \* ruby:2.7-alpine ruby /src/bin/annotate /junits : cat tests/2-tests-1-failure.output && exit 64" run "$PWD/hooks/command" @@ -285,7 +286,7 @@ export annotation_input="tests/tmp/annotation.input" "artifact download \* \* : echo Downloaded artifact \$3 to \$4" stub docker \ - "--log-level error run --rm --volume \* --volume \* --env \* --env \* --env \* ruby:2.7-alpine ruby /src/bin/annotate /junits : echo '
Failure
' && exit 64" + "--log-level error run --rm --volume \* --volume \* --env \* --env \* --env \* ruby:2.7-alpine ruby /src/bin/annotate /junits : cat tests/2-tests-1-failure.output && exit 64" run "$PWD/hooks/command" @@ -311,7 +312,7 @@ export annotation_input="tests/tmp/annotation.input" "artifact download \* \* : echo Downloaded artifact \$3 to \$4" stub docker \ - "--log-level error run --rm --volume \* --volume \* --env \* --env \* --env \* ruby:2.7-alpine ruby /src/bin/annotate /junits : echo '
Failure
' && exit 147" + "--log-level error run --rm --volume \* --volume \* --env \* --env \* --env \* ruby:2.7-alpine ruby /src/bin/annotate /junits : cat tests/2-tests-1-failure.output && exit 147" run "$PWD/hooks/command" From a4100ee8c9041e4e5855c7e5bd4c88e1d33999a4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20A=2E=20Bellone?= Date: Fri, 28 Oct 2022 12:58:03 -0300 Subject: [PATCH 5/7] Better wording of message --- hooks/command | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hooks/command b/hooks/command index f548c92..44be66d 100755 --- a/hooks/command +++ b/hooks/command @@ -86,7 +86,7 @@ if [ $has_errors -eq 0 ]; then if [[ "${BUILDKITE_PLUGIN_JUNIT_ANNOTATE_MIN_TESTS:-0}" -gt "${TOTAL_TESTS}" ]]; then create_annotation=1 fail_build=1 - echo ":warning: Less than ${BUILDKITE_PLUGIN_JUNIT_ANNOTATE_MIN_TESTS} tests run" + echo ":warning: Less than ${BUILDKITE_PLUGIN_JUNIT_ANNOTATE_MIN_TESTS} tests analyzed" fi elif ! check_size; then echo "--- :warning: Failures too large to annotate" From b7f231ac2e494693d41048b65e6f4a1f3200d6d1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20A=2E=20Bellone?= Date: Fri, 28 Oct 2022 12:58:14 -0300 Subject: [PATCH 6/7] Added tests for min-tests feature --- tests/command.bats | 77 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 77 insertions(+) diff --git a/tests/command.bats b/tests/command.bats index e7f1366..e63dc51 100644 --- a/tests/command.bats +++ b/tests/command.bats @@ -344,4 +344,81 @@ export annotation_input="tests/tmp/annotation.input" unstub mktemp unstub buildkite-agent +} + +@test "creates annotation with no failures but min tests triggers" { + export BUILDKITE_PLUGIN_JUNIT_ANNOTATE_ARTIFACTS="junits/*.xml" + export BUILDKITE_PLUGIN_JUNIT_ANNOTATE_MIN_TESTS=1 + + stub mktemp \ + "-d \* : mkdir -p '$artifacts_tmp'; echo '$artifacts_tmp'" \ + "-d \* : mkdir -p '$annotation_tmp'; echo '$annotation_tmp'" + + stub buildkite-agent \ + "artifact download \* \* : echo Downloaded artifact \$3 to \$4" \ + "annotate --context \* --style \* : cat >'${annotation_input}'; echo Annotation added with context \$3 and style \$5, content saved" + + stub docker \ + "--log-level error run --rm --volume \* --volume \* --env \* --env \* --env \* ruby:2.7-alpine ruby /src/bin/annotate /junits : echo 'Total tests: 0'" + + run "$PWD/hooks/command" + + assert_failure + assert_output --partial "Total tests: 0" + assert_output --partial "Less than 1 tests analyzed" + assert_equal "$(cat "${annotation_input}")" 'Total tests: 0' + + unstub mktemp + unstub buildkite-agent + unstub docker +} + +@test "no failures and min-tests ok does not create annotation" { + export BUILDKITE_PLUGIN_JUNIT_ANNOTATE_ARTIFACTS="junits/*.xml" + export BUILDKITE_PLUGIN_JUNIT_ANNOTATE_MIN_TESTS=12 + + stub mktemp \ + "-d \* : mkdir -p '$artifacts_tmp'; echo '$artifacts_tmp'" \ + "-d \* : mkdir -p '$annotation_tmp'; echo '$annotation_tmp'" + + stub buildkite-agent \ + "artifact download \* \* : echo Downloaded artifact \$3 to \$4" + + stub docker \ + "--log-level error run --rm --volume \* --volume \* --env \* --env \* --env \* ruby:2.7-alpine ruby /src/bin/annotate /junits : echo 'Total tests: 100'" + + run "$PWD/hooks/command" + + assert_success + assert_output --partial "Total tests: 100" + refute_output --partial "Less than 12 tests analyzed" + + unstub mktemp + unstub buildkite-agent + unstub docker +} + +@test "min-tests doesn't interfere with actual failures" { + export BUILDKITE_PLUGIN_JUNIT_ANNOTATE_ARTIFACTS="junits/*.xml" + export BUILDKITE_PLUGIN_JUNIT_ANNOTATE_MIN_TESTS=10000 + + stub mktemp \ + "-d \* : mkdir -p '$artifacts_tmp'; echo '$artifacts_tmp'" \ + "-d \* : mkdir -p '$annotation_tmp'; echo '$annotation_tmp'" + + stub buildkite-agent \ + "artifact download \* \* : echo Downloaded artifact \$3 to \$4" \ + "annotate --context \* --style \* : cat >'${annotation_input}'; echo Annotation added with context \$3 and style \$5, content saved" + + stub docker \ + "--log-level error run --rm --volume \* --volume \* --env \* --env \* --env \* ruby:2.7-alpine ruby /src/bin/annotate /junits : cat tests/2-tests-1-failure.output && exit 64" + + run "$PWD/hooks/command" + + assert_success + assert_output --partial "Total tests: 2" + + unstub mktemp + unstub buildkite-agent + unstub docker } \ No newline at end of file From 3b34ab39607a1a24097f403361d1892ffdfad631 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20A=2E=20Bellone?= Date: Fri, 28 Oct 2022 17:31:31 -0300 Subject: [PATCH 7/7] Update version in preparation for next release --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 9808f02..3667f1a 100644 --- a/README.md +++ b/README.md @@ -14,7 +14,7 @@ steps: - wait: ~ continue_on_failure: true - plugins: - - junit-annotate#v2.1.0: + - junit-annotate#v2.2.0: artifacts: tmp/junit-*.xml ``` @@ -92,7 +92,7 @@ To test your plugin in your builds prior to opening a pull request, you can refe steps: - label: Annotate plugins: - - YourGithubHandle/junit-annotate#v2.1.0: + - YourGithubHandle/junit-annotate#v2.2.0: ... ```