From dff4ed30a67b8da4698140d597b624b218945cec Mon Sep 17 00:00:00 2001 From: Jon Smock Date: Fri, 11 Oct 2024 14:52:44 +0000 Subject: [PATCH] Fix failure exit code regression dctest should exit 1 if any test fails. Change the final assertion in core.cljs to be a positive one: unless we see a :passed result, exit 1. This should make it more difficult to introduce this kind of bug while refactoring in the future. We didn't catch this regression in GHA, because we were swallowing all exit codes and only checking the results file counts. Move integration test running into ./test/runexamples, so we can test all examples locally more easily. Add exit code checking. Also support running all examples with nbb as well as with a built image. --- .github/workflows/push.yaml | 64 ++++++++++----------------------- src/dctest/core.cljs | 4 +-- src/dctest/outcome.cljs | 3 ++ test/runexamples | 70 +++++++++++++++++++++++++++++++++++++ 4 files changed, 93 insertions(+), 48 deletions(-) create mode 100755 test/runexamples diff --git a/.github/workflows/push.yaml b/.github/workflows/push.yaml index 0f44062..f40b789 100644 --- a/.github/workflows/push.yaml +++ b/.github/workflows/push.yaml @@ -24,9 +24,22 @@ jobs: - name: Run Tests run: ./test/runtests + test-examples-node: + name: Test Examples (Node) + runs-on: ubuntu-latest + steps: + - name: Checkout Repository + uses: actions/checkout@v4 + with: { submodules: 'recursive', fetch-depth: 0 } + + - name: Install NPM + run: npm install + + - name: Run Examples + run: ./test/runexamples test-examples: - name: Test Examples + name: Test Examples (Docker) runs-on: ubuntu-latest steps: - name: Checkout Repository @@ -37,51 +50,10 @@ jobs: run: | docker build -t dctest . - - name: Setup Fixtures - run: | - docker compose -f examples/docker-compose.yaml up -d - - name: Install NPM (for schema validation) run: npm install - - name: Run Intro Examples - run: | - # Some examples demonstate failure. - # Swallow exit code and check the result counts. - docker run --rm \ - -v /var/run/docker.sock:/var/run/docker.sock \ - -v $(pwd)/examples:/app/examples \ - dctest --results-file /app/examples/results.json examples /app/examples/00-intro.yaml \ - || true - jq --exit-status '[.summary.passed == 5, .summary.failed == 0] | all' examples/results.json && \ - node ./scripts/validateSchema.js examples/results.json schemas/results-file.yaml - - - name: Run Intro + Fail Examples with --continue-on-error - run: | - # Some examples demonstate failure. - # Swallow exit code and check the result counts. - docker run --rm \ - -v /var/run/docker.sock:/var/run/docker.sock \ - -v $(pwd)/examples:/app/examples \ - dctest --continue-on-error --results-file /app/examples/results.json examples /app/examples/00-intro.yaml /app/examples/01-fails.yaml \ - || true - jq --exit-status '[.summary.passed == 6, .summary.failed == 4] | all' examples/results.json && \ - node ./scripts/validateSchema.js examples/results.json schemas/results-file.yaml - - - name: Run Dependency Examples - run: | - # Some examples demonstate failure. - # Swallow exit code and check the result counts. - docker run --rm \ - -v /var/run/docker.sock:/var/run/docker.sock \ - -v $(pwd)/examples:/app/examples \ - dctest --results-file /app/examples/results.json examples /app/examples/02-deps.yaml \ - || true - jq --exit-status '[.summary.passed == 12, .summary.failed == 0] | all' examples/results.json && \ - node ./scripts/validateSchema.js examples/results.json schemas/results-file.yaml - - - - name: Setup Fixtures - if: always() - run: | - docker compose -f examples/docker-compose.yaml down -t 1 --remove-orphans + - name: Run Examples + env: + DCTEST_IMAGE: dctest + run: ./test/runexamples diff --git a/src/dctest/core.cljs b/src/dctest/core.cljs index bd63c00..4b3cfb9 100644 --- a/src/dctest/core.cljs +++ b/src/dctest/core.cljs @@ -7,7 +7,7 @@ [clojure.string :as S] [clojure.walk :refer [postwalk]] [dctest.expressions :as expr] - [dctest.outcome :refer [failure? pending? pending-> short-outcome + [dctest.outcome :refer [failure? passed? pending? pending-> short-outcome fail! pass! skip!]] [dctest.util :as util :refer [obj->str js->map log indent indent-print-table-str]] [promesa.core :as P] @@ -487,4 +487,4 @@ Options: (write-results-file opts summary) (print-results opts summary) - (when (failure? summary) (fatal 1)))) + (when-not (passed? (:summary summary)) (fatal 1)))) diff --git a/src/dctest/outcome.cljs b/src/dctest/outcome.cljs index 1f40666..e7515f0 100644 --- a/src/dctest/outcome.cljs +++ b/src/dctest/outcome.cljs @@ -6,6 +6,9 @@ (defn failure? [{:keys [outcome]}] (= :failed outcome)) +(defn passed? [{:keys [outcome]}] + (= :passed outcome)) + (defn pending? [{:keys [outcome]}] (= :pending outcome)) diff --git a/test/runexamples b/test/runexamples new file mode 100755 index 0000000..44a383c --- /dev/null +++ b/test/runexamples @@ -0,0 +1,70 @@ +#!/usr/bin/env bash + +PROJECT=${PROJECT:-${USER}} +DCTEST_IMAGE=${DCTEST_IMAGE} + +repo_root=$(dirname $(readlink -f "${0}"))/.. +examples_dir=${repo_root}/examples + +results_dir=$(mktemp -d) + +dc() { docker compose -p ${PROJECT} -f ${examples_dir}/docker-compose.yaml "${@}"; } +up() { dc up -d; } +down() { dc down; } + +fail() { + msg=${1} + down + echo $msg + exit 1 +} + +check() { + passed=${1}; shift + failed=${1}; shift + example_files=${@} + + if [ -z "${DCTEST_IMAGE}" ]; then + cmd="${repo_root}/dctest --continue-on-error --results-file ${results_dir}/results.json ${PROJECT}" + for example in ${example_files}; do + cmd="${cmd} ${examples_dir}/${example}" + done + else + cmd="docker run --rm -v /var/run/docker.sock:/var/run/docker.sock" + cmd="${cmd} -v ${repo_root}/examples/:/app/examples -v ${results_dir}/:/app/results" + cmd="${cmd} ${DCTEST_IMAGE} --continue-on-error --results-file /app/results/results.json ${PROJECT}" + for example in ${example_files}; do + cmd="${cmd} /app/examples/${example}" + done + fi + + echo "Running: ${cmd}" + ${cmd} + exitcode=$? + + if [ "${failed}" == "0" ]; then + expected_exitcode="0" + else + expected_exitcode="1" + fi + + [ "${expected_exitcode}" == "${exitcode}" ] \ + || fail "Exit code ${exitcode} doesn't match expected ${expected_exitcode}" + + jq --exit-status '[.summary.passed == '${passed}', .summary.failed == '${failed}'] | all' ${results_dir}/results.json \ + || fail "Results file does not match expected passed/fail rate" + + node ${repo_root}/scripts/validateSchema.js ${results_dir}/results.json ${repo_root}/schemas/results-file.yaml \ + || fail "Results file does not match schema" +} + + +# TESTS + +up + +check 5 0 00-intro.yaml +check 6 4 00-intro.yaml 01-fails.yaml +check 12 0 02-deps.yaml + +down