Fix failure exit code regression

dctest should exit 1 if any test fails. Change the final assertion in
core.cljs to be a positive one:  unless we see a :passed result, exit 1.
This should make it more difficult to introduce this kind of bug while
refactoring in the future.

We didn't catch this regression in GHA, because we were swallowing all
exit codes and only checking the results file counts.

Move integration test running into ./test/runexamples, so we can test
all examples locally more easily.  Add exit code checking.  Also support
running all examples with nbb as well as with a built image.

.github/workflows/push.yaml

@@ -24,9 +24,22 @@ jobs:
       - name: Run Tests
         run: ./test/runtests
 
+  test-examples-node:
+    name: Test Examples (Node)
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout Repository
+        uses: actions/checkout@v4
+        with: { submodules: 'recursive', fetch-depth: 0 }
+
+      - name: Install NPM
+        run: npm install
+
+      - name: Run Examples
+        run: ./test/runexamples
 
   test-examples:
-    name: Test Examples
+    name: Test Examples (Docker)
     runs-on: ubuntu-latest
     steps:
       - name: Checkout Repository
@@ -37,51 +50,10 @@ jobs:
         run: |
           docker build -t dctest .
 
-      - name: Setup Fixtures
-        run: |
-          docker compose -f examples/docker-compose.yaml up -d
-
       - name: Install NPM (for schema validation)
         run: npm install
 
-      - name: Run Intro Examples
-        run: |
-          # Some examples demonstate failure.
-          # Swallow exit code and check the result counts.
-          docker run --rm \
-            -v /var/run/docker.sock:/var/run/docker.sock \
-            -v $(pwd)/examples:/app/examples \
-            dctest --results-file /app/examples/results.json examples /app/examples/00-intro.yaml \
-            || true
-          jq --exit-status '[.summary.passed == 5, .summary.failed == 0] | all' examples/results.json && \
-            node ./scripts/validateSchema.js examples/results.json schemas/results-file.yaml
-
-      - name: Run Intro + Fail Examples with --continue-on-error
-        run: |
-          # Some examples demonstate failure.
-          # Swallow exit code and check the result counts.
-          docker run --rm \
-            -v /var/run/docker.sock:/var/run/docker.sock \
-            -v $(pwd)/examples:/app/examples \
-            dctest --continue-on-error --results-file /app/examples/results.json examples /app/examples/00-intro.yaml /app/examples/01-fails.yaml \
-            || true
-          jq --exit-status '[.summary.passed == 6, .summary.failed == 4] | all' examples/results.json && \
-            node ./scripts/validateSchema.js examples/results.json schemas/results-file.yaml
-
-      - name: Run Dependency Examples
-        run: |
-          # Some examples demonstate failure.
-          # Swallow exit code and check the result counts.
-          docker run --rm \
-            -v /var/run/docker.sock:/var/run/docker.sock \
-            -v $(pwd)/examples:/app/examples \
-            dctest --results-file /app/examples/results.json examples /app/examples/02-deps.yaml \
-            || true
-          jq --exit-status '[.summary.passed == 12, .summary.failed == 0] | all' examples/results.json && \
-            node ./scripts/validateSchema.js examples/results.json schemas/results-file.yaml
-
-
-      - name: Setup Fixtures
-        if: always()
-        run: |
-          docker compose -f examples/docker-compose.yaml down -t 1 --remove-orphans
+      - name: Run Examples
+        env:
+          DCTEST_IMAGE: dctest
+        run: ./test/runexamples

src/dctest/core.cljs

@@ -7,7 +7,7 @@
             [clojure.string :as S]
             [clojure.walk :refer [postwalk]]
             [dctest.expressions :as expr]
-            [dctest.outcome :refer [failure? pending? pending-> short-outcome
+            [dctest.outcome :refer [failure? passed? pending? pending-> short-outcome
                                     fail! pass! skip!]]
             [dctest.util :as util :refer [obj->str js->map log indent indent-print-table-str]]
             [promesa.core :as P]
@@ -487,4 +487,4 @@ Options:
     (write-results-file opts summary)
     (print-results opts summary)
 
-    (when (failure? summary) (fatal 1))))
+    (when-not (passed? (:summary summary)) (fatal 1))))

src/dctest/outcome.cljs

@@ -6,6 +6,9 @@
 (defn failure? [{:keys [outcome]}]
   (= :failed outcome))
 
+(defn passed? [{:keys [outcome]}]
+  (= :passed outcome))
+
 (defn pending? [{:keys [outcome]}]
   (= :pending outcome))
 

test/runexamples

@@ -0,0 +1,70 @@
+#!/usr/bin/env bash
+
+PROJECT=${PROJECT:-${USER}}
+DCTEST_IMAGE=${DCTEST_IMAGE}
+
+repo_root=$(dirname $(readlink -f "${0}"))/..
+examples_dir=${repo_root}/examples
+
+results_dir=$(mktemp -d)
+
+dc() { docker compose -p ${PROJECT} -f ${examples_dir}/docker-compose.yaml "${@}"; }
+up() { dc up -d; }
+down() { dc down; }
+
+fail() {
+  msg=${1}
+  down
+  echo $msg
+  exit 1
+}
+
+check() {
+  passed=${1}; shift
+  failed=${1}; shift
+  example_files=${@}
+
+  if [ -z "${DCTEST_IMAGE}" ]; then
+    cmd="${repo_root}/dctest --continue-on-error --results-file ${results_dir}/results.json ${PROJECT}"
+    for example in ${example_files}; do
+      cmd="${cmd} ${examples_dir}/${example}"
+    done
+  else
+    cmd="docker run --rm -v /var/run/docker.sock:/var/run/docker.sock"
+    cmd="${cmd} -v ${repo_root}/examples/:/app/examples -v ${results_dir}/:/app/results"
+    cmd="${cmd} ${DCTEST_IMAGE} --continue-on-error --results-file /app/results/results.json ${PROJECT}"
+    for example in ${example_files}; do
+      cmd="${cmd} /app/examples/${example}"
+    done
+  fi
+
+  echo "Running: ${cmd}"
+  ${cmd}
+  exitcode=$?
+
+  if [ "${failed}" == "0" ]; then
+    expected_exitcode="0"
+  else
+    expected_exitcode="1"
+  fi
+
+  [ "${expected_exitcode}" == "${exitcode}" ] \
+    || fail "Exit code ${exitcode} doesn't match expected ${expected_exitcode}"
+
+  jq --exit-status '[.summary.passed == '${passed}', .summary.failed == '${failed}'] | all' ${results_dir}/results.json \
+    || fail "Results file does not match expected passed/fail rate"
+
+  node ${repo_root}/scripts/validateSchema.js ${results_dir}/results.json ${repo_root}/schemas/results-file.yaml \
+    || fail "Results file does not match schema"
+}
+
+
+# TESTS
+
+up
+
+check 5  0 00-intro.yaml
+check 6  4 00-intro.yaml 01-fails.yaml
+check 12 0 02-deps.yaml
+
+down