Fix failure exit code regression

dctest should exit 1 if any test fails. We should be checking this instead of swallowing all exit codes in GHA.
Viasat · Oct 11, 2024 · 1a505ec · 1a505ec
1 parent afc0af8
commit 1a505ec
Show file tree

Hide file tree

Showing 3 changed files with 25 additions and 23 deletions.
diff --git a/.github/workflows/push.yaml b/.github/workflows/push.yaml
@@ -46,39 +46,38 @@ jobs:
 
       - name: Run Intro Examples
         run: |
-          # Some examples demonstate failure.
-          # Swallow exit code and check the result counts.
           docker run --rm \
             -v /var/run/docker.sock:/var/run/docker.sock \
             -v $(pwd)/examples:/app/examples \
-            dctest --results-file /app/examples/results.json examples /app/examples/00-intro.yaml \
-            || true
-          jq --exit-status '[.summary.passed == 5, .summary.failed == 0] | all' examples/results.json && \
-            node ./scripts/validateSchema.js examples/results.json schemas/results-file.yaml
+            dctest --results-file /app/examples/results.json examples /app/examples/00-intro.yaml
+
+          jq --exit-status '[.summary.passed == 5, .summary.failed == 0] | all' examples/results.json
+          node ./scripts/validateSchema.js examples/results.json schemas/results-file.yaml
 
       - name: Run Intro + Fail Examples with --continue-on-error
         run: |
-          # Some examples demonstate failure.
-          # Swallow exit code and check the result counts.
-          docker run --rm \
-            -v /var/run/docker.sock:/var/run/docker.sock \
-            -v $(pwd)/examples:/app/examples \
-            dctest --continue-on-error --results-file /app/examples/results.json examples /app/examples/00-intro.yaml /app/examples/01-fails.yaml \
-            || true
-          jq --exit-status '[.summary.passed == 6, .summary.failed == 4] | all' examples/results.json && \
-            node ./scripts/validateSchema.js examples/results.json schemas/results-file.yaml
+          exitcode=$(docker run --rm \
+                       -v /var/run/docker.sock:/var/run/docker.sock \
+                       -v $(pwd)/examples:/app/examples \
+                       dctest --continue-on-error --results-file /app/examples/results.json examples /app/examples/00-intro.yaml /app/examples/01-fails.yaml \
+                       && echo -n "$?" || echo -n "$?")
+          if [ "exitcode" != "1" ]; then
+            echo "dctest exited ${exitcode} instead of code 1 on failure"
+            exit 1
+          fi
+
+          jq --exit-status '[.summary.passed == 6, .summary.failed == 4] | all' examples/results.json
+          node ./scripts/validateSchema.js examples/results.json schemas/results-file.yaml
 
       - name: Run Dependency Examples
         run: |
-          # Some examples demonstate failure.
-          # Swallow exit code and check the result counts.
           docker run --rm \
             -v /var/run/docker.sock:/var/run/docker.sock \
             -v $(pwd)/examples:/app/examples \
-            dctest --results-file /app/examples/results.json examples /app/examples/02-deps.yaml \
-            || true
-          jq --exit-status '[.summary.passed == 12, .summary.failed == 0] | all' examples/results.json && \
-            node ./scripts/validateSchema.js examples/results.json schemas/results-file.yaml
+            dctest --results-file /app/examples/results.json examples /app/examples/02-deps.yaml
+
+          jq --exit-status '[.summary.passed == 12, .summary.failed == 0] | all' examples/results.json
+          node ./scripts/validateSchema.js examples/results.json schemas/results-file.yaml
 
 
       - name: Setup Fixtures

diff --git a/src/dctest/core.cljs b/src/dctest/core.cljs
@@ -7,7 +7,7 @@
             [clojure.string :as S]
             [clojure.walk :refer [postwalk]]
             [dctest.expressions :as expr]
-            [dctest.outcome :refer [failure? pending? pending-> short-outcome
+            [dctest.outcome :refer [failure? passed? pending? pending-> short-outcome
                                     fail! pass! skip!]]
             [dctest.util :as util :refer [obj->str js->map log indent indent-print-table-str]]
             [promesa.core :as P]
@@ -487,4 +487,4 @@ Options:
     (write-results-file opts summary)
     (print-results opts summary)
 
-    (when (failure? summary) (fatal 1))))
+    (when-not (passed? (:summary summary)) (fatal 1))))
diff --git a/src/dctest/outcome.cljs b/src/dctest/outcome.cljs
@@ -6,6 +6,9 @@
 (defn failure? [{:keys [outcome]}]
   (= :failed outcome))
 
+(defn passed? [{:keys [outcome]}]
+  (= :passed outcome))
+
 (defn pending? [{:keys [outcome]}]
   (= :pending outcome))