Fix failure exit code regression

dctest should exit 1 if any test fails. We should be checking this instead of swallowing all exit codes in GHA.
Viasat · Oct 11, 2024 · 80bc6c4 · 80bc6c4
1 parent afc0af8
commit 80bc6c4
Show file tree

Hide file tree

Showing 3 changed files with 24 additions and 22 deletions.
diff --git a/.github/workflows/push.yaml b/.github/workflows/push.yaml
@@ -46,39 +46,38 @@ jobs:
 
       - name: Run Intro Examples
         run: |
-          # Some examples demonstate failure.
-          # Swallow exit code and check the result counts.
           docker run --rm \
             -v /var/run/docker.sock:/var/run/docker.sock \
             -v $(pwd)/examples:/app/examples \
-            dctest --results-file /app/examples/results.json examples /app/examples/00-intro.yaml \
-            || true
-          jq --exit-status '[.summary.passed == 5, .summary.failed == 0] | all' examples/results.json && \
-            node ./scripts/validateSchema.js examples/results.json schemas/results-file.yaml
+            dctest --results-file /app/examples/results.json examples /app/examples/00-intro.yaml
+
+          jq --exit-status '[.summary.passed == 5, .summary.failed == 0] | all' examples/results.json
+          node ./scripts/validateSchema.js examples/results.json schemas/results-file.yaml
 
       - name: Run Intro + Fail Examples with --continue-on-error
         run: |
-          # Some examples demonstate failure.
-          # Swallow exit code and check the result counts.
-          docker run --rm \
-            -v /var/run/docker.sock:/var/run/docker.sock \
-            -v $(pwd)/examples:/app/examples \
-            dctest --continue-on-error --results-file /app/examples/results.json examples /app/examples/00-intro.yaml /app/examples/01-fails.yaml \
-            || true
-          jq --exit-status '[.summary.passed == 6, .summary.failed == 4] | all' examples/results.json && \
-            node ./scripts/validateSchema.js examples/results.json schemas/results-file.yaml
+          exitcode=$(docker run --rm \
+                       -v /var/run/docker.sock:/var/run/docker.sock \
+                       -v $(pwd)/examples:/app/examples \
+                       dctest --continue-on-error --results-file /app/examples/results.json examples /app/examples/00-intro.yaml /app/examples/01-fails.yaml \
+                       && echo "$?" || echo "$?")
+          if [ "exitcode" != "1" ]; then
+            echo "dctest exited ${exitcode} instead of code 1 on failure"
+            exit 1
+          fi
+
+          jq --exit-status '[.summary.passed == 6, .summary.failed == 4] | all' examples/results.json
+          node ./scripts/validateSchema.js examples/results.json schemas/results-file.yaml
 
       - name: Run Dependency Examples
         run: |
-          # Some examples demonstate failure.
-          # Swallow exit code and check the result counts.
           docker run --rm \
             -v /var/run/docker.sock:/var/run/docker.sock \
             -v $(pwd)/examples:/app/examples \
-            dctest --results-file /app/examples/results.json examples /app/examples/02-deps.yaml \
-            || true
-          jq --exit-status '[.summary.passed == 12, .summary.failed == 0] | all' examples/results.json && \
-            node ./scripts/validateSchema.js examples/results.json schemas/results-file.yaml
+            dctest --results-file /app/examples/results.json examples /app/examples/02-deps.yaml
+
+          jq --exit-status '[.summary.passed == 12, .summary.failed == 0] | all' examples/results.json
+          node ./scripts/validateSchema.js examples/results.json schemas/results-file.yaml
 
 
       - name: Setup Fixtures

diff --git a/src/dctest/core.cljs b/src/dctest/core.cljs
@@ -487,4 +487,4 @@ Options:
     (write-results-file opts summary)
     (print-results opts summary)
 
-    (when (failure? summary) (fatal 1))))
+    (when-not (passed? (:summary summary)) (fatal 1))))
diff --git a/src/dctest/outcome.cljs b/src/dctest/outcome.cljs
@@ -6,6 +6,9 @@
 (defn failure? [{:keys [outcome]}]
   (= :failed outcome))
 
+(defn passed? [{:keys [outcome]}]
+  (= :passed outcome))
+
 (defn pending? [{:keys [outcome]}]
   (= :pending outcome))