Correctly select implementations to run via CI (#117)

- **Also run implementations changed in a PR**
- **Fix ignoring runs on GitHub Actions**
- **Adjust Makefile timeout based on runs**
- **Skip cache for implementations modified by PRs**

.github/list-implementations.sh

@@ -0,0 +1,23 @@
+#!/bin/bash
+
+# Get a list of all implementations (with some potentially ignored)
+all_impls=$(make NO_IGNORE=$NO_IGNORE list | sort)
+
+# Add implementations changed in the PR
+if [ "$GITHUB_EVENT_NAME" == "pull_request" ]; then
+  git fetch origin $GITHUB_BASE_REF --depth 1
+  pr_impls=$(git diff --name-only FETCH_HEAD..HEAD | grep '^implementations/' | cut -d/ -f2 | sort -u)
+fi
+
+# Get all implementations that are modified by the PR
+base_impls=$(comm -23 <(echo "$all_impls") <(echo "$pr_impls"))
+
+# Format as JSON like {"include": [{"impl" "foo"}, ...]}
+echo -n '{"include":['
+(echo "$base_impls" | while read impl; do
+  echo -n '{"impl": "'$impl'", "skip_cache": false},'
+done
+echo "$pr_impls" | while read impl; do
+  [ -z "$impl" ] || echo -n '{"impl": "'$impl'", "skip_cache": true},'
+done) | sed 's/,$//' # Remove trailing comma
+echo "]}"

.github/workflows/ci.yml

@@ -8,10 +8,10 @@ on:
         required: true
         default: 3
         type: number
-      ignore:
-        description: 'Ignore implementations marked as excluded'
+      no_ignore:
+        description: "Don't ignore implementations marked as excluded"
         required: false
-        default: true
+        default: false
         type: boolean
   push:
     branches:
@@ -30,8 +30,8 @@ jobs:
       - name: Set implementations
         id: implementations
         env:
-          NO_IGNORE: ${{ (!inputs.ignore || github.event.name == 'schedule') && '1' || '0' }}
-        run: echo implementations=$(make NO_IGNORE=$NO_IGNORE list | tr '\n' ' ' | sed 's/ $//;s/^/{"include":\[{"impl":"/;s/ /"}, {"impl":"/g;s/$/"}]}/') >> $GITHUB_OUTPUT
+          NO_IGNORE: ${{ (inputs.no_ignore || github.event.name == 'schedule') && 'yes' || 'no' }}
+        run: (echo -n "implementations="; ./.github/list-implementations.sh) >> $GITHUB_OUTPUT
 
   benchmark:
     needs: set-implementations
@@ -48,7 +48,7 @@ jobs:
         with:
           path: dist/report.csv
           key: ${{ matrix.impl }}-${{ hashFiles(format('implementations/{0}/**/*', matrix.impl), 'schemas/**/*', 'report.sh') }}-${{ inputs.runs || 3 }}
-          lookup-only: ${{ startsWith(matrix.impl, 'blaze') || github.event_name == 'schedule' }}
+          lookup-only: ${{ matrix.skip_cache || startsWith(matrix.impl, 'blaze') || github.event_name == 'schedule' }}
 
       - name: Run benchmarks
         if: steps.cache-report.outputs.cache-hit != 'true'

Makefile

@@ -1,7 +1,7 @@
 .DEFAULT_GOAL := all
 SCHEMAS = $(notdir $(wildcard schemas/*))
 ALL_IMPLEMENTATIONS = $(notdir $(wildcard implementations/*))
-ifdef NO_IGNORE
+ifeq ($(NO_IGNORE),yes)
 IMPLEMENTATIONS ?= $(ALL_IMPLEMENTATIONS)
 else
 IMPLEMENTATIONS ?= $(filter-out $(patsubst implementations/%/,%,$(dir $(wildcard implementations/*/.benchmark-ignore))), $(ALL_IMPLEMENTATIONS))
@@ -38,7 +38,7 @@ define docker_run
   $(eval $@_INPUT = $(2))
 	rm -f $@
 	for i in $(shell seq 1 $(RUNS)) ; do \
-		timeout -s KILL 5m docker run --rm -v $(CURDIR):/workspace jsonschema-benchmark/$($@_TOOL) $($@_INPUT) > $@.tmp ; \
+		timeout -s KILL $$(( $(RUNS) * 30 + 60 ))s docker run --rm -v $(CURDIR):/workspace jsonschema-benchmark/$($@_TOOL) $($@_INPUT) > $@.tmp ; \
 		STATUS=$$? ; \
 		if [ ! -s $@.tmp ]; then echo "0,0,0" >> $@ ; else cat $@.tmp >> $@ ; fi ; \
 		sed -i "$$ s/$$/,$$STATUS/" $@ ; \
@@ -47,7 +47,7 @@ define docker_run
 endef
 
 list:
-	@echo $(IMPLEMENTATIONS)
+	@echo $(IMPLEMENTATIONS) | tr ' ' '\n'
 
 # Blaze