-
Notifications
You must be signed in to change notification settings - Fork 1
157 lines (145 loc) · 5.48 KB
/
ci.yml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
name: Benchmark
on:
workflow_dispatch:
inputs:
runs:
description: 'Number of experimental runs'
required: true
default: 3
type: number
no_ignore:
description: "Don't ignore implementations marked as excluded"
required: false
default: false
type: boolean
push:
branches:
- main
pull_request:
schedule:
- cron: '5 4 */2 * *'
jobs:
set-implementations:
runs-on: ubuntu-latest
outputs:
implementations: ${{ steps.implementations.outputs.implementations }}
steps:
- uses: actions/checkout@v4
- name: Set implementations
id: implementations
env:
NO_IGNORE: ${{ (inputs.no_ignore || github.event.name == 'schedule') && 'yes' || 'no' }}
run: (echo -n "implementations="; ./.github/list-implementations.sh) >> $GITHUB_OUTPUT
benchmark:
needs: set-implementations
runs-on: ubuntu-latest
env:
BUNDLE_GEMFILE: ${{ github.workspace }}/implementations/json_schemer/Gemfile
strategy:
matrix: ${{ fromJson(needs.set-implementations.outputs.implementations) }}
steps:
- uses: actions/checkout@v4
- name: Restore cached run
uses: actions/cache/restore@v4
id: cache-report
with:
path: dist/report.csv
key: noformat-${{ matrix.impl }}-${{ hashFiles(format('implementations/{0}/**/*', matrix.impl), 'schemas/**/*', 'report.sh') }}-${{ inputs.runs || 3 }}
lookup-only: ${{ matrix.skip_cache || startsWith(matrix.impl, 'blaze') || github.event_name == 'schedule' }}
- name: Install gron
uses: jaxxstorm/[email protected]
with:
repo: tomnomnom/gron
tag: v0.7.1
- name: Run benchmarks
if: steps.cache-report.outputs.cache-hit != 'true'
continue-on-error: true
run: make RUNS=${{ inputs.runs || 3 }} IMPLEMENTATIONS=${{ matrix.impl }} dist/report.csv
- uses: actions/setup-python@v5
with:
python-version: '3.12.5'
- name: Install uv
run: pipx install uv
- name: Validate CSV
run: uv run python .github/csv_min.py > /dev/null
- name: Save output
id: save-output
run: |
{
echo 'report-${{ matrix.impl }}<<EOF'
cat dist/report.csv
echo EOF
} >> $GITHUB_OUTPUT
- name: Cache report
uses: actions/cache/save@v4
if: ${{ !startsWith(matrix.impl, 'blaze') && steps.cache-report.outputs.cache-hit != 'true' }}
with:
path: dist/report.csv
key: ${{ steps.cache-report.outputs.cache-primary-key }}
outputs:
# GitHub Actions does not easily support matrix outputs
# https://github.com/orgs/community/discussions/26639
report-ajv: ${{ steps.save-output.outputs.report-ajv }}
report-ajv-bun: ${{ steps.save-output.outputs.report-ajv-bun }}
report-blaze: ${{ steps.save-output.outputs.report-blaze }}
report-boon: ${{ steps.save-output.outputs.report-boon }}
report-corvus: ${{ steps.save-output.outputs.report-corvus }}
report-go-jsonschema: ${{ steps.save-output.outputs.report-go-jsonschema }}
report-hyperjump: ${{ steps.save-output.outputs.report-hyperjump }}
report-json_schemer: ${{ steps.save-output.outputs.report-json_schemer }}
report-jsoncons: ${{ steps.save-output.outputs.report-jsoncons }}
report-jsonschemadotnet: ${{ steps.save-output.outputs.report-jsonschemadotnet }}
report-kmp-json-schema-validator: ${{ steps.save-output.outputs.report-kmp-json-schema-validator }}
report-networknt: ${{ steps.save-output.outputs.report-networknt }}
report-python-jsonschema: ${{ steps.save-output.outputs.report-python-jsonschema }}
report-schemasafe: ${{ steps.save-output.outputs.report-schemasafe }}
aggregate:
needs: benchmark
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- name: Concatenate output
run: |
mkdir dist
printf "${{ join(needs.benchmark.outputs.*, '\n') }}" | awk 'NR==1 || !/^implementation,/' > dist/report.csv
- run: cat dist/report.csv
- uses: actions/setup-python@v5
with:
python-version: '3.12.5'
- name: Install uv
run: pipx install uv
- name: Process CSV
run: uv run python .github/csv_min.py > dist/report-min.csv
- name: Read CSV
id: csv
uses: juliangruber/read-file-action@v1
with:
path: dist/report-min.csv
- name: Create MD
uses: petems/csv-to-md-table-action@master
id: csv-table-output
with:
csvinput: ${{ steps.csv.outputs.content }}
- name: Output summary
env:
MARKDOWN_TABLE: ${{ steps.csv-table-output.outputs.markdown-table }}
run: echo "$MARKDOWN_TABLE" >> $GITHUB_STEP_SUMMARY
- name: Generate plots
run: mkdir -p dist/results/plots && uv run python plot.py
- name: Upload plots
id: imgur
uses: devicons/[email protected]
with:
path: dist/results/plots/*.png
client_id: ${{secrets.IMGUR_CLIENT_ID}}
- name: Add plots to summary
env:
IMG_URLS: ${{ steps.imgur.outputs.imgur_urls }}
run: python .github/plot_markdown.py >> $GITHUB_STEP_SUMMARY
- uses: actions/upload-artifact@v4
if: always()
with:
name: report
path: |
dist/report.csv
dist/results/plots