Add graph of compile time vs schema size (#128) #593
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Benchmark | |
| on: | |
| workflow_dispatch: | |
| inputs: | |
| runs: | |
| description: 'Number of experimental runs' | |
| required: true | |
| default: 3 | |
| type: number | |
| no_ignore: | |
| description: "Don't ignore implementations marked as excluded" | |
| required: false | |
| default: false | |
| type: boolean | |
| skip_cache: | |
| description: "Skip cache" | |
| required: false | |
| default: false | |
| type: boolean | |
| blaze_branch: | |
| description: "Branch to use for Blaze" | |
| required: false | |
| default: main | |
| type: string | |
| push: | |
| branches: | |
| - main | |
| pull_request: | |
| schedule: | |
| - cron: '5 4 */2 * *' | |
| jobs: | |
| set-implementations: | |
| runs-on: ubuntu-latest | |
| outputs: | |
| implementations: ${{ steps.implementations.outputs.implementations }} | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Set implementations | |
| id: implementations | |
| env: | |
| NO_IGNORE: ${{ (inputs.no_ignore || github.event.name == 'schedule') && 'yes' || 'no' }} | |
| run: (echo -n "implementations="; ./.github/list-implementations.sh) >> $GITHUB_OUTPUT | |
| benchmark: | |
| needs: set-implementations | |
| runs-on: ubuntu-latest | |
| env: | |
| BUNDLE_GEMFILE: ${{ github.workspace }}/implementations/json_schemer/Gemfile | |
| strategy: | |
| matrix: ${{ fromJson(needs.set-implementations.outputs.implementations) }} | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Restore cached run | |
| uses: actions/cache/restore@v4 | |
| id: cache-report | |
| with: | |
| path: dist/report.csv | |
| key: noformat-${{ matrix.impl }}-${{ hashFiles(format('implementations/{0}/**/*', matrix.impl), 'schemas/**/*', 'report.sh') }}-${{ inputs.runs || 3 }} | |
| lookup-only: ${{ matrix.skip_cache || startsWith(matrix.impl, 'blaze') || github.event_name == 'schedule' || inputs.skip_cache }} | |
| - uses: actions/setup-go@v5 | |
| - name: Install gron | |
| run: go install github.com/tomnomnom/gron@latest | |
| - name: Run benchmarks | |
| if: ${{ steps.cache-report.outputs.cache-hit != 'true' || github.event_name == 'schedule' || inputs.skip_cache }} | |
| continue-on-error: ${{ !startsWith(matrix.impl, 'blaze') }} | |
| env: | |
| BLAZE_BRANCH: ${{ inputs.blaze_branch || 'main' }} | |
| run: make RUNS=${{ inputs.runs || 3 }} IMPLEMENTATIONS=${{ matrix.impl }} dist/report.csv | |
| - uses: actions/setup-python@v5 | |
| with: | |
| python-version: '3.12.5' | |
| - name: Install uv | |
| run: pipx install uv | |
| - name: Validate CSV | |
| run: uv run python .github/csv_min.py > /dev/null | |
| - name: Save output | |
| id: save-output | |
| run: | | |
| { | |
| echo 'report-${{ matrix.impl }}<<EOF' | |
| cat dist/report.csv | |
| echo EOF | |
| } >> $GITHUB_OUTPUT | |
| - name: Cache report | |
| uses: actions/cache/save@v4 | |
| if: ${{ !startsWith(matrix.impl, 'blaze') && steps.cache-report.outputs.cache-hit != 'true' }} | |
| with: | |
| path: dist/report.csv | |
| key: ${{ steps.cache-report.outputs.cache-primary-key }} | |
| outputs: | |
| # GitHub Actions does not easily support matrix outputs | |
| # https://github.com/orgs/community/discussions/26639 | |
| report-ajv: ${{ steps.save-output.outputs.report-ajv }} | |
| report-ajv-bun: ${{ steps.save-output.outputs.report-ajv-bun }} | |
| report-blaze: ${{ steps.save-output.outputs.report-blaze }} | |
| report-boon: ${{ steps.save-output.outputs.report-boon }} | |
| report-corvus: ${{ steps.save-output.outputs.report-corvus }} | |
| report-go-jsonschema: ${{ steps.save-output.outputs.report-go-jsonschema }} | |
| report-hyperjump: ${{ steps.save-output.outputs.report-hyperjump }} | |
| report-json_schemer: ${{ steps.save-output.outputs.report-json_schemer }} | |
| report-jsoncons: ${{ steps.save-output.outputs.report-jsoncons }} | |
| report-jsonschemadotnet: ${{ steps.save-output.outputs.report-jsonschemadotnet }} | |
| report-kmp-json-schema-validator: ${{ steps.save-output.outputs.report-kmp-json-schema-validator }} | |
| report-networknt: ${{ steps.save-output.outputs.report-networknt }} | |
| report-python-jsonschema: ${{ steps.save-output.outputs.report-python-jsonschema }} | |
| report-schemasafe: ${{ steps.save-output.outputs.report-schemasafe }} | |
| aggregate: | |
| needs: benchmark | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Concatenate output | |
| run: | | |
| mkdir dist | |
| printf "${{ join(needs.benchmark.outputs.*, '\n') }}" | awk 'NR==1 || !/^implementation,/' > dist/report.csv | |
| - run: cat dist/report.csv | |
| - uses: actions/setup-python@v5 | |
| with: | |
| python-version: '3.12.5' | |
| - name: Install uv | |
| run: pipx install uv | |
| - name: Get jsonschema-strip binary | |
| run: | | |
| git clone --depth 1 --branch main https://github.com/sourcemeta-research/jsonschema-strip | |
| cmake -S jsonschema-strip -B ./jsonschema-strip/build -DCMAKE_BUILD_TYPE:STRING=Release | |
| cmake --build ./jsonschema-strip/build --config Release --target strip | |
| mv ./jsonschema-strip/build/strip ./jsonschema-strip/build/jsonschema-strip | |
| echo "$(pwd)/jsonschema-strip/build" >> $GITHUB_PATH | |
| - uses: actions/setup-go@v5 | |
| - name: Install gron | |
| run: go install github.com/tomnomnom/gron@latest | |
| - name: Process CSV | |
| run: uv run python .github/csv_min.py > dist/report-min.csv | |
| - name: Read CSV | |
| id: csv | |
| uses: juliangruber/read-file-action@v1 | |
| with: | |
| path: dist/report-min.csv | |
| - name: Create MD | |
| uses: petems/csv-to-md-table-action@master | |
| id: csv-table-output | |
| with: | |
| csvinput: ${{ steps.csv.outputs.content }} | |
| - name: Output summary | |
| env: | |
| MARKDOWN_TABLE: ${{ steps.csv-table-output.outputs.markdown-table }} | |
| run: echo "$MARKDOWN_TABLE" >> $GITHUB_STEP_SUMMARY | |
| - name: Generate plots | |
| run: | | |
| ./dataset_summary.sh csv >> dist/summary.csv | |
| mkdir -p dist/results/plots && uv run python plot.py && uv run python plot_compile.py | |
| - name: Upload plots | |
| id: imgur | |
| uses: devicons/[email protected] | |
| with: | |
| path: dist/results/plots/*.png | |
| client_id: ${{secrets.IMGUR_CLIENT_ID}} | |
| - name: Upload compile plot | |
| id: imgur_compile | |
| uses: devicons/[email protected] | |
| with: | |
| path: dist/results/compile.png | |
| client_id: ${{secrets.IMGUR_CLIENT_ID}} | |
| - name: Add plots to summary | |
| env: | |
| IMG_URLS: ${{ steps.imgur.outputs.imgur_urls }} | |
| run: python .github/plot_markdown.py dist/results/plots >> $GITHUB_STEP_SUMMARY | |
| - name: Add plots to summary | |
| env: | |
| IMG_URLS: ${{ steps.imgur_compile.outputs.imgur_urls }} | |
| run: python .github/plot_markdown.py dist/results >> $GITHUB_STEP_SUMMARY | |
| - uses: actions/upload-artifact@v4 | |
| if: always() | |
| with: | |
| name: report | |
| path: | | |
| dist/report.csv | |
| dist/results/plots | |
| dataset-summary: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Get jsonschema-strip binary | |
| run: | | |
| git clone --depth 1 --branch main https://github.com/sourcemeta-research/jsonschema-strip | |
| cmake -S jsonschema-strip -B ./jsonschema-strip/build -DCMAKE_BUILD_TYPE:STRING=Release | |
| cmake --build ./jsonschema-strip/build --config Release --target strip | |
| mv ./jsonschema-strip/build/strip ./jsonschema-strip/build/jsonschema-strip | |
| echo "$(pwd)/jsonschema-strip/build" >> $GITHUB_PATH | |
| - uses: actions/setup-go@v5 | |
| - name: Install gron | |
| run: go install github.com/tomnomnom/gron@latest | |
| - name: Create summary | |
| run: ./dataset_summary.sh md >> $GITHUB_STEP_SUMMARY |