Run Blaze with rpmalloc #594
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Benchmark | |
| on: | |
| workflow_dispatch: | |
| inputs: | |
| runs: | |
| description: 'Number of experimental runs' | |
| required: true | |
| default: 3 | |
| type: number | |
| no_ignore: | |
| description: "Don't ignore implementations marked as excluded" | |
| required: false | |
| default: false | |
| type: boolean | |
| skip_cache: | |
| description: "Skip cache" | |
| required: false | |
| default: false | |
| type: boolean | |
| blaze_branch: | |
| description: "Branch to use for Blaze" | |
| required: false | |
| default: main | |
| type: string | |
| push: | |
| branches: | |
| - main | |
| pull_request: | |
| schedule: | |
| - cron: '5 4 */2 * *' | |
| jobs: | |
| set-implementations: | |
| runs-on: ubuntu-latest | |
| outputs: | |
| implementations: ${{ steps.implementations.outputs.implementations }} | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Set implementations | |
| id: implementations | |
| env: | |
| NO_IGNORE: ${{ (inputs.no_ignore || github.event.name == 'schedule') && 'yes' || 'no' }} | |
| run: (echo -n "implementations="; ./.github/list-implementations.sh) >> $GITHUB_OUTPUT | |
| benchmark: | |
| needs: set-implementations | |
| runs-on: ubuntu-latest | |
| env: | |
| BUNDLE_GEMFILE: ${{ github.workspace }}/implementations/json_schemer/Gemfile | |
| strategy: | |
| matrix: ${{ fromJson(needs.set-implementations.outputs.implementations) }} | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Restore cached run | |
| uses: actions/cache/restore@v4 | |
| id: cache-report | |
| with: | |
| path: dist/report.csv | |
| key: noformat-${{ matrix.impl }}-${{ hashFiles(format('implementations/{0}/**/*', matrix.impl), 'schemas/**/*', 'report.sh') }}-${{ inputs.runs || 3 }} | |
| lookup-only: ${{ matrix.skip_cache || startsWith(matrix.impl, 'blaze') || github.event_name == 'schedule' || inputs.skip_cache }} | |
| - uses: actions/setup-go@v5 | |
| - name: Install gron | |
| run: go install github.com/tomnomnom/gron@latest | |
| - name: Run benchmarks | |
| if: ${{ steps.cache-report.outputs.cache-hit != 'true' || github.event_name == 'schedule' || inputs.skip_cache }} | |
| continue-on-error: ${{ !startsWith(matrix.impl, 'blaze') }} | |
| env: | |
| BLAZE_BRANCH: ${{ inputs.blaze_branch || 'main' }} | |
| run: make RUNS=${{ inputs.runs || 3 }} IMPLEMENTATIONS=${{ matrix.impl }} dist/report.csv | |
| - uses: actions/setup-python@v5 | |
| with: | |
| python-version: '3.12.5' | |
| - name: Install uv | |
| run: pipx install uv | |
| - name: Validate CSV | |
| run: uv run python .github/csv_min.py > /dev/null | |
| - name: Save output | |
| id: save-output | |
| run: | | |
| { | |
| echo 'report-${{ matrix.impl }}<<EOF' | |
| cat dist/report.csv | |
| echo EOF | |
| } >> $GITHUB_OUTPUT | |
| - name: Cache report | |
| uses: actions/cache/save@v4 | |
| if: ${{ !startsWith(matrix.impl, 'blaze') && steps.cache-report.outputs.cache-hit != 'true' }} | |
| with: | |
| path: dist/report.csv | |
| key: ${{ steps.cache-report.outputs.cache-primary-key }} | |
| outputs: | |
| # GitHub Actions does not easily support matrix outputs | |
| # https://github.com/orgs/community/discussions/26639 | |
| report-ajv: ${{ steps.save-output.outputs.report-ajv }} | |
| report-ajv-bun: ${{ steps.save-output.outputs.report-ajv-bun }} | |
| report-blaze: ${{ steps.save-output.outputs.report-blaze }} | |
| report-boon: ${{ steps.save-output.outputs.report-boon }} | |
| report-corvus: ${{ steps.save-output.outputs.report-corvus }} | |
| report-go-jsonschema: ${{ steps.save-output.outputs.report-go-jsonschema }} | |
| report-hyperjump: ${{ steps.save-output.outputs.report-hyperjump }} | |
| report-json_schemer: ${{ steps.save-output.outputs.report-json_schemer }} | |
| report-jsoncons: ${{ steps.save-output.outputs.report-jsoncons }} | |
| report-jsonschemadotnet: ${{ steps.save-output.outputs.report-jsonschemadotnet }} | |
| report-kmp-json-schema-validator: ${{ steps.save-output.outputs.report-kmp-json-schema-validator }} | |
| report-networknt: ${{ steps.save-output.outputs.report-networknt }} | |
| report-python-jsonschema: ${{ steps.save-output.outputs.report-python-jsonschema }} | |
| report-schemasafe: ${{ steps.save-output.outputs.report-schemasafe }} | |
| aggregate: | |
| needs: benchmark | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Concatenate output | |
| run: | | |
| mkdir dist | |
| printf "${{ join(needs.benchmark.outputs.*, '\n') }}" | awk 'NR==1 || !/^implementation,/' > dist/report.csv | |
| - run: cat dist/report.csv | |
| - uses: actions/setup-python@v5 | |
| with: | |
| python-version: '3.12.5' | |
| - name: Install uv | |
| run: pipx install uv | |
| - name: Process CSV | |
| run: uv run python .github/csv_min.py > dist/report-min.csv | |
| - name: Read CSV | |
| id: csv | |
| uses: juliangruber/read-file-action@v1 | |
| with: | |
| path: dist/report-min.csv | |
| - name: Create MD | |
| uses: petems/csv-to-md-table-action@master | |
| id: csv-table-output | |
| with: | |
| csvinput: ${{ steps.csv.outputs.content }} | |
| - name: Output summary | |
| env: | |
| MARKDOWN_TABLE: ${{ steps.csv-table-output.outputs.markdown-table }} | |
| run: echo "$MARKDOWN_TABLE" >> $GITHUB_STEP_SUMMARY | |
| - name: Generate plots | |
| run: mkdir -p dist/results/plots && uv run python plot.py | |
| - name: Upload plots | |
| id: imgur | |
| uses: devicons/[email protected] | |
| with: | |
| path: dist/results/plots/*.png | |
| client_id: ${{secrets.IMGUR_CLIENT_ID}} | |
| - name: Add plots to summary | |
| env: | |
| IMG_URLS: ${{ steps.imgur.outputs.imgur_urls }} | |
| run: python .github/plot_markdown.py >> $GITHUB_STEP_SUMMARY | |
| - uses: actions/upload-artifact@v4 | |
| if: always() | |
| with: | |
| name: report | |
| path: | | |
| dist/report.csv | |
| dist/results/plots | |
| dataset-summary: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Get jsonschema-strip binary | |
| run: | | |
| git clone --depth 1 --branch main https://github.com/sourcemeta-research/jsonschema-strip | |
| cmake -S jsonschema-strip -B ./jsonschema-strip/build -DCMAKE_BUILD_TYPE:STRING=Release | |
| cmake --build ./jsonschema-strip/build --config Release --target strip | |
| mv ./jsonschema-strip/build/strip ./jsonschema-strip/build/jsonschema-strip | |
| echo "$(pwd)/jsonschema-strip/build" >> $GITHUB_PATH | |
| - name: Install moreutils | |
| run: sudo apt-get install moreutils | |
| - name: Strip schemas | |
| run: | | |
| for f in schemas/*/schema.json; do | |
| ./jsonschema-strip/build/jsonschema-strip "$f" | sponge "$f" | |
| done | |
| - name: Create summary | |
| run: ./.github/dataset_summary_table.sh >> $GITHUB_STEP_SUMMARY |