Integer benchmarks #5
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # Run all integer benchmarks on an AWS instance and return parsed results to Slab CI bot. | |
| name: Integer benchmarks | |
| on: | |
| workflow_dispatch: | |
| inputs: | |
| all_precisions: | |
| description: "Run all precisions" | |
| type: boolean | |
| default: false | |
| schedule: | |
| # Weekly benchmarks will be triggered each Saturday at 1a.m. | |
| - cron: '0 1 * * 6' | |
| # Quarterly benchmarks will be triggered right before end of quarter, the 25th of the current month at 4a.m. | |
| # These benchmarks are far longer to execute hence the reason to run them only four time a year. | |
| - cron: '0 4 25 MAR,JUN,SEP,DEC *' | |
| env: | |
| CARGO_TERM_COLOR: always | |
| RESULTS_FILENAME: parsed_benchmark_results_${{ github.sha }}.json | |
| ACTION_RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} | |
| RUST_BACKTRACE: "full" | |
| RUST_MIN_STACK: "8388608" | |
| SLACK_CHANNEL: ${{ secrets.SLACK_CHANNEL }} | |
| SLACK_ICON: https://pbs.twimg.com/profile_images/1274014582265298945/OjBKP9kn_400x400.png | |
| SLACK_USERNAME: ${{ secrets.BOT_USERNAME }} | |
| SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }} | |
| FAST_BENCH: TRUE | |
| jobs: | |
| prepare-matrix: | |
| name: Prepare operations matrix | |
| runs-on: ubuntu-latest | |
| if: github.event_name != 'schedule' || | |
| (github.event_name == 'schedule' && github.repository == 'zama-ai/tfhe-rs') | |
| outputs: | |
| op_flavor: ${{ steps.set_op_flavor.outputs.op_flavor }} | |
| steps: | |
| - name: Weekly benchmarks | |
| if: github.event_name == 'workflow_dispatch' || | |
| github.event.schedule == '0 1 * * 6' | |
| run: | | |
| echo "OP_FLAVOR=[\"default\"]" >> "${GITHUB_ENV}" | |
| - name: Quarterly benchmarks | |
| if: github.event.schedule == '0 4 25 MAR,JUN,SEP,DEC *' | |
| run: | | |
| echo "OP_FLAVOR=[\"default\", \"smart\", \"unchecked\", \"misc\"]" >> "${GITHUB_ENV}" | |
| - name: Set operation flavor output | |
| id: set_op_flavor | |
| run: | | |
| echo "op_flavor=${{ toJSON(env.OP_FLAVOR) }}" >> "${GITHUB_OUTPUT}" | |
| setup-instance: | |
| name: Setup instance (integer-benchmarks) | |
| needs: prepare-matrix | |
| runs-on: ubuntu-latest | |
| outputs: | |
| runner-name: ${{ steps.start-instance.outputs.label }} | |
| steps: | |
| - name: Start instance | |
| id: start-instance | |
| uses: zama-ai/slab-github-runner@c0e7168795bd78f61f61146951ed9d0c73c9b701 | |
| with: | |
| mode: start | |
| github-token: ${{ secrets.SLAB_ACTION_TOKEN }} | |
| slab-url: ${{ secrets.SLAB_BASE_URL }} | |
| job-secret: ${{ secrets.JOB_SECRET }} | |
| backend: aws | |
| profile: bench | |
| integer-benchmarks: | |
| name: Execute integer benchmarks for all operations flavor | |
| needs: [ prepare-matrix, setup-instance ] | |
| runs-on: ${{ needs.setup-instance.outputs.runner-name }} | |
| concurrency: | |
| group: ${{ github.workflow }}_${{ github.ref }} | |
| cancel-in-progress: ${{ github.ref != 'refs/heads/main' }} | |
| continue-on-error: true | |
| timeout-minutes: 1440 # 24 hours | |
| strategy: | |
| max-parallel: 1 | |
| matrix: | |
| command: [ integer, integer_multi_bit] | |
| op_flavor: ${{ fromJson(needs.prepare-matrix.outputs.op_flavor) }} | |
| steps: | |
| - name: Checkout tfhe-rs repo with tags | |
| uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 | |
| with: | |
| fetch-depth: 0 | |
| token: ${{ secrets.FHE_ACTIONS_TOKEN }} | |
| - name: Get benchmark details | |
| run: | | |
| { | |
| echo "BENCH_DATE=$(date --iso-8601=seconds)"; | |
| echo "COMMIT_DATE=$(git --no-pager show -s --format=%cd --date=iso8601-strict ${{ github.sha }})"; | |
| echo "COMMIT_HASH=$(git describe --tags --dirty)"; | |
| } >> "${GITHUB_ENV}" | |
| - name: Set up home | |
| # "Install rust" step require root user to have a HOME directory which is not set. | |
| run: | | |
| echo "HOME=/home/ubuntu" >> "${GITHUB_ENV}" | |
| - name: Install rust | |
| uses: dtolnay/rust-toolchain@7b1c307e0dcbda6122208f10795a713336a9b35a | |
| with: | |
| toolchain: nightly | |
| - name: Checkout Slab repo | |
| uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 | |
| with: | |
| repository: zama-ai/slab | |
| path: slab | |
| token: ${{ secrets.FHE_ACTIONS_TOKEN }} | |
| - name: Should run benchmarks with all precisions | |
| if: inputs.all_precisions | |
| run: | | |
| echo "FAST_BENCH=FALSE" >> "${GITHUB_ENV}" | |
| - name: Run benchmarks with AVX512 | |
| run: | | |
| make BENCH_OP_FLAVOR=${{ matrix.op_flavor }} bench_${{ matrix.command }} | |
| # Run these benchmarks only once | |
| - name: Run compression benchmarks with AVX512 | |
| if: matrix.op_flavor == 'default' && matrix.command == 'integer' | |
| run: | | |
| make bench_integer_compression | |
| - name: Parse results | |
| run: | | |
| python3 ./ci/benchmark_parser.py target/criterion ${{ env.RESULTS_FILENAME }} \ | |
| --database tfhe_rs \ | |
| --hardware "hpc7a.96xlarge" \ | |
| --project-version "${{ env.COMMIT_HASH }}" \ | |
| --branch ${{ github.ref_name }} \ | |
| --commit-date "${{ env.COMMIT_DATE }}" \ | |
| --bench-date "${{ env.BENCH_DATE }}" \ | |
| --walk-subdirs \ | |
| --name-suffix avx512 \ | |
| --throughput | |
| - name: Upload parsed results artifact | |
| uses: actions/upload-artifact@50769540e7f4bd5e21e526ee35c689e35e0d6874 | |
| with: | |
| name: ${{ github.sha }}_${{ matrix.command }}_${{ matrix.op_flavor }} | |
| path: ${{ env.RESULTS_FILENAME }} | |
| - name: Send data to Slab | |
| shell: bash | |
| run: | | |
| python3 slab/scripts/data_sender.py ${{ env.RESULTS_FILENAME }} "${{ secrets.JOB_SECRET }}" \ | |
| --slab-url "${{ secrets.SLAB_URL }}" | |
| - name: Slack Notification | |
| if: ${{ failure() }} | |
| continue-on-error: true | |
| uses: rtCamp/action-slack-notify@4e5fb42d249be6a45a298f3c9543b111b02f7907 | |
| env: | |
| SLACK_COLOR: ${{ job.status }} | |
| SLACK_MESSAGE: "Integer full benchmarks finished with status: ${{ job.status }}. (${{ env.ACTION_RUN_URL }})" | |
| teardown-instance: | |
| name: Teardown instance (integer-benchmarks) | |
| if: ${{ always() && needs.setup-instance.result != 'skipped' }} | |
| needs: [ setup-instance, integer-benchmarks ] | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Stop instance | |
| id: stop-instance | |
| uses: zama-ai/slab-github-runner@c0e7168795bd78f61f61146951ed9d0c73c9b701 | |
| with: | |
| mode: stop | |
| github-token: ${{ secrets.SLAB_ACTION_TOKEN }} | |
| slab-url: ${{ secrets.SLAB_BASE_URL }} | |
| job-secret: ${{ secrets.JOB_SECRET }} | |
| label: ${{ needs.setup-instance.outputs.runner-name }} | |
| - name: Slack Notification | |
| if: ${{ failure() }} | |
| continue-on-error: true | |
| uses: rtCamp/action-slack-notify@4e5fb42d249be6a45a298f3c9543b111b02f7907 | |
| env: | |
| SLACK_COLOR: ${{ job.status }} | |
| SLACK_MESSAGE: "Instance teardown (integer-benchmarks) finished with status: ${{ job.status }}. (${{ env.ACTION_RUN_URL }})" |