chore: consolidate benchmark using workflow_call (#254)

* chore: consolidate benchmark using workflow_call

* feat: switch to using s3 to store results

* chore: delete result files

.github/workflows/bench-rw.yml → .github/workflows/benchmark-call.yml

@@ -1,21 +1,24 @@
-name: Run single page r/w benchmark
+name: "Benchmark: Coordinate Runner & Reporting"
 
 on:
   workflow_dispatch:
-  pull_request:
-    branches: ["main"]
-    paths:
-      - "stark-backend/**"
-      - "db/**"
-  push:
-    branches: ["main"]
-    paths:
-      - "stark-backend/**"
-      - "db/**"
-
-concurrency:
-  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.sha }}
-  cancel-in-progress: true
+    inputs:
+      benchmark_name:
+        type: choice
+        required: true
+        description: The name of the benchmark to run
+        options:
+          - vm_verify_fibair
+          - single_rw
+  workflow_call:
+    inputs:
+      benchmark_name:
+        type: string
+        required: true
+        description: The name of the benchmark to run
+
+env:
+  S3_PATH: s3://axiom-workflow-data-staging-us-east-1/benchmarks/github/results
 
 jobs:
   start-runner:
@@ -56,47 +59,71 @@ jobs:
         with:
           cache-on-failure: true
 
-      - name: Setup gh command line.
-        if: github.event_name == 'push'
-        uses: dev-hanz-ops/[email protected]
+      ######################################################
+      # Run a different benchmark based on benchmark_name:
+      - name: Run benchmark
+        if: inputs.benchmark_name == 'vm_verify_fibair'
+        run: |
+          RUSTFLAGS="-Ctarget-cpu=native" cargo run --release --bin benchmark --features parallel -- vm_verify_fibair -n 16
 
       - name: Run benchmark
+        if: inputs.benchmark_name == 'single_rw'
         run: |
           RUSTFLAGS="-Ctarget-cpu=native" cargo run --release --bin benchmark --features parallel -- rw -r 90 -w 10 --config-folder benchmark/config/single_rw
-
+      ######################################################
+      # If we want this read from main, split this into a different job
       - name: Get previous benchmark value
         id: get-previous-benchmark
         run: |
-          markdown="$(<benchmark/github/results/single_rw.md)"
+          main_sha=$(git rev-parse origin/main)
+          echo "Main branch SHA: $main_sha"
+          echo "main_sha=${main_sha}" >> $GITHUB_ENV
+
+          s5cmd cp "${{ env.S3_PATH }}/main-${{ inputs.benchmark_name }}.md" results.md
+
+          previous_result="$(<results.md)"
+          commit_url="https://github.com/${context.repo.owner}/${context.repo.repo}/commit/${main_sha}"
+
           echo 'previous-benchmark<<EOF' >> $GITHUB_ENV
-          echo "$markdown" >> $GITHUB_ENV
+          echo '# Previous Benchmark' >> $GITHUB_ENV
+          echo "${commit_url}" >> $GITHUB_ENV
+          echo "${previous_result}" >> $GITHUB_ENV
           echo 'EOF' >> $GITHUB_ENV
 
       - name: Update benchmark value
         id: update-benchmark
         run: |
-          mkdir -p benchmark/github/results
-          mv benchmark/tmp/_result.md benchmark/github/results/single_rw.md
-          markdown="$(<benchmark/github/results/single_rw.md)"
+          current_sha=$(git rev-parse HEAD)
+          echo "Current SHA: $current_sha"
+          echo "current_sha=${current_sha}" >> $GITHUB_ENV
+
+          RESULT_PATH=benchmark/tmp/_result.md
+          s5cmd cp $RESULT_PATH "${{ env.S3_PATH }}/${current_sha}-${{ inputs.benchmark_name }}.md"
+
+          new_result="$(<${RESULT_PATH})"
+          echo "${new_result}"
+
+          commit_url="https://github.com/${context.repo.owner}/${context.repo.repo}/commit/${current_sha}"
+
           echo 'new-benchmark<<EOF' >> $GITHUB_ENV
-          echo "$markdown" >> $GITHUB_ENV
+          echo "# New Benchmark" >> $GITHUB_ENV
+          echo "${commit_url}" >> $GITHUB_ENV
+          echo "${new_result}" >> $GITHUB_ENV
           echo 'EOF' >> $GITHUB_ENV
 
       - name: Remove previous comment (if exists)
         if: github.event_name == 'pull_request'
         uses: actions/github-script@v7
-        env:
-          previous_benchmark: ${{ env.previous-benchmark }}
         with:
           script: |
-            const previousBenchmark = process.env['previous_benchmark'];
+            const previousBenchmark = process.env['previous-benchmark'];
             const comments = await github.rest.issues.listComments({
               owner: context.repo.owner,
               repo: context.repo.repo,
               issue_number: context.issue.number
             });
             for (const comment of comments.data) {
-              if (comment.body.startsWith(`# Previous Benchmark\n${previousBenchmark}`)) {
+              if (comment.body.startsWith(previousBenchmark)) {
                 console.log("deleting comment ", comment.id);
                 await github.rest.issues.deleteComment({
                   owner: context.repo.owner,
@@ -109,35 +136,22 @@ jobs:
       - name: Add comment to pull request
         if: github.event_name == 'pull_request'
         uses: actions/github-script@v7
-        env:
-          previous_benchmark: ${{ env.previous-benchmark }}
-          new_benchmark: ${{ env.new-benchmark }}
         with:
           script: |
-            const previousBenchmark = process.env['previous_benchmark'];
-            const newBenchmark = process.env['new_benchmark'];
-
-            console.log(newBenchmark);
+            const previousBenchmark = process.env['previous-benchmark'];
+            const newBenchmark = process.env['new-benchmark'];
 
             github.rest.issues.createComment({
               issue_number: context.issue.number,
               owner: context.repo.owner,
               repo: context.repo.repo,
-              body: `# Previous Benchmark\n${previousBenchmark}\n\n# New Benchmark\n${newBenchmark}\n`
+              body: `${previousBenchmark}\n\n${newBenchmark}`
             });
 
-      - name: Commit the result and make a new PR.
-        if: github.event_name == 'push'
+      - name: Update latest main result in s3
+        if: github.event_name == 'push' && github.ref == 'refs/heads/main'
         run: |
-          export HOME=~
-          git config --global user.name "github-actions[bot]"
-          git config --global user.email "github-actions[bot]@users.noreply.github.com"
-          BRANCH_NAME="benchmark-rw-${{ github.run_id }}"
-          git checkout -b ${BRANCH_NAME}
-          git add benchmark/github/results/single_rw.md
-          git commit -m "Update benchmark"
-          git push origin ${BRANCH_NAME}
-          gh pr create -B main -H ${BRANCH_NAME} --title 'Update benchmark (single page r/w)' --body 'Created by Github action'
+          s5cmd cp "${{ env.S3_PATH }}/${{ env.current_sha }}-${{ inputs.benchmark_name }}.md" "${{ env.S3_PATH }}/main-${{ inputs.benchmark_name }}.md"
 
   stop-runner:
     name: Stop self-hosted EC2 runner

.github/workflows/db.yml

@@ -18,7 +18,7 @@ env:
   AXIOM_FAST_TEST: "1"
 
 jobs:
-  build:
+  tests:
     runs-on: ubuntu-latest
 
     steps:
@@ -32,3 +32,9 @@ jobs:
         working-directory: db/page
         run: |
           cargo t --features parallel
+
+  benchmark:
+    uses: ./.github/workflows/benchmark-call.yml
+    with:
+      benchmark_name: single_rw
+    secrets: inherit

.github/workflows/recursion.yml

@@ -20,7 +20,7 @@ env:
   AXIOM_FAST_TEST: "1"
 
 jobs:
-  build:
+  tests:
     runs-on: ubuntu-latest
 
     steps:
@@ -34,3 +34,9 @@ jobs:
         working-directory: recursion
         run: |
           cargo test --profile=fast --features parallel
+
+  benchmark:
+    uses: ./.github/workflows/benchmark-call.yml
+    with:
+      benchmark_name: vm_verify_fibair
+    secrets: inherit

.github/workflows/vm.yml

@@ -19,7 +19,7 @@ env:
   AXIOM_FAST_TEST: "1"
 
 jobs:
-  build:
+  tests:
     runs-on: ubuntu-latest
 
     steps: