bench: add back profiling mode to benchmarks (#1170)

* chore: add profile.profiling to all bench guest programs

* refactor: BenchmarkCli helper functions for app/agg config

* feat: flamegraphs

* chore: update workflow

* fix: add aggregation toggle

* fix

* fix: quotes

* fix: call vs dispatch

* chore: rename app_log_blowup to leaf_log_blowup

* chore: fix and refactor

* fix: makedirs

* use runs-on magic cache

* debug

* fix: json formatting

* fix: features

* fix: verify_fibair profiling

* feat: update documentation and fix

* last fix

* final fix for real

.github/workflows/benchmark-call.yml

@@ -22,7 +22,7 @@ on:
         required: false
         description: Application level log blowup
         default: 2
-      agg_log_blowup:
+      leaf_log_blowup:
         type: number
         required: false
         description: Aggregation (leaf) level log blowup
@@ -46,6 +46,10 @@ on:
         type: boolean
         required: true
         description: Whether to run the e2e benchmark
+      features:
+        type: string
+        required: false
+        description: Host features, comma separated (aggregation,profiling)
   workflow_call:
     inputs:
       benchmark_name:
@@ -71,7 +75,7 @@ on:
         required: false
         description: Application level log blowup
         default: 2
-      agg_log_blowup:
+      leaf_log_blowup:
         type: number
         required: false
         description: Aggregation (leaf) level log blowup
@@ -95,11 +99,14 @@ on:
         type: boolean
         required: true
         description: Whether to run the e2e benchmark
+      features:
+        type: string
+        required: false
+        description: Host features, comma separated (aggregation,profiling)
 
 env:
   S3_METRICS_PATH: s3://openvm-public-data-sandbox-us-east-1/benchmark/github/metrics
   FEATURE_FLAGS: "bench-metrics,parallel,nightly-features"
-  CMD_ARGS: ""
   INPUT_ARGS: ""
   CARGO_NET_GIT_FETCH_WITH_CLI: "true"
 
@@ -115,7 +122,9 @@ jobs:
       - run-id=${{ github.run_id }}
       - family=m7
       - tag=bench-${{ inputs.benchmark_name }}-${{ github.run_id }}-${{ github.run_number }}-${{ github.run_attempt }}
+      - extras=s3-cache
     steps:
+      - uses: runs-on/action@v1
       ##########################################################################
       # Environment setup                                                      #
       ##########################################################################
@@ -136,11 +145,20 @@ jobs:
 
       - name: Display workflow inputs
         run: echo "${{ toJSON(inputs) }}"
-      - name: Feature flags
+      - name: "Feature flags: memory allocator"
+        run: |
+          ALLOCATOR=${{ inputs.memory_allocator || github.event.inputs.memory_allocator }}
+          echo "FEATURE_FLAGS=$ALLOCATOR,${FEATURE_FLAGS}" >> $GITHUB_ENV
+      - name: "Feature flags: aggregation"
         if: contains(github.event.pull_request.labels.*.name, 'run-benchmark') || (github.event_name == 'push' && github.ref == 'refs/heads/main')
         run: |
           echo "Adding aggregation feature flag"
           echo "FEATURE_FLAGS=${FEATURE_FLAGS},aggregation" >> $GITHUB_ENV
+      - name: "Feature flags: workflow"
+        if: inputs.features || github.event.inputs.features
+        run: |
+          EXTRA_FEATURES=${{ inputs.features || github.event.inputs.features }}
+          echo "FEATURE_FLAGS=${FEATURE_FLAGS},$EXTRA_FEATURES" >> $GITHUB_ENV
 
       - name: Setup e2e (halo2 and arguments)
         run: |
@@ -155,10 +173,8 @@ jobs:
             export PARAMS_DIR=$(pwd)/params
           fi
 
-      - name: Set BIN_NAME and CMD_ARGS
+      - name: Set BIN_NAME
         run: |
-          CMD_ARGS="--features ${FEATURE_FLAGS}"
-          echo "CMD_ARGS=${CMD_ARGS}" >> $GITHUB_ENV
           echo "BIN_NAME=${{ inputs.benchmark_name }}" >> $GITHUB_ENV
 
       - name: Set working directory
@@ -184,21 +200,20 @@ jobs:
 
       - name: Set input args
         run: |
-          INSTANCE_TYPE="--instance_type ${{ inputs.instance_type }}"
-          MEMORY_ALLOCATOR="--memory_allocator ${{ inputs.memory_allocator }}"
+          FEATURES="--features $FEATURE_FLAGS"
           APP_ARG="--app_log_blowup ${{ inputs.app_log_blowup }}"
-          AGG_ARG="--agg_log_blowup ${{ inputs.agg_log_blowup }}"
+          AGG_ARG="--leaf_log_blowup ${{ inputs.leaf_log_blowup }}"
           MAX_SEGMENT_LENGTH="--max_segment_length ${{ inputs.max_segment_length }}"
-          OUTPUT_PATH="--output_path $METRIC_PATH"
-          echo "INPUT_ARGS=${INSTANCE_TYPE} ${MEMORY_ALLOCATOR} ${APP_ARG} ${AGG_ARG} ${MAX_SEGMENT_LENGTH} ${OUTPUT_PATH} ${INPUT_ARGS}" >> $GITHUB_ENV
+          OUTPUT_PATH="--output_path $(pwd)/$METRIC_PATH"
+          echo "INPUT_ARGS=${FEATURES} ${APP_ARG} ${AGG_ARG} ${MAX_SEGMENT_LENGTH} ${OUTPUT_PATH} ${INPUT_ARGS}" >> $GITHUB_ENV
 
       ##########################################################################
       # Find working directory based on benchmark_name and run the benchmark   #
       ##########################################################################
       - name: Run benchmark
         working-directory: ${{ steps.set-working-dir.outputs.working_dir }}
         run: |
-          python3 ${{ steps.set-working-dir.outputs.relative_path }}/ci/scripts/bench.py $BIN_NAME $CMD_ARGS $INPUT_ARGS
+          python3 ${{ steps.set-working-dir.outputs.relative_path }}/ci/scripts/bench.py $BIN_NAME $INPUT_ARGS
 
       ##########################################################################
       # Store metric json file to S3                                           #

.github/workflows/benchmarks.yml

@@ -16,10 +16,21 @@ on:
       - ".github/workflows/benchmarks.yml"
   workflow_dispatch:
     inputs:
+      flamegraphs:
+        type: boolean
+        description: Collect profiling metrics and generate flamegraphs
+        required: false
+        default: false
+      aggregation:
+        type: boolean
+        description: "Run leaf aggregation benchmarks"
+        required: false
+        default: false
       run-benchmark-e2e:
+        type: boolean
         description: "Run end-to-end benchmarks"
         required: false
-        default: "false"
+        default: false
 
 concurrency:
   group: benchmark-${{ github.event.pull_request.number || github.sha }}
@@ -33,6 +44,8 @@ env:
   CARGO_NET_GIT_FETCH_WITH_CLI: "true"
   S3_METRICS_PATH: s3://openvm-public-data-sandbox-us-east-1/benchmark/github/metrics
   S3_MD_PATH: s3://openvm-public-data-sandbox-us-east-1/benchmark/github/results
+  S3_FLAMEGRAPHS_PATH: s3://openvm-public-data-sandbox-us-east-1/benchmark/github/flamegraphs
+  FEATURE_FLAGS: ""
 
 permissions:
   contents: write
@@ -66,8 +79,14 @@ jobs:
           else
             RUN_E2E=${{ github.event.inputs.run-benchmark-e2e || 'false' }}
           fi
+          if [[ "${{ github.event.inputs.aggregation }}" == "true" ]]; then
+            FEATURE_FLAGS="aggregation,${FEATURE_FLAGS}"
+          fi
+          if [[ "${{ github.event.inputs.flamegraphs }}" == "true" ]]; then
+            FEATURE_FLAGS="profiling,${FEATURE_FLAGS}"
+          fi
 
-          matrix=$(jq -c --argjson run_e2e $RUN_E2E '
+          matrix=$(jq -c --argjson run_e2e $RUN_E2E --arg features "$FEATURE_FLAGS" '
             [
               .benchmarks[] |
               .name as $name |
@@ -78,11 +97,12 @@ jobs:
               {
                 name: $name,
                 e2e_bench: $e2e_bench,
+                features: $features,
                 id: $id,
                 instance_type: .instance_type,
                 memory_allocator: .memory_allocator,
                 app_log_blowup: .app_log_blowup,
-                agg_log_blowup: .agg_log_blowup,
+                leaf_log_blowup: .leaf_log_blowup,
                 root_log_blowup: (.root_log_blowup // 0),
                 internal_log_blowup: (.internal_log_blowup // 0),
                 max_segment_length: (.max_segment_length // 1048476)
@@ -107,11 +127,12 @@ jobs:
       instance_type: ${{ matrix.benchmark_run.instance_type }}
       memory_allocator: ${{ matrix.benchmark_run.memory_allocator }}
       app_log_blowup: ${{ matrix.benchmark_run.app_log_blowup }}
-      agg_log_blowup: ${{ matrix.benchmark_run.agg_log_blowup }}
+      leaf_log_blowup: ${{ matrix.benchmark_run.leaf_log_blowup }}
       root_log_blowup: ${{ matrix.benchmark_run.root_log_blowup }}
       internal_log_blowup: ${{ matrix.benchmark_run.internal_log_blowup }}
       max_segment_length: ${{ matrix.benchmark_run.max_segment_length }}
       e2e_bench: ${{ matrix.benchmark_run.e2e_bench }}
+      features: ${{ matrix.benchmark_run.features }}
     secrets: inherit
 
   summarize:
@@ -151,6 +172,7 @@ jobs:
         run: cargo install --force --profile=dev --path .
 
       - name: Set github pages path for dispatch
+        if: github.event_name == 'workflow_dispatch'
         run: |
           BENCHMARK_RESULTS_PATH="benchmarks-dispatch/${{ github.head_ref || github.ref }}"
           echo "BENCHMARK_RESULTS_PATH=${BENCHMARK_RESULTS_PATH}" >> $GITHUB_ENV
@@ -167,15 +189,20 @@ jobs:
           BENCHMARK_RESULTS_PATH="benchmarks"
           echo "BENCHMARK_RESULTS_PATH=${BENCHMARK_RESULTS_PATH}" >> $GITHUB_ENV
 
+      - name: Install inferno-flamegraph
+        if: github.event.inputs.flamegraphs
+        run: cargo install inferno
+
       - name: Download all metric json files from S3
         run: |
-          json_files=$(echo '${{ needs.create-matrix.outputs.matrix }}' | jq -r '
+          matrix=$(echo '${{ needs.create-matrix.outputs.matrix }}')
+          json_files=$(echo $matrix | jq -r '
             .[] |
             "\(.id)-${{ env.CURRENT_SHA }}.json"')
           json_file_list=$(echo -n "$json_files" | paste -sd "," -)
           echo $json_file_list
 
-          prev_json_files=$(echo '${{ needs.create-matrix.outputs.matrix }}' | jq -r '
+          prev_json_files=$(echo $matrix | jq -r '
             .[] |
             "main-\(.id).json"')
           prev_json_file_list=$(echo -n "$prev_json_files" | paste -sd "," -)
@@ -207,33 +234,19 @@ jobs:
           # add metadata to markdown files
           source ci/scripts/utils.sh
           # Parse matrix data into associative arrays
-          while IFS= read -r line; do
+          while IFS= read -r metric_path; do
+            FLAMEGRAPHS=${{ github.event.inputs.flamegraphs || 'false' }}
             # Convert .json to .md
-            md_file="${line%.json}.md"
+            md_path="${metric_path%.json}.md"
 
-            id=$(echo '${{ needs.create-matrix.outputs.matrix }}' | jq -r --arg file "$line" '.[] |
-              select(.id == ($file | split("-")[0])) |
-              {
-                max_segment_length: .max_segment_length,
-                instance_type: .instance_type,
-                memory_allocator: .memory_allocator
-              }')
-
-            if [ ! -z "$id" ]; then
-              max_segment_length=$(echo "$id" | jq -r '.max_segment_length')
-              instance_type=$(echo "$id" | jq -r '.instance_type')
-              memory_allocator=$(echo "$id" | jq -r '.memory_allocator')
-
-              # Call add_metadata for each file with its corresponding data
-              add_metadata \
-                "$md_file" \
-                "$max_segment_length" \
-                "$instance_type" \
-                "$memory_allocator" \
-                "$COMMIT_URL" \
-                "$BENCHMARK_WORKFLOW_URL"
-            fi
-            cp "$md_file" "${TMP_DIR}/"
+            add_metadata_and_flamegraphs \
+              $metric_path \
+              $md_path \
+              $matrix \
+              $COMMIT_URL \
+              $BENCHMARK_WORKFLOW_URL
+
+            cp "$md_path" "${TMP_DIR}/"
           done <<< "$json_files"
 
           echo "" >> summary.md
@@ -244,21 +257,19 @@ jobs:
           cp summary.md ${TMP_DIR}/
 
       ##########################################################################
-      # Update benchmark-results branch with summary upon a PR event           #
+      # Update benchmark-results branch with summary                           #
       ##########################################################################
       - uses: actions/checkout@v4
-        if: github.event_name == 'pull_request' || (github.event_name == 'push' && github.ref == 'refs/heads/main')
         with:
           ref: benchmark-results
 
       - name: Set up git
-        if: github.event_name == 'pull_request' || (github.event_name == 'push' && github.ref == 'refs/heads/main')
         run: |
           git config --global user.email "github-actions[bot]@users.noreply.github.com"
           git config --global user.name "github-actions[bot]"
 
       - name: Update github pages with new bench results
-        if: (github.event_name == 'pull_request' && github.event.pull_request.head.repo.fork == false) || (github.event_name == 'push' && github.ref == 'refs/heads/main')
+        if: (github.event_name == 'pull_request' && github.event.pull_request.head.repo.fork == false) || (github.event_name == 'push' && github.ref == 'refs/heads/main') || (github.event_name == 'workflow_dispatch')
         run: |
           mkdir -p ${BENCHMARK_RESULTS_PATH}
           cp ${TMP_DIR}/*.md ${BENCHMARK_RESULTS_PATH}/
@@ -292,6 +303,11 @@ jobs:
               exit 1
           fi
 
+      - name: Link to Benchmark Results
+        run: |
+          echo "Benchmark results are available at:"
+          echo "https://github.com/${{ github.repository }}/blob/benchmark-results/${BENCHMARK_RESULTS_PATH}/summary.md"
+
       ##########################################################################
       # Update PR comment upon a pull request event                            #
       ##########################################################################

.github/workflows/build.yml

@@ -16,7 +16,9 @@ jobs:
     runs-on:
       - runs-on=${{ github.run_id }}
       - runner=8cpu-linux-x64
+      - extras=s3-cache
     steps:
+      - uses: runs-on/action@v1
       - uses: actions/checkout@v4
       - uses: dtolnay/rust-toolchain@stable
       - uses: Swatinem/rust-cache@v2

.github/workflows/cli.yml

@@ -26,8 +26,10 @@ jobs:
     runs-on:
       - runs-on=${{ github.run_id }}
       - runner=32cpu-linux-arm64
+      - extras=s3-cache
 
     steps:
+      - uses: runs-on/action@v1
       - uses: actions/checkout@v4
       - uses: dtolnay/rust-toolchain@nightly
       - uses: Swatinem/rust-cache@v2

.github/workflows/extension-tests.yml

@@ -39,8 +39,10 @@ jobs:
       - runs-on=${{ github.run_id }}
       - runner=64cpu-linux-arm64
       - tag=extension-${{ matrix.extension.name }}
+      - extras=s3-cache
 
     steps:
+      - uses: runs-on/action@v1
       - uses: actions/checkout@v4
       - uses: dorny/paths-filter@v3
         id: filter

.github/workflows/lints.yml

@@ -16,7 +16,9 @@ jobs:
     runs-on:
       - runs-on=${{ github.run_id }}
       - runner=8cpu-linux-x64
+      - extras=s3-cache
     steps:
+      - uses: runs-on/action@v1
       - uses: actions/checkout@v4
       - uses: dtolnay/rust-toolchain@stable
       - uses: Swatinem/rust-cache@v2

.github/workflows/native-compiler.yml

@@ -23,8 +23,10 @@ jobs:
     runs-on:
       - runs-on=${{ github.run_id }}
       - runner=64cpu-linux-arm64
+      - extras=s3-cache
 
     steps:
+      - uses: runs-on/action@v1
       - uses: actions/checkout@v4
       - uses: dtolnay/rust-toolchain@nightly
       - uses: Swatinem/rust-cache@v2

.github/workflows/primitives.yml

@@ -23,8 +23,10 @@ jobs:
     runs-on:
       - runs-on=${{ github.run_id }}
       - runner=32cpu-linux-arm64
+      - extras=s3-cache
 
     steps:
+      - uses: runs-on/action@v1
       - uses: actions/checkout@v4
       - uses: dtolnay/rust-toolchain@stable
       - uses: Swatinem/rust-cache@v2

.github/workflows/recursion.yml

@@ -24,8 +24,10 @@ jobs:
     runs-on:
       - runs-on=${{ github.run_id }}
       - runner=64cpu-linux-arm64
+      - extras=s3-cache
 
     steps:
+      - uses: runs-on/action@v1
       - uses: actions/checkout@v4
       - uses: dtolnay/rust-toolchain@stable
       - uses: Swatinem/rust-cache@v2

.github/workflows/riscv.yml

@@ -27,8 +27,10 @@ jobs:
       - runs-on=${{ github.run_id }}
       - runner=64cpu-linux-arm64
       - image=arm64-rust-riscv-dev-32gb # for riscv-gnu-toolchain
+      - extras=s3-cache
 
     steps:
+      - uses: runs-on/action@v1
       - uses: actions/checkout@v4
       - uses: dtolnay/rust-toolchain@nightly
       - uses: Swatinem/rust-cache@v2