[omni-bencher] Make all runtimes work

.github/scripts/cmd/cmd.py

@@ -104,7 +104,7 @@ def main():
             print(f'-- listing pallets for benchmark for {runtime["name"]}')
             wasm_file = f"target/{profile}/wbuild/{runtime['package']}/{runtime['package'].replace('-', '_')}.wasm"
             output = os.popen(
-                f"frame-omni-bencher v1 benchmark pallet --no-csv-header --no-storage-info --no-min-squares --no-median-slopes --all --list --runtime={wasm_file}").read()
+                f"frame-omni-bencher v1 benchmark pallet --no-csv-header --no-storage-info --no-min-squares --no-median-slopes --all --list --runtime={wasm_file} {runtime['bench_flags']}").read()
             raw_pallets = output.strip().split('\n')
 
             all_pallets = set()
@@ -182,7 +182,8 @@ def main():
                     f"--repeat=20 " \
                     f"--heap-pages=4096 " \
                     f"{f'--template={template} ' if template else ''}" \
-                    f"--no-storage-info --no-min-squares --no-median-slopes"
+                    f"--no-storage-info --no-min-squares --no-median-slopes " \
+                    f"{config['bench_flags']}"
                 print(f'-- Running: {cmd} \n')
                 status = os.system(cmd)
                 if status != 0 and not args.continue_on_fail:

.github/scripts/cmd/test_cmd.py

@@ -7,21 +7,54 @@
 
 # Mock data for runtimes-matrix.json
 mock_runtimes_matrix = [
-    {"name": "dev", "package": "kitchensink-runtime", "path": "substrate/frame", "header": "substrate/HEADER-APACHE2",  "template": "substrate/.maintain/frame-weight-template.hbs", "bench_features": "runtime-benchmarks,riscv"},
-    {"name": "westend", "package": "westend-runtime", "path": "polkadot/runtime/westend", "header": "polkadot/file_header.txt", "template": "polkadot/xcm/pallet-xcm-benchmarks/template.hbs", "bench_features": "runtime-benchmarks"},
-    {"name": "rococo", "package": "rococo-runtime", "path": "polkadot/runtime/rococo", "header": "polkadot/file_header.txt", "template": "polkadot/xcm/pallet-xcm-benchmarks/template.hbs", "bench_features": "runtime-benchmarks"},
-    {"name": "asset-hub-westend", "package": "asset-hub-westend-runtime", "path": "cumulus/parachains/runtimes/assets/asset-hub-westend", "header": "cumulus/file_header.txt", "template": "cumulus/templates/xcm-bench-template.hbs", "bench_features": "runtime-benchmarks"},
+    {
+        "name": "dev",
+        "package": "kitchensink-runtime",
+        "path": "substrate/frame",
+        "header": "substrate/HEADER-APACHE2",
+        "template": "substrate/.maintain/frame-weight-template.hbs",
+        "bench_features": "runtime-benchmarks,riscv",
+        "bench_flags": "--flag1 --flag2"
+    },
+    {
+        "name": "westend",
+        "package": "westend-runtime",
+        "path": "polkadot/runtime/westend",
+        "header": "polkadot/file_header.txt",
+        "template": "polkadot/xcm/pallet-xcm-benchmarks/template.hbs",
+        "bench_features": "runtime-benchmarks",
+        "bench_flags": "--flag3 --flag4"
+    },
+    {
+        "name": "rococo",
+        "package": "rococo-runtime",
+        "path": "polkadot/runtime/rococo",
+        "header": "polkadot/file_header.txt",
+        "template": "polkadot/xcm/pallet-xcm-benchmarks/template.hbs",
+        "bench_features": "runtime-benchmarks",
+        "bench_flags": ""
+    },
+    {
+        "name": "asset-hub-westend",
+        "package": "asset-hub-westend-runtime",
+        "path": "cumulus/parachains/runtimes/assets/asset-hub-westend",
+        "header": "cumulus/file_header.txt",
+        "template": "cumulus/templates/xcm-bench-template.hbs",
+        "bench_features": "runtime-benchmarks",
+        "bench_flags": "--flag7 --flag8"
+    }
 ]
 
-def get_mock_bench_output(runtime, pallets, output_path, header, template = None):
+def get_mock_bench_output(runtime, pallets, output_path, header, bench_flags, template = None):
     return f"frame-omni-bencher v1 benchmark pallet --extrinsic=* " \
            f"--runtime=target/release/wbuild/{runtime}-runtime/{runtime.replace('-', '_')}_runtime.wasm " \
            f"--pallet={pallets} --header={header} " \
            f"--output={output_path} " \
            f"--wasm-execution=compiled " \
            f"--steps=50 --repeat=20 --heap-pages=4096 " \
            f"{f'--template={template} ' if template else ''}" \
-           f"--no-storage-info --no-min-squares --no-median-slopes"
+           f"--no-storage-info --no-min-squares --no-median-slopes " \
+           f"{bench_flags}"
 
 class TestCmd(unittest.TestCase):
 
@@ -89,10 +122,29 @@ def test_bench_command_normal_execution_all_runtimes(self):
                 call("forklift cargo build -p rococo-runtime --profile release --features=runtime-benchmarks"),
                 call("forklift cargo build -p asset-hub-westend-runtime --profile release --features=runtime-benchmarks"),
 
-                call(get_mock_bench_output('kitchensink', 'pallet_balances', './substrate/frame/balances/src/weights.rs', os.path.abspath('substrate/HEADER-APACHE2'), "substrate/.maintain/frame-weight-template.hbs")),
-                call(get_mock_bench_output('westend', 'pallet_balances', './polkadot/runtime/westend/src/weights', os.path.abspath('polkadot/file_header.txt'))),
+                call(get_mock_bench_output(
+                    runtime='kitchensink',
+                    pallets='pallet_balances',
+                    output_path='./substrate/frame/balances/src/weights.rs',
+                    header=os.path.abspath('substrate/HEADER-APACHE2'),
+                    bench_flags='--flag1 --flag2',
+                    template="substrate/.maintain/frame-weight-template.hbs"
+                )),
+                call(get_mock_bench_output(
+                    runtime='westend',
+                    pallets='pallet_balances',
+                    output_path='./polkadot/runtime/westend/src/weights',
+                    header=os.path.abspath('polkadot/file_header.txt'),
+                    bench_flags='--flag3 --flag4'
+                )),
                 # skips rococo benchmark
-                call(get_mock_bench_output('asset-hub-westend', 'pallet_balances', './cumulus/parachains/runtimes/assets/asset-hub-westend/src/weights', os.path.abspath('cumulus/file_header.txt'))),
+                call(get_mock_bench_output(
+                    runtime='asset-hub-westend',
+                    pallets='pallet_balances',
+                    output_path='./cumulus/parachains/runtimes/assets/asset-hub-westend/src/weights',
+                    header=os.path.abspath('cumulus/file_header.txt'),
+                    bench_flags='--flag7 --flag8'
+                )),
             ]
             self.mock_system.assert_has_calls(expected_calls, any_order=True)
 
@@ -121,8 +173,20 @@ def test_bench_command_normal_execution(self):
                 call("forklift cargo build -p westend-runtime --profile release --features=runtime-benchmarks"),
 
                 # Westend runtime calls
-                call(get_mock_bench_output('westend', 'pallet_balances', './polkadot/runtime/westend/src/weights', header_path)),
-                call(get_mock_bench_output('westend', 'pallet_staking', './polkadot/runtime/westend/src/weights', header_path)),
+                call(get_mock_bench_output(
+                    runtime='westend',
+                    pallets='pallet_balances',
+                    output_path='./polkadot/runtime/westend/src/weights',
+                    header=header_path,
+                    bench_flags='--flag3 --flag4'
+                )),
+                call(get_mock_bench_output(
+                    runtime='westend',
+                    pallets='pallet_staking',
+                    output_path='./polkadot/runtime/westend/src/weights',
+                    header=header_path,
+                    bench_flags='--flag3 --flag4'
+                )),
             ]
             self.mock_system.assert_has_calls(expected_calls, any_order=True)
 
@@ -153,11 +217,12 @@ def test_bench_command_normal_execution_xcm(self):
 
                 # Westend runtime calls
                 call(get_mock_bench_output(
-                    'westend', 
-                    'pallet_xcm_benchmarks::generic', 
-                    './polkadot/runtime/westend/src/weights/xcm', 
-                    header_path, 
-                    "polkadot/xcm/pallet-xcm-benchmarks/template.hbs"
+                    runtime='westend',
+                    pallets='pallet_xcm_benchmarks::generic',
+                    output_path='./polkadot/runtime/westend/src/weights/xcm',
+                    header=header_path,
+                    bench_flags='--flag3 --flag4',
+                    template="polkadot/xcm/pallet-xcm-benchmarks/template.hbs"
                 )),
             ]
             self.mock_system.assert_has_calls(expected_calls, any_order=True)
@@ -188,11 +253,35 @@ def test_bench_command_two_runtimes_two_pallets(self):
                 call("forklift cargo build -p westend-runtime --profile release --features=runtime-benchmarks"),
                 call("forklift cargo build -p rococo-runtime --profile release --features=runtime-benchmarks"),
                 # Westend runtime calls
-                call(get_mock_bench_output('westend', 'pallet_staking', './polkadot/runtime/westend/src/weights', header_path)),
-                call(get_mock_bench_output('westend', 'pallet_balances', './polkadot/runtime/westend/src/weights', header_path)),
+                call(get_mock_bench_output(
+                    runtime='westend',
+                    pallets='pallet_staking',
+                    output_path='./polkadot/runtime/westend/src/weights',
+                    header=header_path,
+                    bench_flags='--flag3 --flag4'
+                )),
+                call(get_mock_bench_output(
+                    runtime='westend',
+                    pallets='pallet_balances',
+                    output_path='./polkadot/runtime/westend/src/weights',
+                    header=header_path,
+                    bench_flags='--flag3 --flag4'
+                )),
                 # Rococo runtime calls
-                call(get_mock_bench_output('rococo', 'pallet_staking', './polkadot/runtime/rococo/src/weights', header_path)),
-                call(get_mock_bench_output('rococo', 'pallet_balances', './polkadot/runtime/rococo/src/weights', header_path)),
+                call(get_mock_bench_output(
+                    runtime='rococo',
+                    pallets='pallet_staking',
+                    output_path='./polkadot/runtime/rococo/src/weights',
+                    header=header_path,
+                    bench_flags=''
+                )),
+                call(get_mock_bench_output(
+                    runtime='rococo',
+                    pallets='pallet_balances',
+                    output_path='./polkadot/runtime/rococo/src/weights',
+                    header=header_path,
+                    bench_flags=''
+                )),
             ]
             self.mock_system.assert_has_calls(expected_calls, any_order=True)
 
@@ -223,11 +312,12 @@ def test_bench_command_one_dev_runtime(self):
                 call("forklift cargo build -p kitchensink-runtime --profile release --features=runtime-benchmarks,riscv"),
                 # Westend runtime calls
                 call(get_mock_bench_output(
-                    'kitchensink', 
-                    'pallet_balances', 
-                    manifest_dir + "/src/weights.rs", 
-                    header_path, 
-                    "substrate/.maintain/frame-weight-template.hbs"
+                    runtime='kitchensink',
+                    pallets='pallet_balances',
+                    output_path=manifest_dir + "/src/weights.rs",
+                    header=header_path,
+                    bench_flags='--flag1 --flag2',
+                    template="substrate/.maintain/frame-weight-template.hbs"
                 )),
             ]
             self.mock_system.assert_has_calls(expected_calls, any_order=True)
@@ -257,10 +347,11 @@ def test_bench_command_one_cumulus_runtime(self):
                 call("forklift cargo build -p asset-hub-westend-runtime --profile release --features=runtime-benchmarks"),
                 # Asset-hub-westend runtime calls
                 call(get_mock_bench_output(
-                    'asset-hub-westend', 
-                    'pallet_assets', 
-                    './cumulus/parachains/runtimes/assets/asset-hub-westend/src/weights', 
-                    header_path
+                    runtime='asset-hub-westend',
+                    pallets='pallet_assets',
+                    output_path='./cumulus/parachains/runtimes/assets/asset-hub-westend/src/weights',
+                    header=header_path,
+                    bench_flags='--flag7 --flag8'
                 )),
             ]
 
@@ -291,17 +382,19 @@ def test_bench_command_one_cumulus_runtime_xcm(self):
                 call("forklift cargo build -p asset-hub-westend-runtime --profile release --features=runtime-benchmarks"),
                 # Asset-hub-westend runtime calls
                 call(get_mock_bench_output(
-                    'asset-hub-westend', 
-                    'pallet_xcm_benchmarks::generic', 
-                    './cumulus/parachains/runtimes/assets/asset-hub-westend/src/weights/xcm', 
-                    header_path, 
-                    "cumulus/templates/xcm-bench-template.hbs"
+                    runtime='asset-hub-westend',
+                    pallets='pallet_xcm_benchmarks::generic',
+                    output_path='./cumulus/parachains/runtimes/assets/asset-hub-westend/src/weights/xcm',
+                    header=header_path,
+                    bench_flags='--flag7 --flag8',
+                    template="cumulus/templates/xcm-bench-template.hbs"
                 )),
                 call(get_mock_bench_output(
-                    'asset-hub-westend', 
-                    'pallet_assets', 
-                    './cumulus/parachains/runtimes/assets/asset-hub-westend/src/weights', 
-                    header_path
+                    runtime='asset-hub-westend',
+                    pallets='pallet_assets',
+                    output_path='./cumulus/parachains/runtimes/assets/asset-hub-westend/src/weights',
+                    header=header_path,
+                    bench_flags='--flag7 --flag8'
                 )),
             ]
 

.github/workflows/check-frame-omni-bencher.yml

@@ -16,21 +16,19 @@ env:
   ARTIFACTS_NAME: frame-omni-bencher-artifacts
 
 jobs:
-
   preflight:
-    # TODO: remove once migration is complete or this workflow is fully stable
-    if: contains(github.event.label.name, 'GHA-migration')
     uses: ./.github/workflows/reusable-preflight.yml
 
   quick-benchmarks-omni:
-    runs-on: ${{ needs.preflight.outputs.RUNNER }}
+    runs-on: parity-benchmark
     needs: [preflight]
     if: ${{ needs.preflight.outputs.changes_rust }}
     env:
       RUSTFLAGS: "-C debug-assertions"
       RUST_BACKTRACE: "full"
       WASM_BUILD_NO_COLOR: 1
       WASM_BUILD_RUSTFLAGS: "-C debug-assertions"
+      RUST_LOG: "frame_omni_bencher=info,polkadot_sdk_frame=info"
     timeout-minutes: 30
     container:
       image: ${{ needs.preflight.outputs.IMAGE }}
@@ -42,33 +40,41 @@ jobs:
           forklift cargo build --locked --quiet --release -p asset-hub-westend-runtime --features runtime-benchmarks
           forklift cargo run --locked --release -p frame-omni-bencher --quiet -- v1 benchmark pallet --runtime target/release/wbuild/asset-hub-westend-runtime/asset_hub_westend_runtime.compact.compressed.wasm --all --steps 2 --repeat 1 --quiet
 
+  runtime-matrix:
+    runs-on: ubuntu-latest
+    needs: [preflight]
+    if: ${{ needs.preflight.outputs.changes_rust }}
+    timeout-minutes: 30
+    outputs:
+      runtime: ${{ steps.runtime.outputs.runtime }}
+    container:
+      image: ${{ needs.preflight.outputs.IMAGE }}
+    name: Extract runtimes from matrix
+    steps:
+      - uses: actions/checkout@v4
+      - id: runtime
+        run: |
+          RUNTIMES=$(jq '[.[] | select(.package != null)]' .github/workflows/runtimes-matrix.json)
+
+          RUNTIMES=$(echo $RUNTIMES | jq -c .)
+          echo "runtime=$RUNTIMES"
+          echo "runtime=$RUNTIMES" >> $GITHUB_OUTPUT
+
   run-frame-omni-bencher:
-    runs-on: ${{ needs.preflight.outputs.RUNNER }}
-    needs: [preflight] # , build-frame-omni-bencher ]
+    runs-on: parity-benchmark
+    needs: [preflight, runtime-matrix]
     if: ${{ needs.preflight.outputs.changes_rust }}
     timeout-minutes: 30
     strategy:
       fail-fast: false # keep running other workflows even if one fails, to see the logs of all possible failures
       matrix:
-        runtime:
-          [
-            westend-runtime,
-            rococo-runtime,
-            asset-hub-rococo-runtime,
-            asset-hub-westend-runtime,
-            bridge-hub-rococo-runtime,
-            bridge-hub-westend-runtime,
-            collectives-westend-runtime,
-            coretime-rococo-runtime,
-            coretime-westend-runtime,
-            people-rococo-runtime,
-            people-westend-runtime,
-            glutton-westend-runtime,
-          ]
+        runtime: ${{ fromJSON(needs.runtime-matrix.outputs.runtime) }}
     container:
       image: ${{ needs.preflight.outputs.IMAGE }}
     env:
-      PACKAGE_NAME: ${{ matrix.runtime }}
+      PACKAGE_NAME: ${{ matrix.runtime.package }}
+      FLAGS: ${{ matrix.runtime.bench_flags }}
+      RUST_LOG: "frame_omni_bencher=info,polkadot_sdk_frame=info"
     steps:
       - name: Checkout
         uses: actions/checkout@v4
@@ -77,10 +83,13 @@ jobs:
         run: |
           RUNTIME_BLOB_NAME=$(echo $PACKAGE_NAME | sed 's/-/_/g').compact.compressed.wasm
           RUNTIME_BLOB_PATH=./target/release/wbuild/$PACKAGE_NAME/$RUNTIME_BLOB_NAME
-          forklift cargo build --release --locked -p $PACKAGE_NAME -p frame-omni-bencher --features runtime-benchmarks
+          forklift cargo build --release --locked -p $PACKAGE_NAME -p frame-omni-bencher --features runtime-benchmarks --quiet
           echo "Running short benchmarking for PACKAGE_NAME=$PACKAGE_NAME and RUNTIME_BLOB_PATH=$RUNTIME_BLOB_PATH"
           ls -lrt $RUNTIME_BLOB_PATH
-          ./target/release/frame-omni-bencher v1 benchmark pallet --runtime $RUNTIME_BLOB_PATH --all --steps 2 --repeat 1
+
+          cmd="./target/release/frame-omni-bencher v1 benchmark pallet --runtime $RUNTIME_BLOB_PATH --all --steps 2 --repeat 1 $FLAGS"
+          echo "Running command: $cmd"
+          eval "$cmd"
   confirm-frame-omni-benchers-passed:
     runs-on: ubuntu-latest
     name: All benchmarks passed