[omni-bencher] Make all runtimes work

.github/workflows/check-frame-omni-bencher.yml

@@ -16,21 +16,19 @@ env:
   ARTIFACTS_NAME: frame-omni-bencher-artifacts
 
 jobs:
-
   preflight:
-    # TODO: remove once migration is complete or this workflow is fully stable
-    if: contains(github.event.label.name, 'GHA-migration')
     uses: ./.github/workflows/reusable-preflight.yml
 
   quick-benchmarks-omni:
-    runs-on: ${{ needs.preflight.outputs.RUNNER }}
+    runs-on: parity-benchmark
     needs: [preflight]
     if: ${{ needs.preflight.outputs.changes_rust }}
     env:
       RUSTFLAGS: "-C debug-assertions"
       RUST_BACKTRACE: "full"
       WASM_BUILD_NO_COLOR: 1
       WASM_BUILD_RUSTFLAGS: "-C debug-assertions"
+      RUST_LOG: "frame_omni_bencher=info,polkadot_sdk_frame=info"
     timeout-minutes: 30
     container:
       image: ${{ needs.preflight.outputs.IMAGE }}
@@ -42,33 +40,41 @@ jobs:
           forklift cargo build --locked --quiet --release -p asset-hub-westend-runtime --features runtime-benchmarks
           forklift cargo run --locked --release -p frame-omni-bencher --quiet -- v1 benchmark pallet --runtime target/release/wbuild/asset-hub-westend-runtime/asset_hub_westend_runtime.compact.compressed.wasm --all --steps 2 --repeat 1 --quiet
 
+  runtime-matrix:
+    runs-on: ubuntu-latest
+    needs: [preflight]
+    if: ${{ needs.preflight.outputs.changes_rust }}
+    timeout-minutes: 30
+    outputs:
+      runtime: ${{ steps.runtime.outputs.runtime }}
+    container:
+      image: ${{ needs.preflight.outputs.IMAGE }}
+    name: Extract runtimes from matrix
+    steps:
+      - uses: actions/checkout@v4
+      - id: runtime
+        run: |
+          RUNTIMES=$(jq '[.[] | select(.package != null)]' .github/workflows/runtimes-matrix.json)
+
+          RUNTIMES=$(echo $RUNTIMES | jq -c .)
+          echo "runtime=$RUNTIMES"
+          echo "runtime=$RUNTIMES" >> $GITHUB_OUTPUT
+
   run-frame-omni-bencher:
-    runs-on: ${{ needs.preflight.outputs.RUNNER }}
-    needs: [preflight] # , build-frame-omni-bencher ]
+    runs-on: parity-benchmark
+    needs: [preflight, runtime-matrix]
     if: ${{ needs.preflight.outputs.changes_rust }}
     timeout-minutes: 30
     strategy:
       fail-fast: false # keep running other workflows even if one fails, to see the logs of all possible failures
       matrix:
-        runtime:
-          [
-            westend-runtime,
-            rococo-runtime,
-            asset-hub-rococo-runtime,
-            asset-hub-westend-runtime,
-            bridge-hub-rococo-runtime,
-            bridge-hub-westend-runtime,
-            collectives-westend-runtime,
-            coretime-rococo-runtime,
-            coretime-westend-runtime,
-            people-rococo-runtime,
-            people-westend-runtime,
-            glutton-westend-runtime,
-          ]
+        runtime: ${{ fromJSON(needs.runtime-matrix.outputs.runtime) }}
     container:
       image: ${{ needs.preflight.outputs.IMAGE }}
     env:
-      PACKAGE_NAME: ${{ matrix.runtime }}
+      PACKAGE_NAME: ${{ matrix.runtime.package }}
+      FLAGS: ${{ matrix.runtime.bench_flags }}
+      RUST_LOG: "frame_omni_bencher=info,polkadot_sdk_frame=info"
     steps:
       - name: Checkout
         uses: actions/checkout@v4
@@ -77,10 +83,13 @@ jobs:
         run: |
           RUNTIME_BLOB_NAME=$(echo $PACKAGE_NAME | sed 's/-/_/g').compact.compressed.wasm
           RUNTIME_BLOB_PATH=./target/release/wbuild/$PACKAGE_NAME/$RUNTIME_BLOB_NAME
-          forklift cargo build --release --locked -p $PACKAGE_NAME -p frame-omni-bencher --features runtime-benchmarks
+          forklift cargo build --release --locked -p $PACKAGE_NAME -p frame-omni-bencher --features runtime-benchmarks --quiet
           echo "Running short benchmarking for PACKAGE_NAME=$PACKAGE_NAME and RUNTIME_BLOB_PATH=$RUNTIME_BLOB_PATH"
           ls -lrt $RUNTIME_BLOB_PATH
-          ./target/release/frame-omni-bencher v1 benchmark pallet --runtime $RUNTIME_BLOB_PATH --all --steps 2 --repeat 1
+
+          cmd="./target/release/frame-omni-bencher v1 benchmark pallet --runtime $RUNTIME_BLOB_PATH --all --steps 2 --repeat 1 $FLAGS"
+          echo "Running command: $cmd"
+          eval "$cmd"
   confirm-frame-omni-benchers-passed:
     runs-on: ubuntu-latest
     name: All benchmarks passed

.github/workflows/runtimes-matrix.json

@@ -4,6 +4,7 @@
     "package": "kitchensink-runtime",
     "path": "substrate/frame",
     "header": "substrate/HEADER-APACHE2",
+    "bench_flags": "--genesis-builder-policy=none --exclude-pallets=pallet_xcm,pallet_xcm_benchmarks::fungible,pallet_xcm_benchmarks::generic,pallet_nomination_pools,pallet_remark,pallet_transaction_storage",
     "template": "substrate/.maintain/frame-weight-template.hbs",
     "bench_features": "runtime-benchmarks,riscv",
     "uri": null,
@@ -84,6 +85,7 @@
     "path": "cumulus/parachains/runtimes/contracts/contracts-rococo",
     "header": "cumulus/file_header.txt",
     "bench_features": "runtime-benchmarks",
+    "bench_flags": "--genesis-builder-policy=none --exclude-pallets=pallet_xcm,pallet_contracts",
     "template": "cumulus/templates/xcm-bench-template.hbs",
     "uri": "wss://rococo-contracts-rpc.polkadot.io:443",
     "is_relay": false
@@ -94,6 +96,7 @@
     "path": "cumulus/parachains/runtimes/coretime/coretime-rococo",
     "header": "cumulus/file_header.txt",
     "bench_features": "runtime-benchmarks",
+    "bench_flags": "--genesis-builder-policy=none --exclude-pallets=pallet_xcm,pallet_xcm_benchmarks::fungible,pallet_xcm_benchmarks::generic",
     "template": "cumulus/templates/xcm-bench-template.hbs",
     "uri": "wss://rococo-coretime-rpc.polkadot.io:443",
     "is_relay": false
@@ -104,6 +107,7 @@
     "path": "cumulus/parachains/runtimes/coretime/coretime-westend",
     "header": "cumulus/file_header.txt",
     "bench_features": "runtime-benchmarks",
+    "bench_flags": "--genesis-builder-policy=none --exclude-pallets=pallet_xcm,pallet_xcm_benchmarks::fungible,pallet_xcm_benchmarks::generic",
     "template": "cumulus/templates/xcm-bench-template.hbs",
     "uri": "wss://westend-coretime-rpc.polkadot.io:443",
     "is_relay": false
@@ -114,6 +118,7 @@
     "path": "cumulus/parachains/runtimes/gluttons/glutton-westend",
     "header": "cumulus/file_header.txt",
     "bench_features": "runtime-benchmarks",
+    "bench_flags": "--genesis-builder-policy=none",
     "template": "cumulus/templates/xcm-bench-template.hbs",
     "uri": null,
     "is_relay": false
@@ -124,6 +129,7 @@
     "path": "cumulus/parachains/runtimes/people/people-rococo",
     "header": "cumulus/file_header.txt",
     "bench_features": "runtime-benchmarks",
+    "bench_flags": "--genesis-builder-policy=none --exclude-pallets=pallet_xcm,pallet_xcm_benchmarks::fungible,pallet_xcm_benchmarks::generic",
     "template": "cumulus/templates/xcm-bench-template.hbs",
     "uri": "wss://rococo-people-rpc.polkadot.io:443",
     "is_relay": false
@@ -134,6 +140,7 @@
     "path": "cumulus/parachains/runtimes/people/people-westend",
     "header": "cumulus/file_header.txt",
     "bench_features": "runtime-benchmarks",
+    "bench_flags": "--genesis-builder-policy=none --exclude-pallets=pallet_xcm,pallet_xcm_benchmarks::fungible,pallet_xcm_benchmarks::generic",
     "template": "cumulus/templates/xcm-bench-template.hbs",
     "uri": "wss://westend-people-rpc.polkadot.io:443",
     "is_relay": false

prdoc/pr_5872.prdoc

@@ -0,0 +1,13 @@
+title: '[omni-bencher] Make all runtimes work'
+doc:
+- audience: Runtime Dev
+  description: |-
+    Changes:
+    - Add `--exclude-pallets` to exclude some pallets from runtimes where we dont have genesis presets yet
+    - Make `--genesis-builder-policy=none` work with `--runtime`
+    - CI: Run the frame-omni-bencher for all runtimes
+crates:
+- name: frame-benchmarking-cli
+  bump: patch
+- name: frame-benchmarking-pallet-pov
+  bump: patch

substrate/utils/frame/benchmarking-cli/src/pallet/command.rs

@@ -36,7 +36,7 @@ use sp_core::{
 		testing::{TestOffchainExt, TestTransactionPoolExt},
 		OffchainDbExt, OffchainWorkerExt, TransactionPoolExt,
 	},
-	traits::{CallContext, CodeExecutor, ReadRuntimeVersionExt},
+	traits::{CallContext, CodeExecutor, ReadRuntimeVersionExt, WrappedRuntimeCode},
 	Hasher,
 };
 use sp_externalities::Extensions;
@@ -47,6 +47,7 @@ use sp_storage::{well_known_keys::CODE, Storage};
 use sp_trie::{proof_size_extension::ProofSizeExt, recorder::Recorder};
 use sp_wasm_interface::HostFunctions;
 use std::{
+	borrow::Cow,
 	collections::{BTreeMap, BTreeSet, HashMap},
 	fmt::Debug,
 	fs,
@@ -502,33 +503,28 @@ impl PalletCmd {
 	}
 
 	fn select_benchmarks_to_run(&self, list: Vec<BenchmarkList>) -> Result<Vec<SelectedBenchmark>> {
-		let pallet = self.pallet.clone().unwrap_or_default();
-		let pallet = pallet.as_bytes();
-
 		let extrinsic = self.extrinsic.clone().unwrap_or_default();
 		let extrinsic_split: Vec<&str> = extrinsic.split(',').collect();
 		let extrinsics: Vec<_> = extrinsic_split.iter().map(|x| x.trim().as_bytes()).collect();
 
 		// Use the benchmark list and the user input to determine the set of benchmarks to run.
 		let mut benchmarks_to_run = Vec::new();
-		list.iter()
-			.filter(|item| pallet.is_empty() || pallet == &b"*"[..] || pallet == &item.pallet[..])
-			.for_each(|item| {
-				for benchmark in &item.benchmarks {
-					let benchmark_name = &benchmark.name;
-					if extrinsic.is_empty() ||
-						extrinsic.as_bytes() == &b"*"[..] ||
-						extrinsics.contains(&&benchmark_name[..])
-					{
-						benchmarks_to_run.push((
-							item.pallet.clone(),
-							benchmark.name.clone(),
-							benchmark.components.clone(),
-							benchmark.pov_modes.clone(),
-						))
-					}
+		list.iter().filter(|item| self.pallet_selected(&item.pallet)).for_each(|item| {
+			for benchmark in &item.benchmarks {
+				let benchmark_name = &benchmark.name;
+				if extrinsic.is_empty() ||
+					extrinsic.as_bytes() == &b"*"[..] ||
+					extrinsics.contains(&&benchmark_name[..])
+				{
+					benchmarks_to_run.push((
+						item.pallet.clone(),
+						benchmark.name.clone(),
+						benchmark.components.clone(),
+						benchmark.pov_modes.clone(),
+					))
 				}
-			});
+			}
+		});
 		// Convert `Vec<u8>` to `String` for better readability.
 		let benchmarks_to_run: Vec<_> = benchmarks_to_run
 			.into_iter()
@@ -558,6 +554,16 @@ impl PalletCmd {
 		Ok(benchmarks_to_run)
 	}
 
+	/// Whether this pallet should be run.
+	fn pallet_selected(&self, pallet: &Vec<u8>) -> bool {
+		let include = self.pallet.clone().unwrap_or_default();
+
+		let included = include.is_empty() || include == "*" || include.as_bytes() == pallet;
+		let excluded = self.exclude_pallets.iter().any(|p| p.as_bytes() == pallet);
+
+		included && !excluded
+	}
+
 	/// Build the genesis storage by either the Genesis Builder API, chain spec or nothing.
 	///
 	/// Behaviour can be controlled by the `--genesis-builder` flag.
@@ -566,8 +572,7 @@ impl PalletCmd {
 		chain_spec: &Option<Box<dyn ChainSpec>>,
 	) -> Result<sp_storage::Storage> {
 		Ok(match (self.genesis_builder, self.runtime.as_ref()) {
-			(Some(GenesisBuilderPolicy::None), Some(_)) => return Err("Cannot use `--genesis-builder=none` with `--runtime` since the runtime would be ignored.".into()),
-			(Some(GenesisBuilderPolicy::None), None) => Storage::default(),
+			(Some(GenesisBuilderPolicy::None), _) => Storage::default(),
 			(Some(GenesisBuilderPolicy::SpecGenesis | GenesisBuilderPolicy::Spec), Some(_)) =>
 					return Err("Cannot use `--genesis-builder=spec-genesis` with `--runtime` since the runtime would be ignored.".into()),
 			(Some(GenesisBuilderPolicy::SpecGenesis | GenesisBuilderPolicy::Spec), None) | (None, None) => {
@@ -690,10 +695,19 @@ impl PalletCmd {
 		&self,
 		state: &'a BenchmarkingState<H>,
 	) -> Result<FetchedCode<'a, BenchmarkingState<H>, H>> {
-		log::info!("Loading WASM from state");
-		let state = sp_state_machine::backend::BackendRuntimeCode::new(state);
-
-		Ok(FetchedCode { state })
+		if let Some(runtime) = self.runtime.as_ref() {
+			log::info!("Loading WASM from file");
+			let code = fs::read(runtime)?;
+			let hash = sp_core::blake2_256(&code).to_vec();
+			let wrapped_code = WrappedRuntimeCode(Cow::Owned(code));
+
+			Ok(FetchedCode::FromFile { wrapped_code, heap_pages: self.heap_pages, hash })
+		} else {
+			log::info!("Loading WASM from state");
+			let state = sp_state_machine::backend::BackendRuntimeCode::new(state);
+
+			Ok(FetchedCode::FromGenesis { state })
+		}
 	}
 
 	/// Allocation strategy for pallet benchmarking.

substrate/utils/frame/benchmarking-cli/src/pallet/mod.rs

@@ -53,6 +53,10 @@ pub struct PalletCmd {
 	#[arg(short, long, required_unless_present_any = ["list", "json_input", "all"], default_value_if("all", "true", Some("*".into())))]
 	pub extrinsic: Option<String>,
 
+	/// Comma separated list of pallets that should be excluded from the benchmark.
+	#[arg(long, value_parser, num_args = 1.., value_delimiter = ',')]
+	pub exclude_pallets: Vec<String>,
+
 	/// Run benchmarks for all pallets and extrinsics.
 	///
 	/// This is equivalent to running `--pallet * --extrinsic *`.

substrate/utils/frame/benchmarking-cli/src/pallet/types.rs

@@ -18,7 +18,7 @@
 //! Various types used by this crate.
 
 use sc_cli::Result;
-use sp_core::traits::RuntimeCode;
+use sp_core::traits::{RuntimeCode, WrappedRuntimeCode};
 use sp_runtime::traits::Hash;
 
 /// How the genesis state for benchmarking should be build.
@@ -43,8 +43,9 @@ pub enum GenesisBuilderPolicy {
 /// A runtime blob that was either fetched from genesis storage or loaded from a file.
 // NOTE: This enum is only needed for the annoying lifetime bounds on `RuntimeCode`. Otherwise we
 // could just directly return the blob.
-pub struct FetchedCode<'a, B, H> {
-	pub state: sp_state_machine::backend::BackendRuntimeCode<'a, B, H>,
+pub enum FetchedCode<'a, B, H> {
+	FromGenesis { state: sp_state_machine::backend::BackendRuntimeCode<'a, B, H> },
+	FromFile { wrapped_code: WrappedRuntimeCode<'a>, heap_pages: Option<u64>, hash: Vec<u8> },
 }
 
 impl<'a, B, H> FetchedCode<'a, B, H>
@@ -54,7 +55,14 @@ where
 {
 	/// The runtime blob.
 	pub fn code(&'a self) -> Result<RuntimeCode<'a>> {
-		self.state.runtime_code().map_err(Into::into)
+		match self {
+			Self::FromGenesis { state } => state.runtime_code().map_err(Into::into),
+			Self::FromFile { wrapped_code, heap_pages, hash } => Ok(RuntimeCode {
+				code_fetcher: wrapped_code,
+				heap_pages: *heap_pages,
+				hash: hash.clone(),
+			}),
+		}
 	}
 }