Str 908: Prover client enable import of built ELF (#648)

* datatool export elf

* update readme

* prover client load elf

* prover client load elf

* guest build and binary flag update

* doc updates

* lint fix

.gitignore

@@ -242,7 +242,7 @@ rocksdb_prover/
 .idea/
 
 # ZkVm ELF
-elf
+elfs/
 
 # macOS specific files
 .DS_Store

bin/datatool/Cargo.toml

@@ -27,6 +27,7 @@ zeroize.workspace = true
 
 [features]
 default = []
-risc0 = ["strata-risc0-guest-builder", "bytemuck"]
-sp1 = ["strata-sp1-guest-builder"]
-sp1-docker = ["sp1", "strata-sp1-guest-builder/docker-build"]
+risc0-builder = ["strata-risc0-guest-builder", "bytemuck"]
+sp1-builder = ["strata-sp1-guest-builder/sp1-dev"]
+sp1-mock-builder = ["sp1-builder", "strata-sp1-guest-builder/mock"]
+sp1-docker-builder = ["sp1-builder", "strata-sp1-guest-builder/docker-build"]

bin/datatool/README.md

@@ -37,20 +37,20 @@ Before proceeding, make sure that you have SP1 correctly set up by following the
 
 To ensure that the RollupParams contain the correct verifying key, build the binary in release mode and confirm that SP1 is set up correctly by following its installation instructions.
 
-For production usage—since SP1 verification key generation is platform and workspace dependent—build the data tool in release mode with the sp1-docker feature:
+For production usage—since SP1 verification key generation is platform and workspace dependent—build the data tool in release mode with the sp1-docker-builder feature:
 
 ```bash
-cargo build --bin strata-datatool -F "sp1-docker" --release
+cargo build --bin strata-datatool -F "sp1-docker-builder" --release
 ```
 
 Because building the guest code in Docker can be time-consuming, you can generate the verification key locally for testing or development using:
 
 ```bash
-cargo build --bin strata-datatool -F "sp1" --release
+cargo build --bin strata-datatool -F "sp1-builder" --release
 ```
 
 Additionally, the generated ELF can be exported after building the datatool as specified above:
 
 ```bash
-strata-datatool genparams --elf-path <ELF-PATH>
+strata-datatool genparams --elf-dir <ELF-PATH>
 ```

bin/datatool/src/util.rs

@@ -76,10 +76,10 @@ pub(super) fn exec_subc(cmd: Subcommand, ctx: &mut CmdContext) -> anyhow::Result
 /// # Errors
 ///
 /// Returns an error if the export process fails.
-fn export_elf(elf_path: &PathBuf) -> anyhow::Result<()> {
-    #[cfg(feature = "sp1")]
+fn export_elf(_elf_path: &PathBuf) -> anyhow::Result<()> {
+    #[cfg(feature = "sp1-builder")]
     {
-        strata_sp1_guest_builder::export_elf(elf_path)?
+        strata_sp1_guest_builder::export_elf(_elf_path)?
     }
 
     Ok(())
@@ -100,7 +100,7 @@ fn export_elf(elf_path: &PathBuf) -> anyhow::Result<()> {
 /// only one ZKVM can be supported at a time.
 fn resolve_rollup_vk() -> RollupVerifyingKey {
     // Use SP1 if only `sp1` feature is enabled
-    #[cfg(all(feature = "sp1", not(feature = "risc0")))]
+    #[cfg(all(feature = "sp1-builder", not(feature = "risc0-builder")))]
     {
         use strata_sp1_guest_builder::GUEST_CHECKPOINT_VK_HASH_STR;
         let vk_buf32: Buf32 = GUEST_CHECKPOINT_VK_HASH_STR
@@ -110,7 +110,7 @@ fn resolve_rollup_vk() -> RollupVerifyingKey {
     }
 
     // Use Risc0 if only `risc0` feature is enabled
-    #[cfg(all(feature = "risc0", not(feature = "sp1")))]
+    #[cfg(all(feature = "risc0-builder", not(feature = "sp1-builder")))]
     {
         use strata_risc0_guest_builder::GUEST_RISC0_CHECKPOINT_ID;
         let vk_u8: [u8; 32] = bytemuck::cast(GUEST_RISC0_CHECKPOINT_ID);
@@ -119,7 +119,7 @@ fn resolve_rollup_vk() -> RollupVerifyingKey {
     }
 
     // Panic if both `sp1` and `risc0` feature are enabled
-    #[cfg(all(feature = "risc0", feature = "sp1"))]
+    #[cfg(all(feature = "risc0-builder", feature = "sp1-builder"))]
     {
         panic!(
             "Conflicting ZKVM features: both 'sp1' and 'risc0' are enabled. \
@@ -128,7 +128,7 @@ fn resolve_rollup_vk() -> RollupVerifyingKey {
     }
 
     // If neither `risc0` nor `sp1` is enabled, use the Native verifying key
-    #[cfg(all(not(feature = "risc0"), not(feature = "sp1")))]
+    #[cfg(all(not(feature = "risc0-builder"), not(feature = "sp1-builder")))]
     {
         RollupVerifyingKey::NativeVerifyingKey(Buf32::zero())
     }

bin/prover-client/Cargo.toml

@@ -35,6 +35,7 @@ async-trait.workspace = true
 bincode.workspace = true
 bitcoin = { workspace = true, features = ["rand-std"] }
 borsh.workspace = true
+cfg-if.workspace = true
 hex.workspace = true
 jsonrpsee = { workspace = true, features = ["http-client"] }
 musig2.workspace = true
@@ -62,8 +63,14 @@ strata-test-utils.workspace = true
 
 [features]
 default = []
-sp1 = ["zkaleido-sp1-adapter/prover", "strata-sp1-guest-builder/prover"]
-sp1-mock = ["sp1", "zkaleido-sp1-adapter/mock", "strata-sp1-guest-builder/mock"]
+sp1 = ["zkaleido-sp1-adapter/prover"]
+sp1-mock = ["sp1", "zkaleido-sp1-adapter/mock"]
+sp1-builder = ["sp1", "strata-sp1-guest-builder/prover"]
+sp1-mock-builder = [
+  "sp1-builder",
+  "zkaleido-sp1-adapter/mock",
+  "strata-sp1-guest-builder/mock",
+]
 
 risc0 = ["zkaleido-risc0-adapter/prover", "strata-risc0-guest-builder/prover"]
 risc0-mock = ["risc0", "zkaleido-risc0-adapter/mock"]

bin/prover-client/src/hosts/sp1.rs

@@ -1,26 +1,109 @@
 use std::sync::LazyLock;
 
+use cfg_if::cfg_if;
 use strata_primitives::proof::ProofContext;
+#[cfg(feature = "sp1-builder")]
 use strata_sp1_guest_builder::*;
 use zkaleido_sp1_adapter::SP1Host;
 
-pub static BTC_BLOCKSPACE_HOST: LazyLock<SP1Host> =
-    std::sync::LazyLock::new(|| SP1Host::new_from_bytes(&GUEST_BTC_BLOCKSPACE_PK));
+cfg_if! {
+    if #[cfg(not(feature = "sp1-builder"))] {
+        use std::env;
+        pub static ELF_BASE_PATH: LazyLock<String> = LazyLock::new(|| {
+            env::var("ELF_BASE_PATH").unwrap_or_else(|_| "elfs/sp1".to_string())
+        });
+    }
+}
 
-pub static L1_BATCH_HOST: LazyLock<SP1Host> =
-    std::sync::LazyLock::new(|| SP1Host::new_from_bytes(&GUEST_L1_BATCH_PK));
+// BTC_BLOCKSPACE_HOST
+cfg_if! {
+    if #[cfg(feature = "sp1-builder")] {
+        pub static BTC_BLOCKSPACE_HOST: LazyLock<SP1Host> =
+            LazyLock::new(|| SP1Host::new_from_bytes(&GUEST_BTC_BLOCKSPACE_PK));
+    } else {
+        pub static BTC_BLOCKSPACE_HOST: LazyLock<SP1Host> = LazyLock::new(|| {
+            let elf_path = format!("{}/guest-btc-blockspace.elf", &*ELF_BASE_PATH);
+            let elf = std::fs::read(&elf_path)
+                .expect(&format!("Failed to read ELF file from {}", elf_path));
+            SP1Host::init(&elf)
+        });
+    }
+}
 
-pub static EVM_EE_STF_HOST: LazyLock<SP1Host> =
-    std::sync::LazyLock::new(|| SP1Host::new_from_bytes(&GUEST_EVM_EE_STF_PK));
+// L1_BATCH_HOST
+cfg_if! {
+    if #[cfg(feature = "sp1-builder")] {
+        pub static L1_BATCH_HOST: LazyLock<SP1Host> =
+            LazyLock::new(|| SP1Host::new_from_bytes(&GUEST_L1_BATCH_PK));
+    } else {
+        pub static L1_BATCH_HOST: LazyLock<SP1Host> = LazyLock::new(|| {
+            let elf_path = format!("{}/guest-l1-batch.elf", &*ELF_BASE_PATH);
+            let elf = std::fs::read(&elf_path)
+                .expect(&format!("Failed to read ELF file from {}", elf_path));
+            SP1Host::init(&elf)
+        });
+    }
+}
 
-pub static CL_STF_HOST: LazyLock<SP1Host> =
-    std::sync::LazyLock::new(|| SP1Host::new_from_bytes(&GUEST_CL_STF_PK));
+// EVM_EE_STF_HOST
+cfg_if! {
+    if #[cfg(feature = "sp1-builder")] {
+        pub static EVM_EE_STF_HOST: LazyLock<SP1Host> =
+            LazyLock::new(|| SP1Host::new_from_bytes(&GUEST_EVM_EE_STF_PK));
+    } else {
+        pub static EVM_EE_STF_HOST: LazyLock<SP1Host> = LazyLock::new(|| {
+            let elf_path = format!("{}/guest-evm-ee-stf.elf", &*ELF_BASE_PATH);
+            let elf = std::fs::read(&elf_path)
+                .expect(&format!("Failed to read ELF file from {}", elf_path));
+            SP1Host::init(&elf)
+        });
+    }
+}
 
-pub static CL_AGG_HOST: LazyLock<SP1Host> =
-    std::sync::LazyLock::new(|| SP1Host::new_from_bytes(&GUEST_CL_AGG_PK));
+// CL_STF_HOST
+cfg_if! {
+    if #[cfg(feature = "sp1-builder")] {
+        pub static CL_STF_HOST: LazyLock<SP1Host> =
+            LazyLock::new(|| SP1Host::new_from_bytes(&GUEST_CL_STF_PK));
+    } else {
+        pub static CL_STF_HOST: LazyLock<SP1Host> = LazyLock::new(|| {
+            let elf_path = format!("{}/guest-cl-stf.elf", &*ELF_BASE_PATH);
+            let elf = std::fs::read(&elf_path)
+                .expect(&format!("Failed to read ELF file from {}", elf_path));
+            SP1Host::init(&elf)
+        });
+    }
+}
 
-pub static CHECKPOINT_HOST: LazyLock<SP1Host> =
-    std::sync::LazyLock::new(|| SP1Host::new_from_bytes(&GUEST_CHECKPOINT_PK));
+// CL_AGG_HOST
+cfg_if! {
+    if #[cfg(feature = "sp1-builder")] {
+        pub static CL_AGG_HOST: LazyLock<SP1Host> =
+            LazyLock::new(|| SP1Host::new_from_bytes(&GUEST_CL_AGG_PK));
+    } else {
+        pub static CL_AGG_HOST: LazyLock<SP1Host> = LazyLock::new(|| {
+            let elf_path = format!("{}/guest-cl-agg.elf", &*ELF_BASE_PATH);
+            let elf = std::fs::read(&elf_path)
+                .expect(&format!("Failed to read ELF file from {}", elf_path));
+            SP1Host::init(&elf)
+        });
+    }
+}
+
+// CHECKPOINT_HOST
+cfg_if! {
+    if #[cfg(feature = "sp1-builder")] {
+        pub static CHECKPOINT_HOST: LazyLock<SP1Host> =
+            LazyLock::new(|| SP1Host::new_from_bytes(&GUEST_CHECKPOINT_PK));
+    } else {
+        pub static CHECKPOINT_HOST: LazyLock<SP1Host> = LazyLock::new(|| {
+            let elf_path = format!("{}/guest-checkpoint.elf", &*ELF_BASE_PATH);
+            let elf = std::fs::read(&elf_path)
+                .expect(&format!("Failed to read ELF file from {}", elf_path));
+            SP1Host::init(&elf)
+        });
+    }
+}
 
 /// Returns a reference to the appropriate `SP1Host` instance based on the given [`ProofContext`].
 ///

functional-tests/run_test.sh

@@ -12,7 +12,7 @@ fi
 # Conditionally run cargo build based on PROVER_TEST
 if [ ! -z $PROVER_TEST ]; then
     echo "Running on sp1-mock mode"
-    cargo build --release -F sp1-mock
+    cargo build --release -F sp1-mock-builder
 	export PATH=$(realpath ../target/release/):$PATH
 else
     echo "Running on seq mode"

provers/sp1/Cargo.toml

@@ -13,6 +13,7 @@ zkaleido-sp1-adapter = { git = "https://github.com/alpenlabs/zkaleido", tag = "v
 bincode.workspace = true
 cargo_metadata = "0.19.1"
 sha2.workspace = true
+cfg-if.workspace = true
 sp1-helper = { git = "https://github.com/succinctlabs/sp1.git", rev = "6c5a7f2846cd3610ecd38b1641f0e370fd07ee83" }
 sp1-sdk = "4.0.0"
 
@@ -21,3 +22,4 @@ default = ["prover"]
 mock = []
 prover = ["zkaleido-sp1-adapter"]
 docker-build = []
+sp1-dev = []

provers/sp1/build.rs

@@ -4,16 +4,17 @@ use std::{
     path::{Path, PathBuf},
 };
 
-#[cfg(not(debug_assertions))]
-use bincode::{deserialize, serialize};
-#[cfg(not(debug_assertions))]
-use cargo_metadata::MetadataCommand;
-#[cfg(not(debug_assertions))]
-use sha2::{Digest, Sha256};
-#[cfg(not(debug_assertions))]
-use sp1_helper::{build_program_with_args, BuildArgs};
-#[cfg(not(debug_assertions))]
-use sp1_sdk::{HashableKey, ProverClient, SP1VerifyingKey};
+use cfg_if::cfg_if;
+
+cfg_if! {
+    if #[cfg(all(feature = "sp1-dev", not(debug_assertions)))] {
+        use bincode::{deserialize, serialize};
+        use cargo_metadata::MetadataCommand;
+        use sha2::{Digest, Sha256};
+        use sp1_helper::{build_program_with_args, BuildArgs};
+        use sp1_sdk::{HashableKey, ProverClient, SP1VerifyingKey};
+    }
+}
 
 // Guest program names
 const EVM_EE_STF: &str = "guest-evm-ee-stf";
@@ -167,7 +168,7 @@ fn get_output_dir() -> PathBuf {
 }
 
 /// Checks if the cache is valid by comparing the expected ID with the saved ID.
-#[cfg(not(debug_assertions))]
+#[cfg(all(feature = "sp1-dev", not(debug_assertions)))]
 fn is_cache_valid(expected_id: &[u8; 32], paths: &[PathBuf; 4]) -> bool {
     // Check if any required files are missing
     if paths.iter().any(|path| !path.exists()) {
@@ -184,7 +185,7 @@ fn is_cache_valid(expected_id: &[u8; 32], paths: &[PathBuf; 4]) -> bool {
 }
 
 /// Ensures the cache is valid and returns the ELF contents and SP1 Verifying Key.
-#[cfg(not(debug_assertions))]
+#[cfg(all(feature = "sp1-dev", not(debug_assertions)))]
 fn ensure_cache_validity(program: &str) -> Result<SP1VerifyingKey, String> {
     let cache_dir = format!("{}/cache", program);
     let paths = ["elf", "id", "vk", "pk"]
@@ -221,8 +222,19 @@ fn ensure_cache_validity(program: &str) -> Result<SP1VerifyingKey, String> {
 }
 
 /// Generates the ELF contents and VK hash for a given program.
-#[cfg(not(debug_assertions))]
+#[cfg(all(feature = "sp1-dev", not(debug_assertions)))]
 fn generate_elf_contents_and_vk_hash(program: &str) -> ([u32; 8], String) {
+    // Check if the Clippy linter is enabled by examining the "RUSTC_WORKSPACE_WRAPPER" environment
+    // variable. If it contains "clippy-driver", Clippy is active; in that case, return mock ELF
+    // contents and VK hash.
+    let is_clippy_enabled = std::env::var("RUSTC_WORKSPACE_WRAPPER")
+        .map(|val| val.contains("clippy-driver"))
+        .unwrap_or(false);
+
+    if is_clippy_enabled {
+        return get_mock_elf_contents_and_vk_hash();
+    }
+
     let mut build_args = BuildArgs {
         ..Default::default()
     };
@@ -261,14 +273,18 @@ fn generate_elf_contents_and_vk_hash(program: &str) -> ([u32; 8], String) {
 
 #[cfg(debug_assertions)]
 fn generate_elf_contents_and_vk_hash(_program: &str) -> ([u32; 8], String) {
+    get_mock_elf_contents_and_vk_hash()
+}
+
+fn get_mock_elf_contents_and_vk_hash() -> ([u32; 8], String) {
     (
         [0u32; 8],
         "0x0000000000000000000000000000000000000000000000000000000000000000".to_owned(),
     )
 }
 
 /// Copies the compiled ELF file of the specified program to its cache directory.
-#[cfg(not(debug_assertions))]
+#[cfg(all(feature = "sp1-dev", not(debug_assertions)))]
 fn migrate_elf(program: &str) {
     // Get the build directory from the environment
     let sp1_build_dir =