From dd86be94fe66997f2ca23bd62169e536f62d3b51 Mon Sep 17 00:00:00 2001 From: Zhang Zhuo Date: Mon, 26 Aug 2024 15:55:12 +0800 Subject: [PATCH 1/4] refactor(prover): move params out of prover/verifier --- prover/src/aggregator/prover.rs | 18 +++-- prover/src/aggregator/verifier.rs | 18 +++-- prover/src/common/prover.rs | 15 ++-- prover/src/common/prover/aggregation.rs | 2 +- prover/src/common/prover/chunk.rs | 2 +- prover/src/common/prover/compression.rs | 2 +- prover/src/common/prover/evm.rs | 2 +- prover/src/common/prover/inner.rs | 2 +- prover/src/common/prover/mock.rs | 2 +- prover/src/common/prover/recursion.rs | 2 +- prover/src/common/prover/utils.rs | 24 +------ prover/src/common/verifier.rs | 19 ++--- prover/src/common/verifier/evm.rs | 4 +- prover/src/common/verifier/utils.rs | 4 +- prover/src/inner/prover.rs | 14 ++-- prover/src/inner/prover/mock.rs | 2 +- prover/src/inner/verifier.rs | 31 +++++---- prover/src/test/batch.rs | 93 +++++++++++++++---------- prover/src/test/chunk.rs | 33 +++++---- prover/src/test/inner.rs | 42 ++++++----- prover/src/zkevm/prover.rs | 31 ++++++--- prover/src/zkevm/verifier.rs | 24 ++++--- 22 files changed, 205 insertions(+), 181 deletions(-) diff --git a/prover/src/aggregator/prover.rs b/prover/src/aggregator/prover.rs index 33e8f820e8..bc2f3f6df7 100644 --- a/prover/src/aggregator/prover.rs +++ b/prover/src/aggregator/prover.rs @@ -1,14 +1,15 @@ -use std::{env, iter::repeat}; +use std::{collections::BTreeMap, env, iter::repeat}; use aggregator::{BatchHash, BatchHeader, ChunkInfo, MAX_AGG_SNARKS}; use anyhow::{bail, Result}; use eth_types::H256; +use halo2_proofs::{halo2curves::bn256::Bn256, poly::kzg::commitment::ParamsKZG}; use sha2::{Digest, Sha256}; use snark_verifier_sdk::Snark; use crate::{ common, - config::{LayerId, AGG_DEGREES}, + config::LayerId, consts::{BATCH_KECCAK_ROW, BATCH_VK_FILENAME, BUNDLE_VK_FILENAME, CHUNK_PROTOCOL_FILENAME}, io::{force_to_read, try_to_read}, proof::BundleProof, @@ -17,20 +18,23 @@ use crate::{ }; #[derive(Debug)] -pub struct Prover { +pub struct Prover<'params> { // Make it public for testing with inner functions (unnecessary for FFI). - pub prover_impl: common::Prover, + pub prover_impl: common::Prover<'params>, pub chunk_protocol: Vec, raw_vk_batch: Option>, raw_vk_bundle: Option>, } -impl Prover { - pub fn from_dirs(params_dir: &str, assets_dir: &str) -> Self { +impl<'params> Prover<'params> { + pub fn from_dirs( + params_map: &'params BTreeMap>, + assets_dir: &str, + ) -> Self { log::debug!("set env KECCAK_ROWS={}", BATCH_KECCAK_ROW.to_string()); env::set_var("KECCAK_ROWS", BATCH_KECCAK_ROW.to_string()); - let prover_impl = common::Prover::from_params_dir(params_dir, &AGG_DEGREES); + let prover_impl = common::Prover::from_params(params_map); let chunk_protocol = force_to_read(assets_dir, &CHUNK_PROTOCOL_FILENAME); let raw_vk_batch = try_to_read(assets_dir, &BATCH_VK_FILENAME); diff --git a/prover/src/aggregator/verifier.rs b/prover/src/aggregator/verifier.rs index b4c55bba48..7932f27cb7 100644 --- a/prover/src/aggregator/verifier.rs +++ b/prover/src/aggregator/verifier.rs @@ -13,18 +13,18 @@ use halo2_proofs::{ }; use snark_verifier_sdk::verify_evm_calldata; use snark_verifier_sdk::Snark; -use std::env; +use std::{collections::BTreeMap, env}; #[derive(Debug)] -pub struct Verifier { +pub struct Verifier<'params> { // Make it public for testing with inner functions (unnecessary for FFI). - pub inner: common::Verifier, + pub inner: common::Verifier<'params, CompressionCircuit>, deployment_code: Option>, } -impl Verifier { +impl<'params> Verifier<'params> { pub fn new( - params: ParamsKZG, + params: &'params ParamsKZG, vk: VerifyingKey, deployment_code: Vec, ) -> Self { @@ -36,12 +36,16 @@ impl Verifier { } } - pub fn from_dirs(params_dir: &str, assets_dir: &str) -> Self { + pub fn from_params_map( + params_map: &'params BTreeMap>, + assets_dir: &str, + ) -> Self { let raw_vk = force_to_read(assets_dir, &batch_vk_filename()); let deployment_code = try_to_read(assets_dir, &DEPLOYMENT_CODE_FILENAME); env::set_var("COMPRESSION_CONFIG", &*LAYER4_CONFIG_PATH); - let inner = common::Verifier::from_params_dir(params_dir, *LAYER4_DEGREE, &raw_vk); + let params = params_map.get(&*LAYER4_DEGREE).expect("should be loaded"); + let inner = common::Verifier::from_params(params, &raw_vk); Self { inner, diff --git a/prover/src/common/prover.rs b/prover/src/common/prover.rs index bd7dd5916a..91ae2f8347 100644 --- a/prover/src/common/prover.rs +++ b/prover/src/common/prover.rs @@ -16,22 +16,22 @@ mod recursion; mod utils; #[derive(Debug)] -pub struct Prover { +pub struct Prover<'params> { // degree -> params (use BTreeMap to find proper degree for params downsize) - params_map: BTreeMap>, + pub(crate) params_map: &'params BTreeMap>, // Cached id -> pk pk_map: HashMap>, } -impl Prover { - pub fn from_params(params_map: BTreeMap>) -> Self { +impl<'params> Prover<'params> { + pub fn from_params(params_map: &'params BTreeMap>) -> Self { Self { params_map, pk_map: HashMap::new(), } } - pub fn from_params_dir(params_dir: &str, degrees: &[u32]) -> Self { + pub fn load_params_map(params_dir: &str, degrees: &[u32]) -> BTreeMap> { let degrees = BTreeSet::from_iter(degrees); let max_degree = **degrees.last().unwrap(); @@ -63,9 +63,6 @@ impl Prover { params_map.insert(*d, params); } - Self { - params_map, - pk_map: HashMap::new(), - } + params_map } } diff --git a/prover/src/common/prover/aggregation.rs b/prover/src/common/prover/aggregation.rs index 1384d02039..4d4ca2bc1b 100644 --- a/prover/src/common/prover/aggregation.rs +++ b/prover/src/common/prover/aggregation.rs @@ -10,7 +10,7 @@ use rand::Rng; use snark_verifier_sdk::Snark; use std::env; -impl Prover { +impl<'params> Prover<'params> { pub fn gen_agg_snark( &mut self, id: &str, diff --git a/prover/src/common/prover/chunk.rs b/prover/src/common/prover/chunk.rs index 32b0aefe2d..cc80ee371f 100644 --- a/prover/src/common/prover/chunk.rs +++ b/prover/src/common/prover/chunk.rs @@ -5,7 +5,7 @@ use anyhow::{anyhow, Result}; use snark_verifier_sdk::Snark; use zkevm_circuits::evm_circuit::witness::Block; -impl Prover { +impl<'params> Prover<'params> { pub fn load_or_gen_final_chunk_snark( &mut self, name: &str, diff --git a/prover/src/common/prover/compression.rs b/prover/src/common/prover/compression.rs index 4fd3de12b5..80078f84d1 100644 --- a/prover/src/common/prover/compression.rs +++ b/prover/src/common/prover/compression.rs @@ -10,7 +10,7 @@ use rand::Rng; use snark_verifier_sdk::Snark; use std::env; -impl Prover { +impl<'params> Prover<'params> { pub fn gen_comp_snark( &mut self, id: &str, diff --git a/prover/src/common/prover/evm.rs b/prover/src/common/prover/evm.rs index 5dc863748a..c019898092 100644 --- a/prover/src/common/prover/evm.rs +++ b/prover/src/common/prover/evm.rs @@ -11,7 +11,7 @@ use rand::Rng; use snark_verifier_sdk::{gen_evm_proof_shplonk, CircuitExt, Snark}; use std::env; -impl Prover { +impl<'params> Prover<'params> { pub fn load_or_gen_comp_evm_proof( &mut self, name: &str, diff --git a/prover/src/common/prover/inner.rs b/prover/src/common/prover/inner.rs index 3d4adcccbc..ae87ea5746 100644 --- a/prover/src/common/prover/inner.rs +++ b/prover/src/common/prover/inner.rs @@ -10,7 +10,7 @@ use rand::Rng; use snark_verifier_sdk::{gen_snark_shplonk, Snark}; use zkevm_circuits::evm_circuit::witness::Block; -impl Prover { +impl<'params> Prover<'params> { pub fn gen_inner_snark( &mut self, id: &str, diff --git a/prover/src/common/prover/mock.rs b/prover/src/common/prover/mock.rs index 9a84af12b8..595f4641cd 100644 --- a/prover/src/common/prover/mock.rs +++ b/prover/src/common/prover/mock.rs @@ -6,7 +6,7 @@ use std::sync::LazyLock; pub static MOCK_PROVE: LazyLock = LazyLock::new(|| read_env_var("MOCK_PROVE", false)); -impl Prover { +impl<'params> Prover<'params> { pub fn assert_if_mock_prover>(id: &str, degree: u32, circuit: &C) { if !*MOCK_PROVE { return; diff --git a/prover/src/common/prover/recursion.rs b/prover/src/common/prover/recursion.rs index 6046a85838..168b5641ca 100644 --- a/prover/src/common/prover/recursion.rs +++ b/prover/src/common/prover/recursion.rs @@ -14,7 +14,7 @@ use crate::{ use super::Prover; -impl Prover { +impl<'params> Prover<'params> { pub fn gen_recursion_snark( &mut self, id: &str, diff --git a/prover/src/common/prover/utils.rs b/prover/src/common/prover/utils.rs index 4100eed401..8bf8363cb3 100644 --- a/prover/src/common/prover/utils.rs +++ b/prover/src/common/prover/utils.rs @@ -4,12 +4,12 @@ use anyhow::Result; use halo2_proofs::{ halo2curves::bn256::{Bn256, Fr, G1Affine}, plonk::{keygen_pk2, Circuit, ProvingKey}, - poly::{commitment::Params, kzg::commitment::ParamsKZG}, + poly::kzg::commitment::ParamsKZG, }; use rand::Rng; use snark_verifier_sdk::{gen_snark_shplonk, CircuitExt, Snark}; -impl Prover { +impl<'params> Prover<'params> { pub fn gen_snark>( &mut self, id: &str, @@ -32,25 +32,7 @@ impl Prover { Ok(snark) } - pub fn params(&mut self, degree: u32) -> &ParamsKZG { - if self.params_map.contains_key(°ree) { - return &self.params_map[°ree]; - } - - log::warn!("Optimization: download params{degree} to params dir"); - - log::info!("Before generate params of {degree}"); - let mut new_params = self - .params_map - .range(degree..) - .next() - .unwrap_or_else(|| panic!("Must have params of degree-{degree}")) - .1 - .clone(); - new_params.downsize(degree); - log::info!("After generate params of {degree}"); - - self.params_map.insert(degree, new_params); + pub fn params(&self, degree: u32) -> &ParamsKZG { &self.params_map[°ree] } diff --git a/prover/src/common/verifier.rs b/prover/src/common/verifier.rs index 65f42c3e29..053d6fccc6 100644 --- a/prover/src/common/verifier.rs +++ b/prover/src/common/verifier.rs @@ -1,4 +1,4 @@ -use crate::{io::deserialize_vk, utils::load_params}; +use crate::io::deserialize_vk; use halo2_proofs::{ halo2curves::bn256::{Bn256, Fr, G1Affine}, plonk::VerifyingKey, @@ -11,14 +11,14 @@ mod evm; mod utils; #[derive(Debug)] -pub struct Verifier> { - params: ParamsKZG, +pub struct Verifier<'params, C: CircuitExt> { + params: &'params ParamsKZG, vk: VerifyingKey, phantom: PhantomData, } -impl> Verifier { - pub fn new(params: ParamsKZG, vk: VerifyingKey) -> Self { +impl<'params, C: CircuitExt> Verifier<'params, C> { + pub fn new(params: &'params ParamsKZG, vk: VerifyingKey) -> Self { Self { params, vk, @@ -26,18 +26,11 @@ impl> Verifier { } } - pub fn from_params(params: ParamsKZG, raw_vk: &[u8]) -> Self { + pub fn from_params(params: &'params ParamsKZG, raw_vk: &[u8]) -> Self { let vk = deserialize_vk::(raw_vk); - Self::new(params, vk) } - pub fn from_params_dir(params_dir: &str, degree: u32, vk: &[u8]) -> Self { - let params = load_params(params_dir, degree, None).unwrap(); - - Self::from_params(params, vk) - } - pub fn verify_snark(&self, snark: Snark) -> bool { verify_snark_shplonk::(self.params.verifier_params(), snark, &self.vk) } diff --git a/prover/src/common/verifier/evm.rs b/prover/src/common/verifier/evm.rs index f2337facf4..783eea2d00 100644 --- a/prover/src/common/verifier/evm.rs +++ b/prover/src/common/verifier/evm.rs @@ -3,8 +3,8 @@ use crate::EvmProof; use halo2_proofs::halo2curves::bn256::Fr; use snark_verifier_sdk::CircuitExt; -impl> Verifier { +impl<'params, C: CircuitExt> Verifier<'params, C> { pub fn gen_evm_verifier(&self, evm_proof: &EvmProof, output_dir: Option<&str>) { - crate::evm::gen_evm_verifier::(&self.params, &self.vk, evm_proof, output_dir) + crate::evm::gen_evm_verifier::(self.params, &self.vk, evm_proof, output_dir) } } diff --git a/prover/src/common/verifier/utils.rs b/prover/src/common/verifier/utils.rs index a77ff2e3a4..b5883feec1 100644 --- a/prover/src/common/verifier/utils.rs +++ b/prover/src/common/verifier/utils.rs @@ -6,9 +6,9 @@ use halo2_proofs::{ }; use snark_verifier_sdk::CircuitExt; -impl> Verifier { +impl<'params, C: CircuitExt> Verifier<'params, C> { pub fn params(&self) -> &ParamsKZG { - &self.params + self.params } pub fn vk(&self) -> &VerifyingKey { diff --git a/prover/src/inner/prover.rs b/prover/src/inner/prover.rs index 9cfb49f244..2d9471df18 100644 --- a/prover/src/inner/prover.rs +++ b/prover/src/inner/prover.rs @@ -14,14 +14,14 @@ use std::marker::PhantomData; mod mock; #[derive(Debug)] -pub struct Prover { +pub struct Prover<'params, C: TargetCircuit> { // Make it public for testing with inner functions (unnecessary for FFI). - pub prover_impl: common::Prover, + pub prover_impl: common::Prover<'params>, phantom: PhantomData, } -impl From for Prover { - fn from(prover_impl: common::Prover) -> Self { +impl<'params, C: TargetCircuit> From> for Prover<'params, C> { + fn from(prover_impl: common::Prover<'params>) -> Self { Self { prover_impl, phantom: PhantomData, @@ -29,9 +29,9 @@ impl From for Prover { } } -impl Prover { - pub fn from_params_dir(params_dir: &str) -> Self { - common::Prover::from_params_dir(params_dir, &[*INNER_DEGREE]).into() +impl<'params, C: TargetCircuit> Prover<'params, C> { + pub fn degrees() -> Vec { + vec![*INNER_DEGREE] } pub fn gen_inner_snark(&mut self, id: &str, block_traces: Vec) -> Result { diff --git a/prover/src/inner/prover/mock.rs b/prover/src/inner/prover/mock.rs index ce777e1492..e023b06f4f 100644 --- a/prover/src/inner/prover/mock.rs +++ b/prover/src/inner/prover/mock.rs @@ -10,7 +10,7 @@ use halo2_proofs::{dev::MockProver, halo2curves::bn256::Fr}; use snark_verifier_sdk::CircuitExt; use zkevm_circuits::witness::Block; -impl Prover { +impl<'params, C: TargetCircuit> Prover<'params, C> { pub fn mock_prove_target_circuit(block_trace: BlockTrace) -> anyhow::Result<()> { Self::mock_prove_target_circuit_chunk(vec![block_trace]) } diff --git a/prover/src/inner/verifier.rs b/prover/src/inner/verifier.rs index 858f00a1c8..0b3a5be138 100644 --- a/prover/src/inner/verifier.rs +++ b/prover/src/inner/verifier.rs @@ -1,35 +1,38 @@ -use crate::{ - common, config::INNER_DEGREE, io::deserialize_vk, utils::load_params, - zkevm::circuit::TargetCircuit, -}; -use halo2_proofs::plonk::keygen_vk; +use std::collections::BTreeMap; + +use crate::{common, config::INNER_DEGREE, io::deserialize_vk, zkevm::circuit::TargetCircuit}; +use halo2_proofs::{halo2curves::bn256::Bn256, plonk::keygen_vk, poly::kzg::commitment::ParamsKZG}; use snark_verifier_sdk::Snark; #[derive(Debug)] -pub struct Verifier { +pub struct Verifier<'params, C: TargetCircuit> { // Make it public for testing with inner functions (unnecessary for FFI). - pub inner: common::Verifier, + pub inner: common::Verifier<'params, C::Inner>, } -impl From> for Verifier { - fn from(inner: common::Verifier) -> Self { +impl<'params, C: TargetCircuit> From> for Verifier<'params, C> { + fn from(inner: common::Verifier<'params, C::Inner>) -> Self { Self { inner } } } -impl Verifier { - pub fn from_params_dir(params_dir: &str, raw_vk: Option<&[u8]>) -> Self { - let params = load_params(params_dir, *INNER_DEGREE, None).unwrap(); +impl<'params, C: TargetCircuit> Verifier<'params, C> { + pub fn from_params_map( + params_map: &'params BTreeMap>, + raw_vk: Option<&[u8]>, + ) -> Self { + let params = params_map.get(&*INNER_DEGREE).expect("should be loaded"); let vk = raw_vk.map_or_else( || { let dummy_circuit = C::dummy_inner_circuit().expect("gen dummy circuit"); - keygen_vk(¶ms, &dummy_circuit).unwrap() + keygen_vk(params, &dummy_circuit).unwrap() }, deserialize_vk::, ); - common::Verifier::new(params, vk).into() + let verifier = common::Verifier::new(params, vk); + verifier.into() } pub fn verify_inner_snark(&self, snark: Snark) -> bool { diff --git a/prover/src/test/batch.rs b/prover/src/test/batch.rs index 95f5ed0eba..dd273437d7 100644 --- a/prover/src/test/batch.rs +++ b/prover/src/test/batch.rs @@ -1,58 +1,60 @@ +use halo2_proofs::{halo2curves::bn256::Bn256, poly::kzg::commitment::ParamsKZG}; + use crate::{ aggregator::{Prover, Verifier}, - config::LayerId, + config::{LayerId, AGG_DEGREES}, consts::DEPLOYMENT_CODE_FILENAME, io::force_to_read, types::BundleProvingTask, utils::read_env_var, BatchProvingTask, }; -use std::sync::{LazyLock, Mutex}; +use std::{ + collections::BTreeMap, + sync::{LazyLock, Mutex}, +}; -static BATCH_PROVER: LazyLock> = LazyLock::new(|| { - let assets_dir = read_env_var("SCROLL_PROVER_ASSETS_DIR", "./test_assets".to_string()); +static PARAMS_MAP: LazyLock>> = LazyLock::new(|| { let params_dir = read_env_var("SCROLL_PROVER_PARAMS_DIR", "./test_params".to_string()); - - let prover = Prover::from_dirs(¶ms_dir, &assets_dir); - log::info!("Constructed batch-prover"); - - Mutex::new(prover) + crate::common::Prover::load_params_map(¶ms_dir, &AGG_DEGREES) }); -static BATCH_VERIFIER: LazyLock> = LazyLock::new(|| { +static BATCH_PROVER: LazyLock> = LazyLock::new(|| { let assets_dir = read_env_var("SCROLL_PROVER_ASSETS_DIR", "./test_assets".to_string()); - let mut prover = BATCH_PROVER.lock().expect("poisoned batch-prover"); - let params = prover.prover_impl.params(LayerId::Layer4.degree()).clone(); - - let deployment_code = force_to_read(&assets_dir, &DEPLOYMENT_CODE_FILENAME); - - let pk = prover - .prover_impl - .pk(LayerId::Layer4.id()) - .expect("Failed to get batch-prove PK"); - let vk = pk.get_vk().clone(); - - let verifier = Verifier::new(params, vk, deployment_code); - log::info!("Constructed batch-verifier"); + let prover = Prover::from_dirs(&PARAMS_MAP, &assets_dir); + log::info!("Constructed batch-prover"); - Mutex::new(verifier) + Mutex::new(prover) }); pub fn batch_prove(test: &str, batch: BatchProvingTask) { log::info!("{test}: batch-prove BEGIN"); - let proof = BATCH_PROVER - .lock() - .expect("poisoned batch-prover") + let mut prover = BATCH_PROVER.lock().expect("poisoned batch-prover"); + let proof = prover .gen_batch_proof(batch, None, None) .unwrap_or_else(|err| panic!("{test}: failed to generate batch proof: {err}")); log::info!("{test}: generated batch proof"); - let verified = BATCH_VERIFIER - .lock() - .expect("poisoned batch-verifier") - .verify_batch_proof(&proof); + let verifier = { + let assets_dir = read_env_var("SCROLL_PROVER_ASSETS_DIR", "./test_assets".to_string()); + + let params = prover.prover_impl.params(LayerId::Layer4.degree()); + + let deployment_code = force_to_read(&assets_dir, &DEPLOYMENT_CODE_FILENAME); + + let pk = prover + .prover_impl + .pk(LayerId::Layer4.id()) + .expect("Failed to get batch-prove PK"); + let vk = pk.get_vk().clone(); + + let verifier = Verifier::new(params, vk, deployment_code); + log::info!("Constructed batch-verifier"); + verifier + }; + let verified = verifier.verify_batch_proof(&proof); assert!(verified, "{test}: failed to verify batch proof"); log::info!("{test}: batch-prove END"); @@ -61,17 +63,32 @@ pub fn batch_prove(test: &str, batch: BatchProvingTask) { pub fn bundle_prove(test: &str, bundle: BundleProvingTask) { log::info!("{test}: bundle-prove BEGIN"); - let proof = BATCH_PROVER - .lock() - .expect("poisoned batch-prover") + let mut prover = BATCH_PROVER.lock().expect("poisoned batch-prover"); + let proof = prover .gen_bundle_proof(bundle, None, None) .unwrap_or_else(|err| panic!("{test}: failed to generate bundle proof: {err}")); log::info!("{test}: generated bundle proof"); - let verified = BATCH_VERIFIER - .lock() - .expect("poisoned batch-verifier") - .verify_bundle_proof(proof); + // FIXME: clean redundant code + let verifier = { + let assets_dir = read_env_var("SCROLL_PROVER_ASSETS_DIR", "./test_assets".to_string()); + + let params = prover.prover_impl.params(LayerId::Layer4.degree()); + + let deployment_code = force_to_read(&assets_dir, &DEPLOYMENT_CODE_FILENAME); + + let pk = prover + .prover_impl + .pk(LayerId::Layer4.id()) + .expect("Failed to get batch-prove PK"); + let vk = pk.get_vk().clone(); + + let verifier = Verifier::new(params, vk, deployment_code); + log::info!("Constructed batch-verifier"); + verifier + }; + + let verified = verifier.verify_bundle_proof(proof); assert!(verified, "{test}: failed to verify bundle proof"); log::info!("{test}: bundle-prove END"); diff --git a/prover/src/test/chunk.rs b/prover/src/test/chunk.rs index caaa443c98..0f4bffcf1d 100644 --- a/prover/src/test/chunk.rs +++ b/prover/src/test/chunk.rs @@ -1,27 +1,27 @@ +use halo2_proofs::{halo2curves::bn256::Bn256, poly::kzg::commitment::ParamsKZG}; + use crate::{ + config::ZKEVM_DEGREES, utils::read_env_var, zkevm::{Prover, Verifier}, ChunkProof, ChunkProvingTask, }; -use std::sync::{LazyLock, Mutex}; +use std::{ + collections::BTreeMap, + sync::{LazyLock, Mutex}, +}; -static CHUNK_PROVER: LazyLock> = LazyLock::new(|| { +static PARAMS_MAP: LazyLock>> = LazyLock::new(|| { let params_dir = read_env_var("SCROLL_PROVER_PARAMS_DIR", "./test_params".to_string()); - let assets_dir = read_env_var("SCROLL_PROVER_ASSETS_DIR", "./test_assets".to_string()); - let prover = Prover::from_dirs(¶ms_dir, &assets_dir); - log::info!("Constructed chunk-prover"); - - Mutex::new(prover) + crate::common::Prover::load_params_map(¶ms_dir, &ZKEVM_DEGREES) }); -static CHUNK_VERIFIER: LazyLock> = LazyLock::new(|| { - let params_dir = read_env_var("SCROLL_PROVER_PARAMS_DIR", "./test_params".to_string()); +static CHUNK_PROVER: LazyLock> = LazyLock::new(|| { let assets_dir = read_env_var("SCROLL_PROVER_ASSETS_DIR", "./test_assets".to_string()); + let prover = Prover::from_params_map(&PARAMS_MAP, &assets_dir); + log::info!("Constructed chunk-prover"); - let verifier = Verifier::from_dirs(¶ms_dir, &assets_dir); - log::info!("Constructed chunk-verifier"); - - Mutex::new(verifier) + Mutex::new(prover) }); pub fn chunk_prove(desc: &str, chunk: ChunkProvingTask) -> ChunkProof { @@ -34,7 +34,12 @@ pub fn chunk_prove(desc: &str, chunk: ChunkProvingTask) -> ChunkProof { .unwrap_or_else(|err| panic!("{desc}: failed to generate chunk snark: {err}")); log::info!("{desc}: generated chunk proof"); - let verifier = CHUNK_VERIFIER.lock().expect("poisoned chunk-verifier"); + let verifier = { + let assets_dir = read_env_var("SCROLL_PROVER_ASSETS_DIR", "./test_assets".to_string()); + let verifier = Verifier::from_params_map(prover.prover_impl.params_map, &assets_dir); + log::info!("Constructed chunk-verifier"); + verifier + }; let verified = verifier.verify_chunk_proof(proof.clone()); assert!(verified, "{desc}: failed to verify chunk snark"); diff --git a/prover/src/test/inner.rs b/prover/src/test/inner.rs index b78978e94e..1727d15029 100644 --- a/prover/src/test/inner.rs +++ b/prover/src/test/inner.rs @@ -1,3 +1,5 @@ +use halo2_proofs::{halo2curves::bn256::Bn256, poly::kzg::commitment::ParamsKZG}; + use crate::{ common::{Prover, Verifier}, config::{LayerId, INNER_DEGREE}, @@ -5,31 +7,23 @@ use crate::{ zkevm::circuit::{SuperCircuit, TargetCircuit}, WitnessBlock, }; -use std::sync::{LazyLock, Mutex}; +use std::{ + collections::BTreeMap, + sync::{LazyLock, Mutex}, +}; -static INNER_PROVER: LazyLock> = LazyLock::new(|| { +static PARAMS_MAP: LazyLock>> = LazyLock::new(|| { let params_dir = read_env_var("SCROLL_PROVER_PARAMS_DIR", "./test_params".to_string()); - let prover = Prover::from_params_dir(¶ms_dir, &[*INNER_DEGREE]); + crate::common::Prover::load_params_map(¶ms_dir, &[*INNER_DEGREE]) +}); + +static INNER_PROVER: LazyLock> = LazyLock::new(|| { + let prover = Prover::from_params(&PARAMS_MAP); log::info!("Constructed inner-prover"); Mutex::new(prover) }); -static INNER_VERIFIER: LazyLock::Inner>>> = - LazyLock::new(|| { - let mut prover = INNER_PROVER.lock().expect("poisoned inner-prover"); - let params = prover.params(*INNER_DEGREE).clone(); - - let inner_id = LayerId::Inner.id().to_string(); - let pk = prover.pk(&inner_id).expect("Failed to get inner-prove PK"); - let vk = pk.get_vk().clone(); - - let verifier = Verifier::new(params, vk); - log::info!("Constructed inner-verifier"); - - Mutex::new(verifier) - }); - pub fn inner_prove(test: &str, witness_block: &WitnessBlock) { log::info!("{test}: inner-prove BEGIN"); @@ -41,7 +35,17 @@ pub fn inner_prove(test: &str, witness_block: &WitnessBlock) { .unwrap_or_else(|err| panic!("{test}: failed to generate inner snark: {err}")); log::info!("{test}: generated inner snark"); - let verifier = INNER_VERIFIER.lock().expect("poisoned inner-verifier"); + let verifier: Verifier<'_, ::Inner> = { + let params = prover.params(*INNER_DEGREE); + + let inner_id = LayerId::Inner.id().to_string(); + let pk = prover.pk(&inner_id).expect("Failed to get inner-prove PK"); + let vk = pk.get_vk().clone(); + + let verifier = Verifier::new(params, vk); + log::info!("Constructed inner-verifier"); + verifier + }; let verified = verifier.verify_snark(snark); assert!(verified, "{test}: failed to verify inner snark"); diff --git a/prover/src/zkevm/prover.rs b/prover/src/zkevm/prover.rs index 895f84ba33..e5f46f574b 100644 --- a/prover/src/zkevm/prover.rs +++ b/prover/src/zkevm/prover.rs @@ -1,3 +1,5 @@ +use std::collections::BTreeMap; + use crate::{ common, config::{LayerId, ZKEVM_DEGREES}, @@ -11,21 +13,28 @@ use crate::{ }; use aggregator::ChunkInfo; use anyhow::Result; +use halo2_proofs::{halo2curves::bn256::Bn256, poly::kzg::commitment::ParamsKZG}; #[derive(Debug)] -pub struct Prover { +pub struct Prover<'params> { // Make it public for testing with inner functions (unnecessary for FFI). - pub prover_impl: common::Prover, - verifier: Option, + pub prover_impl: common::Prover<'params>, + verifier: Option>, raw_vk: Option>, } -impl Prover { - pub fn from_dirs(params_dir: &str, assets_dir: &str) -> Self { - let prover_impl = common::Prover::from_params_dir(params_dir, &ZKEVM_DEGREES); +impl<'params> Prover<'params> { + pub fn degrees() -> Vec { + (*ZKEVM_DEGREES).clone() + } + pub fn from_params_map( + params_map: &'params BTreeMap>, + assets_dir: &str, + ) -> Self { + let prover_impl = common::Prover::from_params(params_map); let raw_vk = try_to_read(assets_dir, &CHUNK_VK_FILENAME); - let verifier = if raw_vk.is_none() { + let _verifier = if raw_vk.is_none() { log::warn!( "zkevm-prover: {} doesn't exist in {}", *CHUNK_VK_FILENAME, @@ -33,13 +42,15 @@ impl Prover { ); None } else { - Some(super::verifier::Verifier::from_dirs(params_dir, assets_dir)) + Some(super::verifier::Verifier::from_params_map( + prover_impl.params_map, + assets_dir, + )) }; - Self { prover_impl, raw_vk, - verifier, + verifier: None, } } diff --git a/prover/src/zkevm/verifier.rs b/prover/src/zkevm/verifier.rs index 3748cc9f48..81b914cef8 100644 --- a/prover/src/zkevm/verifier.rs +++ b/prover/src/zkevm/verifier.rs @@ -11,30 +11,34 @@ use halo2_proofs::{ plonk::VerifyingKey, poly::kzg::commitment::ParamsKZG, }; -use std::env; +use std::{collections::BTreeMap, env}; #[derive(Debug)] -pub struct Verifier { +pub struct Verifier<'params> { // Make it public for testing with inner functions (unnecessary for FFI). - pub inner: common::Verifier, + pub inner: common::Verifier<'params, CompressionCircuit>, } -impl From> for Verifier { - fn from(inner: common::Verifier) -> Self { +impl<'params> From> for Verifier<'params> { + fn from(inner: common::Verifier<'params, CompressionCircuit>) -> Self { Self { inner } } } -impl Verifier { - pub fn new(params: ParamsKZG, vk: VerifyingKey) -> Self { +impl<'params> Verifier<'params> { + pub fn new(params: &'params ParamsKZG, vk: VerifyingKey) -> Self { common::Verifier::new(params, vk).into() } - pub fn from_dirs(params_dir: &str, assets_dir: &str) -> Self { + pub fn from_params_map( + params_map: &'params BTreeMap>, + assets_dir: &str, + ) -> Self { let raw_vk = force_to_read(assets_dir, &chunk_vk_filename()); - env::set_var("COMPRESSION_CONFIG", &*LAYER2_CONFIG_PATH); - common::Verifier::from_params_dir(params_dir, *LAYER2_DEGREE, &raw_vk).into() + let params = params_map.get(&*LAYER2_DEGREE).expect("should be loaded"); + let verifier = common::Verifier::from_params(params, &raw_vk); + verifier.into() } pub fn verify_chunk_proof(&self, proof: ChunkProof) -> bool { From 87e0bfa8869e91f16a380f5f24ee66bf1c951d69 Mon Sep 17 00:00:00 2001 From: Zhang Zhuo Date: Mon, 26 Aug 2024 16:03:38 +0800 Subject: [PATCH 2/4] clean --- prover/src/aggregator/prover.rs | 9 ++++++--- prover/src/aggregator/verifier.rs | 2 +- prover/src/common/prover.rs | 4 ++-- prover/src/test/batch.rs | 2 +- prover/src/test/chunk.rs | 4 ++-- prover/src/test/inner.rs | 2 +- prover/src/zkevm/prover.rs | 10 +++++----- prover/src/zkevm/verifier.rs | 2 +- 8 files changed, 19 insertions(+), 16 deletions(-) diff --git a/prover/src/aggregator/prover.rs b/prover/src/aggregator/prover.rs index bc2f3f6df7..267dce1098 100644 --- a/prover/src/aggregator/prover.rs +++ b/prover/src/aggregator/prover.rs @@ -9,7 +9,7 @@ use snark_verifier_sdk::Snark; use crate::{ common, - config::LayerId, + config::{LayerId, AGG_DEGREES}, consts::{BATCH_KECCAK_ROW, BATCH_VK_FILENAME, BUNDLE_VK_FILENAME, CHUNK_PROTOCOL_FILENAME}, io::{force_to_read, try_to_read}, proof::BundleProof, @@ -27,14 +27,17 @@ pub struct Prover<'params> { } impl<'params> Prover<'params> { - pub fn from_dirs( + pub fn degrees() -> Vec { + (*AGG_DEGREES).clone() + } + pub fn from_params_and_assets( params_map: &'params BTreeMap>, assets_dir: &str, ) -> Self { log::debug!("set env KECCAK_ROWS={}", BATCH_KECCAK_ROW.to_string()); env::set_var("KECCAK_ROWS", BATCH_KECCAK_ROW.to_string()); - let prover_impl = common::Prover::from_params(params_map); + let prover_impl = common::Prover::from_params_map(params_map); let chunk_protocol = force_to_read(assets_dir, &CHUNK_PROTOCOL_FILENAME); let raw_vk_batch = try_to_read(assets_dir, &BATCH_VK_FILENAME); diff --git a/prover/src/aggregator/verifier.rs b/prover/src/aggregator/verifier.rs index 7932f27cb7..0b3bc252f1 100644 --- a/prover/src/aggregator/verifier.rs +++ b/prover/src/aggregator/verifier.rs @@ -36,7 +36,7 @@ impl<'params> Verifier<'params> { } } - pub fn from_params_map( + pub fn from_params_and_assets( params_map: &'params BTreeMap>, assets_dir: &str, ) -> Self { diff --git a/prover/src/common/prover.rs b/prover/src/common/prover.rs index 91ae2f8347..b3df0582e9 100644 --- a/prover/src/common/prover.rs +++ b/prover/src/common/prover.rs @@ -18,13 +18,13 @@ mod utils; #[derive(Debug)] pub struct Prover<'params> { // degree -> params (use BTreeMap to find proper degree for params downsize) - pub(crate) params_map: &'params BTreeMap>, + pub params_map: &'params BTreeMap>, // Cached id -> pk pk_map: HashMap>, } impl<'params> Prover<'params> { - pub fn from_params(params_map: &'params BTreeMap>) -> Self { + pub fn from_params_map(params_map: &'params BTreeMap>) -> Self { Self { params_map, pk_map: HashMap::new(), diff --git a/prover/src/test/batch.rs b/prover/src/test/batch.rs index dd273437d7..15ecf06248 100644 --- a/prover/src/test/batch.rs +++ b/prover/src/test/batch.rs @@ -22,7 +22,7 @@ static PARAMS_MAP: LazyLock>> = LazyLock::new(|| static BATCH_PROVER: LazyLock> = LazyLock::new(|| { let assets_dir = read_env_var("SCROLL_PROVER_ASSETS_DIR", "./test_assets".to_string()); - let prover = Prover::from_dirs(&PARAMS_MAP, &assets_dir); + let prover = Prover::from_params_and_assets(&PARAMS_MAP, &assets_dir); log::info!("Constructed batch-prover"); Mutex::new(prover) diff --git a/prover/src/test/chunk.rs b/prover/src/test/chunk.rs index 0f4bffcf1d..4d7c29d1e4 100644 --- a/prover/src/test/chunk.rs +++ b/prover/src/test/chunk.rs @@ -18,7 +18,7 @@ static PARAMS_MAP: LazyLock>> = LazyLock::new(|| static CHUNK_PROVER: LazyLock> = LazyLock::new(|| { let assets_dir = read_env_var("SCROLL_PROVER_ASSETS_DIR", "./test_assets".to_string()); - let prover = Prover::from_params_map(&PARAMS_MAP, &assets_dir); + let prover = Prover::from_params_and_assets(&PARAMS_MAP, &assets_dir); log::info!("Constructed chunk-prover"); Mutex::new(prover) @@ -36,7 +36,7 @@ pub fn chunk_prove(desc: &str, chunk: ChunkProvingTask) -> ChunkProof { let verifier = { let assets_dir = read_env_var("SCROLL_PROVER_ASSETS_DIR", "./test_assets".to_string()); - let verifier = Verifier::from_params_map(prover.prover_impl.params_map, &assets_dir); + let verifier = Verifier::from_params_and_assets(prover.prover_impl.params_map, &assets_dir); log::info!("Constructed chunk-verifier"); verifier }; diff --git a/prover/src/test/inner.rs b/prover/src/test/inner.rs index 1727d15029..fe6e90d5df 100644 --- a/prover/src/test/inner.rs +++ b/prover/src/test/inner.rs @@ -18,7 +18,7 @@ static PARAMS_MAP: LazyLock>> = LazyLock::new(|| }); static INNER_PROVER: LazyLock> = LazyLock::new(|| { - let prover = Prover::from_params(&PARAMS_MAP); + let prover = Prover::from_params_map(&PARAMS_MAP); log::info!("Constructed inner-prover"); Mutex::new(prover) diff --git a/prover/src/zkevm/prover.rs b/prover/src/zkevm/prover.rs index e5f46f574b..2008615396 100644 --- a/prover/src/zkevm/prover.rs +++ b/prover/src/zkevm/prover.rs @@ -27,14 +27,14 @@ impl<'params> Prover<'params> { pub fn degrees() -> Vec { (*ZKEVM_DEGREES).clone() } - pub fn from_params_map( + pub fn from_params_and_assets( params_map: &'params BTreeMap>, assets_dir: &str, ) -> Self { - let prover_impl = common::Prover::from_params(params_map); + let prover_impl = common::Prover::from_params_map(params_map); let raw_vk = try_to_read(assets_dir, &CHUNK_VK_FILENAME); - let _verifier = if raw_vk.is_none() { + let verifier = if raw_vk.is_none() { log::warn!( "zkevm-prover: {} doesn't exist in {}", *CHUNK_VK_FILENAME, @@ -42,7 +42,7 @@ impl<'params> Prover<'params> { ); None } else { - Some(super::verifier::Verifier::from_params_map( + Some(super::verifier::Verifier::from_params_and_assets( prover_impl.params_map, assets_dir, )) @@ -50,7 +50,7 @@ impl<'params> Prover<'params> { Self { prover_impl, raw_vk, - verifier: None, + verifier, } } diff --git a/prover/src/zkevm/verifier.rs b/prover/src/zkevm/verifier.rs index 81b914cef8..f382cd1199 100644 --- a/prover/src/zkevm/verifier.rs +++ b/prover/src/zkevm/verifier.rs @@ -30,7 +30,7 @@ impl<'params> Verifier<'params> { common::Verifier::new(params, vk).into() } - pub fn from_params_map( + pub fn from_params_and_assets( params_map: &'params BTreeMap>, assets_dir: &str, ) -> Self { From 1668cb9b341271c8e6d1362cbe9460ab2a8ee60a Mon Sep 17 00:00:00 2001 From: Rohit Narurkar Date: Mon, 26 Aug 2024 18:11:33 +0100 Subject: [PATCH 3/4] remove redundant function 'degrees' --- prover/src/aggregator/prover.rs | 5 +---- prover/src/zkevm/prover.rs | 15 +++------------ 2 files changed, 4 insertions(+), 16 deletions(-) diff --git a/prover/src/aggregator/prover.rs b/prover/src/aggregator/prover.rs index 156894a79f..e9cdfe9090 100644 --- a/prover/src/aggregator/prover.rs +++ b/prover/src/aggregator/prover.rs @@ -9,7 +9,7 @@ use snark_verifier_sdk::Snark; use crate::{ common, - config::{LayerId, AGG_DEGREES}, + config::LayerId, consts::{BATCH_KECCAK_ROW, BATCH_VK_FILENAME, BUNDLE_VK_FILENAME, CHUNK_PROTOCOL_FILENAME}, io::{force_to_read, try_to_read}, proof::BundleProof, @@ -27,9 +27,6 @@ pub struct Prover<'params> { } impl<'params> Prover<'params> { - pub fn degrees() -> Vec { - (*AGG_DEGREES).clone() - } pub fn from_params_and_assets( params_map: &'params BTreeMap>, assets_dir: &str, diff --git a/prover/src/zkevm/prover.rs b/prover/src/zkevm/prover.rs index 2008615396..4d9f71b8b4 100644 --- a/prover/src/zkevm/prover.rs +++ b/prover/src/zkevm/prover.rs @@ -1,15 +1,9 @@ use std::collections::BTreeMap; use crate::{ - common, - config::{LayerId, ZKEVM_DEGREES}, - consts::CHUNK_VK_FILENAME, - io::try_to_read, - proof::compare_chunk_info, - types::ChunkProvingTask, - utils::chunk_trace_to_witness_block, - zkevm::circuit::calculate_row_usage_of_witness_block, - ChunkProof, + common, config::LayerId, consts::CHUNK_VK_FILENAME, io::try_to_read, proof::compare_chunk_info, + types::ChunkProvingTask, utils::chunk_trace_to_witness_block, + zkevm::circuit::calculate_row_usage_of_witness_block, ChunkProof, }; use aggregator::ChunkInfo; use anyhow::Result; @@ -24,9 +18,6 @@ pub struct Prover<'params> { } impl<'params> Prover<'params> { - pub fn degrees() -> Vec { - (*ZKEVM_DEGREES).clone() - } pub fn from_params_and_assets( params_map: &'params BTreeMap>, assets_dir: &str, From 339fe409d62ebb6504ea45cbfb7161fa5cdfb4f3 Mon Sep 17 00:00:00 2001 From: Rohit Narurkar Date: Mon, 26 Aug 2024 18:11:53 +0100 Subject: [PATCH 4/4] add sanity check (decoded blob == batch bytes) --- aggregator/src/eip4844.rs | 33 +++++++++++++++++++++++++++++++++ prover/src/aggregator/prover.rs | 14 +++++++++++++- 2 files changed, 46 insertions(+), 1 deletion(-) diff --git a/aggregator/src/eip4844.rs b/aggregator/src/eip4844.rs index 22f764c682..5f4a29a5dc 100644 --- a/aggregator/src/eip4844.rs +++ b/aggregator/src/eip4844.rs @@ -54,3 +54,36 @@ pub fn get_blob_bytes(batch_bytes: &[u8]) -> Vec { blob_bytes } + +/// Given the blob's bytes, take into account the first byte, i.e. enable_encoding? and either spit +/// out the raw bytes or zstd decode them. +pub fn decode_blob(blob_bytes: &[u8]) -> std::io::Result> { + let enable_encoding = blob_bytes[0].eq(&1); + + // If not encoded, spit out the rest of the bytes, as it is. + if !enable_encoding { + return Ok(blob_bytes[1..].to_vec()); + } + + // The bytes following the first byte represent the zstd-encoded bytes. + let mut encoded_bytes = blob_bytes[1..].to_vec(); + let mut encoded_len = encoded_bytes.len(); + let mut decoded_bytes = Vec::with_capacity(5 * 4096 * 32); + loop { + let mut decoder = zstd_encoder::zstd::stream::read::Decoder::new(encoded_bytes.as_slice())?; + decoder.include_magicbytes(false)?; + decoder.window_log_max(30)?; + + decoded_bytes.clear(); + + if std::io::copy(&mut decoder, &mut decoded_bytes).is_ok() { + break; + } + + // The error above means we need to truncate the suffix 0-byte. + encoded_len -= 1; + encoded_bytes.truncate(encoded_len); + } + + Ok(decoded_bytes) +} diff --git a/prover/src/aggregator/prover.rs b/prover/src/aggregator/prover.rs index e9cdfe9090..fcfe4831ee 100644 --- a/prover/src/aggregator/prover.rs +++ b/prover/src/aggregator/prover.rs @@ -1,6 +1,8 @@ use std::{collections::BTreeMap, env, iter::repeat}; -use aggregator::{BatchHash, BatchHeader, ChunkInfo, MAX_AGG_SNARKS}; +use aggregator::{ + eip4844::decode_blob, BatchData, BatchHash, BatchHeader, ChunkInfo, MAX_AGG_SNARKS, +}; use anyhow::{bail, Result}; use eth_types::H256; use halo2_proofs::{halo2curves::bn256::Bn256, poly::kzg::commitment::ParamsKZG}; @@ -212,6 +214,16 @@ impl<'params> Prover<'params> { let batch_hash = batch_header.batch_hash(); let batch_info: BatchHash = BatchHash::construct(&chunk_hashes, batch_header, &batch.blob_bytes); + let batch_data: BatchData = BatchData::from(&batch_info); + + // sanity check: + // - conditionally decoded blob should match batch data. + let batch_bytes = batch_data.get_batch_data_bytes(); + let decoded_blob_bytes = decode_blob(&batch.blob_bytes)?; + assert_eq!( + batch_bytes, decoded_blob_bytes, + "BatchProvingTask(sanity) mismatch batch bytes and decoded blob bytes", + ); let layer3_snark = self.prover_impl.load_or_gen_agg_snark( name,