From a8515285c9360387639488cf636e2600cd58114a Mon Sep 17 00:00:00 2001 From: Rohit Narurkar Date: Tue, 9 Jul 2024 15:47:28 +0100 Subject: [PATCH 01/17] wip --- Cargo.lock | 11 +++++++++++ Cargo.toml | 3 ++- prover2/Cargo.toml | 13 +++++++++++++ prover2/src/lib.rs | 4 ++++ prover2/src/prover.rs | 0 prover2/src/types.rs | 0 prover2/src/util.rs | 0 prover2/src/verifier.rs | 0 8 files changed, 30 insertions(+), 1 deletion(-) create mode 100644 prover2/Cargo.toml create mode 100644 prover2/src/lib.rs create mode 100644 prover2/src/prover.rs create mode 100644 prover2/src/types.rs create mode 100644 prover2/src/util.rs create mode 100644 prover2/src/verifier.rs diff --git a/Cargo.lock b/Cargo.lock index 0e7394dd37..7e19fca543 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3492,6 +3492,17 @@ dependencies = [ "zkevm-circuits", ] +[[package]] +name = "prover2" +version = "0.11.0" +dependencies = [ + "aggregator", + "serde", + "snark-verifier", + "snark-verifier-sdk", + "zkevm-circuits", +] + [[package]] name = "psm" version = "0.1.21" diff --git a/Cargo.toml b/Cargo.toml index 76e6357800..70e2f3a6db 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -12,7 +12,8 @@ members = [ "mock", "testool", "aggregator", - "prover" + "prover", + "prover2" ] resolver = "2" diff --git a/prover2/Cargo.toml b/prover2/Cargo.toml new file mode 100644 index 0000000000..2cba262087 --- /dev/null +++ b/prover2/Cargo.toml @@ -0,0 +1,13 @@ +[package] +name = "prover2" +version.workspace = true +edition.workspace = true +license.workspace = true + +[dependencies] +aggregator = { path = "../aggregator" } +zkevm-circuits = { path = "../zkevm-circuits" } + +serde.workspace = true +snark-verifier.workspace = true +snark-verifier-sdk.workspace = true diff --git a/prover2/src/lib.rs b/prover2/src/lib.rs new file mode 100644 index 0000000000..d3a165ccfa --- /dev/null +++ b/prover2/src/lib.rs @@ -0,0 +1,4 @@ +mod prover; +mod types; +mod util; +mod verifier; diff --git a/prover2/src/prover.rs b/prover2/src/prover.rs new file mode 100644 index 0000000000..e69de29bb2 diff --git a/prover2/src/types.rs b/prover2/src/types.rs new file mode 100644 index 0000000000..e69de29bb2 diff --git a/prover2/src/util.rs b/prover2/src/util.rs new file mode 100644 index 0000000000..e69de29bb2 diff --git a/prover2/src/verifier.rs b/prover2/src/verifier.rs new file mode 100644 index 0000000000..e69de29bb2 From d2cc189fd660b1a989634aca1693ea421c5fb131 Mon Sep 17 00:00:00 2001 From: Rohit Narurkar Date: Thu, 11 Jul 2024 03:19:15 +0100 Subject: [PATCH 02/17] more wip: prover config and some other types --- Cargo.lock | 4 ++ prover2/Cargo.toml | 5 ++ prover2/src/error.rs | 12 ++++ prover2/src/lib.rs | 8 +++ prover2/src/prover.rs | 0 prover2/src/prover/config.rs | 106 +++++++++++++++++++++++++++++++++++ prover2/src/prover/mod.rs | 22 ++++++++ prover2/src/prover/params.rs | 15 +++++ prover2/src/types.rs | 0 prover2/src/types/layer.rs | 26 +++++++++ prover2/src/types/mod.rs | 31 ++++++++++ prover2/src/types/proof.rs | 10 ++++ prover2/src/types/task.rs | 8 +++ prover2/src/util.rs | 20 +++++++ prover2/src/verifier.rs | 1 + 15 files changed, 268 insertions(+) create mode 100644 prover2/src/error.rs delete mode 100644 prover2/src/prover.rs create mode 100644 prover2/src/prover/config.rs create mode 100644 prover2/src/prover/mod.rs create mode 100644 prover2/src/prover/params.rs delete mode 100644 prover2/src/types.rs create mode 100644 prover2/src/types/layer.rs create mode 100644 prover2/src/types/mod.rs create mode 100644 prover2/src/types/proof.rs create mode 100644 prover2/src/types/task.rs diff --git a/Cargo.lock b/Cargo.lock index 7e19fca543..ee29a85940 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3497,9 +3497,13 @@ name = "prover2" version = "0.11.0" dependencies = [ "aggregator", + "halo2_proofs", "serde", + "serde_json", "snark-verifier", "snark-verifier-sdk", + "thiserror", + "tracing", "zkevm-circuits", ] diff --git a/prover2/Cargo.toml b/prover2/Cargo.toml index 2cba262087..b7b0b33b57 100644 --- a/prover2/Cargo.toml +++ b/prover2/Cargo.toml @@ -8,6 +8,11 @@ license.workspace = true aggregator = { path = "../aggregator" } zkevm-circuits = { path = "../zkevm-circuits" } +halo2_proofs.workspace = true serde.workspace = true +serde_json.workspace = true snark-verifier.workspace = true snark-verifier-sdk.workspace = true + +thiserror = "1.0" +tracing = "0.1.40" diff --git a/prover2/src/error.rs b/prover2/src/error.rs new file mode 100644 index 0000000000..0a9c6c5e19 --- /dev/null +++ b/prover2/src/error.rs @@ -0,0 +1,12 @@ +use thiserror::Error; + +/// Represents error variants possibly encountered during the proof generation process. +#[derive(Debug, Error)] +pub enum ProverError { + /// Error encountered while doing I/O operations. + #[error(transparent)] + IoError(#[from] std::io::Error), + /// Error encountered during serialization/deserialization of JSON. + #[error(transparent)] + SerdeJsonError(#[from] serde_json::Error), +} diff --git a/prover2/src/lib.rs b/prover2/src/lib.rs index d3a165ccfa..d15201906b 100644 --- a/prover2/src/lib.rs +++ b/prover2/src/lib.rs @@ -1,4 +1,12 @@ +mod error; +pub use error::ProverError; + mod prover; +pub use prover::{config::ProverConfig, params::Params, Prover}; + mod types; +pub use types::{layer::ProofLayer, proof::Proof}; + mod util; + mod verifier; diff --git a/prover2/src/prover.rs b/prover2/src/prover.rs deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/prover2/src/prover/config.rs b/prover2/src/prover/config.rs new file mode 100644 index 0000000000..66b49c640b --- /dev/null +++ b/prover2/src/prover/config.rs @@ -0,0 +1,106 @@ +use std::{ + collections::BTreeMap, env::current_dir, fs::File, io::BufReader, marker::PhantomData, + path::PathBuf, +}; + +use halo2_proofs::{ + halo2curves::bn256::{Bn256, G1Affine}, + plonk::ProvingKey, + poly::kzg::commitment::ParamsKZG, + SerdeFormat, +}; + +use crate::{ + types::ProverType, + util::{kzg_params_path, non_native_params_path}, + Params, ProofLayer, ProverError, +}; + +/// Configuration for a generic prover. +#[derive(Default)] +pub struct ProverConfig { + /// KZG setup parameters by proof layer. + pub kzg_params: BTreeMap>, + /// Config parameters for non-native field arithmetics by proof layer. + pub non_native_params: BTreeMap, + /// Proving keys by proof layer. + pub pks: BTreeMap>, + /// Optional directory to locate KZG setup parameters. + pub kzg_params_dir: Option, + /// Optional directory to locate non-native field arithmetic config params. + pub non_native_params_dir: Option, + /// Optional directory to cache proofs. + pub cache_dir: Option, + + _prover_type: PhantomData, +} + +impl ProverConfig { + /// Returns prover config after inserting the non-native field arithmetic config for the given + /// proof layer. + pub fn with_params(mut self, layer: ProofLayer, params: Params) -> Self { + self.non_native_params.insert(layer, params); + self + } + + /// Returns prover config after inserting the proving key for the given proof layer. + pub fn with_pk(mut self, layer: ProofLayer, pk: ProvingKey) -> Self { + self.pks.insert(layer, pk); + self + } + + /// Returns prover config with a directory to load KZG params from. + pub fn with_kzg_params_dir(mut self, dir: impl Into) -> Self { + self.kzg_params_dir = Some(dir.into()); + self + } + + /// Returns prover config with a directory to load non-native field arithmetic config params + /// from. + pub fn with_non_native_params_dir(mut self, dir: impl Into) -> Self { + self.non_native_params_dir = Some(dir.into()); + self + } + + /// Returns prover config with a cache directory configured. + pub fn with_cache_dir(mut self, dir: impl Into) -> Self { + self.cache_dir = Some(dir.into()); + self + } +} + +impl ProverConfig { + pub fn load(mut self) -> Result { + // The proof layers that this prover needs to generate proofs for. + let proof_layers = Type::layers(); + + // Use the configured directories or fallback to current working directory. + let non_native_params_dir = self.non_native_params_dir.clone().unwrap_or(current_dir()?); + let kzg_params_dir = self.kzg_params_dir.clone().unwrap_or(current_dir()?); + + // Read and store non-native field arithmetic config params for each layer. + for layer in proof_layers { + let params_file = File::open(non_native_params_path( + non_native_params_dir.as_path(), + layer, + ))?; + let params = serde_json::from_reader(params_file)?; + self.non_native_params.insert(layer, params); + } + + // Read and store KZG setup params for each layer. + for (&layer, non_native_params) in self.non_native_params.iter() { + let params_file = File::open(kzg_params_path( + kzg_params_dir.as_path(), + non_native_params.degree, + ))?; + let params = ParamsKZG::::read_custom( + &mut BufReader::new(params_file), + SerdeFormat::RawBytesUnchecked, + )?; + self.kzg_params.insert(layer, params); + } + + Ok(self) + } +} diff --git a/prover2/src/prover/mod.rs b/prover2/src/prover/mod.rs new file mode 100644 index 0000000000..ccf4954e56 --- /dev/null +++ b/prover2/src/prover/mod.rs @@ -0,0 +1,22 @@ +use crate::{Proof, ProverConfig, ProverError}; + +pub mod config; +pub mod params; + +/// A generic prover that is capable of generating proofs for given tasks. +pub struct Prover { + /// Config for the prover. + pub config: ProverConfig, +} + +impl Prover { + pub fn new(config: ProverConfig) -> Self { + Self { config } + } +} + +impl Prover { + pub fn gen_proof() -> Result { + unimplemented!() + } +} diff --git a/prover2/src/prover/params.rs b/prover2/src/prover/params.rs new file mode 100644 index 0000000000..88473f096b --- /dev/null +++ b/prover2/src/prover/params.rs @@ -0,0 +1,15 @@ +use serde::{Deserialize, Serialize}; +use snark_verifier::loader::halo2::halo2_ecc::fields::fp::FpStrategy; + +/// Parameters to configure the non-native field arithmetic chip. +#[derive(Serialize, Deserialize)] +pub struct Params { + pub strategy: FpStrategy, + pub degree: u32, + pub num_advice: Vec, + pub num_lookup_advice: Vec, + pub num_fixed: usize, + pub lookup_bits: usize, + pub limb_bits: usize, + pub num_limbs: usize, +} diff --git a/prover2/src/types.rs b/prover2/src/types.rs deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/prover2/src/types/layer.rs b/prover2/src/types/layer.rs new file mode 100644 index 0000000000..1a4d48ab57 --- /dev/null +++ b/prover2/src/types/layer.rs @@ -0,0 +1,26 @@ +use serde::{Deserialize, Serialize}; + +#[derive(Clone, Copy, Serialize, Deserialize, Eq, PartialEq, PartialOrd, Ord)] +pub enum ProofLayer { + Layer0, + Layer1, + Layer2, + Layer3, + Layer4, + Layer5, + Layer6, +} + +impl ToString for ProofLayer { + fn to_string(&self) -> String { + String::from(match self { + Self::Layer0 => "layer0", + Self::Layer1 => "layer1", + Self::Layer2 => "layer2", + Self::Layer3 => "layer3", + Self::Layer4 => "layer4", + Self::Layer5 => "layer5", + Self::Layer6 => "layer6", + }) + } +} diff --git a/prover2/src/types/mod.rs b/prover2/src/types/mod.rs new file mode 100644 index 0000000000..28ed8e4df5 --- /dev/null +++ b/prover2/src/types/mod.rs @@ -0,0 +1,31 @@ +use crate::ProofLayer; + +pub mod layer; +pub mod proof; +pub mod task; + +pub trait ProverType { + fn layers() -> Vec; +} + +pub struct ProverTypeChunk; +pub struct ProverTypeBatch; +pub struct ProverTypeBundle; + +impl ProverType for ProverTypeChunk { + fn layers() -> Vec { + vec![ProofLayer::Layer0, ProofLayer::Layer1, ProofLayer::Layer2] + } +} + +impl ProverType for ProverTypeBatch { + fn layers() -> Vec { + vec![ProofLayer::Layer3, ProofLayer::Layer4] + } +} + +impl ProverType for ProverTypeBundle { + fn layers() -> Vec { + vec![ProofLayer::Layer5, ProofLayer::Layer6] + } +} diff --git a/prover2/src/types/proof.rs b/prover2/src/types/proof.rs new file mode 100644 index 0000000000..cb26430c13 --- /dev/null +++ b/prover2/src/types/proof.rs @@ -0,0 +1,10 @@ +use serde::{Deserialize, Serialize}; + +use super::layer::ProofLayer; + +/// Describes an output from a [`Prover`]'s proof generation process when given a [`ProvingTask`]. +#[derive(Serialize, Deserialize)] +pub struct Proof { + /// The proof layer. + layer: ProofLayer, +} diff --git a/prover2/src/types/task.rs b/prover2/src/types/task.rs new file mode 100644 index 0000000000..b0a2f95f45 --- /dev/null +++ b/prover2/src/types/task.rs @@ -0,0 +1,8 @@ +use serde::{de::DeserializeOwned, ser::Serialize}; + +/// Describes the behaviour to be supported by a type that can be used as an input (or a task) to +/// instruct a [`Prover`] to generate a proof. +trait ProvingTask: Serialize + DeserializeOwned { + /// An identifier for the proving task. + fn id(&self) -> String; +} diff --git a/prover2/src/util.rs b/prover2/src/util.rs index e69de29bb2..03225f8463 100644 --- a/prover2/src/util.rs +++ b/prover2/src/util.rs @@ -0,0 +1,20 @@ +use std::path::{Path, PathBuf}; + +use crate::ProofLayer; + +/// The config parameters for non native field arithmetics is a *.config file. +const NON_NATIVE_PARAMS_EXT: &str = ".config"; + +/// The path to the config parameters for a given proof layer. +/// +/// /{layer}.config +pub fn non_native_params_path(dir: &Path, layer: ProofLayer) -> PathBuf { + dir.join(format!("{}{NON_NATIVE_PARAMS_EXT}", layer.to_string())) +} + +/// The path to the KZG params by degree. +/// +/// /params{degree} +pub fn kzg_params_path(dir: &Path, degree: u32) -> PathBuf { + dir.join(format!("params{degree}")) +} diff --git a/prover2/src/verifier.rs b/prover2/src/verifier.rs index e69de29bb2..8b13789179 100644 --- a/prover2/src/verifier.rs +++ b/prover2/src/verifier.rs @@ -0,0 +1 @@ + From be66942fb33079be45f014f2764c0606e07e3d69 Mon Sep 17 00:00:00 2001 From: Rohit Narurkar Date: Thu, 11 Jul 2024 12:22:38 +0100 Subject: [PATCH 03/17] wip: update setup prover config --- prover2/src/lib.rs | 2 +- prover2/src/prover/config.rs | 59 +++++++++++++++++++++++++----------- prover2/src/prover/mod.rs | 14 ++++++++- prover2/src/util.rs | 51 +++++++++++++++++++++++++++++-- 4 files changed, 103 insertions(+), 23 deletions(-) diff --git a/prover2/src/lib.rs b/prover2/src/lib.rs index d15201906b..af466a601b 100644 --- a/prover2/src/lib.rs +++ b/prover2/src/lib.rs @@ -2,7 +2,7 @@ mod error; pub use error::ProverError; mod prover; -pub use prover::{config::ProverConfig, params::Params, Prover}; +pub use prover::{config::ProverConfig, params::Params, BatchProver, BundleProver, ChunkProver}; mod types; pub use types::{layer::ProofLayer, proof::Proof}; diff --git a/prover2/src/prover/config.rs b/prover2/src/prover/config.rs index 66b49c640b..d0f1504c23 100644 --- a/prover2/src/prover/config.rs +++ b/prover2/src/prover/config.rs @@ -1,5 +1,8 @@ use std::{ - collections::BTreeMap, env::current_dir, fs::File, io::BufReader, marker::PhantomData, + collections::BTreeMap, + fs::{create_dir, create_dir_all, File}, + io::BufReader, + marker::PhantomData, path::PathBuf, }; @@ -12,7 +15,11 @@ use halo2_proofs::{ use crate::{ types::ProverType, - util::{kzg_params_path, non_native_params_path}, + util::{ + default_cache_dir, default_kzg_params_dir, default_non_native_params_dir, kzg_params_path, + non_native_params_path as nn_params_path, CACHE_PATH_EVM, CACHE_PATH_PI, CACHE_PATH_PROOFS, + CACHE_PATH_TASKS, + }, Params, ProofLayer, ProverError, }; @@ -22,7 +29,7 @@ pub struct ProverConfig { /// KZG setup parameters by proof layer. pub kzg_params: BTreeMap>, /// Config parameters for non-native field arithmetics by proof layer. - pub non_native_params: BTreeMap, + pub nn_params: BTreeMap, /// Proving keys by proof layer. pub pks: BTreeMap>, /// Optional directory to locate KZG setup parameters. @@ -38,8 +45,14 @@ pub struct ProverConfig { impl ProverConfig { /// Returns prover config after inserting the non-native field arithmetic config for the given /// proof layer. - pub fn with_params(mut self, layer: ProofLayer, params: Params) -> Self { - self.non_native_params.insert(layer, params); + pub fn with_nn_params(mut self, layer: ProofLayer, params: Params) -> Self { + self.nn_params.insert(layer, params); + self + } + + /// Returns prover config after inserting KZG setup params for the given proof layer. + pub fn with_kzg_params(mut self, layer: ProofLayer, params: ParamsKZG) -> Self { + self.kzg_params.insert(layer, params); self } @@ -70,30 +83,33 @@ impl ProverConfig { } impl ProverConfig { - pub fn load(mut self) -> Result { + /// Setup the prover config by reading relevant config files from storage. + pub fn setup(mut self) -> Result { // The proof layers that this prover needs to generate proofs for. let proof_layers = Type::layers(); // Use the configured directories or fallback to current working directory. - let non_native_params_dir = self.non_native_params_dir.clone().unwrap_or(current_dir()?); - let kzg_params_dir = self.kzg_params_dir.clone().unwrap_or(current_dir()?); + let nn_params_dir = self + .non_native_params_dir + .clone() + .unwrap_or(default_non_native_params_dir()?); + let kzg_params_dir = self + .kzg_params_dir + .clone() + .unwrap_or(default_kzg_params_dir()?); + let cache_dir = self.cache_dir.clone().unwrap_or(default_cache_dir()?); // Read and store non-native field arithmetic config params for each layer. for layer in proof_layers { - let params_file = File::open(non_native_params_path( - non_native_params_dir.as_path(), - layer, - ))?; + let params_file = File::open(nn_params_path(nn_params_dir.as_path(), layer))?; let params = serde_json::from_reader(params_file)?; - self.non_native_params.insert(layer, params); + self.nn_params.insert(layer, params); } // Read and store KZG setup params for each layer. - for (&layer, non_native_params) in self.non_native_params.iter() { - let params_file = File::open(kzg_params_path( - kzg_params_dir.as_path(), - non_native_params.degree, - ))?; + for (&layer, nn_params) in self.nn_params.iter() { + let params_file = + File::open(kzg_params_path(kzg_params_dir.as_path(), nn_params.degree))?; let params = ParamsKZG::::read_custom( &mut BufReader::new(params_file), SerdeFormat::RawBytesUnchecked, @@ -101,6 +117,13 @@ impl ProverConfig { self.kzg_params.insert(layer, params); } + // Setup the cache directory's structure. + create_dir_all(cache_dir.as_path())?; + create_dir(cache_dir.join(CACHE_PATH_TASKS))?; + create_dir(cache_dir.join(CACHE_PATH_PROOFS))?; + create_dir(cache_dir.join(CACHE_PATH_PI))?; + create_dir(cache_dir.join(CACHE_PATH_EVM))?; + Ok(self) } } diff --git a/prover2/src/prover/mod.rs b/prover2/src/prover/mod.rs index ccf4954e56..8237c0c26b 100644 --- a/prover2/src/prover/mod.rs +++ b/prover2/src/prover/mod.rs @@ -1,8 +1,20 @@ -use crate::{Proof, ProverConfig, ProverError}; +use crate::{ + types::{ProverTypeBatch, ProverTypeBundle, ProverTypeChunk}, + Proof, ProverConfig, ProverError, +}; pub mod config; pub mod params; +/// Convenience type for chunk prover. +pub type ChunkProver = Prover; + +/// Convenience type for batch prover. +pub type BatchProver = Prover; + +/// Convenience type for bundle prover. +pub type BundleProver = Prover; + /// A generic prover that is capable of generating proofs for given tasks. pub struct Prover { /// Config for the prover. diff --git a/prover2/src/util.rs b/prover2/src/util.rs index 03225f8463..c77261ed31 100644 --- a/prover2/src/util.rs +++ b/prover2/src/util.rs @@ -1,9 +1,33 @@ -use std::path::{Path, PathBuf}; +use std::{ + env::current_dir, + path::{Path, PathBuf}, +}; use crate::ProofLayer; -/// The config parameters for non native field arithmetics is a *.config file. -const NON_NATIVE_PARAMS_EXT: &str = ".config"; +/// The config parameters for non native field arithmetics are in a *.config file. +pub const NON_NATIVE_PARAMS_EXT: &str = ".config"; + +/// The config parameters for non native field arithmetics are by default in this directory. +pub const NON_NATIVE_PARAMS_DIR: &str = ".config"; + +/// The KZG setup parameters are by default in this directory. +pub const KZG_PARAMS_DIR: &str = ".params"; + +/// The default directory used for cached data. +pub const CACHE_PATH: &str = ".cache"; + +/// The directory within cache to store proving tasks in JSON format. +pub const CACHE_PATH_TASKS: &str = "tasks"; + +/// The directory within cache to store proof outputs. +pub const CACHE_PATH_PROOFS: &str = "proofs"; + +/// The directory within cache to store public input data. +pub const CACHE_PATH_PI: &str = "pi"; + +/// The directory within cache to store Verifier contract code. +pub const CACHE_PATH_EVM: &str = "evm"; /// The path to the config parameters for a given proof layer. /// @@ -18,3 +42,24 @@ pub fn non_native_params_path(dir: &Path, layer: ProofLayer) -> PathBuf { pub fn kzg_params_path(dir: &Path, degree: u32) -> PathBuf { dir.join(format!("params{degree}")) } + +/// The default path to find non-native field arithmetic config params. +/// +/// /.config +pub fn default_non_native_params_dir() -> Result { + Ok(current_dir()?.join(NON_NATIVE_PARAMS_DIR)) +} + +/// The default path to find KZG setup parameters. +/// +/// /.params +pub fn default_kzg_params_dir() -> Result { + Ok(current_dir()?.join(KZG_PARAMS_DIR)) +} + +/// The default path to the cache directory. +/// +/// /.cache +pub fn default_cache_dir() -> Result { + Ok(current_dir()?.join(CACHE_PATH)) +} From 88c701202cca4575ef23c2ec1cb42a9979ec6173 Mon Sep 17 00:00:00 2001 From: Rohit Narurkar Date: Thu, 11 Jul 2024 13:47:56 +0100 Subject: [PATCH 04/17] wip: better err handling and basic setup test --- Cargo.lock | 1 + prover2/Cargo.toml | 1 + prover2/src/error.rs | 23 +++++++++--- prover2/src/prover/config.rs | 53 +++++++++++++++++++++------- prover2/src/types/mod.rs | 3 ++ prover2/src/util/fs.rs | 48 +++++++++++++++++++++++++ prover2/src/{util.rs => util/mod.rs} | 22 ++++++++---- 7 files changed, 127 insertions(+), 24 deletions(-) create mode 100644 prover2/src/util/fs.rs rename prover2/src/{util.rs => util/mod.rs} (75%) diff --git a/Cargo.lock b/Cargo.lock index ee29a85940..eadb83abed 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3497,6 +3497,7 @@ name = "prover2" version = "0.11.0" dependencies = [ "aggregator", + "anyhow", "halo2_proofs", "serde", "serde_json", diff --git a/prover2/Cargo.toml b/prover2/Cargo.toml index b7b0b33b57..5785618899 100644 --- a/prover2/Cargo.toml +++ b/prover2/Cargo.toml @@ -8,6 +8,7 @@ license.workspace = true aggregator = { path = "../aggregator" } zkevm-circuits = { path = "../zkevm-circuits" } +anyhow.workspace = true halo2_proofs.workspace = true serde.workspace = true serde_json.workspace = true diff --git a/prover2/src/error.rs b/prover2/src/error.rs index 0a9c6c5e19..603227162f 100644 --- a/prover2/src/error.rs +++ b/prover2/src/error.rs @@ -1,12 +1,27 @@ +use std::path::PathBuf; + use thiserror::Error; /// Represents error variants possibly encountered during the proof generation process. #[derive(Debug, Error)] pub enum ProverError { - /// Error encountered while doing I/O operations. + /// Error occurred while doing other I/O operations. #[error(transparent)] - IoError(#[from] std::io::Error), + OtherIo(#[from] std::io::Error), + /// Error encountered while reading from or writing to files. + #[error("an error occurred while reading/writing {path}: {source}")] + IoReadWrite { + /// The path we tried to read from or write to. + path: PathBuf, + /// The source error. + source: std::io::Error, + }, /// Error encountered during serialization/deserialization of JSON. - #[error(transparent)] - SerdeJsonError(#[from] serde_json::Error), + #[error("an error occurred while reading/writing json {path}: {source}")] + ReadWriteJson { + /// The path of the file we tried to serialize/deserialize. + path: PathBuf, + /// The source error. + source: serde_json::Error, + }, } diff --git a/prover2/src/prover/config.rs b/prover2/src/prover/config.rs index d0f1504c23..3dd59b7168 100644 --- a/prover2/src/prover/config.rs +++ b/prover2/src/prover/config.rs @@ -1,7 +1,6 @@ use std::{ collections::BTreeMap, - fs::{create_dir, create_dir_all, File}, - io::BufReader, + fs::{create_dir, create_dir_all}, marker::PhantomData, path::PathBuf, }; @@ -10,15 +9,14 @@ use halo2_proofs::{ halo2curves::bn256::{Bn256, G1Affine}, plonk::ProvingKey, poly::kzg::commitment::ParamsKZG, - SerdeFormat, }; use crate::{ types::ProverType, util::{ default_cache_dir, default_kzg_params_dir, default_non_native_params_dir, kzg_params_path, - non_native_params_path as nn_params_path, CACHE_PATH_EVM, CACHE_PATH_PI, CACHE_PATH_PROOFS, - CACHE_PATH_TASKS, + non_native_params_path as nn_params_path, read_json, read_kzg_params, CACHE_PATH_EVM, + CACHE_PATH_PI, CACHE_PATH_PROOFS, CACHE_PATH_TASKS, }, Params, ProofLayer, ProverError, }; @@ -101,19 +99,15 @@ impl ProverConfig { // Read and store non-native field arithmetic config params for each layer. for layer in proof_layers { - let params_file = File::open(nn_params_path(nn_params_dir.as_path(), layer))?; - let params = serde_json::from_reader(params_file)?; + let params_path = nn_params_path(nn_params_dir.as_path(), layer); + let params = read_json(params_path.as_path())?; self.nn_params.insert(layer, params); } // Read and store KZG setup params for each layer. for (&layer, nn_params) in self.nn_params.iter() { - let params_file = - File::open(kzg_params_path(kzg_params_dir.as_path(), nn_params.degree))?; - let params = ParamsKZG::::read_custom( - &mut BufReader::new(params_file), - SerdeFormat::RawBytesUnchecked, - )?; + let params_path = kzg_params_path(kzg_params_dir.as_path(), nn_params.degree); + let params = read_kzg_params(params_path.as_path())?; self.kzg_params.insert(layer, params); } @@ -127,3 +121,36 @@ impl ProverConfig { Ok(self) } } + +#[cfg(test)] +mod tests { + use std::env::current_dir; + + use crate::{ + types::{ProverTypeBatch, ProverTypeBundle, ProverTypeChunk}, + ProverConfig, + }; + + #[test] + fn setup_prover() -> anyhow::Result<()> { + let test_dir = current_dir()?.join("test_data"); + + let _chunk_prover_config = ProverConfig::::default() + .with_non_native_params_dir(test_dir.join(".config")) + .with_kzg_params_dir(test_dir.join(".params")) + .with_cache_dir(test_dir.join(".cache")) + .setup()?; + let _batch_prover_config = ProverConfig::::default() + .with_non_native_params_dir(test_dir.join(".config")) + .with_kzg_params_dir(test_dir.join(".params")) + .with_cache_dir(test_dir.join(".cache")) + .setup()?; + let _bundle_prover_config = ProverConfig::::default() + .with_non_native_params_dir(test_dir.join(".config")) + .with_kzg_params_dir(test_dir.join(".params")) + .with_cache_dir(test_dir.join(".cache")) + .setup()?; + + Ok(()) + } +} diff --git a/prover2/src/types/mod.rs b/prover2/src/types/mod.rs index 28ed8e4df5..f214892adc 100644 --- a/prover2/src/types/mod.rs +++ b/prover2/src/types/mod.rs @@ -8,8 +8,11 @@ pub trait ProverType { fn layers() -> Vec; } +#[derive(Default)] pub struct ProverTypeChunk; +#[derive(Default)] pub struct ProverTypeBatch; +#[derive(Default)] pub struct ProverTypeBundle; impl ProverType for ProverTypeChunk { diff --git a/prover2/src/util/fs.rs b/prover2/src/util/fs.rs new file mode 100644 index 0000000000..d5a5f24d5c --- /dev/null +++ b/prover2/src/util/fs.rs @@ -0,0 +1,48 @@ +use std::{ + fs::{self, File}, + io::BufReader, + path::Path, +}; + +use halo2_proofs::{halo2curves::bn256::Bn256, poly::kzg::commitment::ParamsKZG, SerdeFormat}; +use serde::de::DeserializeOwned; + +use crate::{ + ProverError, + ProverError::{IoReadWrite, ReadWriteJson}, +}; + +/// Wrapper functionality for opening a file. +pub fn open(path: &Path) -> Result { + File::open(path).map_err(|source| IoReadWrite { + source, + path: path.into(), + }) +} + +/// Wrapper functionality for reading bytes from a file. +pub fn read(path: impl AsRef) -> Result, ProverError> { + let path = path.as_ref(); + fs::read(path).map_err(|source| IoReadWrite { + source, + path: path.into(), + }) +} + +/// Wrapper functionality for reading a JSON file. +pub fn read_json(path: &Path) -> Result { + let bytes = read(path)?; + serde_json::from_slice(&bytes).map_err(|source| ReadWriteJson { + source, + path: path.into(), + }) +} + +/// Read KZG setup parameters that are in a custom serde format. +pub fn read_kzg_params(path: &Path) -> Result, ProverError> { + let f = open(path)?; + Ok(ParamsKZG::::read_custom( + &mut BufReader::new(f), + SerdeFormat::RawBytesUnchecked, + )?) +} diff --git a/prover2/src/util.rs b/prover2/src/util/mod.rs similarity index 75% rename from prover2/src/util.rs rename to prover2/src/util/mod.rs index c77261ed31..65c801ad77 100644 --- a/prover2/src/util.rs +++ b/prover2/src/util/mod.rs @@ -3,7 +3,10 @@ use std::{ path::{Path, PathBuf}, }; -use crate::ProofLayer; +use crate::{ProofLayer, ProverError}; + +mod fs; +pub use fs::{read_json, read_kzg_params}; /// The config parameters for non native field arithmetics are in a *.config file. pub const NON_NATIVE_PARAMS_EXT: &str = ".config"; @@ -43,23 +46,28 @@ pub fn kzg_params_path(dir: &Path, degree: u32) -> PathBuf { dir.join(format!("params{degree}")) } +/// Wrapper functionality for current working directory. +pub fn pwd() -> Result { + Ok(current_dir()?) +} + /// The default path to find non-native field arithmetic config params. /// /// /.config -pub fn default_non_native_params_dir() -> Result { - Ok(current_dir()?.join(NON_NATIVE_PARAMS_DIR)) +pub fn default_non_native_params_dir() -> Result { + Ok(pwd()?.join(NON_NATIVE_PARAMS_DIR)) } /// The default path to find KZG setup parameters. /// /// /.params -pub fn default_kzg_params_dir() -> Result { - Ok(current_dir()?.join(KZG_PARAMS_DIR)) +pub fn default_kzg_params_dir() -> Result { + Ok(pwd()?.join(KZG_PARAMS_DIR)) } /// The default path to the cache directory. /// /// /.cache -pub fn default_cache_dir() -> Result { - Ok(current_dir()?.join(CACHE_PATH)) +pub fn default_cache_dir() -> Result { + Ok(pwd()?.join(CACHE_PATH)) } From c3a4523b577d0790d5dc7132157af9a82d460d45 Mon Sep 17 00:00:00 2001 From: Rohit Narurkar Date: Thu, 11 Jul 2024 17:08:14 +0100 Subject: [PATCH 05/17] setup tracing --- .gitignore | 3 +- Cargo.lock | 86 ++++++++++++++++++++++++++++++- prover2/.gitignore | 1 + prover2/Cargo.toml | 5 +- prover2/Makefile | 9 ++++ prover2/scripts/download_setup.sh | 15 ++++++ prover2/src/error.rs | 16 ++++++ prover2/src/prover/config.rs | 61 ++++++++++++++++------ prover2/src/prover/params.rs | 2 +- prover2/src/types/layer.rs | 2 +- prover2/src/types/mod.rs | 8 +-- prover2/src/util/env.rs | 35 +++++++++++++ prover2/src/util/mod.rs | 9 ++++ 13 files changed, 225 insertions(+), 27 deletions(-) create mode 100644 prover2/.gitignore create mode 100644 prover2/Makefile create mode 100644 prover2/scripts/download_setup.sh create mode 100644 prover2/src/util/env.rs diff --git a/.gitignore b/.gitignore index 8ea299a653..7caf13d3bd 100644 --- a/.gitignore +++ b/.gitignore @@ -5,7 +5,6 @@ .idea *.log *.json -*.sh *.txt *.srs -tmp \ No newline at end of file +tmp diff --git a/Cargo.lock b/Cargo.lock index eadb83abed..155dd93a3e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2728,6 +2728,15 @@ dependencies = [ "winapi", ] +[[package]] +name = "matchers" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8263075bb86c5a1b1427b5ae862e8889656f126e9f77c484496e8b47cf5c5558" +dependencies = [ + "regex-automata 0.1.10", +] + [[package]] name = "maybe-rayon" version = "0.1.1" @@ -2832,6 +2841,16 @@ version = "1.0.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "650eef8c711430f1a879fdd01d4745a7deea475becfb90269c06775983bbf086" +[[package]] +name = "nu-ansi-term" +version = "0.46.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77a8165726e8236064dbb45459242600304b42a5ea24ee2948e18e023bf7ba84" +dependencies = [ + "overload", + "winapi", +] + [[package]] name = "num" version = "0.4.2" @@ -3024,6 +3043,12 @@ dependencies = [ "syn 1.0.109", ] +[[package]] +name = "overload" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b15813163c1d831bf4a13c3610c05c0d03b39feb07f7e09fa234dac9b15aaf39" + [[package]] name = "pairing" version = "0.23.0" @@ -3505,6 +3530,7 @@ dependencies = [ "snark-verifier-sdk", "thiserror", "tracing", + "tracing-subscriber", "zkevm-circuits", ] @@ -3625,10 +3651,19 @@ checksum = "c117dbdfde9c8308975b6a18d71f3f385c89461f7b3fb054288ecf2a2058ba4c" dependencies = [ "aho-corasick", "memchr", - "regex-automata", + "regex-automata 0.4.6", "regex-syntax 0.8.3", ] +[[package]] +name = "regex-automata" +version = "0.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c230d73fb8d8c1b9c0b3135c5142a8acee3a0558fb8db5cf1cb65f8d7862132" +dependencies = [ + "regex-syntax 0.6.29", +] + [[package]] name = "regex-automata" version = "0.4.6" @@ -4288,6 +4323,15 @@ dependencies = [ "cfg-if 1.0.0", ] +[[package]] +name = "sharded-slab" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f40ca3c46823713e0d4209592e8d6e826aa57e928f09752619fc696c499637f6" +dependencies = [ + "lazy_static", +] + [[package]] name = "signature" version = "2.2.0" @@ -4689,6 +4733,16 @@ dependencies = [ "winapi", ] +[[package]] +name = "thread_local" +version = "1.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b9ef9bad013ada3808854ceac7b46812a6465ba368859a37e2100283d2d719c" +dependencies = [ + "cfg-if 1.0.0", + "once_cell", +] + [[package]] name = "threadpool" version = "1.8.1" @@ -4897,6 +4951,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c06d3da6113f116aaee68e4d601191614c9053067f9ab7f6edbcb161237daa54" dependencies = [ "once_cell", + "valuable", ] [[package]] @@ -4909,6 +4964,35 @@ dependencies = [ "tracing", ] +[[package]] +name = "tracing-log" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee855f1f400bd0e5c02d150ae5de3840039a3f54b025156404e34c23c03f47c3" +dependencies = [ + "log", + "once_cell", + "tracing-core", +] + +[[package]] +name = "tracing-subscriber" +version = "0.3.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ad0f048c97dbd9faa9b7df56362b8ebcaa52adb06b498c050d2f4e32f90a7a8b" +dependencies = [ + "matchers", + "nu-ansi-term", + "once_cell", + "regex", + "sharded-slab", + "smallvec", + "thread_local", + "tracing", + "tracing-core", + "tracing-log", +] + [[package]] name = "try-lock" version = "0.2.5" diff --git a/prover2/.gitignore b/prover2/.gitignore new file mode 100644 index 0000000000..74c3f9fc83 --- /dev/null +++ b/prover2/.gitignore @@ -0,0 +1 @@ +test_data/ diff --git a/prover2/Cargo.toml b/prover2/Cargo.toml index 5785618899..707c3e722e 100644 --- a/prover2/Cargo.toml +++ b/prover2/Cargo.toml @@ -16,4 +16,7 @@ snark-verifier.workspace = true snark-verifier-sdk.workspace = true thiserror = "1.0" -tracing = "0.1.40" +tracing = "0.1" + +[dev-dependencies] +tracing-subscriber = { version = "0.3", features = ["env-filter", "std"] } diff --git a/prover2/Makefile b/prover2/Makefile new file mode 100644 index 0000000000..503096bc9a --- /dev/null +++ b/prover2/Makefile @@ -0,0 +1,9 @@ + +help: ## Display this help screen + @grep -h \ + -E '^[a-zA-Z_-]+:.*?## .*$$' $(MAKEFILE_LIST) | \ + awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}' +# usage: +# make download-setup -e degree=20 dir=./test_data/.params +download-setup: + sh scripts/download_setup.sh ${degree} ${dir} diff --git a/prover2/scripts/download_setup.sh b/prover2/scripts/download_setup.sh new file mode 100644 index 0000000000..d463724e3c --- /dev/null +++ b/prover2/scripts/download_setup.sh @@ -0,0 +1,15 @@ +set -x +set -e + +# Set degree to env SCROLL_PROVER_MAX_DEGREE, first input or default value 26. +degree="${SCROLL_PROVER_MAX_DEGREE:-${1:-26}}" + +# Set the output dir to second input or default as `./integration/params`. +dir="${2:-"./test_data/.params"}" +mkdir -p "$dir" + +file="$dir"/params"${degree}" +rm -f "$file" + +# degree 1 - 26 +axel -ac https://circuit-release.s3.us-west-2.amazonaws.com/setup/params"$degree" -o "$file" diff --git a/prover2/src/error.rs b/prover2/src/error.rs index 603227162f..2d914d4623 100644 --- a/prover2/src/error.rs +++ b/prover2/src/error.rs @@ -24,4 +24,20 @@ pub enum ProverError { /// The source error. source: serde_json::Error, }, + /// Error encountered while reading variable from the process environment. + #[error("an error occurred while reading variable from the environment {key}: {source}")] + EnvVar { + /// The key tried to be read. + key: String, + /// The source error. + source: std::env::VarError, + }, + /// Error encountered while parsing a string. + #[error("an error occurred while parsing string {src}: {err}")] + Parse { + /// Source string that we tried to parse. + src: String, + /// Parsing error. + err: String, + }, } diff --git a/prover2/src/prover/config.rs b/prover2/src/prover/config.rs index 3dd59b7168..389b7b0a0e 100644 --- a/prover2/src/prover/config.rs +++ b/prover2/src/prover/config.rs @@ -1,8 +1,5 @@ use std::{ - collections::BTreeMap, - fs::{create_dir, create_dir_all}, - marker::PhantomData, - path::PathBuf, + collections::BTreeMap, fs::create_dir_all, iter::once, marker::PhantomData, path::PathBuf, }; use halo2_proofs::{ @@ -10,19 +7,21 @@ use halo2_proofs::{ plonk::ProvingKey, poly::kzg::commitment::ParamsKZG, }; +use tracing::{debug, info, instrument, trace}; use crate::{ types::ProverType, util::{ default_cache_dir, default_kzg_params_dir, default_non_native_params_dir, kzg_params_path, - non_native_params_path as nn_params_path, read_json, read_kzg_params, CACHE_PATH_EVM, - CACHE_PATH_PI, CACHE_PATH_PROOFS, CACHE_PATH_TASKS, + non_native_params_path as nn_params_path, read_env_or_default, read_json, read_kzg_params, + CACHE_PATH_EVM, CACHE_PATH_PI, CACHE_PATH_PROOFS, CACHE_PATH_TASKS, DEFAULT_DEGREE_LAYER0, + ENV_DEGREE_LAYER0, }, Params, ProofLayer, ProverError, }; /// Configuration for a generic prover. -#[derive(Default)] +#[derive(Default, Debug)] pub struct ProverConfig { /// KZG setup parameters by proof layer. pub kzg_params: BTreeMap>, @@ -82,7 +81,10 @@ impl ProverConfig { impl ProverConfig { /// Setup the prover config by reading relevant config files from storage. + #[instrument(name = "ProverConfig::setup", skip(self))] pub fn setup(mut self) -> Result { + info!("setting up ProverConfig"); + // The proof layers that this prover needs to generate proofs for. let proof_layers = Type::layers(); @@ -98,25 +100,45 @@ impl ProverConfig { let cache_dir = self.cache_dir.clone().unwrap_or(default_cache_dir()?); // Read and store non-native field arithmetic config params for each layer. + trace!("loading non-native field arithmetic params"); for layer in proof_layers { - let params_path = nn_params_path(nn_params_dir.as_path(), layer); - let params = read_json(params_path.as_path())?; - self.nn_params.insert(layer, params); + // Layer0 (SuperCircuit) does not have non-native field arithmetics. + if layer != ProofLayer::Layer0 { + let params_path = nn_params_path(nn_params_dir.as_path(), layer); + debug!("reading config params for {:?}: {:?}", layer, params_path); + let params = read_json(params_path.as_path())?; + self.nn_params.insert(layer, params); + } } // Read and store KZG setup params for each layer. - for (&layer, nn_params) in self.nn_params.iter() { - let params_path = kzg_params_path(kzg_params_dir.as_path(), nn_params.degree); + trace!("loading KZG setup params"); + for (&layer, degree) in self + .nn_params + .iter() + .map(|(k, v)| (k, v.degree)) + .chain(once(( + &ProofLayer::Layer0, + read_env_or_default(ENV_DEGREE_LAYER0, DEFAULT_DEGREE_LAYER0), + ))) + { + let params_path = kzg_params_path(kzg_params_dir.as_path(), degree); + debug!( + "reading kzg params for {:?} (degree = {:?}): {:?}", + layer, degree, params_path + ); let params = read_kzg_params(params_path.as_path())?; self.kzg_params.insert(layer, params); } // Setup the cache directory's structure. - create_dir_all(cache_dir.as_path())?; - create_dir(cache_dir.join(CACHE_PATH_TASKS))?; - create_dir(cache_dir.join(CACHE_PATH_PROOFS))?; - create_dir(cache_dir.join(CACHE_PATH_PI))?; - create_dir(cache_dir.join(CACHE_PATH_EVM))?; + trace!("setting up cache"); + create_dir_all(cache_dir.join(CACHE_PATH_TASKS))?; + create_dir_all(cache_dir.join(CACHE_PATH_PROOFS))?; + create_dir_all(cache_dir.join(CACHE_PATH_PI))?; + create_dir_all(cache_dir.join(CACHE_PATH_EVM))?; + + info!("setup ProverConfig"); Ok(self) } @@ -133,6 +155,11 @@ mod tests { #[test] fn setup_prover() -> anyhow::Result<()> { + tracing_subscriber::fmt() + .with_env_filter(tracing_subscriber::EnvFilter::from_default_env()) + .pretty() + .init(); + let test_dir = current_dir()?.join("test_data"); let _chunk_prover_config = ProverConfig::::default() diff --git a/prover2/src/prover/params.rs b/prover2/src/prover/params.rs index 88473f096b..69509fa5db 100644 --- a/prover2/src/prover/params.rs +++ b/prover2/src/prover/params.rs @@ -2,7 +2,7 @@ use serde::{Deserialize, Serialize}; use snark_verifier::loader::halo2::halo2_ecc::fields::fp::FpStrategy; /// Parameters to configure the non-native field arithmetic chip. -#[derive(Serialize, Deserialize)] +#[derive(Debug, Serialize, Deserialize)] pub struct Params { pub strategy: FpStrategy, pub degree: u32, diff --git a/prover2/src/types/layer.rs b/prover2/src/types/layer.rs index 1a4d48ab57..dfb16ad4ba 100644 --- a/prover2/src/types/layer.rs +++ b/prover2/src/types/layer.rs @@ -1,6 +1,6 @@ use serde::{Deserialize, Serialize}; -#[derive(Clone, Copy, Serialize, Deserialize, Eq, PartialEq, PartialOrd, Ord)] +#[derive(Clone, Copy, Debug, Serialize, Deserialize, Eq, PartialEq, PartialOrd, Ord)] pub enum ProofLayer { Layer0, Layer1, diff --git a/prover2/src/types/mod.rs b/prover2/src/types/mod.rs index f214892adc..a404f1a798 100644 --- a/prover2/src/types/mod.rs +++ b/prover2/src/types/mod.rs @@ -4,15 +4,15 @@ pub mod layer; pub mod proof; pub mod task; -pub trait ProverType { +pub trait ProverType: std::fmt::Debug { fn layers() -> Vec; } -#[derive(Default)] +#[derive(Default, Debug)] pub struct ProverTypeChunk; -#[derive(Default)] +#[derive(Default, Debug)] pub struct ProverTypeBatch; -#[derive(Default)] +#[derive(Default, Debug)] pub struct ProverTypeBundle; impl ProverType for ProverTypeChunk { diff --git a/prover2/src/util/env.rs b/prover2/src/util/env.rs new file mode 100644 index 0000000000..7803841f79 --- /dev/null +++ b/prover2/src/util/env.rs @@ -0,0 +1,35 @@ +use std::{env::var, str::FromStr}; + +use crate::{ + ProverError, + ProverError::{EnvVar, Parse}, +}; + +/// Wrapper to read variable from the environment. +pub fn read_env(key: &str) -> Result { + var(key).map_err(|source| EnvVar { + source, + key: key.into(), + }) +} + +/// Read variable from the environment and parse to a generic type. +pub fn read_env_as(key: &str) -> Result +where + T::Err: std::error::Error, +{ + let src = read_env(key)?; + + src.parse::().map_err(|e| Parse { + src, + err: e.to_string(), + }) +} + +/// Read variable from the environment if set, otherwise return the provided default value. +pub fn read_env_or_default(key: &str, default: T) -> T +where + T::Err: std::error::Error, +{ + read_env_as::(key).unwrap_or(default) +} diff --git a/prover2/src/util/mod.rs b/prover2/src/util/mod.rs index 65c801ad77..db3dafe95a 100644 --- a/prover2/src/util/mod.rs +++ b/prover2/src/util/mod.rs @@ -8,6 +8,9 @@ use crate::{ProofLayer, ProverError}; mod fs; pub use fs::{read_json, read_kzg_params}; +mod env; +pub use env::read_env_or_default; + /// The config parameters for non native field arithmetics are in a *.config file. pub const NON_NATIVE_PARAMS_EXT: &str = ".config"; @@ -32,6 +35,12 @@ pub const CACHE_PATH_PI: &str = "pi"; /// The directory within cache to store Verifier contract code. pub const CACHE_PATH_EVM: &str = "evm"; +/// The environment variable to be set to configure custom degree for the super circuit (layer0). +pub const ENV_DEGREE_LAYER0: &str = "SUPER_CIRCUIT_DEGREE"; + +/// The default degree for the super circuit (layer0). +pub const DEFAULT_DEGREE_LAYER0: u32 = 20; + /// The path to the config parameters for a given proof layer. /// /// /{layer}.config From b5e2c7df3e4097ae7f1e35b574fa1f9f5299cb97 Mon Sep 17 00:00:00 2001 From: Rohit Narurkar Date: Thu, 11 Jul 2024 19:04:04 +0100 Subject: [PATCH 06/17] doc --- prover2/src/prover/mod.rs | 1 + prover2/src/types/layer.rs | 10 ++++++++++ prover2/src/types/mod.rs | 5 +++++ 3 files changed, 16 insertions(+) diff --git a/prover2/src/prover/mod.rs b/prover2/src/prover/mod.rs index 8237c0c26b..938c9d1897 100644 --- a/prover2/src/prover/mod.rs +++ b/prover2/src/prover/mod.rs @@ -22,6 +22,7 @@ pub struct Prover { } impl Prover { + /// Construct a new prover. pub fn new(config: ProverConfig) -> Self { Self { config } } diff --git a/prover2/src/types/layer.rs b/prover2/src/types/layer.rs index dfb16ad4ba..8035d57baf 100644 --- a/prover2/src/types/layer.rs +++ b/prover2/src/types/layer.rs @@ -1,13 +1,23 @@ use serde::{Deserialize, Serialize}; +/// Various layers in the proof generation process. #[derive(Clone, Copy, Debug, Serialize, Deserialize, Eq, PartialEq, PartialOrd, Ord)] pub enum ProofLayer { + /// The super circuit (ZkEVM) layer. This is the innermost proof layer. Layer0, + /// The compression layer on top of layer0. Layer1, + /// The compression layer on top of layer1. The proof from this layer is the [`ChunkProof`]. Layer2, + /// The batch circuit layer. At this layer, we batch multiple [`ChunkProof`]s. Layer3, + /// The compression layer on top of layer3. The proof from this layer is the [`BatchProof`]. Layer4, + /// The recursion circuit layer. At this layer, we construct proofs recursively over a previous + /// SNARK from the recursion circuit. Layer5, + /// The compression layer on top of layer5. The proof from this layer is the [`BundleProof`], + /// which is verified in EVM. Layer6, } diff --git a/prover2/src/types/mod.rs b/prover2/src/types/mod.rs index a404f1a798..c9a44d7b80 100644 --- a/prover2/src/types/mod.rs +++ b/prover2/src/types/mod.rs @@ -8,10 +8,15 @@ pub trait ProverType: std::fmt::Debug { fn layers() -> Vec; } +/// The chunk prover that constructs proofs at layer0, layer1 and layer2. #[derive(Default, Debug)] pub struct ProverTypeChunk; + +/// The batch prover that constructs proofs at layer3 and layer4. #[derive(Default, Debug)] pub struct ProverTypeBatch; + +/// The bundle prover that constructs proofs at layer5 and layer6. #[derive(Default, Debug)] pub struct ProverTypeBundle; From 32ac529dd2dca8315a950787ab361239fba5be53 Mon Sep 17 00:00:00 2001 From: Rohit Narurkar Date: Thu, 11 Jul 2024 23:45:57 +0100 Subject: [PATCH 07/17] wip: store degrees in prover config --- prover2/src/lib.rs | 2 +- prover2/src/prover/config.rs | 39 ++++++++++++++++++++---------------- prover2/src/prover/mod.rs | 17 +++++++++++++--- prover2/src/types/mod.rs | 13 +++++++++++- prover2/src/types/task.rs | 31 ++++++++++++++++++++++++++-- 5 files changed, 78 insertions(+), 24 deletions(-) diff --git a/prover2/src/lib.rs b/prover2/src/lib.rs index af466a601b..5005dc8bd1 100644 --- a/prover2/src/lib.rs +++ b/prover2/src/lib.rs @@ -5,7 +5,7 @@ mod prover; pub use prover::{config::ProverConfig, params::Params, BatchProver, BundleProver, ChunkProver}; mod types; -pub use types::{layer::ProofLayer, proof::Proof}; +pub use types::{layer::ProofLayer, proof::Proof, task::ProvingTask}; mod util; diff --git a/prover2/src/prover/config.rs b/prover2/src/prover/config.rs index 389b7b0a0e..3995ddc712 100644 --- a/prover2/src/prover/config.rs +++ b/prover2/src/prover/config.rs @@ -23,6 +23,8 @@ use crate::{ /// Configuration for a generic prover. #[derive(Default, Debug)] pub struct ProverConfig { + /// Polynomial degree used by proof generation layer. + pub degrees: BTreeMap, /// KZG setup parameters by proof layer. pub kzg_params: BTreeMap>, /// Config parameters for non-native field arithmetics by proof layer. @@ -32,7 +34,7 @@ pub struct ProverConfig { /// Optional directory to locate KZG setup parameters. pub kzg_params_dir: Option, /// Optional directory to locate non-native field arithmetic config params. - pub non_native_params_dir: Option, + pub nn_params_dir: Option, /// Optional directory to cache proofs. pub cache_dir: Option, @@ -67,8 +69,8 @@ impl ProverConfig { /// Returns prover config with a directory to load non-native field arithmetic config params /// from. - pub fn with_non_native_params_dir(mut self, dir: impl Into) -> Self { - self.non_native_params_dir = Some(dir.into()); + pub fn with_nn_params_dir(mut self, dir: impl Into) -> Self { + self.nn_params_dir = Some(dir.into()); self } @@ -90,7 +92,7 @@ impl ProverConfig { // Use the configured directories or fallback to current working directory. let nn_params_dir = self - .non_native_params_dir + .nn_params_dir .clone() .unwrap_or(default_non_native_params_dir()?); let kzg_params_dir = self @@ -106,22 +108,20 @@ impl ProverConfig { if layer != ProofLayer::Layer0 { let params_path = nn_params_path(nn_params_dir.as_path(), layer); debug!("reading config params for {:?}: {:?}", layer, params_path); - let params = read_json(params_path.as_path())?; + let params = read_json::(params_path.as_path())?; + self.degrees.insert(layer, params.degree); self.nn_params.insert(layer, params); } + + if layer == ProofLayer::Layer0 { + let layer0_degree = read_env_or_default(ENV_DEGREE_LAYER0, DEFAULT_DEGREE_LAYER0); + self.degrees.insert(ProofLayer::Layer0, layer0_degree); + } } // Read and store KZG setup params for each layer. trace!("loading KZG setup params"); - for (&layer, degree) in self - .nn_params - .iter() - .map(|(k, v)| (k, v.degree)) - .chain(once(( - &ProofLayer::Layer0, - read_env_or_default(ENV_DEGREE_LAYER0, DEFAULT_DEGREE_LAYER0), - ))) - { + for (&layer, °ree) in self.degrees.iter() { let params_path = kzg_params_path(kzg_params_dir.as_path(), degree); debug!( "reading kzg params for {:?} (degree = {:?}): {:?}", @@ -138,6 +138,11 @@ impl ProverConfig { create_dir_all(cache_dir.join(CACHE_PATH_PI))?; create_dir_all(cache_dir.join(CACHE_PATH_EVM))?; + // Update directories in self. + self.nn_params_dir.replace(nn_params_dir); + self.kzg_params_dir.replace(kzg_params_dir); + self.cache_dir.replace(cache_dir); + info!("setup ProverConfig"); Ok(self) @@ -163,17 +168,17 @@ mod tests { let test_dir = current_dir()?.join("test_data"); let _chunk_prover_config = ProverConfig::::default() - .with_non_native_params_dir(test_dir.join(".config")) + .with_nn_params_dir(test_dir.join(".config")) .with_kzg_params_dir(test_dir.join(".params")) .with_cache_dir(test_dir.join(".cache")) .setup()?; let _batch_prover_config = ProverConfig::::default() - .with_non_native_params_dir(test_dir.join(".config")) + .with_nn_params_dir(test_dir.join(".config")) .with_kzg_params_dir(test_dir.join(".params")) .with_cache_dir(test_dir.join(".cache")) .setup()?; let _bundle_prover_config = ProverConfig::::default() - .with_non_native_params_dir(test_dir.join(".config")) + .with_nn_params_dir(test_dir.join(".config")) .with_kzg_params_dir(test_dir.join(".params")) .with_cache_dir(test_dir.join(".cache")) .setup()?; diff --git a/prover2/src/prover/mod.rs b/prover2/src/prover/mod.rs index 938c9d1897..8e2228e056 100644 --- a/prover2/src/prover/mod.rs +++ b/prover2/src/prover/mod.rs @@ -1,5 +1,7 @@ +use tracing::instrument; + use crate::{ - types::{ProverTypeBatch, ProverTypeBundle, ProverTypeChunk}, + types::{ProverType, ProverTypeBatch, ProverTypeBundle, ProverTypeChunk}, Proof, ProverConfig, ProverError, }; @@ -16,6 +18,7 @@ pub type BatchProver = Prover; pub type BundleProver = Prover; /// A generic prover that is capable of generating proofs for given tasks. +#[derive(Debug)] pub struct Prover { /// Config for the prover. pub config: ProverConfig, @@ -28,8 +31,16 @@ impl Prover { } } -impl Prover { - pub fn gen_proof() -> Result { +impl Prover { + /// Generate a proof for the given task. + #[instrument(name = "Prover::gen_proof", skip(self))] + pub fn gen_proof(&mut self, _task: Type::Task) -> Result { + // try to fetch proof from cache. + + // generate SNARK. + + // write proof to cache. + unimplemented!() } } diff --git a/prover2/src/types/mod.rs b/prover2/src/types/mod.rs index c9a44d7b80..10f61f6bad 100644 --- a/prover2/src/types/mod.rs +++ b/prover2/src/types/mod.rs @@ -1,10 +1,15 @@ -use crate::ProofLayer; +use crate::{ProofLayer, ProvingTask}; + +use self::task::{BatchProvingTask, BundleProvingTask, ChunkProvingTask}; pub mod layer; pub mod proof; pub mod task; pub trait ProverType: std::fmt::Debug { + type Task: ProvingTask; + + /// The prover supports proof generation at the following layers. fn layers() -> Vec; } @@ -21,18 +26,24 @@ pub struct ProverTypeBatch; pub struct ProverTypeBundle; impl ProverType for ProverTypeChunk { + type Task = ChunkProvingTask; + fn layers() -> Vec { vec![ProofLayer::Layer0, ProofLayer::Layer1, ProofLayer::Layer2] } } impl ProverType for ProverTypeBatch { + type Task = BatchProvingTask; + fn layers() -> Vec { vec![ProofLayer::Layer3, ProofLayer::Layer4] } } impl ProverType for ProverTypeBundle { + type Task = BundleProvingTask; + fn layers() -> Vec { vec![ProofLayer::Layer5, ProofLayer::Layer6] } diff --git a/prover2/src/types/task.rs b/prover2/src/types/task.rs index b0a2f95f45..a6ce91d043 100644 --- a/prover2/src/types/task.rs +++ b/prover2/src/types/task.rs @@ -1,8 +1,35 @@ -use serde::{de::DeserializeOwned, ser::Serialize}; +use serde::{de::DeserializeOwned, Deserialize, Serialize}; /// Describes the behaviour to be supported by a type that can be used as an input (or a task) to /// instruct a [`Prover`] to generate a proof. -trait ProvingTask: Serialize + DeserializeOwned { +pub trait ProvingTask: Serialize + DeserializeOwned + std::fmt::Debug { /// An identifier for the proving task. fn id(&self) -> String; } + +#[derive(Debug, Serialize, Deserialize)] +pub struct ChunkProvingTask; + +#[derive(Debug, Serialize, Deserialize)] +pub struct BatchProvingTask; + +#[derive(Debug, Serialize, Deserialize)] +pub struct BundleProvingTask; + +impl ProvingTask for ChunkProvingTask { + fn id(&self) -> String { + "chunk".into() + } +} + +impl ProvingTask for BatchProvingTask { + fn id(&self) -> String { + "batch".into() + } +} + +impl ProvingTask for BundleProvingTask { + fn id(&self) -> String { + "bundle".into() + } +} From b34f6163001c8188bd885403926de6be831e665d Mon Sep 17 00:00:00 2001 From: Rohit Narurkar Date: Fri, 12 Jul 2024 00:42:22 +0100 Subject: [PATCH 08/17] wip: small update --- prover2/src/prover/config.rs | 19 ++++++++++++++----- prover2/src/prover/mod.rs | 14 ++++++++------ prover2/src/util/mod.rs | 6 ++++++ 3 files changed, 28 insertions(+), 11 deletions(-) diff --git a/prover2/src/prover/config.rs b/prover2/src/prover/config.rs index 3995ddc712..6eb127fa15 100644 --- a/prover2/src/prover/config.rs +++ b/prover2/src/prover/config.rs @@ -1,6 +1,4 @@ -use std::{ - collections::BTreeMap, fs::create_dir_all, iter::once, marker::PhantomData, path::PathBuf, -}; +use std::{collections::BTreeMap, fs::create_dir_all, marker::PhantomData, path::PathBuf}; use halo2_proofs::{ halo2curves::bn256::{Bn256, G1Affine}, @@ -14,8 +12,8 @@ use crate::{ util::{ default_cache_dir, default_kzg_params_dir, default_non_native_params_dir, kzg_params_path, non_native_params_path as nn_params_path, read_env_or_default, read_json, read_kzg_params, - CACHE_PATH_EVM, CACHE_PATH_PI, CACHE_PATH_PROOFS, CACHE_PATH_TASKS, DEFAULT_DEGREE_LAYER0, - ENV_DEGREE_LAYER0, + CACHE_PATH_EVM, CACHE_PATH_PI, CACHE_PATH_PROOFS, CACHE_PATH_SNARKS, CACHE_PATH_TASKS, + DEFAULT_DEGREE_LAYER0, ENV_DEGREE_LAYER0, JSON_EXT, }, Params, ProofLayer, ProverError, }; @@ -134,6 +132,7 @@ impl ProverConfig { // Setup the cache directory's structure. trace!("setting up cache"); create_dir_all(cache_dir.join(CACHE_PATH_TASKS))?; + create_dir_all(cache_dir.join(CACHE_PATH_SNARKS))?; create_dir_all(cache_dir.join(CACHE_PATH_PROOFS))?; create_dir_all(cache_dir.join(CACHE_PATH_PI))?; create_dir_all(cache_dir.join(CACHE_PATH_EVM))?; @@ -149,6 +148,16 @@ impl ProverConfig { } } +impl ProverConfig { + /// Returns the path to a proof with the given identifier if caching is enabled in the prover's + /// config. + pub fn path_proof(&self, id: &str) -> Option { + self.cache_dir + .as_ref() + .map(|dir| dir.join(CACHE_PATH_PROOFS).join(id).join(JSON_EXT)) + } +} + #[cfg(test)] mod tests { use std::env::current_dir; diff --git a/prover2/src/prover/mod.rs b/prover2/src/prover/mod.rs index 8e2228e056..ff79fa048b 100644 --- a/prover2/src/prover/mod.rs +++ b/prover2/src/prover/mod.rs @@ -34,12 +34,14 @@ impl Prover { impl Prover { /// Generate a proof for the given task. #[instrument(name = "Prover::gen_proof", skip(self))] - pub fn gen_proof(&mut self, _task: Type::Task) -> Result { - // try to fetch proof from cache. - - // generate SNARK. - - // write proof to cache. + pub fn gen_proof(&mut self, task: Type::Task) -> Result { + // generate SNARKs for the different layers. + // + // - re-use SNARK if cache hit and early return + // - gen SNARK + // - write SNARK to cache + + // dump outermost SNARK proof to cache. unimplemented!() } diff --git a/prover2/src/util/mod.rs b/prover2/src/util/mod.rs index db3dafe95a..c8df4884e4 100644 --- a/prover2/src/util/mod.rs +++ b/prover2/src/util/mod.rs @@ -11,6 +11,9 @@ pub use fs::{read_json, read_kzg_params}; mod env; pub use env::read_env_or_default; +/// The extension used for JSON files. +pub const JSON_EXT: &str = ".json"; + /// The config parameters for non native field arithmetics are in a *.config file. pub const NON_NATIVE_PARAMS_EXT: &str = ".config"; @@ -26,6 +29,9 @@ pub const CACHE_PATH: &str = ".cache"; /// The directory within cache to store proving tasks in JSON format. pub const CACHE_PATH_TASKS: &str = "tasks"; +/// The directory within cache to store SNARKs generated at intermediate proving layers. +pub const CACHE_PATH_SNARKS: &str = "snarks"; + /// The directory within cache to store proof outputs. pub const CACHE_PATH_PROOFS: &str = "proofs"; From 3ae53887f49a1a4b4589f7fe882fa3196b06fe6b Mon Sep 17 00:00:00 2001 From: Rohit Narurkar Date: Fri, 12 Jul 2024 02:29:56 +0100 Subject: [PATCH 09/17] wip: extend prover type trait --- Cargo.lock | 1 + prover2/Cargo.toml | 1 + prover2/src/error.rs | 3 ++ prover2/src/prover/config.rs | 12 ++--- prover2/src/prover/mod.rs | 5 +- prover2/src/types/mod.rs | 97 ++++++++++++++++++++++++++++++++---- prover2/src/types/task.rs | 4 +- 7 files changed, 103 insertions(+), 20 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 155dd93a3e..9956d9611b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3523,6 +3523,7 @@ version = "0.11.0" dependencies = [ "aggregator", "anyhow", + "gadgets", "halo2_proofs", "serde", "serde_json", diff --git a/prover2/Cargo.toml b/prover2/Cargo.toml index 707c3e722e..867de64deb 100644 --- a/prover2/Cargo.toml +++ b/prover2/Cargo.toml @@ -6,6 +6,7 @@ license.workspace = true [dependencies] aggregator = { path = "../aggregator" } +gadgets = { path = "../gadgets" } zkevm-circuits = { path = "../zkevm-circuits" } anyhow.workspace = true diff --git a/prover2/src/error.rs b/prover2/src/error.rs index 2d914d4623..0e11abb9ca 100644 --- a/prover2/src/error.rs +++ b/prover2/src/error.rs @@ -40,4 +40,7 @@ pub enum ProverError { /// Parsing error. err: String, }, + /// Custom error. + #[error("custom error: {0}")] + Custom(String), } diff --git a/prover2/src/prover/config.rs b/prover2/src/prover/config.rs index 6eb127fa15..247d6c8455 100644 --- a/prover2/src/prover/config.rs +++ b/prover2/src/prover/config.rs @@ -162,8 +162,10 @@ impl ProverConfig { mod tests { use std::env::current_dir; + use aggregator::MAX_AGG_SNARKS; + use crate::{ - types::{ProverTypeBatch, ProverTypeBundle, ProverTypeChunk}, + types::{ProverTypeBatch, ProverTypeChunk}, ProverConfig, }; @@ -181,12 +183,8 @@ mod tests { .with_kzg_params_dir(test_dir.join(".params")) .with_cache_dir(test_dir.join(".cache")) .setup()?; - let _batch_prover_config = ProverConfig::::default() - .with_nn_params_dir(test_dir.join(".config")) - .with_kzg_params_dir(test_dir.join(".params")) - .with_cache_dir(test_dir.join(".cache")) - .setup()?; - let _bundle_prover_config = ProverConfig::::default() + + let _batch_prover_config = ProverConfig::>::default() .with_nn_params_dir(test_dir.join(".config")) .with_kzg_params_dir(test_dir.join(".params")) .with_cache_dir(test_dir.join(".cache")) diff --git a/prover2/src/prover/mod.rs b/prover2/src/prover/mod.rs index ff79fa048b..f9fdf016e8 100644 --- a/prover2/src/prover/mod.rs +++ b/prover2/src/prover/mod.rs @@ -1,3 +1,4 @@ +use aggregator::MAX_AGG_SNARKS; use tracing::instrument; use crate::{ @@ -12,7 +13,7 @@ pub mod params; pub type ChunkProver = Prover; /// Convenience type for batch prover. -pub type BatchProver = Prover; +pub type BatchProver = Prover>; /// Convenience type for bundle prover. pub type BundleProver = Prover; @@ -40,6 +41,8 @@ impl Prover { // - re-use SNARK if cache hit and early return // - gen SNARK // - write SNARK to cache + let _base_layer = Type::base_layer()?; + let _compression_layers = Type::compression_layers(); // dump outermost SNARK proof to cache. diff --git a/prover2/src/types/mod.rs b/prover2/src/types/mod.rs index 10f61f6bad..b634578437 100644 --- a/prover2/src/types/mod.rs +++ b/prover2/src/types/mod.rs @@ -1,16 +1,65 @@ -use crate::{ProofLayer, ProvingTask}; +use aggregator::{AggregationCircuit, CompressionCircuit}; +use halo2_proofs::{ + halo2curves::bn256::{Bn256, Fr}, + plonk::Circuit, + poly::kzg::commitment::ParamsKZG, +}; +use snark_verifier_sdk::{CircuitExt, Snark}; +use zkevm_circuits::super_circuit::params::ScrollSuperCircuit; -use self::task::{BatchProvingTask, BundleProvingTask, ChunkProvingTask}; +use crate::{ProofLayer, ProverError, ProvingTask}; pub mod layer; + pub mod proof; + pub mod task; +use task::{BatchProvingTask, ChunkProvingTask}; pub trait ProverType: std::fmt::Debug { + /// The name of the prover. + const NAME: &'static str; + + /// The proving task that provides the relevant values required by the prover type to build its + /// base circuit. type Task: ProvingTask; + /// The circuit used at the base layer of this prover type. + type BaseCircuit: Circuit + CircuitExt; + + /// The compression circuit used to compress the base layer SNARK one or more times before + /// finally producing the outermost layer's SNARK. + type CompressionCircuit: Circuit + CircuitExt; + /// The prover supports proof generation at the following layers. fn layers() -> Vec; + + /// Returns the base layer. + fn base_layer() -> Result { + Self::layers() + .first() + .ok_or(ProverError::Custom(format!( + "no base layer for {}", + Self::NAME + ))) + .copied() + } + + /// Returns the subsequent layers after the base layer, i.e. the layers where the previous + /// layer's SNARK is compressed. + fn compression_layers() -> Vec { + Self::layers()[1..].to_vec() + } + + /// Builds the base circuit given witness in the proving task. + fn build_base(task: Self::Task) -> Self::BaseCircuit; + + /// Builds the compression circuit given the previous layer's SNARK. + fn build_compression( + kzg_params: &ParamsKZG, + prev_snark: Snark, + layer: ProofLayer, + ) -> Self::CompressionCircuit; } /// The chunk prover that constructs proofs at layer0, layer1 and layer2. @@ -19,32 +68,60 @@ pub struct ProverTypeChunk; /// The batch prover that constructs proofs at layer3 and layer4. #[derive(Default, Debug)] -pub struct ProverTypeBatch; +pub struct ProverTypeBatch; /// The bundle prover that constructs proofs at layer5 and layer6. #[derive(Default, Debug)] pub struct ProverTypeBundle; impl ProverType for ProverTypeChunk { + const NAME: &'static str = "ChunkProver"; + type Task = ChunkProvingTask; + type BaseCircuit = ScrollSuperCircuit; + + type CompressionCircuit = CompressionCircuit; + fn layers() -> Vec { vec![ProofLayer::Layer0, ProofLayer::Layer1, ProofLayer::Layer2] } + + fn build_base(_task: Self::Task) -> Self::BaseCircuit { + unimplemented!() + } + + fn build_compression( + _params: &ParamsKZG, + _prev_snark: Snark, + _layer: ProofLayer, + ) -> Self::CompressionCircuit { + unimplemented!() + } } -impl ProverType for ProverTypeBatch { - type Task = BatchProvingTask; +impl ProverType for ProverTypeBatch { + const NAME: &'static str = "BatchProver"; + + type Task = BatchProvingTask; + + type BaseCircuit = AggregationCircuit; + + type CompressionCircuit = CompressionCircuit; fn layers() -> Vec { vec![ProofLayer::Layer3, ProofLayer::Layer4] } -} -impl ProverType for ProverTypeBundle { - type Task = BundleProvingTask; + fn build_base(_task: Self::Task) -> Self::BaseCircuit { + unimplemented!() + } - fn layers() -> Vec { - vec![ProofLayer::Layer5, ProofLayer::Layer6] + fn build_compression( + _params: &ParamsKZG, + _prev_snark: Snark, + _layer: ProofLayer, + ) -> Self::CompressionCircuit { + unimplemented!() } } diff --git a/prover2/src/types/task.rs b/prover2/src/types/task.rs index a6ce91d043..d281d1a88e 100644 --- a/prover2/src/types/task.rs +++ b/prover2/src/types/task.rs @@ -11,7 +11,7 @@ pub trait ProvingTask: Serialize + DeserializeOwned + std::fmt::Debug { pub struct ChunkProvingTask; #[derive(Debug, Serialize, Deserialize)] -pub struct BatchProvingTask; +pub struct BatchProvingTask; #[derive(Debug, Serialize, Deserialize)] pub struct BundleProvingTask; @@ -22,7 +22,7 @@ impl ProvingTask for ChunkProvingTask { } } -impl ProvingTask for BatchProvingTask { +impl ProvingTask for BatchProvingTask { fn id(&self) -> String { "batch".into() } From 1fe7d6ab70d020a4f6cc6981a9ad8197b5f6066c Mon Sep 17 00:00:00 2001 From: Rohit Narurkar Date: Fri, 12 Jul 2024 03:02:32 +0100 Subject: [PATCH 10/17] wip: base and compression --- prover2/src/error.rs | 6 ++++++ prover2/src/prover/config.rs | 9 +++++++++ prover2/src/prover/mod.rs | 31 ++++++++++++++++++++++--------- 3 files changed, 37 insertions(+), 9 deletions(-) diff --git a/prover2/src/error.rs b/prover2/src/error.rs index 0e11abb9ca..1ec55be7ee 100644 --- a/prover2/src/error.rs +++ b/prover2/src/error.rs @@ -2,6 +2,8 @@ use std::path::PathBuf; use thiserror::Error; +use crate::ProofLayer; + /// Represents error variants possibly encountered during the proof generation process. #[derive(Debug, Error)] pub enum ProverError { @@ -40,6 +42,10 @@ pub enum ProverError { /// Parsing error. err: String, }, + /// Error that indicates the KZG setup parameters for specified layer are missing from prover + /// config. + #[error("prover {0} missing KZG setup params for {1:?}")] + MissingKzgParams(String, ProofLayer), /// Custom error. #[error("custom error: {0}")] Custom(String), diff --git a/prover2/src/prover/config.rs b/prover2/src/prover/config.rs index 247d6c8455..5b8787ef0f 100644 --- a/prover2/src/prover/config.rs +++ b/prover2/src/prover/config.rs @@ -158,6 +158,15 @@ impl ProverConfig { } } +impl ProverConfig { + /// Returns the KZG setup parameters for the proof layer. + pub fn kzg_params(&self, layer: ProofLayer) -> Result<&ParamsKZG, ProverError> { + self.kzg_params + .get(&layer) + .ok_or(ProverError::MissingKzgParams(Type::NAME.into(), layer)) + } +} + #[cfg(test)] mod tests { use std::env::current_dir; diff --git a/prover2/src/prover/mod.rs b/prover2/src/prover/mod.rs index f9fdf016e8..e888e31839 100644 --- a/prover2/src/prover/mod.rs +++ b/prover2/src/prover/mod.rs @@ -1,9 +1,11 @@ use aggregator::MAX_AGG_SNARKS; +use halo2_proofs::{halo2curves::bn256::Fr, plonk::Circuit}; +use snark_verifier_sdk::{CircuitExt, Snark}; use tracing::instrument; use crate::{ types::{ProverType, ProverTypeBatch, ProverTypeBundle, ProverTypeChunk}, - Proof, ProverConfig, ProverError, + Proof, ProofLayer, ProverConfig, ProverError, }; pub mod config; @@ -36,16 +38,27 @@ impl Prover { /// Generate a proof for the given task. #[instrument(name = "Prover::gen_proof", skip(self))] pub fn gen_proof(&mut self, task: Type::Task) -> Result { - // generate SNARKs for the different layers. - // - // - re-use SNARK if cache hit and early return - // - gen SNARK - // - write SNARK to cache - let _base_layer = Type::base_layer()?; - let _compression_layers = Type::compression_layers(); + // Generate SNARKs for all the layers at which the prover operates. We start from the base + // layer, i.e. the innermost layer and compress SNARKs until we have the SNARK of the + // outermost layer for this prover. + let base_layer = Type::base_layer()?; + let base_circuit = Type::build_base(task); + let mut snark = self.gen_snark(base_layer, &base_circuit)?; - // dump outermost SNARK proof to cache. + for layer in Type::compression_layers() { + let kzg_params = self.config.kzg_params(layer)?; + let compression_circuit = Type::build_compression(kzg_params, snark, layer); + snark = self.gen_snark(layer, &compression_circuit)?; + } unimplemented!() } + + /// Generate a SNARK for the given circuit. + fn gen_snark(&mut self, _layer: ProofLayer, _circuit: &C) -> Result + where + C: Circuit + CircuitExt, + { + unimplemented!() + } } From 85cabf4f62b75b796861464a524eeafb68365a09 Mon Sep 17 00:00:00 2001 From: Rohit Narurkar Date: Fri, 12 Jul 2024 18:58:10 +0100 Subject: [PATCH 11/17] more progress --- Cargo.lock | 7 +++ prover2/Cargo.toml | 7 +++ prover2/src/error.rs | 12 ++++ prover2/src/prover/config.rs | 51 ++++++++++++--- prover2/src/prover/mod.rs | 60 ++++++++++++++---- prover2/src/types/layer.rs | 2 +- prover2/src/types/mod.rs | 47 ++++++++++---- prover2/src/types/proof.rs | 91 ++++++++++++++++++++++++++- prover2/src/util/dir.rs | 76 +++++++++++++++++++++++ prover2/src/util/{fs.rs => io.rs} | 22 ++++++- prover2/src/util/mod.rs | 100 ++++++++---------------------- prover2/src/util/serde.rs | 15 +++++ 12 files changed, 381 insertions(+), 109 deletions(-) create mode 100644 prover2/src/util/dir.rs rename prover2/src/util/{fs.rs => io.rs} (65%) create mode 100644 prover2/src/util/serde.rs diff --git a/Cargo.lock b/Cargo.lock index 9956d9611b..5ddde44887 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3523,8 +3523,15 @@ version = "0.11.0" dependencies = [ "aggregator", "anyhow", + "base64 0.13.1", + "eth-types", + "ethers-core", "gadgets", + "git-version", "halo2_proofs", + "once_cell", + "rand", + "rand_xorshift", "serde", "serde_json", "snark-verifier", diff --git a/prover2/Cargo.toml b/prover2/Cargo.toml index 867de64deb..25c0ae6474 100644 --- a/prover2/Cargo.toml +++ b/prover2/Cargo.toml @@ -6,16 +6,23 @@ license.workspace = true [dependencies] aggregator = { path = "../aggregator" } +eth-types = { path = "../eth-types" } gadgets = { path = "../gadgets" } zkevm-circuits = { path = "../zkevm-circuits" } anyhow.workspace = true +base64.workspace = true +ethers-core.workspace = true halo2_proofs.workspace = true +once_cell.workspace = true +rand.workspace = true +rand_xorshift.workspace = true serde.workspace = true serde_json.workspace = true snark-verifier.workspace = true snark-verifier-sdk.workspace = true +git-version = "0.3.5" thiserror = "1.0" tracing = "0.1" diff --git a/prover2/src/error.rs b/prover2/src/error.rs index 1ec55be7ee..788cdb10be 100644 --- a/prover2/src/error.rs +++ b/prover2/src/error.rs @@ -10,6 +10,9 @@ pub enum ProverError { /// Error occurred while doing other I/O operations. #[error(transparent)] OtherIo(#[from] std::io::Error), + /// Error occurred while doing other serialization/deserialization operations. + #[error(transparent)] + OtherSerde(#[from] serde_json::Error), /// Error encountered while reading from or writing to files. #[error("an error occurred while reading/writing {path}: {source}")] IoReadWrite { @@ -46,6 +49,15 @@ pub enum ProverError { /// config. #[error("prover {0} missing KZG setup params for {1:?}")] MissingKzgParams(String, ProofLayer), + /// Error that indicates the proving key for specified layer is missing from prover config. + #[error("prover {0} missing proving key for {1:?}")] + MissingProvingKey(String, ProofLayer), + /// Error that has occurred during keygen process. + #[error("an error occurred during keygen process for prover={0} layer={1:?}: {2}")] + Keygen(String, ProofLayer, halo2_proofs::plonk::Error), + /// Error that has occurred during SNARK generation process. + #[error("an error occurred during SNARK generation for task={0} layer={1:?}: {2}")] + GenSnark(String, ProofLayer, halo2_proofs::plonk::Error), /// Custom error. #[error("custom error: {0}")] Custom(String), diff --git a/prover2/src/prover/config.rs b/prover2/src/prover/config.rs index 5b8787ef0f..d5ecc4de2a 100644 --- a/prover2/src/prover/config.rs +++ b/prover2/src/prover/config.rs @@ -1,8 +1,8 @@ -use std::{collections::BTreeMap, fs::create_dir_all, marker::PhantomData, path::PathBuf}; +use std::{collections::HashMap, fs::create_dir_all, marker::PhantomData, path::PathBuf}; use halo2_proofs::{ - halo2curves::bn256::{Bn256, G1Affine}, - plonk::ProvingKey, + halo2curves::bn256::{Bn256, Fr, G1Affine}, + plonk::{keygen_pk2, Circuit, ProvingKey}, poly::kzg::commitment::ParamsKZG, }; use tracing::{debug, info, instrument, trace}; @@ -22,13 +22,13 @@ use crate::{ #[derive(Default, Debug)] pub struct ProverConfig { /// Polynomial degree used by proof generation layer. - pub degrees: BTreeMap, + pub degrees: HashMap, /// KZG setup parameters by proof layer. - pub kzg_params: BTreeMap>, + pub kzg_params: HashMap>, /// Config parameters for non-native field arithmetics by proof layer. - pub nn_params: BTreeMap, + pub nn_params: HashMap, /// Proving keys by proof layer. - pub pks: BTreeMap>, + pub pks: HashMap>, /// Optional directory to locate KZG setup parameters. pub kzg_params_dir: Option, /// Optional directory to locate non-native field arithmetic config params. @@ -79,6 +79,9 @@ impl ProverConfig { } } +/// Convenience type. +type KzgParamsAndPk<'a> = (&'a ParamsKZG, &'a ProvingKey); + impl ProverConfig { /// Setup the prover config by reading relevant config files from storage. #[instrument(name = "ProverConfig::setup", skip(self))] @@ -146,6 +149,25 @@ impl ProverConfig { Ok(self) } + + /// Returns the proving key for the proof layer. + pub fn gen_proving_key>( + &mut self, + layer: ProofLayer, + circuit: &C, + ) -> Result { + if self.pks.contains_key(&layer) { + return Ok((&self.kzg_params[&layer], &self.pks[&layer])); + } + + // Generate proving key for the circuit and insert into the cached map. + let kzg_params = self.kzg_params(layer)?; + let pk = keygen_pk2(kzg_params, circuit) + .map_err(|e| ProverError::Keygen(Type::NAME.to_string(), layer, e))?; + self.pks.insert(layer, pk); + + Ok((&self.kzg_params[&layer], &self.pks[&layer])) + } } impl ProverConfig { @@ -156,6 +178,13 @@ impl ProverConfig { .as_ref() .map(|dir| dir.join(CACHE_PATH_PROOFS).join(id).join(JSON_EXT)) } + + /// Returns the path to cache the generated SNARK. + pub fn path_snark(&self, id: &str, layer: ProofLayer) -> Option { + self.cache_dir + .as_ref() + .map(|dir| dir.join(CACHE_PATH_SNARKS).join(format!("{layer:?}-{id}"))) + } } impl ProverConfig { @@ -165,6 +194,14 @@ impl ProverConfig { .get(&layer) .ok_or(ProverError::MissingKzgParams(Type::NAME.into(), layer)) } + + /// Returns the proving key for the proof layer. + pub fn proving_key(&self, layer: ProofLayer) -> Result<&ProvingKey, ProverError> { + self.pks.get(&layer).ok_or(ProverError::MissingProvingKey( + Type::NAME.to_string(), + layer, + )) + } } #[cfg(test)] diff --git a/prover2/src/prover/mod.rs b/prover2/src/prover/mod.rs index e888e31839..459ceb2b35 100644 --- a/prover2/src/prover/mod.rs +++ b/prover2/src/prover/mod.rs @@ -1,11 +1,12 @@ use aggregator::MAX_AGG_SNARKS; -use halo2_proofs::{halo2curves::bn256::Fr, plonk::Circuit}; +use halo2_proofs::halo2curves::bn256::Fr; use snark_verifier_sdk::{CircuitExt, Snark}; use tracing::instrument; use crate::{ types::{ProverType, ProverTypeBatch, ProverTypeBundle, ProverTypeChunk}, - Proof, ProofLayer, ProverConfig, ProverError, + util::{gen_rng, read_json, write_json}, + Proof, ProofLayer, ProverConfig, ProverError, ProvingTask, }; pub mod config; @@ -37,28 +38,61 @@ impl Prover { impl Prover { /// Generate a proof for the given task. #[instrument(name = "Prover::gen_proof", skip(self))] - pub fn gen_proof(&mut self, task: Type::Task) -> Result { - // Generate SNARKs for all the layers at which the prover operates. We start from the base - // layer, i.e. the innermost layer and compress SNARKs until we have the SNARK of the - // outermost layer for this prover. + pub fn gen_proof( + &mut self, + task: Type::Task, + ) -> Result, ProverError> { + // Early return if the proof for the given task is already available in cache. + let id = task.id(); + let path_proof = self.config.path_proof(&id); + if let Some(path) = &path_proof { + if let Ok(proof) = read_json(path) { + return Ok(proof); + } + } + + // Generate SNARKs for all the layers at which the prover operates. + // + // We start from the base layer, i.e. the innermost layer for the prover. let base_layer = Type::base_layer()?; - let base_circuit = Type::build_base(task); - let mut snark = self.gen_snark(base_layer, &base_circuit)?; + let (base_circuit, aux_data) = Type::build_base(task); + let mut snark = self.gen_snark(&id, base_layer, base_circuit)?; + // The base layer's SNARK is compressed for every layer of compression. for layer in Type::compression_layers() { let kzg_params = self.config.kzg_params(layer)?; let compression_circuit = Type::build_compression(kzg_params, snark, layer); - snark = self.gen_snark(layer, &compression_circuit)?; + snark = self.gen_snark(&id, layer, compression_circuit)?; + } + + // We have the final compressed SNARK for the proof generation process under the prover. + let outermost_layer = Type::outermost_layer()?; + let pk = self.config.proving_key(outermost_layer)?; + let proof = Proof::new(outermost_layer, snark, pk, aux_data)?; + + // Dump the proof if caching is enabled. + if let Some(path) = &path_proof { + write_json(path, &proof)?; } - unimplemented!() + Ok(proof) } /// Generate a SNARK for the given circuit. - fn gen_snark(&mut self, _layer: ProofLayer, _circuit: &C) -> Result + fn gen_snark( + &mut self, + id: &str, + layer: ProofLayer, + circuit: C, + ) -> Result where - C: Circuit + CircuitExt, + C: CircuitExt, { - unimplemented!() + let path = self.config.path_snark(id, layer); + let (kzg_params, pk) = self.config.gen_proving_key(layer, &circuit)?; + let mut rng = gen_rng(); + + snark_verifier_sdk::gen_snark_shplonk(kzg_params, pk, circuit, &mut rng, path) + .map_err(|e| ProverError::GenSnark(id.into(), layer, e)) } } diff --git a/prover2/src/types/layer.rs b/prover2/src/types/layer.rs index 8035d57baf..c8d26b47cb 100644 --- a/prover2/src/types/layer.rs +++ b/prover2/src/types/layer.rs @@ -1,7 +1,7 @@ use serde::{Deserialize, Serialize}; /// Various layers in the proof generation process. -#[derive(Clone, Copy, Debug, Serialize, Deserialize, Eq, PartialEq, PartialOrd, Ord)] +#[derive(Clone, Copy, Debug, Serialize, Deserialize, Eq, PartialEq, PartialOrd, Ord, Hash)] pub enum ProofLayer { /// The super circuit (ZkEVM) layer. This is the innermost proof layer. Layer0, diff --git a/prover2/src/types/mod.rs b/prover2/src/types/mod.rs index b634578437..48ace60aba 100644 --- a/prover2/src/types/mod.rs +++ b/prover2/src/types/mod.rs @@ -1,9 +1,10 @@ -use aggregator::{AggregationCircuit, CompressionCircuit}; +use aggregator::{AggregationCircuit, ChunkInfo, CompressionCircuit}; +use ethers_core::types::H256; use halo2_proofs::{ halo2curves::bn256::{Bn256, Fr}, - plonk::Circuit, poly::kzg::commitment::ParamsKZG, }; +use serde::{de::DeserializeOwned, Deserialize, Serialize}; use snark_verifier_sdk::{CircuitExt, Snark}; use zkevm_circuits::super_circuit::params::ScrollSuperCircuit; @@ -25,11 +26,15 @@ pub trait ProverType: std::fmt::Debug { type Task: ProvingTask; /// The circuit used at the base layer of this prover type. - type BaseCircuit: Circuit + CircuitExt; + type BaseCircuit: CircuitExt; /// The compression circuit used to compress the base layer SNARK one or more times before /// finally producing the outermost layer's SNARK. - type CompressionCircuit: Circuit + CircuitExt; + type CompressionCircuit: CircuitExt; + + /// The auxiliary data attached to the final proof from the prover type. For instance, a [`ChunkProver`] needs to attach the [`ChunkInfo`], + /// which is then used by the [`BatchProver`] to construct its [`BatchProvingTask`]. + type ProofAuxData: Serialize + DeserializeOwned; /// The prover supports proof generation at the following layers. fn layers() -> Vec; @@ -38,10 +43,16 @@ pub trait ProverType: std::fmt::Debug { fn base_layer() -> Result { Self::layers() .first() - .ok_or(ProverError::Custom(format!( - "no base layer for {}", - Self::NAME - ))) + .ok_or(ProverError::Custom(format!("no layer for {}", Self::NAME))) + .copied() + } + + /// Returns the outermost layer. This is generally the last compression layer of the prover + /// type. + fn outermost_layer() -> Result { + Self::layers() + .last() + .ok_or(ProverError::Custom(format!("no layer for {}", Self::NAME))) .copied() } @@ -52,7 +63,7 @@ pub trait ProverType: std::fmt::Debug { } /// Builds the base circuit given witness in the proving task. - fn build_base(task: Self::Task) -> Self::BaseCircuit; + fn build_base(task: Self::Task) -> (Self::BaseCircuit, Self::ProofAuxData); /// Builds the compression circuit given the previous layer's SNARK. fn build_compression( @@ -74,6 +85,16 @@ pub struct ProverTypeBatch; #[derive(Default, Debug)] pub struct ProverTypeBundle; +#[derive(Serialize, Deserialize)] +pub struct ChunkProofAuxData { + chunk_infos: Vec, +} + +#[derive(Serialize, Deserialize)] +pub struct BatchProofAuxData { + batch_hash: H256, +} + impl ProverType for ProverTypeChunk { const NAME: &'static str = "ChunkProver"; @@ -83,11 +104,13 @@ impl ProverType for ProverTypeChunk { type CompressionCircuit = CompressionCircuit; + type ProofAuxData = ChunkProofAuxData; + fn layers() -> Vec { vec![ProofLayer::Layer0, ProofLayer::Layer1, ProofLayer::Layer2] } - fn build_base(_task: Self::Task) -> Self::BaseCircuit { + fn build_base(_task: Self::Task) -> (Self::BaseCircuit, Self::ProofAuxData) { unimplemented!() } @@ -109,11 +132,13 @@ impl ProverType for ProverTypeBatch { type CompressionCircuit = CompressionCircuit; + type ProofAuxData = BatchProofAuxData; + fn layers() -> Vec { vec![ProofLayer::Layer3, ProofLayer::Layer4] } - fn build_base(_task: Self::Task) -> Self::BaseCircuit { + fn build_base(_task: Self::Task) -> (Self::BaseCircuit, Self::ProofAuxData) { unimplemented!() } diff --git a/prover2/src/types/proof.rs b/prover2/src/types/proof.rs index cb26430c13..9911966475 100644 --- a/prover2/src/types/proof.rs +++ b/prover2/src/types/proof.rs @@ -1,10 +1,97 @@ +use eth_types::base64; +use halo2_proofs::{ + halo2curves::bn256::{Fr, G1Affine}, + plonk::{Circuit, ProvingKey, VerifyingKey}, + SerdeFormat, +}; use serde::{Deserialize, Serialize}; +use snark_verifier_sdk::Snark; -use super::layer::ProofLayer; +use crate::{ + util::{deserialize_be, serialize_be, GIT_VERSION}, + ProofLayer, ProverError, +}; /// Describes an output from a [`Prover`]'s proof generation process when given a [`ProvingTask`]. #[derive(Serialize, Deserialize)] -pub struct Proof { +pub struct Proof { + /// Version of the source code (git describe --abbrev=8) used for proof generation. + git_version: String, /// The proof layer. layer: ProofLayer, + /// Verification key for this SNARK proof. + #[serde(with = "base64")] + vk: Vec, + /// The public instances (flattened bytes) to the SNARK. + #[serde(with = "base64")] + instances: Vec, + /// The protocol computed for SNARK. + #[serde(with = "base64")] + protocol: Vec, + /// The inner proof. + #[serde(with = "base64")] + proof: Vec, + /// Auxiliary data to attach with the proof. This data would generally be required by the next + /// layer's proof generation process. + #[serde(flatten)] + aux: Aux, +} + +impl Proof { + /// Construct a new [`Proof`] given the SNARK for the proof layer and some auxiliary data. + pub fn new( + layer: ProofLayer, + snark: Snark, + pk: &ProvingKey, + aux: Aux, + ) -> Result { + let git_version = GIT_VERSION.to_string(); + let vk = pk.get_vk().to_bytes(SerdeFormat::Processed); + let protocol = serde_json::to_vec(&snark.protocol)?; + let instances = snark.instances[0] + .iter() + .flat_map(serialize_be) + .collect::>(); + let proof = snark.proof; + + Ok(Self { + git_version, + layer, + vk, + protocol, + instances, + proof, + aux, + }) + } +} + +impl Proof { + /// Deserialize and return the verifying key. + pub fn verifying_key>(&self) -> Result, ProverError> { + Ok(VerifyingKey::from_bytes::( + &self.vk, + SerdeFormat::Processed, + )?) + } +} + +impl TryInto for Proof { + type Error = serde_json::Error; + + fn try_into(self) -> Result { + let protocol = serde_json::from_slice(&self.protocol)?; + let instances = self + .instances + .chunks_exact(32) + .map(deserialize_be) + .collect::>(); + let proof = self.proof; + + Ok(Snark { + protocol, + instances: vec![instances], + proof, + }) + } } diff --git a/prover2/src/util/dir.rs b/prover2/src/util/dir.rs new file mode 100644 index 0000000000..a765d53ccc --- /dev/null +++ b/prover2/src/util/dir.rs @@ -0,0 +1,76 @@ +use std::{ + env::current_dir, + path::{Path, PathBuf}, +}; + +use crate::{ProofLayer, ProverError}; + +/// The extension used for JSON files. +pub const JSON_EXT: &str = ".json"; + +/// The config parameters for non native field arithmetics are in a *.config file. +pub const NON_NATIVE_PARAMS_EXT: &str = ".config"; + +/// The config parameters for non native field arithmetics are by default in this directory. +pub const NON_NATIVE_PARAMS_DIR: &str = ".config"; + +/// The KZG setup parameters are by default in this directory. +pub const KZG_PARAMS_DIR: &str = ".params"; + +/// The default directory used for cached data. +pub const CACHE_PATH: &str = ".cache"; + +/// The directory within cache to store proving tasks in JSON format. +pub const CACHE_PATH_TASKS: &str = "tasks"; + +/// The directory within cache to store SNARKs generated at intermediate proving layers. +pub const CACHE_PATH_SNARKS: &str = "snarks"; + +/// The directory within cache to store proof outputs. +pub const CACHE_PATH_PROOFS: &str = "proofs"; + +/// The directory within cache to store public input data. +pub const CACHE_PATH_PI: &str = "pi"; + +/// The directory within cache to store Verifier contract code. +pub const CACHE_PATH_EVM: &str = "evm"; + +/// The path to the config parameters for a given proof layer. +/// +/// /{layer}.config +pub fn non_native_params_path(dir: &Path, layer: ProofLayer) -> PathBuf { + dir.join(format!("{}{NON_NATIVE_PARAMS_EXT}", layer.to_string())) +} + +/// The path to the KZG params by degree. +/// +/// /params{degree} +pub fn kzg_params_path(dir: &Path, degree: u32) -> PathBuf { + dir.join(format!("params{degree}")) +} + +/// Wrapper functionality for current working directory. +pub fn pwd() -> Result { + Ok(current_dir()?) +} + +/// The default path to find non-native field arithmetic config params. +/// +/// /.config +pub fn default_non_native_params_dir() -> Result { + Ok(pwd()?.join(NON_NATIVE_PARAMS_DIR)) +} + +/// The default path to find KZG setup parameters. +/// +/// /.params +pub fn default_kzg_params_dir() -> Result { + Ok(pwd()?.join(KZG_PARAMS_DIR)) +} + +/// The default path to the cache directory. +/// +/// /.cache +pub fn default_cache_dir() -> Result { + Ok(pwd()?.join(CACHE_PATH)) +} diff --git a/prover2/src/util/fs.rs b/prover2/src/util/io.rs similarity index 65% rename from prover2/src/util/fs.rs rename to prover2/src/util/io.rs index d5a5f24d5c..ae5d7e87ea 100644 --- a/prover2/src/util/fs.rs +++ b/prover2/src/util/io.rs @@ -5,7 +5,7 @@ use std::{ }; use halo2_proofs::{halo2curves::bn256::Bn256, poly::kzg::commitment::ParamsKZG, SerdeFormat}; -use serde::de::DeserializeOwned; +use serde::{de::DeserializeOwned, Serialize}; use crate::{ ProverError, @@ -29,6 +29,21 @@ pub fn read(path: impl AsRef) -> Result, ProverError> { }) } +/// Wrapper functionality to write bytes to a file. +pub fn write(path: impl AsRef, data: &[u8]) -> Result<(), ProverError> { + let path = path.as_ref(); + fs::write(path, data).map_err(|source| IoReadWrite { + source, + path: path.into(), + }) +} + +/// Wrapper functionality to create a file and write to it. +pub fn create_and_write(path: &Path, data: &[u8]) -> Result<(), ProverError> { + File::create(path)?; + write(path, data) +} + /// Wrapper functionality for reading a JSON file. pub fn read_json(path: &Path) -> Result { let bytes = read(path)?; @@ -38,6 +53,11 @@ pub fn read_json(path: &Path) -> Result { }) } +pub fn write_json(path: &Path, data: &T) -> Result<(), ProverError> { + let bytes = serde_json::to_vec(data)?; + create_and_write(path, &bytes) +} + /// Read KZG setup parameters that are in a custom serde format. pub fn read_kzg_params(path: &Path) -> Result, ProverError> { let f = open(path)?; diff --git a/prover2/src/util/mod.rs b/prover2/src/util/mod.rs index c8df4884e4..75eaf8f791 100644 --- a/prover2/src/util/mod.rs +++ b/prover2/src/util/mod.rs @@ -1,45 +1,23 @@ -use std::{ - env::current_dir, - path::{Path, PathBuf}, +use git_version::git_version; +use once_cell::sync::Lazy; +use rand::{Rng, SeedableRng}; +use rand_xorshift::XorShiftRng; + +mod dir; +pub use dir::{ + default_cache_dir, default_kzg_params_dir, default_non_native_params_dir, kzg_params_path, + non_native_params_path, CACHE_PATH_EVM, CACHE_PATH_PI, CACHE_PATH_PROOFS, CACHE_PATH_SNARKS, + CACHE_PATH_TASKS, JSON_EXT, }; -use crate::{ProofLayer, ProverError}; - -mod fs; -pub use fs::{read_json, read_kzg_params}; +mod io; +pub use io::{read_json, read_kzg_params, write_json}; mod env; pub use env::read_env_or_default; -/// The extension used for JSON files. -pub const JSON_EXT: &str = ".json"; - -/// The config parameters for non native field arithmetics are in a *.config file. -pub const NON_NATIVE_PARAMS_EXT: &str = ".config"; - -/// The config parameters for non native field arithmetics are by default in this directory. -pub const NON_NATIVE_PARAMS_DIR: &str = ".config"; - -/// The KZG setup parameters are by default in this directory. -pub const KZG_PARAMS_DIR: &str = ".params"; - -/// The default directory used for cached data. -pub const CACHE_PATH: &str = ".cache"; - -/// The directory within cache to store proving tasks in JSON format. -pub const CACHE_PATH_TASKS: &str = "tasks"; - -/// The directory within cache to store SNARKs generated at intermediate proving layers. -pub const CACHE_PATH_SNARKS: &str = "snarks"; - -/// The directory within cache to store proof outputs. -pub const CACHE_PATH_PROOFS: &str = "proofs"; - -/// The directory within cache to store public input data. -pub const CACHE_PATH_PI: &str = "pi"; - -/// The directory within cache to store Verifier contract code. -pub const CACHE_PATH_EVM: &str = "evm"; +mod serde; +pub use serde::{deserialize_be, serialize_be}; /// The environment variable to be set to configure custom degree for the super circuit (layer0). pub const ENV_DEGREE_LAYER0: &str = "SUPER_CIRCUIT_DEGREE"; @@ -47,42 +25,16 @@ pub const ENV_DEGREE_LAYER0: &str = "SUPER_CIRCUIT_DEGREE"; /// The default degree for the super circuit (layer0). pub const DEFAULT_DEGREE_LAYER0: u32 = 20; -/// The path to the config parameters for a given proof layer. -/// -/// /{layer}.config -pub fn non_native_params_path(dir: &Path, layer: ProofLayer) -> PathBuf { - dir.join(format!("{}{NON_NATIVE_PARAMS_EXT}", layer.to_string())) -} - -/// The path to the KZG params by degree. -/// -/// /params{degree} -pub fn kzg_params_path(dir: &Path, degree: u32) -> PathBuf { - dir.join(format!("params{degree}")) -} - -/// Wrapper functionality for current working directory. -pub fn pwd() -> Result { - Ok(current_dir()?) -} - -/// The default path to find non-native field arithmetic config params. -/// -/// /.config -pub fn default_non_native_params_dir() -> Result { - Ok(pwd()?.join(NON_NATIVE_PARAMS_DIR)) -} - -/// The default path to find KZG setup parameters. -/// -/// /.params -pub fn default_kzg_params_dir() -> Result { - Ok(pwd()?.join(KZG_PARAMS_DIR)) -} - -/// The default path to the cache directory. -/// -/// /.cache -pub fn default_cache_dir() -> Result { - Ok(pwd()?.join(CACHE_PATH)) +/// Git version (git describe) of the source code. +pub static GIT_VERSION: Lazy<&str> = Lazy::new(|| { + git_version!(args = ["--abbrev=8", "--always"]) + .split('-') + .last() + .expect("git describe should not fail") +}); + +/// Seed and return a random number generator. +pub fn gen_rng() -> impl Rng + Send { + let seed = [0u8; 16]; + XorShiftRng::from_seed(seed) } diff --git a/prover2/src/util/serde.rs b/prover2/src/util/serde.rs new file mode 100644 index 0000000000..97b7eb4c10 --- /dev/null +++ b/prover2/src/util/serde.rs @@ -0,0 +1,15 @@ +use halo2_proofs::halo2curves::bn256::Fr; +use snark_verifier::loader::halo2::halo2_ecc::halo2_base::utils::ScalarField; + +/// Serialize a field element [`Fr`] to big-endian bytes. +pub fn serialize_be(field: &Fr) -> Vec { + field.to_bytes().into_iter().rev().collect() +} + +/// Deserialize a field element [`Fr`] from big-endian bytes. +pub fn deserialize_be(be_bytes: &[u8]) -> Fr { + let mut le_bytes = [0u8; 32]; + le_bytes.copy_from_slice(be_bytes); + le_bytes.reverse(); + Fr::from_bytes_le(&le_bytes) +} From 87a00d79d45a6ef6c0a9d37cd92f4a1041acc801 Mon Sep 17 00:00:00 2001 From: Rohit Narurkar Date: Sun, 14 Jul 2024 00:58:22 +0100 Subject: [PATCH 12/17] prover type can be halo2 or evm --- prover2/src/prover/mod.rs | 117 ++++++++++++++++++++++++++++++++----- prover2/src/types/proof.rs | 25 +++++++- 2 files changed, 125 insertions(+), 17 deletions(-) diff --git a/prover2/src/prover/mod.rs b/prover2/src/prover/mod.rs index 459ceb2b35..65f563401e 100644 --- a/prover2/src/prover/mod.rs +++ b/prover2/src/prover/mod.rs @@ -13,29 +13,30 @@ pub mod config; pub mod params; /// Convenience type for chunk prover. -pub type ChunkProver = Prover; +pub type ChunkProver = Prover; /// Convenience type for batch prover. -pub type BatchProver = Prover>; +pub type BatchProver = Prover, false>; -/// Convenience type for bundle prover. -pub type BundleProver = Prover; +/// Convenience type for bundle prover. Since bundling of batches is also the final layer, we wish +/// to verify the proof in EVM. +pub type BundleProver = Prover; /// A generic prover that is capable of generating proofs for given tasks. #[derive(Debug)] -pub struct Prover { +pub struct Prover { /// Config for the prover. pub config: ProverConfig, } -impl Prover { +impl Prover { /// Construct a new prover. pub fn new(config: ProverConfig) -> Self { Self { config } } } -impl Prover { +impl Prover { /// Generate a proof for the given task. #[instrument(name = "Prover::gen_proof", skip(self))] pub fn gen_proof( @@ -44,6 +45,7 @@ impl Prover { ) -> Result, ProverError> { // Early return if the proof for the given task is already available in cache. let id = task.id(); + let path_proof = self.config.path_proof(&id); if let Some(path) = &path_proof { if let Ok(proof) = read_json(path) { @@ -51,35 +53,118 @@ impl Prover { } } + let proof = if EVM_VERIFY { + self.gen_proof_evm(task)? + } else { + self.gen_proof_halo2(task)? + }; + + // Dump the proof if caching is enabled. + if let Some(path) = &path_proof { + write_json(path, &proof)?; + } + + Ok(proof) + } + + /// Generate a halo2 proof for the given task. The poseidon hash function is used for + /// fiat-shamir transform on the transcript. + #[instrument(name = "Prover::gen_proof_halo2", skip(self))] + fn gen_proof_halo2( + &mut self, + task: Type::Task, + ) -> Result, ProverError> { + let id = task.id(); + // Generate SNARKs for all the layers at which the prover operates. // // We start from the base layer, i.e. the innermost layer for the prover. - let base_layer = Type::base_layer()?; - let (base_circuit, aux_data) = Type::build_base(task); - let mut snark = self.gen_snark(&id, base_layer, base_circuit)?; + let (mut snark, aux_data) = self.gen_base_snark(task)?; // The base layer's SNARK is compressed for every layer of compression. for layer in Type::compression_layers() { let kzg_params = self.config.kzg_params(layer)?; let compression_circuit = Type::build_compression(kzg_params, snark, layer); - snark = self.gen_snark(&id, layer, compression_circuit)?; + snark = self.gen_halo2_snark(&id, layer, compression_circuit)?; } // We have the final compressed SNARK for the proof generation process under the prover. let outermost_layer = Type::outermost_layer()?; let pk = self.config.proving_key(outermost_layer)?; - let proof = Proof::new(outermost_layer, snark, pk, aux_data)?; + let proof = Proof::new_from_snark(outermost_layer, snark, pk, aux_data)?; - // Dump the proof if caching is enabled. - if let Some(path) = &path_proof { - write_json(path, &proof)?; + Ok(proof) + } + + /// Generate an EVM-verifiable proof for the given task. The Keccak256 hash function is used + /// for fiat-shamir transform on the transcript. + #[instrument(name = "Prover::gen_proof_evm", skip(self))] + fn gen_proof_evm( + &mut self, + task: Type::Task, + ) -> Result, ProverError> { + let id = task.id(); + + // Generate SNARKs for all the layers at which the prover operates. + // + // We start from the base layer, i.e. the innermost layer for the prover. + let (mut snark, aux_data) = self.gen_base_snark(task)?; + + // The base layer's SNARK is compressed for every layer of compression, except the last + // (final) layer. The final layer of compression is supposed to be EVM-verifiable. + for &layer in Type::compression_layers().iter().rev().skip(1).rev() { + let kzg_params = self.config.kzg_params(layer)?; + let compression_circuit = Type::build_compression(kzg_params, snark, layer); + snark = self.gen_halo2_snark(&id, layer, compression_circuit)?; } + // We have the final compressed SNARK for the proof generation process under the prover. + let outermost_layer = Type::outermost_layer()?; + let kzg_params = self.config.kzg_params(outermost_layer)?; + let compression_circuit = Type::build_compression(kzg_params, snark, outermost_layer); + let instances = compression_circuit.instances(); + let (kzg_params, pk) = self + .config + .gen_proving_key(outermost_layer, &compression_circuit)?; + let mut rng = gen_rng(); + let raw_proof = snark_verifier_sdk::gen_evm_proof_shplonk( + kzg_params, + pk, + compression_circuit, + instances.clone(), + &mut rng, + ); + + let proof = Proof::new_from_raw(outermost_layer, &instances[0], &raw_proof, pk, aux_data); + Ok(proof) } + /// Generates a SNARK for the base circuit of a prover. THe base circuit is generally a circuit + /// with larger number of advice columns while being of lower degree. The SNARK of the base + /// circuit is then compressed using the compression layer to produce a proof that's cheaper to + /// verify. + #[instrument(name = "Prover::gen_base_snark", skip(self))] + fn gen_base_snark( + &mut self, + task: Type::Task, + ) -> Result<(Snark, Type::ProofAuxData), ProverError> { + let id = task.id(); + + // Generate SNARKs for all the layers at which the prover operates. + // + // We start from the base layer, i.e. the innermost layer for the prover. + let base_layer = Type::base_layer()?; + let (base_circuit, aux_data) = Type::build_base(task); + + Ok(( + self.gen_halo2_snark(&id, base_layer, base_circuit)?, + aux_data, + )) + } + /// Generate a SNARK for the given circuit. - fn gen_snark( + fn gen_halo2_snark( &mut self, id: &str, layer: ProofLayer, diff --git a/prover2/src/types/proof.rs b/prover2/src/types/proof.rs index 9911966475..81006ad96f 100644 --- a/prover2/src/types/proof.rs +++ b/prover2/src/types/proof.rs @@ -39,7 +39,7 @@ pub struct Proof { impl Proof { /// Construct a new [`Proof`] given the SNARK for the proof layer and some auxiliary data. - pub fn new( + pub fn new_from_snark( layer: ProofLayer, snark: Snark, pk: &ProvingKey, @@ -64,6 +64,29 @@ impl Proof { aux, }) } + + /// Construct a new [`Proof`] given the raw proof and instances for an EVM-verifiable proof. + pub fn new_from_raw( + layer: ProofLayer, + instances: &[Fr], + proof: &[u8], + pk: &ProvingKey, + aux: Aux, + ) -> Self { + let git_version = GIT_VERSION.to_string(); + let vk = pk.get_vk().to_bytes(SerdeFormat::Processed); + let instances = instances.iter().flat_map(serialize_be).collect::>(); + + Self { + git_version, + layer, + vk, + protocol: vec![], + instances, + proof: proof.to_vec(), + aux, + } + } } impl Proof { From 94b9bd1566964afb4158959f5f80d5633a79e2db Mon Sep 17 00:00:00 2001 From: Rohit Narurkar Date: Sun, 14 Jul 2024 18:24:33 +0100 Subject: [PATCH 13/17] add logging --- prover2/src/prover/mod.rs | 43 ++++++++++++++++++++++++++------------ prover2/src/types/mod.rs | 12 +++++------ prover2/src/types/proof.rs | 10 ++++----- 3 files changed, 41 insertions(+), 24 deletions(-) diff --git a/prover2/src/prover/mod.rs b/prover2/src/prover/mod.rs index 65f563401e..b0ad294006 100644 --- a/prover2/src/prover/mod.rs +++ b/prover2/src/prover/mod.rs @@ -1,7 +1,7 @@ use aggregator::MAX_AGG_SNARKS; use halo2_proofs::halo2curves::bn256::Fr; use snark_verifier_sdk::{CircuitExt, Snark}; -use tracing::instrument; +use tracing::{info, instrument, trace}; use crate::{ types::{ProverType, ProverTypeBatch, ProverTypeBundle, ProverTypeChunk}, @@ -42,28 +42,36 @@ impl Prover { pub fn gen_proof( &mut self, task: Type::Task, - ) -> Result, ProverError> { + ) -> Result, ProverError> { + info!(name = "gen proof", prover = ?Type::NAME.to_string(), ?task); + // Early return if the proof for the given task is already available in cache. let id = task.id(); let path_proof = self.config.path_proof(&id); if let Some(path) = &path_proof { + trace!(name = "read proof from cache", ?path); if let Ok(proof) = read_json(path) { + trace!(name = "early return cache hit", ?proof); return Ok(proof); } } let proof = if EVM_VERIFY { + trace!(name = "gen evm proof", ?task); self.gen_proof_evm(task)? } else { + trace!(name = "gen halo2 proof", ?task); self.gen_proof_halo2(task)? }; // Dump the proof if caching is enabled. if let Some(path) = &path_proof { + trace!(name = "dump proof", ?path, ?proof); write_json(path, &proof)?; } + info!(name = "gen proof OK", prover = ?Type::NAME.to_string(), ?proof); Ok(proof) } @@ -73,19 +81,24 @@ impl Prover { fn gen_proof_halo2( &mut self, task: Type::Task, - ) -> Result, ProverError> { + ) -> Result, ProverError> { let id = task.id(); // Generate SNARKs for all the layers at which the prover operates. // // We start from the base layer, i.e. the innermost layer for the prover. - let (mut snark, aux_data) = self.gen_base_snark(task)?; + let layer = Type::base_layer()?; + trace!(name = "gen base snark", ?layer); + let (mut snark, aux_data) = self.gen_base_snark(layer, task)?; + trace!(name = "gen base snark OK", ?aux_data); // The base layer's SNARK is compressed for every layer of compression. for layer in Type::compression_layers() { let kzg_params = self.config.kzg_params(layer)?; let compression_circuit = Type::build_compression(kzg_params, snark, layer); + trace!(name = "gen compression snark", ?layer); snark = self.gen_halo2_snark(&id, layer, compression_circuit)?; + trace!(name = "gen compression snark OK", ?layer); } // We have the final compressed SNARK for the proof generation process under the prover. @@ -102,20 +115,25 @@ impl Prover { fn gen_proof_evm( &mut self, task: Type::Task, - ) -> Result, ProverError> { + ) -> Result, ProverError> { let id = task.id(); // Generate SNARKs for all the layers at which the prover operates. // // We start from the base layer, i.e. the innermost layer for the prover. - let (mut snark, aux_data) = self.gen_base_snark(task)?; + let layer = Type::base_layer()?; + trace!(name = "gen base snark", ?layer); + let (mut snark, aux_data) = self.gen_base_snark(layer, task)?; + trace!(name = "gen base snark OK", ?aux_data); // The base layer's SNARK is compressed for every layer of compression, except the last // (final) layer. The final layer of compression is supposed to be EVM-verifiable. for &layer in Type::compression_layers().iter().rev().skip(1).rev() { let kzg_params = self.config.kzg_params(layer)?; let compression_circuit = Type::build_compression(kzg_params, snark, layer); + trace!(name = "gen compression snark", ?layer); snark = self.gen_halo2_snark(&id, layer, compression_circuit)?; + trace!(name = "gen compression snark OK", ?layer); } // We have the final compressed SNARK for the proof generation process under the prover. @@ -127,6 +145,7 @@ impl Prover { .config .gen_proving_key(outermost_layer, &compression_circuit)?; let mut rng = gen_rng(); + trace!(name = "gen evm proof", layer = ?outermost_layer); let raw_proof = snark_verifier_sdk::gen_evm_proof_shplonk( kzg_params, pk, @@ -134,6 +153,7 @@ impl Prover { instances.clone(), &mut rng, ); + trace!(name = "gen evm proof OK", layer = ?outermost_layer); let proof = Proof::new_from_raw(outermost_layer, &instances[0], &raw_proof, pk, aux_data); @@ -144,9 +164,9 @@ impl Prover { /// with larger number of advice columns while being of lower degree. The SNARK of the base /// circuit is then compressed using the compression layer to produce a proof that's cheaper to /// verify. - #[instrument(name = "Prover::gen_base_snark", skip(self))] fn gen_base_snark( &mut self, + base_layer: ProofLayer, task: Type::Task, ) -> Result<(Snark, Type::ProofAuxData), ProverError> { let id = task.id(); @@ -154,13 +174,10 @@ impl Prover { // Generate SNARKs for all the layers at which the prover operates. // // We start from the base layer, i.e. the innermost layer for the prover. - let base_layer = Type::base_layer()?; - let (base_circuit, aux_data) = Type::build_base(task); + let (base_circuit, aux_data) = Type::build_base(&task); + let snark = self.gen_halo2_snark(&id, base_layer, base_circuit)?; - Ok(( - self.gen_halo2_snark(&id, base_layer, base_circuit)?, - aux_data, - )) + Ok((snark, aux_data)) } /// Generate a SNARK for the given circuit. diff --git a/prover2/src/types/mod.rs b/prover2/src/types/mod.rs index 48ace60aba..6fcc1f43bb 100644 --- a/prover2/src/types/mod.rs +++ b/prover2/src/types/mod.rs @@ -34,7 +34,7 @@ pub trait ProverType: std::fmt::Debug { /// The auxiliary data attached to the final proof from the prover type. For instance, a [`ChunkProver`] needs to attach the [`ChunkInfo`], /// which is then used by the [`BatchProver`] to construct its [`BatchProvingTask`]. - type ProofAuxData: Serialize + DeserializeOwned; + type ProofAuxData: Serialize + DeserializeOwned + std::fmt::Debug; /// The prover supports proof generation at the following layers. fn layers() -> Vec; @@ -63,7 +63,7 @@ pub trait ProverType: std::fmt::Debug { } /// Builds the base circuit given witness in the proving task. - fn build_base(task: Self::Task) -> (Self::BaseCircuit, Self::ProofAuxData); + fn build_base(task: &Self::Task) -> (Self::BaseCircuit, Self::ProofAuxData); /// Builds the compression circuit given the previous layer's SNARK. fn build_compression( @@ -85,12 +85,12 @@ pub struct ProverTypeBatch; #[derive(Default, Debug)] pub struct ProverTypeBundle; -#[derive(Serialize, Deserialize)] +#[derive(Serialize, Deserialize, Debug)] pub struct ChunkProofAuxData { chunk_infos: Vec, } -#[derive(Serialize, Deserialize)] +#[derive(Serialize, Deserialize, Debug)] pub struct BatchProofAuxData { batch_hash: H256, } @@ -110,7 +110,7 @@ impl ProverType for ProverTypeChunk { vec![ProofLayer::Layer0, ProofLayer::Layer1, ProofLayer::Layer2] } - fn build_base(_task: Self::Task) -> (Self::BaseCircuit, Self::ProofAuxData) { + fn build_base(_task: &Self::Task) -> (Self::BaseCircuit, Self::ProofAuxData) { unimplemented!() } @@ -138,7 +138,7 @@ impl ProverType for ProverTypeBatch { vec![ProofLayer::Layer3, ProofLayer::Layer4] } - fn build_base(_task: Self::Task) -> (Self::BaseCircuit, Self::ProofAuxData) { + fn build_base(_task: &Self::Task) -> (Self::BaseCircuit, Self::ProofAuxData) { unimplemented!() } diff --git a/prover2/src/types/proof.rs b/prover2/src/types/proof.rs index 81006ad96f..b26a66dc08 100644 --- a/prover2/src/types/proof.rs +++ b/prover2/src/types/proof.rs @@ -13,8 +13,8 @@ use crate::{ }; /// Describes an output from a [`Prover`]'s proof generation process when given a [`ProvingTask`]. -#[derive(Serialize, Deserialize)] -pub struct Proof { +#[derive(Serialize, Deserialize, Debug)] +pub struct Proof { /// Version of the source code (git describe --abbrev=8) used for proof generation. git_version: String, /// The proof layer. @@ -37,7 +37,7 @@ pub struct Proof { aux: Aux, } -impl Proof { +impl Proof { /// Construct a new [`Proof`] given the SNARK for the proof layer and some auxiliary data. pub fn new_from_snark( layer: ProofLayer, @@ -89,7 +89,7 @@ impl Proof { } } -impl Proof { +impl Proof { /// Deserialize and return the verifying key. pub fn verifying_key>(&self) -> Result, ProverError> { Ok(VerifyingKey::from_bytes::( @@ -99,7 +99,7 @@ impl Proof { } } -impl TryInto for Proof { +impl TryInto for Proof { type Error = serde_json::Error; fn try_into(self) -> Result { From 52c82bab924ba9a7b8e94ef5f529c9853cdfa908 Mon Sep 17 00:00:00 2001 From: Rohit Narurkar Date: Mon, 15 Jul 2024 12:09:00 +0100 Subject: [PATCH 14/17] fix docs --- prover2/src/lib.rs | 11 ++--- prover2/src/prover/config.rs | 9 ++-- prover2/src/prover/params.rs | 2 +- prover2/src/types/layer.rs | 25 ++++++++--- prover2/src/types/mod.rs | 83 ++++++++++++++++++++++++++++++------ prover2/src/types/proof.rs | 33 ++++++++------ prover2/src/types/task.rs | 25 +++++++++-- 7 files changed, 144 insertions(+), 44 deletions(-) diff --git a/prover2/src/lib.rs b/prover2/src/lib.rs index 5005dc8bd1..86d874ab2f 100644 --- a/prover2/src/lib.rs +++ b/prover2/src/lib.rs @@ -1,12 +1,13 @@ mod error; -pub use error::ProverError; -mod prover; -pub use prover::{config::ProverConfig, params::Params, BatchProver, BundleProver, ChunkProver}; +pub mod prover; -mod types; -pub use types::{layer::ProofLayer, proof::Proof, task::ProvingTask}; +pub mod types; mod util; mod verifier; + +pub use error::ProverError; +pub use prover::{config::ProverConfig, BatchProver, BundleProver, ChunkProver, Prover}; +pub use types::{layer::ProofLayer, proof::Proof, task::ProvingTask}; diff --git a/prover2/src/prover/config.rs b/prover2/src/prover/config.rs index d5ecc4de2a..30ab95365b 100644 --- a/prover2/src/prover/config.rs +++ b/prover2/src/prover/config.rs @@ -8,6 +8,7 @@ use halo2_proofs::{ use tracing::{debug, info, instrument, trace}; use crate::{ + prover::params::NonNativeParams, types::ProverType, util::{ default_cache_dir, default_kzg_params_dir, default_non_native_params_dir, kzg_params_path, @@ -15,7 +16,7 @@ use crate::{ CACHE_PATH_EVM, CACHE_PATH_PI, CACHE_PATH_PROOFS, CACHE_PATH_SNARKS, CACHE_PATH_TASKS, DEFAULT_DEGREE_LAYER0, ENV_DEGREE_LAYER0, JSON_EXT, }, - Params, ProofLayer, ProverError, + ProofLayer, ProverError, }; /// Configuration for a generic prover. @@ -26,7 +27,7 @@ pub struct ProverConfig { /// KZG setup parameters by proof layer. pub kzg_params: HashMap>, /// Config parameters for non-native field arithmetics by proof layer. - pub nn_params: HashMap, + pub nn_params: HashMap, /// Proving keys by proof layer. pub pks: HashMap>, /// Optional directory to locate KZG setup parameters. @@ -42,7 +43,7 @@ pub struct ProverConfig { impl ProverConfig { /// Returns prover config after inserting the non-native field arithmetic config for the given /// proof layer. - pub fn with_nn_params(mut self, layer: ProofLayer, params: Params) -> Self { + pub fn with_nn_params(mut self, layer: ProofLayer, params: NonNativeParams) -> Self { self.nn_params.insert(layer, params); self } @@ -109,7 +110,7 @@ impl ProverConfig { if layer != ProofLayer::Layer0 { let params_path = nn_params_path(nn_params_dir.as_path(), layer); debug!("reading config params for {:?}: {:?}", layer, params_path); - let params = read_json::(params_path.as_path())?; + let params = read_json::(params_path.as_path())?; self.degrees.insert(layer, params.degree); self.nn_params.insert(layer, params); } diff --git a/prover2/src/prover/params.rs b/prover2/src/prover/params.rs index 69509fa5db..bd3ac51eaf 100644 --- a/prover2/src/prover/params.rs +++ b/prover2/src/prover/params.rs @@ -3,7 +3,7 @@ use snark_verifier::loader::halo2::halo2_ecc::fields::fp::FpStrategy; /// Parameters to configure the non-native field arithmetic chip. #[derive(Debug, Serialize, Deserialize)] -pub struct Params { +pub struct NonNativeParams { pub strategy: FpStrategy, pub degree: u32, pub num_advice: Vec, diff --git a/prover2/src/types/layer.rs b/prover2/src/types/layer.rs index c8d26b47cb..cbd128ee95 100644 --- a/prover2/src/types/layer.rs +++ b/prover2/src/types/layer.rs @@ -3,21 +3,34 @@ use serde::{Deserialize, Serialize}; /// Various layers in the proof generation process. #[derive(Clone, Copy, Debug, Serialize, Deserialize, Eq, PartialEq, PartialOrd, Ord, Hash)] pub enum ProofLayer { - /// The super circuit (ZkEVM) layer. This is the innermost proof layer. + /// The [`SuperCircuit`][super_circuit] (ZkEVM) layer. This is the innermost proof layer. + /// + /// [super_circuit]: zkevm_circuits::super_circuit::SuperCircuit Layer0, /// The compression layer on top of layer0. Layer1, - /// The compression layer on top of layer1. The proof from this layer is the [`ChunkProof`]. + /// The compression layer on top of layer1. The proof from this layer is the [`Proof`][proof] returned + /// by the [`ChunkProver`][gen_proof]. + /// + /// [proof]: crate::Proof + /// [gen_proof]: crate::prover::ChunkProver::gen_proof Layer2, - /// The batch circuit layer. At this layer, we batch multiple [`ChunkProof`]s. + /// The batch circuit layer. At this layer, we batch multiple `ChunkProof`s. Layer3, - /// The compression layer on top of layer3. The proof from this layer is the [`BatchProof`]. + /// The compression layer on top of layer3. The proof from this layer is the [`Proof`][proof] returned + /// by the [`BatchProver`][gen_proof]. + /// + /// [proof]: crate::Proof + /// [gen_proof]: crate::prover::BatchProver::gen_proof Layer4, /// The recursion circuit layer. At this layer, we construct proofs recursively over a previous /// SNARK from the recursion circuit. Layer5, - /// The compression layer on top of layer5. The proof from this layer is the [`BundleProof`], - /// which is verified in EVM. + /// The compression layer on top of layer5. The proof from this layer is the [`Proof`][proof] returned + /// by the [`BundleProver`][gen_proof]. + /// + /// [proof]: crate::Proof + /// [gen_proof]: crate::prover::BundleProver::gen_proof Layer6, } diff --git a/prover2/src/types/mod.rs b/prover2/src/types/mod.rs index 6fcc1f43bb..91e7724e56 100644 --- a/prover2/src/types/mod.rs +++ b/prover2/src/types/mod.rs @@ -17,6 +17,25 @@ pub mod proof; pub mod task; use task::{BatchProvingTask, ChunkProvingTask}; +/// Defines the behaviour that a prover type must implement. +/// +/// The proof generation process involves the following steps: +/// 1. [`ProverTypeChunk`]: prove the EVM execution trace from a list of blocks ([`layer0`][layer0]) +/// and compress it twice ([`layer1`][layer1] and [`layer2`][layer2]), where the outermost proof +/// (`layer2`) is the `ChunkProof`. +/// 2. [`ProverTypeBatch`]: prove the batching of one or more chunks ([`layer3`][layer3]) +/// and compress it ([`layer4`][layer4]) where this outermost proof (`layer4`) is the `BatchProof`. +/// 3. [`ProverTypeBundle`]: prove the bundling of one or more batches recursively ([`layer5`][layer5]) +/// and compress it ([`layer6`][layer6]) where this outermost proof (`layer6`) is the `BundleProof`. +/// The `BundleProof` is an EVM-verifiable proof. +/// +/// [layer0]: crate::ProofLayer::Layer0 +/// [layer1]: crate::ProofLayer::Layer1 +/// [layer2]: crate::ProofLayer::Layer2 +/// [layer3]: crate::ProofLayer::Layer3 +/// [layer4]: crate::ProofLayer::Layer4 +/// [layer5]: crate::ProofLayer::Layer5 +/// [layer6]: crate::ProofLayer::Layer6 pub trait ProverType: std::fmt::Debug { /// The name of the prover. const NAME: &'static str; @@ -28,18 +47,31 @@ pub trait ProverType: std::fmt::Debug { /// The circuit used at the base layer of this prover type. type BaseCircuit: CircuitExt; - /// The compression circuit used to compress the base layer SNARK one or more times before - /// finally producing the outermost layer's SNARK. + /// The [`Compression Circuit`][compr_circuit] used to compress the base layer + /// [`SNARK`][snark] one or more times before finally producing the outermost + /// layer's SNARK. + /// + /// [snark]: snark_verifier_sdk::Snark + /// [compr_circuit]: aggregator::CompressionCircuit type CompressionCircuit: CircuitExt; - /// The auxiliary data attached to the final proof from the prover type. For instance, a [`ChunkProver`] needs to attach the [`ChunkInfo`], - /// which is then used by the [`BatchProver`] to construct its [`BatchProvingTask`]. + /// The auxiliary data attached to the [`Proof`][proof] from the prover type. + /// For instance, a [`ChunkProver`][chunk_prover] needs to attach the [`ChunkInfo`], + /// which is then used by the [`BatchProver`][batch_prover] to construct its + /// [`BatchProvingTask`]. + /// + /// [proof]: crate::Proof + /// [chunk_prover]: crate::prover::ChunkProver + /// [batch_prover]: crate::prover::BatchProver type ProofAuxData: Serialize + DeserializeOwned + std::fmt::Debug; /// The prover supports proof generation at the following layers. fn layers() -> Vec; - /// Returns the base layer. + /// Returns the base layer. The base layer's proof is generated by building the + /// [`BaseCircuit`][base_circuit]. + /// + /// [base_circuit]: Self::BaseCircuit fn base_layer() -> Result { Self::layers() .first() @@ -62,10 +94,14 @@ pub trait ProverType: std::fmt::Debug { Self::layers()[1..].to_vec() } - /// Builds the base circuit given witness in the proving task. + /// Builds the [`BaseCircuit`][base_circuit] given witness in the proving task. + /// + /// [base_circuit]: Self::BaseCircuit fn build_base(task: &Self::Task) -> (Self::BaseCircuit, Self::ProofAuxData); - /// Builds the compression circuit given the previous layer's SNARK. + /// Builds the [`CompressionCircuit`][compr_circuit] given the previous layer's SNARK. + /// + /// [compr_circuit]: Self::CompressionCircuit fn build_compression( kzg_params: &ParamsKZG, prev_snark: Snark, @@ -73,26 +109,49 @@ pub trait ProverType: std::fmt::Debug { ) -> Self::CompressionCircuit; } -/// The chunk prover that constructs proofs at layer0, layer1 and layer2. +/// The chunk prover that constructs proofs at [`layer0`][layer0], [`layer1`][layer1] and [`layer2`][layer2]. +/// +/// [layer0]: crate::ProofLayer::Layer0 +/// [layer1]: crate::ProofLayer::Layer1 +/// [layer2]: crate::ProofLayer::Layer2 #[derive(Default, Debug)] pub struct ProverTypeChunk; -/// The batch prover that constructs proofs at layer3 and layer4. +/// The batch prover that constructs proofs at [`layer3`][layer3] and [`layer4`][layer4]. +/// +/// [layer3]: crate::ProofLayer::Layer3 +/// [layer4]: crate::ProofLayer::Layer4 #[derive(Default, Debug)] pub struct ProverTypeBatch; -/// The bundle prover that constructs proofs at layer5 and layer6. +/// The bundle prover that constructs proofs at [`layer5`][layer5] and [`layer6`][layer6]. +/// +/// [layer5]: crate::ProofLayer::Layer5 +/// [layer6]: crate::ProofLayer::Layer6 #[derive(Default, Debug)] pub struct ProverTypeBundle; +/// Auxiliary data attached to a [`ChunkProof`][proof] +/// +/// [proof]: crate::Proof #[derive(Serialize, Deserialize, Debug)] pub struct ChunkProofAuxData { - chunk_infos: Vec, + /// The chunk's information, that is eventually needed by the [`BatchProver`][batch_prover]'s + /// proof generation process. + /// + /// [batch_prover]: crate::prover::BatchProver::gen_proof + pub chunk_info: ChunkInfo, } +/// Auxiliary data attached to a [`BatchProof`][proof] +/// +/// [proof]: crate::Proof #[derive(Serialize, Deserialize, Debug)] pub struct BatchProofAuxData { - batch_hash: H256, + /// The hash of the [`BatchHeader`][batch_header] + /// + /// [batch_header]: aggregator::BatchHeader + pub batch_hash: H256, } impl ProverType for ProverTypeChunk { diff --git a/prover2/src/types/proof.rs b/prover2/src/types/proof.rs index b26a66dc08..047c2a743d 100644 --- a/prover2/src/types/proof.rs +++ b/prover2/src/types/proof.rs @@ -12,33 +12,40 @@ use crate::{ ProofLayer, ProverError, }; -/// Describes an output from a [`Prover`]'s proof generation process when given a [`ProvingTask`]. +/// Describes an output from a [`Prover`][prover]'s proof generation process when +/// given a [`ProvingTask`][proving_task]. +/// +/// [prover]: crate::prover::Prover +/// [proving_task]: crate::ProvingTask #[derive(Serialize, Deserialize, Debug)] pub struct Proof { /// Version of the source code (git describe --abbrev=8) used for proof generation. - git_version: String, + pub git_version: String, /// The proof layer. - layer: ProofLayer, - /// Verification key for this SNARK proof. + pub layer: ProofLayer, + /// The raw [`VerificationKey`][vk] for this [`SNARK`][snark] proof. + /// + /// [vk]: halo2_proofs::plonk::VerifyingKey + /// [snark]: snark_verifier_sdk::Snark #[serde(with = "base64")] - vk: Vec, + pub vk: Vec, /// The public instances (flattened bytes) to the SNARK. #[serde(with = "base64")] - instances: Vec, + pub instances: Vec, /// The protocol computed for SNARK. #[serde(with = "base64")] - protocol: Vec, + pub protocol: Vec, /// The inner proof. #[serde(with = "base64")] - proof: Vec, - /// Auxiliary data to attach with the proof. This data would generally be required by the next - /// layer's proof generation process. + pub proof: Vec, + /// Auxiliary data to attach with the proof. This data would generally be required + /// by the next layer's proof generation process. #[serde(flatten)] - aux: Aux, + pub aux: Aux, } impl Proof { - /// Construct a new [`Proof`] given the SNARK for the proof layer and some auxiliary data. + /// Construct a new proof given the SNARK for the proof layer and some auxiliary data. pub fn new_from_snark( layer: ProofLayer, snark: Snark, @@ -65,7 +72,7 @@ impl Proof { }) } - /// Construct a new [`Proof`] given the raw proof and instances for an EVM-verifiable proof. + /// Construct a new proof given the raw proof and instances for an EVM-verifiable proof. pub fn new_from_raw( layer: ProofLayer, instances: &[Fr], diff --git a/prover2/src/types/task.rs b/prover2/src/types/task.rs index d281d1a88e..4e38c7aeda 100644 --- a/prover2/src/types/task.rs +++ b/prover2/src/types/task.rs @@ -1,18 +1,37 @@ use serde::{de::DeserializeOwned, Deserialize, Serialize}; -/// Describes the behaviour to be supported by a type that can be used as an input (or a task) to -/// instruct a [`Prover`] to generate a proof. +/// Describes the behaviour to be supported by a [`ProverType`][prover_type] that +/// can be used as an input (or a task) to instruct a [`Prover`][prover] to generate +/// a proof. +/// +/// [prover_type]: crate::types::ProverType +/// [prover]: crate::prover::Prover pub trait ProvingTask: Serialize + DeserializeOwned + std::fmt::Debug { - /// An identifier for the proving task. + /// A unique identifier for the proving task. fn id(&self) -> String; } +/// A [`ProvingTask`] used to build the base circuit, i.e. [`SuperCircuit`][super_circuit], +/// for the [`ChunkProver`][chunk_prover]. +/// +/// [super_circuit]: zkevm_circuits::super_circuit::SuperCircuit +/// [chunk_prover]: crate::types::ProverTypeChunk #[derive(Debug, Serialize, Deserialize)] pub struct ChunkProvingTask; +/// A [`ProvingTask`] used to build the base circuit, i.e. [`BatchCircuit`][batch_circuit], +/// for the [`BatchProver`][batch_prover]. +/// +/// [batch_circuit]: aggregator::BatchCircuit +/// [batch_prover]: crate::types::ProverTypeBatch #[derive(Debug, Serialize, Deserialize)] pub struct BatchProvingTask; +/// A [`ProvingTask`] used to build the base circuit, i.e. [`RecursionCircuit`][recursion_circuit], +/// for the [`BundleProver`][bundle_prover]. +/// +/// [recursion_circuit]: aggregator::RecursionCircuit +/// [bundle_prover]: crate::types::ProverTypeBundle #[derive(Debug, Serialize, Deserialize)] pub struct BundleProvingTask; From 6ba7b597273d4a3df8b00bcffef80e9e3a6572ca Mon Sep 17 00:00:00 2001 From: Rohit Narurkar Date: Mon, 15 Jul 2024 18:05:17 +0100 Subject: [PATCH 15/17] doc/refactor/test --- prover2/Makefile | 6 +++- prover2/configs/layer1.config | 14 ++++++++ prover2/configs/layer2.config | 14 ++++++++ prover2/configs/layer3.config | 14 ++++++++ prover2/configs/layer4.config | 14 ++++++++ prover2/configs/layer5.config | 14 ++++++++ prover2/configs/layer6.config | 14 ++++++++ prover2/scripts/setup_test_data.sh | 11 +++++++ prover2/src/error.rs | 2 +- prover2/src/lib.rs | 11 +++---- prover2/src/prover/config.rs | 52 ++++++++++++++++-------------- prover2/src/prover/mod.rs | 14 ++++++-- prover2/src/types/layer.rs | 6 ++-- prover2/src/types/mod.rs | 39 +++++++++++----------- prover2/src/types/proof.rs | 5 +-- prover2/src/util/dir.rs | 21 ++++-------- prover2/src/util/mod.rs | 6 ++-- 17 files changed, 182 insertions(+), 75 deletions(-) create mode 100644 prover2/configs/layer1.config create mode 100644 prover2/configs/layer2.config create mode 100644 prover2/configs/layer3.config create mode 100644 prover2/configs/layer4.config create mode 100644 prover2/configs/layer5.config create mode 100644 prover2/configs/layer6.config create mode 100644 prover2/scripts/setup_test_data.sh diff --git a/prover2/Makefile b/prover2/Makefile index 503096bc9a..fbe2ffb50c 100644 --- a/prover2/Makefile +++ b/prover2/Makefile @@ -1,4 +1,3 @@ - help: ## Display this help screen @grep -h \ -E '^[a-zA-Z_-]+:.*?## .*$$' $(MAKEFILE_LIST) | \ @@ -7,3 +6,8 @@ help: ## Display this help screen # make download-setup -e degree=20 dir=./test_data/.params download-setup: sh scripts/download_setup.sh ${degree} ${dir} + +# usage: +# make setup-test-data -e dir=./test_data/.configs +setup-test-data: + sh scripts/setup_test_data.sh ${dir} diff --git a/prover2/configs/layer1.config b/prover2/configs/layer1.config new file mode 100644 index 0000000000..0c40c6951f --- /dev/null +++ b/prover2/configs/layer1.config @@ -0,0 +1,14 @@ +{ + "strategy": "Simple", + "degree": 24, + "num_advice": [ + 15 + ], + "num_lookup_advice": [ + 2 + ], + "num_fixed": 1, + "lookup_bits": 20, + "limb_bits": 88, + "num_limbs": 3 +} diff --git a/prover2/configs/layer2.config b/prover2/configs/layer2.config new file mode 100644 index 0000000000..736faa2699 --- /dev/null +++ b/prover2/configs/layer2.config @@ -0,0 +1,14 @@ +{ + "strategy": "Simple", + "degree": 25, + "num_advice": [ + 1 + ], + "num_lookup_advice": [ + 1 + ], + "num_fixed": 1, + "lookup_bits": 24, + "limb_bits": 88, + "num_limbs": 3 +} diff --git a/prover2/configs/layer3.config b/prover2/configs/layer3.config new file mode 100644 index 0000000000..be4fea746c --- /dev/null +++ b/prover2/configs/layer3.config @@ -0,0 +1,14 @@ +{ + "strategy": "Simple", + "degree": 21, + "num_advice": [ + 63 + ], + "num_lookup_advice": [ + 8 + ], + "num_fixed": 2, + "lookup_bits": 20, + "limb_bits": 88, + "num_limbs": 3 +} diff --git a/prover2/configs/layer4.config b/prover2/configs/layer4.config new file mode 100644 index 0000000000..ec9ac98c2b --- /dev/null +++ b/prover2/configs/layer4.config @@ -0,0 +1,14 @@ +{ + "strategy": "Simple", + "degree": 26, + "num_advice": [ + 2 + ], + "num_lookup_advice": [ + 1 + ], + "num_fixed": 1, + "lookup_bits": 25, + "limb_bits": 88, + "num_limbs": 3 +} diff --git a/prover2/configs/layer5.config b/prover2/configs/layer5.config new file mode 100644 index 0000000000..dc976d773d --- /dev/null +++ b/prover2/configs/layer5.config @@ -0,0 +1,14 @@ +{ + "strategy": "Simple", + "degree": 21, + "num_advice": [ + 4 + ], + "num_lookup_advice": [ + 1 + ], + "num_fixed": 1, + "lookup_bits": 20, + "limb_bits" :88, + "num_limbs": 3 +} \ No newline at end of file diff --git a/prover2/configs/layer6.config b/prover2/configs/layer6.config new file mode 100644 index 0000000000..dd752cb3ce --- /dev/null +++ b/prover2/configs/layer6.config @@ -0,0 +1,14 @@ +{ + "strategy": "Simple", + "degree": 26, + "num_advice": [ + 1 + ], + "num_lookup_advice": [ + 1 + ], + "num_fixed": 1, + "lookup_bits": 25, + "limb_bits": 88, + "num_limbs": 3 +} diff --git a/prover2/scripts/setup_test_data.sh b/prover2/scripts/setup_test_data.sh new file mode 100644 index 0000000000..dcb681f28d --- /dev/null +++ b/prover2/scripts/setup_test_data.sh @@ -0,0 +1,11 @@ +set -x +set -e + +dir="${1:-"./test_data/.configs"}" +mkdir -p "$dir" + +for ((i = 1; i < 7; ++i)); do + template="./configs/layer${i}.config" + file="$dir/layer${i}.config" + jq ".degree = ${i}" ${template} > ${file} +done diff --git a/prover2/src/error.rs b/prover2/src/error.rs index 788cdb10be..098dbf1093 100644 --- a/prover2/src/error.rs +++ b/prover2/src/error.rs @@ -2,7 +2,7 @@ use std::path::PathBuf; use thiserror::Error; -use crate::ProofLayer; +use crate::types::layer::ProofLayer; /// Represents error variants possibly encountered during the proof generation process. #[derive(Debug, Error)] diff --git a/prover2/src/lib.rs b/prover2/src/lib.rs index 86d874ab2f..f15fdf7a33 100644 --- a/prover2/src/lib.rs +++ b/prover2/src/lib.rs @@ -1,13 +1,10 @@ mod error; - -pub mod prover; - -pub mod types; +pub use error::ProverError; mod util; mod verifier; -pub use error::ProverError; -pub use prover::{config::ProverConfig, BatchProver, BundleProver, ChunkProver, Prover}; -pub use types::{layer::ProofLayer, proof::Proof, task::ProvingTask}; +pub mod prover; + +pub mod types; diff --git a/prover2/src/prover/config.rs b/prover2/src/prover/config.rs index 30ab95365b..115ee85f36 100644 --- a/prover2/src/prover/config.rs +++ b/prover2/src/prover/config.rs @@ -5,18 +5,18 @@ use halo2_proofs::{ plonk::{keygen_pk2, Circuit, ProvingKey}, poly::kzg::commitment::ParamsKZG, }; -use tracing::{debug, info, instrument, trace}; +use tracing::{info, instrument, trace, warn}; use crate::{ prover::params::NonNativeParams, - types::ProverType, + types::{layer::ProofLayer, ProverType}, util::{ - default_cache_dir, default_kzg_params_dir, default_non_native_params_dir, kzg_params_path, + default_kzg_params_dir, default_non_native_params_dir, kzg_params_path, non_native_params_path as nn_params_path, read_env_or_default, read_json, read_kzg_params, CACHE_PATH_EVM, CACHE_PATH_PI, CACHE_PATH_PROOFS, CACHE_PATH_SNARKS, CACHE_PATH_TASKS, DEFAULT_DEGREE_LAYER0, ENV_DEGREE_LAYER0, JSON_EXT, }, - ProofLayer, ProverError, + ProverError, }; /// Configuration for a generic prover. @@ -87,7 +87,7 @@ impl ProverConfig { /// Setup the prover config by reading relevant config files from storage. #[instrument(name = "ProverConfig::setup", skip(self))] pub fn setup(mut self) -> Result { - info!("setting up ProverConfig"); + info!(name = "setup prover config", prover_type = ?Type::NAME.to_string()); // The proof layers that this prover needs to generate proofs for. let proof_layers = Type::layers(); @@ -101,52 +101,57 @@ impl ProverConfig { .kzg_params_dir .clone() .unwrap_or(default_kzg_params_dir()?); - let cache_dir = self.cache_dir.clone().unwrap_or(default_cache_dir()?); + trace!(name = "config directories", non_native_params = ?nn_params_dir, kzg_params = ?kzg_params_dir, caching = ?self.cache_dir); + if self.cache_dir.is_none() { + warn!(name = "setup prover without caching"); + } // Read and store non-native field arithmetic config params for each layer. - trace!("loading non-native field arithmetic params"); for layer in proof_layers { // Layer0 (SuperCircuit) does not have non-native field arithmetics. if layer != ProofLayer::Layer0 { let params_path = nn_params_path(nn_params_dir.as_path(), layer); - debug!("reading config params for {:?}: {:?}", layer, params_path); + trace!(name = "read config for non-native field arithmetics", ?layer, path = ?params_path); let params = read_json::(params_path.as_path())?; self.degrees.insert(layer, params.degree); self.nn_params.insert(layer, params); } if layer == ProofLayer::Layer0 { + trace!(name = "read environment variable", ?layer, key = ?ENV_DEGREE_LAYER0, default = ?DEFAULT_DEGREE_LAYER0); let layer0_degree = read_env_or_default(ENV_DEGREE_LAYER0, DEFAULT_DEGREE_LAYER0); + trace!(name = "configured degree", ?layer, degree = ?layer0_degree); self.degrees.insert(ProofLayer::Layer0, layer0_degree); } } // Read and store KZG setup params for each layer. - trace!("loading KZG setup params"); for (&layer, °ree) in self.degrees.iter() { let params_path = kzg_params_path(kzg_params_dir.as_path(), degree); - debug!( - "reading kzg params for {:?} (degree = {:?}): {:?}", - layer, degree, params_path + trace!( + name = "read KZG setup parameters", + ?layer, + ?degree, + path = ?params_path, ); let params = read_kzg_params(params_path.as_path())?; self.kzg_params.insert(layer, params); } // Setup the cache directory's structure. - trace!("setting up cache"); - create_dir_all(cache_dir.join(CACHE_PATH_TASKS))?; - create_dir_all(cache_dir.join(CACHE_PATH_SNARKS))?; - create_dir_all(cache_dir.join(CACHE_PATH_PROOFS))?; - create_dir_all(cache_dir.join(CACHE_PATH_PI))?; - create_dir_all(cache_dir.join(CACHE_PATH_EVM))?; + if let Some(ref cache_dir) = self.cache_dir { + create_dir_all(cache_dir.join(CACHE_PATH_TASKS))?; + create_dir_all(cache_dir.join(CACHE_PATH_SNARKS))?; + create_dir_all(cache_dir.join(CACHE_PATH_PROOFS))?; + create_dir_all(cache_dir.join(CACHE_PATH_PI))?; + create_dir_all(cache_dir.join(CACHE_PATH_EVM))?; + } // Update directories in self. self.nn_params_dir.replace(nn_params_dir); self.kzg_params_dir.replace(kzg_params_dir); - self.cache_dir.replace(cache_dir); - info!("setup ProverConfig"); + info!(name = "setup prover config OK", prover_type = ?Type::NAME.to_string()); Ok(self) } @@ -212,8 +217,8 @@ mod tests { use aggregator::MAX_AGG_SNARKS; use crate::{ + prover::ProverConfig, types::{ProverTypeBatch, ProverTypeChunk}, - ProverConfig, }; #[test] @@ -226,15 +231,14 @@ mod tests { let test_dir = current_dir()?.join("test_data"); let _chunk_prover_config = ProverConfig::::default() - .with_nn_params_dir(test_dir.join(".config")) + .with_nn_params_dir(test_dir.join(".configs")) .with_kzg_params_dir(test_dir.join(".params")) .with_cache_dir(test_dir.join(".cache")) .setup()?; let _batch_prover_config = ProverConfig::>::default() - .with_nn_params_dir(test_dir.join(".config")) + .with_nn_params_dir(test_dir.join(".configs")) .with_kzg_params_dir(test_dir.join(".params")) - .with_cache_dir(test_dir.join(".cache")) .setup()?; Ok(()) diff --git a/prover2/src/prover/mod.rs b/prover2/src/prover/mod.rs index b0ad294006..671cfbb6d8 100644 --- a/prover2/src/prover/mod.rs +++ b/prover2/src/prover/mod.rs @@ -4,12 +4,22 @@ use snark_verifier_sdk::{CircuitExt, Snark}; use tracing::{info, instrument, trace}; use crate::{ - types::{ProverType, ProverTypeBatch, ProverTypeBundle, ProverTypeChunk}, + prover::config::ProverConfig, + types::{ + layer::ProofLayer, proof::Proof, task::ProvingTask, ProverType, ProverTypeBatch, + ProverTypeBundle, ProverTypeChunk, + }, util::{gen_rng, read_json, write_json}, - Proof, ProofLayer, ProverConfig, ProverError, ProvingTask, + ProverError, }; +/// Includes the configuration setup related to a [`Prover`]. pub mod config; + +/// Config parameters for non-native field arithmetics, that will be used to configure the +/// [`FpConfig`][fp_config] chip from halo2-lib. +/// +/// [fp_config]: snark_verifier::loader::halo2::halo2_ecc::fields::fp::FpConfig pub mod params; /// Convenience type for chunk prover. diff --git a/prover2/src/types/layer.rs b/prover2/src/types/layer.rs index cbd128ee95..facd3bf17d 100644 --- a/prover2/src/types/layer.rs +++ b/prover2/src/types/layer.rs @@ -12,7 +12,7 @@ pub enum ProofLayer { /// The compression layer on top of layer1. The proof from this layer is the [`Proof`][proof] returned /// by the [`ChunkProver`][gen_proof]. /// - /// [proof]: crate::Proof + /// [proof]: crate::types::proof::Proof /// [gen_proof]: crate::prover::ChunkProver::gen_proof Layer2, /// The batch circuit layer. At this layer, we batch multiple `ChunkProof`s. @@ -20,7 +20,7 @@ pub enum ProofLayer { /// The compression layer on top of layer3. The proof from this layer is the [`Proof`][proof] returned /// by the [`BatchProver`][gen_proof]. /// - /// [proof]: crate::Proof + /// [proof]: crate::types::proof::Proof /// [gen_proof]: crate::prover::BatchProver::gen_proof Layer4, /// The recursion circuit layer. At this layer, we construct proofs recursively over a previous @@ -29,7 +29,7 @@ pub enum ProofLayer { /// The compression layer on top of layer5. The proof from this layer is the [`Proof`][proof] returned /// by the [`BundleProver`][gen_proof]. /// - /// [proof]: crate::Proof + /// [proof]: crate::types::proof::Proof /// [gen_proof]: crate::prover::BundleProver::gen_proof Layer6, } diff --git a/prover2/src/types/mod.rs b/prover2/src/types/mod.rs index 91e7724e56..1a90d453b6 100644 --- a/prover2/src/types/mod.rs +++ b/prover2/src/types/mod.rs @@ -8,7 +8,10 @@ use serde::{de::DeserializeOwned, Deserialize, Serialize}; use snark_verifier_sdk::{CircuitExt, Snark}; use zkevm_circuits::super_circuit::params::ScrollSuperCircuit; -use crate::{ProofLayer, ProverError, ProvingTask}; +use crate::{ + types::{layer::ProofLayer, task::ProvingTask}, + ProverError, +}; pub mod layer; @@ -29,13 +32,13 @@ use task::{BatchProvingTask, ChunkProvingTask}; /// and compress it ([`layer6`][layer6]) where this outermost proof (`layer6`) is the `BundleProof`. /// The `BundleProof` is an EVM-verifiable proof. /// -/// [layer0]: crate::ProofLayer::Layer0 -/// [layer1]: crate::ProofLayer::Layer1 -/// [layer2]: crate::ProofLayer::Layer2 -/// [layer3]: crate::ProofLayer::Layer3 -/// [layer4]: crate::ProofLayer::Layer4 -/// [layer5]: crate::ProofLayer::Layer5 -/// [layer6]: crate::ProofLayer::Layer6 +/// [layer0]: crate::types::layer::ProofLayer::Layer0 +/// [layer1]: crate::types::layer::ProofLayer::Layer1 +/// [layer2]: crate::types::layer::ProofLayer::Layer2 +/// [layer3]: crate::types::layer::ProofLayer::Layer3 +/// [layer4]: crate::types::layer::ProofLayer::Layer4 +/// [layer5]: crate::types::layer::ProofLayer::Layer5 +/// [layer6]: crate::types::layer::ProofLayer::Layer6 pub trait ProverType: std::fmt::Debug { /// The name of the prover. const NAME: &'static str; @@ -60,7 +63,7 @@ pub trait ProverType: std::fmt::Debug { /// which is then used by the [`BatchProver`][batch_prover] to construct its /// [`BatchProvingTask`]. /// - /// [proof]: crate::Proof + /// [proof]: crate::types::proof::Proof /// [chunk_prover]: crate::prover::ChunkProver /// [batch_prover]: crate::prover::BatchProver type ProofAuxData: Serialize + DeserializeOwned + std::fmt::Debug; @@ -111,29 +114,29 @@ pub trait ProverType: std::fmt::Debug { /// The chunk prover that constructs proofs at [`layer0`][layer0], [`layer1`][layer1] and [`layer2`][layer2]. /// -/// [layer0]: crate::ProofLayer::Layer0 -/// [layer1]: crate::ProofLayer::Layer1 -/// [layer2]: crate::ProofLayer::Layer2 +/// [layer0]: crate::types::layer::ProofLayer::Layer0 +/// [layer1]: crate::types::layer::ProofLayer::Layer1 +/// [layer2]: crate::types::layer::ProofLayer::Layer2 #[derive(Default, Debug)] pub struct ProverTypeChunk; /// The batch prover that constructs proofs at [`layer3`][layer3] and [`layer4`][layer4]. /// -/// [layer3]: crate::ProofLayer::Layer3 -/// [layer4]: crate::ProofLayer::Layer4 +/// [layer3]: crate::types::layer::ProofLayer::Layer3 +/// [layer4]: crate::types::layer::ProofLayer::Layer4 #[derive(Default, Debug)] pub struct ProverTypeBatch; /// The bundle prover that constructs proofs at [`layer5`][layer5] and [`layer6`][layer6]. /// -/// [layer5]: crate::ProofLayer::Layer5 -/// [layer6]: crate::ProofLayer::Layer6 +/// [layer5]: crate::types::layer::ProofLayer::Layer5 +/// [layer6]: crate::types::layer::ProofLayer::Layer6 #[derive(Default, Debug)] pub struct ProverTypeBundle; /// Auxiliary data attached to a [`ChunkProof`][proof] /// -/// [proof]: crate::Proof +/// [proof]: crate::types::proof::Proof #[derive(Serialize, Deserialize, Debug)] pub struct ChunkProofAuxData { /// The chunk's information, that is eventually needed by the [`BatchProver`][batch_prover]'s @@ -145,7 +148,7 @@ pub struct ChunkProofAuxData { /// Auxiliary data attached to a [`BatchProof`][proof] /// -/// [proof]: crate::Proof +/// [proof]: crate::types::proof::Proof #[derive(Serialize, Deserialize, Debug)] pub struct BatchProofAuxData { /// The hash of the [`BatchHeader`][batch_header] diff --git a/prover2/src/types/proof.rs b/prover2/src/types/proof.rs index 047c2a743d..7525ebc4e5 100644 --- a/prover2/src/types/proof.rs +++ b/prover2/src/types/proof.rs @@ -8,15 +8,16 @@ use serde::{Deserialize, Serialize}; use snark_verifier_sdk::Snark; use crate::{ + types::layer::ProofLayer, util::{deserialize_be, serialize_be, GIT_VERSION}, - ProofLayer, ProverError, + ProverError, }; /// Describes an output from a [`Prover`][prover]'s proof generation process when /// given a [`ProvingTask`][proving_task]. /// /// [prover]: crate::prover::Prover -/// [proving_task]: crate::ProvingTask +/// [proving_task]: crate::types::task::ProvingTask #[derive(Serialize, Deserialize, Debug)] pub struct Proof { /// Version of the source code (git describe --abbrev=8) used for proof generation. diff --git a/prover2/src/util/dir.rs b/prover2/src/util/dir.rs index a765d53ccc..60c2e6f2c1 100644 --- a/prover2/src/util/dir.rs +++ b/prover2/src/util/dir.rs @@ -3,7 +3,10 @@ use std::{ path::{Path, PathBuf}, }; -use crate::{ProofLayer, ProverError}; +use crate::{types::layer::ProofLayer, ProverError}; + +/// Test data directory. +pub const TEST_DATA_DIR: &str = "test_data"; /// The extension used for JSON files. pub const JSON_EXT: &str = ".json"; @@ -12,14 +15,11 @@ pub const JSON_EXT: &str = ".json"; pub const NON_NATIVE_PARAMS_EXT: &str = ".config"; /// The config parameters for non native field arithmetics are by default in this directory. -pub const NON_NATIVE_PARAMS_DIR: &str = ".config"; +pub const NON_NATIVE_PARAMS_DIR: &str = ".configs"; /// The KZG setup parameters are by default in this directory. pub const KZG_PARAMS_DIR: &str = ".params"; -/// The default directory used for cached data. -pub const CACHE_PATH: &str = ".cache"; - /// The directory within cache to store proving tasks in JSON format. pub const CACHE_PATH_TASKS: &str = "tasks"; @@ -58,19 +58,12 @@ pub fn pwd() -> Result { /// /// /.config pub fn default_non_native_params_dir() -> Result { - Ok(pwd()?.join(NON_NATIVE_PARAMS_DIR)) + Ok(pwd()?.join(TEST_DATA_DIR).join(NON_NATIVE_PARAMS_DIR)) } /// The default path to find KZG setup parameters. /// /// /.params pub fn default_kzg_params_dir() -> Result { - Ok(pwd()?.join(KZG_PARAMS_DIR)) -} - -/// The default path to the cache directory. -/// -/// /.cache -pub fn default_cache_dir() -> Result { - Ok(pwd()?.join(CACHE_PATH)) + Ok(pwd()?.join(TEST_DATA_DIR).join(KZG_PARAMS_DIR)) } diff --git a/prover2/src/util/mod.rs b/prover2/src/util/mod.rs index 75eaf8f791..892c36f905 100644 --- a/prover2/src/util/mod.rs +++ b/prover2/src/util/mod.rs @@ -5,9 +5,9 @@ use rand_xorshift::XorShiftRng; mod dir; pub use dir::{ - default_cache_dir, default_kzg_params_dir, default_non_native_params_dir, kzg_params_path, - non_native_params_path, CACHE_PATH_EVM, CACHE_PATH_PI, CACHE_PATH_PROOFS, CACHE_PATH_SNARKS, - CACHE_PATH_TASKS, JSON_EXT, + default_kzg_params_dir, default_non_native_params_dir, kzg_params_path, non_native_params_path, + CACHE_PATH_EVM, CACHE_PATH_PI, CACHE_PATH_PROOFS, CACHE_PATH_SNARKS, CACHE_PATH_TASKS, + JSON_EXT, }; mod io; From bbb5c942df6c89db781e7e8143942015e1ec23ea Mon Sep 17 00:00:00 2001 From: Rohit Narurkar Date: Mon, 15 Jul 2024 22:51:12 +0100 Subject: [PATCH 16/17] gen/cache evm verifier --- prover2/src/prover/config.rs | 13 +++++++++++++ prover2/src/prover/mod.rs | 26 ++++++++++++++++++++++++-- prover2/src/util/mod.rs | 6 +++--- prover2/src/util/serde.rs | 4 ++++ 4 files changed, 44 insertions(+), 5 deletions(-) diff --git a/prover2/src/prover/config.rs b/prover2/src/prover/config.rs index 115ee85f36..963e9446d4 100644 --- a/prover2/src/prover/config.rs +++ b/prover2/src/prover/config.rs @@ -191,6 +191,19 @@ impl ProverConfig { .as_ref() .map(|dir| dir.join(CACHE_PATH_SNARKS).join(format!("{layer:?}-{id}"))) } + + /// Returns the paths to dump the EVM plonk verifier contract (in YUL) and the deployment code + /// for the verifier contract. + pub fn path_evm(&self, commit_ref: &str) -> Option<(PathBuf, PathBuf)> { + self.cache_dir.as_ref().map(|dir| { + ( + dir.join(CACHE_PATH_EVM) + .join(format!("evm_verifier_{commit_ref}.yul")), + dir.join(CACHE_PATH_EVM) + .join(format!("evm_verifier_{commit_ref}.bin")), + ) + }) + } } impl ProverConfig { diff --git a/prover2/src/prover/mod.rs b/prover2/src/prover/mod.rs index 671cfbb6d8..c81ebc5672 100644 --- a/prover2/src/prover/mod.rs +++ b/prover2/src/prover/mod.rs @@ -1,5 +1,7 @@ use aggregator::MAX_AGG_SNARKS; -use halo2_proofs::halo2curves::bn256::Fr; +use ethers_core::utils::keccak256; +use halo2_proofs::halo2curves::bn256::{Bn256, Fr}; +use snark_verifier::pcs::kzg::{Bdfg21, Kzg}; use snark_verifier_sdk::{CircuitExt, Snark}; use tracing::{info, instrument, trace}; @@ -9,7 +11,7 @@ use crate::{ layer::ProofLayer, proof::Proof, task::ProvingTask, ProverType, ProverTypeBatch, ProverTypeBundle, ProverTypeChunk, }, - util::{gen_rng, read_json, write_json}, + util::{gen_rng, read_json, write, write_json, GIT_VERSION}, ProverError, }; @@ -127,6 +129,8 @@ impl Prover { task: Type::Task, ) -> Result, ProverError> { let id = task.id(); + let commit_ref = *GIT_VERSION; + let evm_paths = self.config.path_evm(commit_ref); // Generate SNARKs for all the layers at which the prover operates. // @@ -151,6 +155,7 @@ impl Prover { let kzg_params = self.config.kzg_params(outermost_layer)?; let compression_circuit = Type::build_compression(kzg_params, snark, outermost_layer); let instances = compression_circuit.instances(); + let num_instances = compression_circuit.num_instance(); let (kzg_params, pk) = self .config .gen_proving_key(outermost_layer, &compression_circuit)?; @@ -165,6 +170,23 @@ impl Prover { ); trace!(name = "gen evm proof OK", layer = ?outermost_layer); + // At this point we know the verifying key, KZG setup parameters and the proof. We can + // build the Plonk verifier contract (EVM-verifier). + if let Some((evm_yul_path, evm_deployment_code_path)) = evm_paths { + trace!(name = "gen evm verifier", path = ?evm_yul_path); + let evm_deployment_code = snark_verifier_sdk::gen_evm_verifier::< + Type::CompressionCircuit, + Kzg, + >( + kzg_params, + pk.get_vk(), + num_instances, + Some(evm_yul_path.as_path()), + ); + trace!(name = "write evm verifier deployment code", path = ?evm_deployment_code_path, code_hash = ?keccak256(&evm_deployment_code)); + write(evm_deployment_code_path.as_path(), &evm_deployment_code)?; + } + let proof = Proof::new_from_raw(outermost_layer, &instances[0], &raw_proof, pk, aux_data); Ok(proof) diff --git a/prover2/src/util/mod.rs b/prover2/src/util/mod.rs index 892c36f905..9be2eda911 100644 --- a/prover2/src/util/mod.rs +++ b/prover2/src/util/mod.rs @@ -10,12 +10,12 @@ pub use dir::{ JSON_EXT, }; -mod io; -pub use io::{read_json, read_kzg_params, write_json}; - mod env; pub use env::read_env_or_default; +mod io; +pub use io::{read_json, read_kzg_params, write, write_json}; + mod serde; pub use serde::{deserialize_be, serialize_be}; diff --git a/prover2/src/util/serde.rs b/prover2/src/util/serde.rs index 97b7eb4c10..8caf460bad 100644 --- a/prover2/src/util/serde.rs +++ b/prover2/src/util/serde.rs @@ -2,11 +2,15 @@ use halo2_proofs::halo2curves::bn256::Fr; use snark_verifier::loader::halo2::halo2_ecc::halo2_base::utils::ScalarField; /// Serialize a field element [`Fr`] to big-endian bytes. +/// +/// [`Fr`]: halo2_proofs::halo2curves::bn256::Fr pub fn serialize_be(field: &Fr) -> Vec { field.to_bytes().into_iter().rev().collect() } /// Deserialize a field element [`Fr`] from big-endian bytes. +/// +/// [`Fr`]: halo2_proofs::halo2curves::bn256::Fr pub fn deserialize_be(be_bytes: &[u8]) -> Fr { let mut le_bytes = [0u8; 32]; le_bytes.copy_from_slice(be_bytes); From c319165ba59418c1be3735e04359ed37fbf865d8 Mon Sep 17 00:00:00 2001 From: Rohit Narurkar Date: Tue, 30 Jul 2024 14:44:38 +0100 Subject: [PATCH 17/17] fix(doc): post-merge against develop --- Cargo.lock | 2 +- prover2/src/types/mod.rs | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 90d5b5fdc2..df91c588ca 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3546,7 +3546,7 @@ dependencies = [ [[package]] name = "prover2" -version = "0.11.0" +version = "0.12.0" dependencies = [ "aggregator", "anyhow", diff --git a/prover2/src/types/mod.rs b/prover2/src/types/mod.rs index 1a90d453b6..b6ad824516 100644 --- a/prover2/src/types/mod.rs +++ b/prover2/src/types/mod.rs @@ -1,4 +1,4 @@ -use aggregator::{AggregationCircuit, ChunkInfo, CompressionCircuit}; +use aggregator::{BatchCircuit, ChunkInfo, CompressionCircuit}; use ethers_core::types::H256; use halo2_proofs::{ halo2curves::bn256::{Bn256, Fr}, @@ -190,7 +190,7 @@ impl ProverType for ProverTypeBatch { type Task = BatchProvingTask; - type BaseCircuit = AggregationCircuit; + type BaseCircuit = BatchCircuit; type CompressionCircuit = CompressionCircuit;