diff --git a/.gitignore b/.gitignore index 8ea299a653..7caf13d3bd 100644 --- a/.gitignore +++ b/.gitignore @@ -5,7 +5,6 @@ .idea *.log *.json -*.sh *.txt *.srs -tmp \ No newline at end of file +tmp diff --git a/Cargo.lock b/Cargo.lock index fe058a3e7c..a6f768d6ba 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2755,6 +2755,15 @@ dependencies = [ "winapi", ] +[[package]] +name = "matchers" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8263075bb86c5a1b1427b5ae862e8889656f126e9f77c484496e8b47cf5c5558" +dependencies = [ + "regex-automata 0.1.10", +] + [[package]] name = "maybe-rayon" version = "0.1.1" @@ -2859,6 +2868,16 @@ version = "1.0.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "650eef8c711430f1a879fdd01d4745a7deea475becfb90269c06775983bbf086" +[[package]] +name = "nu-ansi-term" +version = "0.46.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77a8165726e8236064dbb45459242600304b42a5ea24ee2948e18e023bf7ba84" +dependencies = [ + "overload", + "winapi", +] + [[package]] name = "num" version = "0.4.2" @@ -3051,6 +3070,12 @@ dependencies = [ "syn 1.0.109", ] +[[package]] +name = "overload" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b15813163c1d831bf4a13c3610c05c0d03b39feb07f7e09fa234dac9b15aaf39" + [[package]] name = "pairing" version = "0.23.0" @@ -3519,6 +3544,31 @@ dependencies = [ "zkevm-circuits", ] +[[package]] +name = "prover2" +version = "0.12.0" +dependencies = [ + "aggregator", + "anyhow", + "base64 0.13.1", + "eth-types", + "ethers-core", + "gadgets", + "git-version", + "halo2_proofs", + "once_cell", + "rand", + "rand_xorshift", + "serde", + "serde_json", + "snark-verifier", + "snark-verifier-sdk", + "thiserror", + "tracing", + "tracing-subscriber", + "zkevm-circuits", +] + [[package]] name = "psm" version = "0.1.21" @@ -3636,10 +3686,19 @@ checksum = "c117dbdfde9c8308975b6a18d71f3f385c89461f7b3fb054288ecf2a2058ba4c" dependencies = [ "aho-corasick", "memchr", - "regex-automata", + "regex-automata 0.4.6", "regex-syntax 0.8.3", ] +[[package]] +name = "regex-automata" +version = "0.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c230d73fb8d8c1b9c0b3135c5142a8acee3a0558fb8db5cf1cb65f8d7862132" +dependencies = [ + "regex-syntax 0.6.29", +] + [[package]] name = "regex-automata" version = "0.4.6" @@ -4301,6 +4360,15 @@ dependencies = [ "cfg-if 1.0.0", ] +[[package]] +name = "sharded-slab" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f40ca3c46823713e0d4209592e8d6e826aa57e928f09752619fc696c499637f6" +dependencies = [ + "lazy_static", +] + [[package]] name = "signature" version = "2.2.0" @@ -4702,6 +4770,16 @@ dependencies = [ "winapi", ] +[[package]] +name = "thread_local" +version = "1.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b9ef9bad013ada3808854ceac7b46812a6465ba368859a37e2100283d2d719c" +dependencies = [ + "cfg-if 1.0.0", + "once_cell", +] + [[package]] name = "threadpool" version = "1.8.1" @@ -4910,6 +4988,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c06d3da6113f116aaee68e4d601191614c9053067f9ab7f6edbcb161237daa54" dependencies = [ "once_cell", + "valuable", ] [[package]] @@ -4922,6 +5001,35 @@ dependencies = [ "tracing", ] +[[package]] +name = "tracing-log" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee855f1f400bd0e5c02d150ae5de3840039a3f54b025156404e34c23c03f47c3" +dependencies = [ + "log", + "once_cell", + "tracing-core", +] + +[[package]] +name = "tracing-subscriber" +version = "0.3.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ad0f048c97dbd9faa9b7df56362b8ebcaa52adb06b498c050d2f4e32f90a7a8b" +dependencies = [ + "matchers", + "nu-ansi-term", + "once_cell", + "regex", + "sharded-slab", + "smallvec", + "thread_local", + "tracing", + "tracing-core", + "tracing-log", +] + [[package]] name = "try-lock" version = "0.2.5" diff --git a/Cargo.toml b/Cargo.toml index a375c91d2f..b041e7b7c2 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -12,7 +12,8 @@ members = [ "mock", "testool", "aggregator", - "prover" + "prover", + "prover2" ] resolver = "2" diff --git a/prover2/.gitignore b/prover2/.gitignore new file mode 100644 index 0000000000..74c3f9fc83 --- /dev/null +++ b/prover2/.gitignore @@ -0,0 +1 @@ +test_data/ diff --git a/prover2/Cargo.toml b/prover2/Cargo.toml new file mode 100644 index 0000000000..25c0ae6474 --- /dev/null +++ b/prover2/Cargo.toml @@ -0,0 +1,30 @@ +[package] +name = "prover2" +version.workspace = true +edition.workspace = true +license.workspace = true + +[dependencies] +aggregator = { path = "../aggregator" } +eth-types = { path = "../eth-types" } +gadgets = { path = "../gadgets" } +zkevm-circuits = { path = "../zkevm-circuits" } + +anyhow.workspace = true +base64.workspace = true +ethers-core.workspace = true +halo2_proofs.workspace = true +once_cell.workspace = true +rand.workspace = true +rand_xorshift.workspace = true +serde.workspace = true +serde_json.workspace = true +snark-verifier.workspace = true +snark-verifier-sdk.workspace = true + +git-version = "0.3.5" +thiserror = "1.0" +tracing = "0.1" + +[dev-dependencies] +tracing-subscriber = { version = "0.3", features = ["env-filter", "std"] } diff --git a/prover2/Makefile b/prover2/Makefile new file mode 100644 index 0000000000..fbe2ffb50c --- /dev/null +++ b/prover2/Makefile @@ -0,0 +1,13 @@ +help: ## Display this help screen + @grep -h \ + -E '^[a-zA-Z_-]+:.*?## .*$$' $(MAKEFILE_LIST) | \ + awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}' +# usage: +# make download-setup -e degree=20 dir=./test_data/.params +download-setup: + sh scripts/download_setup.sh ${degree} ${dir} + +# usage: +# make setup-test-data -e dir=./test_data/.configs +setup-test-data: + sh scripts/setup_test_data.sh ${dir} diff --git a/prover2/configs/layer1.config b/prover2/configs/layer1.config new file mode 100644 index 0000000000..0c40c6951f --- /dev/null +++ b/prover2/configs/layer1.config @@ -0,0 +1,14 @@ +{ + "strategy": "Simple", + "degree": 24, + "num_advice": [ + 15 + ], + "num_lookup_advice": [ + 2 + ], + "num_fixed": 1, + "lookup_bits": 20, + "limb_bits": 88, + "num_limbs": 3 +} diff --git a/prover2/configs/layer2.config b/prover2/configs/layer2.config new file mode 100644 index 0000000000..736faa2699 --- /dev/null +++ b/prover2/configs/layer2.config @@ -0,0 +1,14 @@ +{ + "strategy": "Simple", + "degree": 25, + "num_advice": [ + 1 + ], + "num_lookup_advice": [ + 1 + ], + "num_fixed": 1, + "lookup_bits": 24, + "limb_bits": 88, + "num_limbs": 3 +} diff --git a/prover2/configs/layer3.config b/prover2/configs/layer3.config new file mode 100644 index 0000000000..be4fea746c --- /dev/null +++ b/prover2/configs/layer3.config @@ -0,0 +1,14 @@ +{ + "strategy": "Simple", + "degree": 21, + "num_advice": [ + 63 + ], + "num_lookup_advice": [ + 8 + ], + "num_fixed": 2, + "lookup_bits": 20, + "limb_bits": 88, + "num_limbs": 3 +} diff --git a/prover2/configs/layer4.config b/prover2/configs/layer4.config new file mode 100644 index 0000000000..ec9ac98c2b --- /dev/null +++ b/prover2/configs/layer4.config @@ -0,0 +1,14 @@ +{ + "strategy": "Simple", + "degree": 26, + "num_advice": [ + 2 + ], + "num_lookup_advice": [ + 1 + ], + "num_fixed": 1, + "lookup_bits": 25, + "limb_bits": 88, + "num_limbs": 3 +} diff --git a/prover2/configs/layer5.config b/prover2/configs/layer5.config new file mode 100644 index 0000000000..dc976d773d --- /dev/null +++ b/prover2/configs/layer5.config @@ -0,0 +1,14 @@ +{ + "strategy": "Simple", + "degree": 21, + "num_advice": [ + 4 + ], + "num_lookup_advice": [ + 1 + ], + "num_fixed": 1, + "lookup_bits": 20, + "limb_bits" :88, + "num_limbs": 3 +} \ No newline at end of file diff --git a/prover2/configs/layer6.config b/prover2/configs/layer6.config new file mode 100644 index 0000000000..dd752cb3ce --- /dev/null +++ b/prover2/configs/layer6.config @@ -0,0 +1,14 @@ +{ + "strategy": "Simple", + "degree": 26, + "num_advice": [ + 1 + ], + "num_lookup_advice": [ + 1 + ], + "num_fixed": 1, + "lookup_bits": 25, + "limb_bits": 88, + "num_limbs": 3 +} diff --git a/prover2/scripts/download_setup.sh b/prover2/scripts/download_setup.sh new file mode 100644 index 0000000000..d463724e3c --- /dev/null +++ b/prover2/scripts/download_setup.sh @@ -0,0 +1,15 @@ +set -x +set -e + +# Set degree to env SCROLL_PROVER_MAX_DEGREE, first input or default value 26. +degree="${SCROLL_PROVER_MAX_DEGREE:-${1:-26}}" + +# Set the output dir to second input or default as `./integration/params`. +dir="${2:-"./test_data/.params"}" +mkdir -p "$dir" + +file="$dir"/params"${degree}" +rm -f "$file" + +# degree 1 - 26 +axel -ac https://circuit-release.s3.us-west-2.amazonaws.com/setup/params"$degree" -o "$file" diff --git a/prover2/scripts/setup_test_data.sh b/prover2/scripts/setup_test_data.sh new file mode 100644 index 0000000000..dcb681f28d --- /dev/null +++ b/prover2/scripts/setup_test_data.sh @@ -0,0 +1,11 @@ +set -x +set -e + +dir="${1:-"./test_data/.configs"}" +mkdir -p "$dir" + +for ((i = 1; i < 7; ++i)); do + template="./configs/layer${i}.config" + file="$dir/layer${i}.config" + jq ".degree = ${i}" ${template} > ${file} +done diff --git a/prover2/src/error.rs b/prover2/src/error.rs new file mode 100644 index 0000000000..098dbf1093 --- /dev/null +++ b/prover2/src/error.rs @@ -0,0 +1,64 @@ +use std::path::PathBuf; + +use thiserror::Error; + +use crate::types::layer::ProofLayer; + +/// Represents error variants possibly encountered during the proof generation process. +#[derive(Debug, Error)] +pub enum ProverError { + /// Error occurred while doing other I/O operations. + #[error(transparent)] + OtherIo(#[from] std::io::Error), + /// Error occurred while doing other serialization/deserialization operations. + #[error(transparent)] + OtherSerde(#[from] serde_json::Error), + /// Error encountered while reading from or writing to files. + #[error("an error occurred while reading/writing {path}: {source}")] + IoReadWrite { + /// The path we tried to read from or write to. + path: PathBuf, + /// The source error. + source: std::io::Error, + }, + /// Error encountered during serialization/deserialization of JSON. + #[error("an error occurred while reading/writing json {path}: {source}")] + ReadWriteJson { + /// The path of the file we tried to serialize/deserialize. + path: PathBuf, + /// The source error. + source: serde_json::Error, + }, + /// Error encountered while reading variable from the process environment. + #[error("an error occurred while reading variable from the environment {key}: {source}")] + EnvVar { + /// The key tried to be read. + key: String, + /// The source error. + source: std::env::VarError, + }, + /// Error encountered while parsing a string. + #[error("an error occurred while parsing string {src}: {err}")] + Parse { + /// Source string that we tried to parse. + src: String, + /// Parsing error. + err: String, + }, + /// Error that indicates the KZG setup parameters for specified layer are missing from prover + /// config. + #[error("prover {0} missing KZG setup params for {1:?}")] + MissingKzgParams(String, ProofLayer), + /// Error that indicates the proving key for specified layer is missing from prover config. + #[error("prover {0} missing proving key for {1:?}")] + MissingProvingKey(String, ProofLayer), + /// Error that has occurred during keygen process. + #[error("an error occurred during keygen process for prover={0} layer={1:?}: {2}")] + Keygen(String, ProofLayer, halo2_proofs::plonk::Error), + /// Error that has occurred during SNARK generation process. + #[error("an error occurred during SNARK generation for task={0} layer={1:?}: {2}")] + GenSnark(String, ProofLayer, halo2_proofs::plonk::Error), + /// Custom error. + #[error("custom error: {0}")] + Custom(String), +} diff --git a/prover2/src/lib.rs b/prover2/src/lib.rs new file mode 100644 index 0000000000..f15fdf7a33 --- /dev/null +++ b/prover2/src/lib.rs @@ -0,0 +1,10 @@ +mod error; +pub use error::ProverError; + +mod util; + +mod verifier; + +pub mod prover; + +pub mod types; diff --git a/prover2/src/prover/config.rs b/prover2/src/prover/config.rs new file mode 100644 index 0000000000..963e9446d4 --- /dev/null +++ b/prover2/src/prover/config.rs @@ -0,0 +1,259 @@ +use std::{collections::HashMap, fs::create_dir_all, marker::PhantomData, path::PathBuf}; + +use halo2_proofs::{ + halo2curves::bn256::{Bn256, Fr, G1Affine}, + plonk::{keygen_pk2, Circuit, ProvingKey}, + poly::kzg::commitment::ParamsKZG, +}; +use tracing::{info, instrument, trace, warn}; + +use crate::{ + prover::params::NonNativeParams, + types::{layer::ProofLayer, ProverType}, + util::{ + default_kzg_params_dir, default_non_native_params_dir, kzg_params_path, + non_native_params_path as nn_params_path, read_env_or_default, read_json, read_kzg_params, + CACHE_PATH_EVM, CACHE_PATH_PI, CACHE_PATH_PROOFS, CACHE_PATH_SNARKS, CACHE_PATH_TASKS, + DEFAULT_DEGREE_LAYER0, ENV_DEGREE_LAYER0, JSON_EXT, + }, + ProverError, +}; + +/// Configuration for a generic prover. +#[derive(Default, Debug)] +pub struct ProverConfig { + /// Polynomial degree used by proof generation layer. + pub degrees: HashMap, + /// KZG setup parameters by proof layer. + pub kzg_params: HashMap>, + /// Config parameters for non-native field arithmetics by proof layer. + pub nn_params: HashMap, + /// Proving keys by proof layer. + pub pks: HashMap>, + /// Optional directory to locate KZG setup parameters. + pub kzg_params_dir: Option, + /// Optional directory to locate non-native field arithmetic config params. + pub nn_params_dir: Option, + /// Optional directory to cache proofs. + pub cache_dir: Option, + + _prover_type: PhantomData, +} + +impl ProverConfig { + /// Returns prover config after inserting the non-native field arithmetic config for the given + /// proof layer. + pub fn with_nn_params(mut self, layer: ProofLayer, params: NonNativeParams) -> Self { + self.nn_params.insert(layer, params); + self + } + + /// Returns prover config after inserting KZG setup params for the given proof layer. + pub fn with_kzg_params(mut self, layer: ProofLayer, params: ParamsKZG) -> Self { + self.kzg_params.insert(layer, params); + self + } + + /// Returns prover config after inserting the proving key for the given proof layer. + pub fn with_pk(mut self, layer: ProofLayer, pk: ProvingKey) -> Self { + self.pks.insert(layer, pk); + self + } + + /// Returns prover config with a directory to load KZG params from. + pub fn with_kzg_params_dir(mut self, dir: impl Into) -> Self { + self.kzg_params_dir = Some(dir.into()); + self + } + + /// Returns prover config with a directory to load non-native field arithmetic config params + /// from. + pub fn with_nn_params_dir(mut self, dir: impl Into) -> Self { + self.nn_params_dir = Some(dir.into()); + self + } + + /// Returns prover config with a cache directory configured. + pub fn with_cache_dir(mut self, dir: impl Into) -> Self { + self.cache_dir = Some(dir.into()); + self + } +} + +/// Convenience type. +type KzgParamsAndPk<'a> = (&'a ParamsKZG, &'a ProvingKey); + +impl ProverConfig { + /// Setup the prover config by reading relevant config files from storage. + #[instrument(name = "ProverConfig::setup", skip(self))] + pub fn setup(mut self) -> Result { + info!(name = "setup prover config", prover_type = ?Type::NAME.to_string()); + + // The proof layers that this prover needs to generate proofs for. + let proof_layers = Type::layers(); + + // Use the configured directories or fallback to current working directory. + let nn_params_dir = self + .nn_params_dir + .clone() + .unwrap_or(default_non_native_params_dir()?); + let kzg_params_dir = self + .kzg_params_dir + .clone() + .unwrap_or(default_kzg_params_dir()?); + trace!(name = "config directories", non_native_params = ?nn_params_dir, kzg_params = ?kzg_params_dir, caching = ?self.cache_dir); + if self.cache_dir.is_none() { + warn!(name = "setup prover without caching"); + } + + // Read and store non-native field arithmetic config params for each layer. + for layer in proof_layers { + // Layer0 (SuperCircuit) does not have non-native field arithmetics. + if layer != ProofLayer::Layer0 { + let params_path = nn_params_path(nn_params_dir.as_path(), layer); + trace!(name = "read config for non-native field arithmetics", ?layer, path = ?params_path); + let params = read_json::(params_path.as_path())?; + self.degrees.insert(layer, params.degree); + self.nn_params.insert(layer, params); + } + + if layer == ProofLayer::Layer0 { + trace!(name = "read environment variable", ?layer, key = ?ENV_DEGREE_LAYER0, default = ?DEFAULT_DEGREE_LAYER0); + let layer0_degree = read_env_or_default(ENV_DEGREE_LAYER0, DEFAULT_DEGREE_LAYER0); + trace!(name = "configured degree", ?layer, degree = ?layer0_degree); + self.degrees.insert(ProofLayer::Layer0, layer0_degree); + } + } + + // Read and store KZG setup params for each layer. + for (&layer, °ree) in self.degrees.iter() { + let params_path = kzg_params_path(kzg_params_dir.as_path(), degree); + trace!( + name = "read KZG setup parameters", + ?layer, + ?degree, + path = ?params_path, + ); + let params = read_kzg_params(params_path.as_path())?; + self.kzg_params.insert(layer, params); + } + + // Setup the cache directory's structure. + if let Some(ref cache_dir) = self.cache_dir { + create_dir_all(cache_dir.join(CACHE_PATH_TASKS))?; + create_dir_all(cache_dir.join(CACHE_PATH_SNARKS))?; + create_dir_all(cache_dir.join(CACHE_PATH_PROOFS))?; + create_dir_all(cache_dir.join(CACHE_PATH_PI))?; + create_dir_all(cache_dir.join(CACHE_PATH_EVM))?; + } + + // Update directories in self. + self.nn_params_dir.replace(nn_params_dir); + self.kzg_params_dir.replace(kzg_params_dir); + + info!(name = "setup prover config OK", prover_type = ?Type::NAME.to_string()); + + Ok(self) + } + + /// Returns the proving key for the proof layer. + pub fn gen_proving_key>( + &mut self, + layer: ProofLayer, + circuit: &C, + ) -> Result { + if self.pks.contains_key(&layer) { + return Ok((&self.kzg_params[&layer], &self.pks[&layer])); + } + + // Generate proving key for the circuit and insert into the cached map. + let kzg_params = self.kzg_params(layer)?; + let pk = keygen_pk2(kzg_params, circuit) + .map_err(|e| ProverError::Keygen(Type::NAME.to_string(), layer, e))?; + self.pks.insert(layer, pk); + + Ok((&self.kzg_params[&layer], &self.pks[&layer])) + } +} + +impl ProverConfig { + /// Returns the path to a proof with the given identifier if caching is enabled in the prover's + /// config. + pub fn path_proof(&self, id: &str) -> Option { + self.cache_dir + .as_ref() + .map(|dir| dir.join(CACHE_PATH_PROOFS).join(id).join(JSON_EXT)) + } + + /// Returns the path to cache the generated SNARK. + pub fn path_snark(&self, id: &str, layer: ProofLayer) -> Option { + self.cache_dir + .as_ref() + .map(|dir| dir.join(CACHE_PATH_SNARKS).join(format!("{layer:?}-{id}"))) + } + + /// Returns the paths to dump the EVM plonk verifier contract (in YUL) and the deployment code + /// for the verifier contract. + pub fn path_evm(&self, commit_ref: &str) -> Option<(PathBuf, PathBuf)> { + self.cache_dir.as_ref().map(|dir| { + ( + dir.join(CACHE_PATH_EVM) + .join(format!("evm_verifier_{commit_ref}.yul")), + dir.join(CACHE_PATH_EVM) + .join(format!("evm_verifier_{commit_ref}.bin")), + ) + }) + } +} + +impl ProverConfig { + /// Returns the KZG setup parameters for the proof layer. + pub fn kzg_params(&self, layer: ProofLayer) -> Result<&ParamsKZG, ProverError> { + self.kzg_params + .get(&layer) + .ok_or(ProverError::MissingKzgParams(Type::NAME.into(), layer)) + } + + /// Returns the proving key for the proof layer. + pub fn proving_key(&self, layer: ProofLayer) -> Result<&ProvingKey, ProverError> { + self.pks.get(&layer).ok_or(ProverError::MissingProvingKey( + Type::NAME.to_string(), + layer, + )) + } +} + +#[cfg(test)] +mod tests { + use std::env::current_dir; + + use aggregator::MAX_AGG_SNARKS; + + use crate::{ + prover::ProverConfig, + types::{ProverTypeBatch, ProverTypeChunk}, + }; + + #[test] + fn setup_prover() -> anyhow::Result<()> { + tracing_subscriber::fmt() + .with_env_filter(tracing_subscriber::EnvFilter::from_default_env()) + .pretty() + .init(); + + let test_dir = current_dir()?.join("test_data"); + + let _chunk_prover_config = ProverConfig::::default() + .with_nn_params_dir(test_dir.join(".configs")) + .with_kzg_params_dir(test_dir.join(".params")) + .with_cache_dir(test_dir.join(".cache")) + .setup()?; + + let _batch_prover_config = ProverConfig::>::default() + .with_nn_params_dir(test_dir.join(".configs")) + .with_kzg_params_dir(test_dir.join(".params")) + .setup()?; + + Ok(()) + } +} diff --git a/prover2/src/prover/mod.rs b/prover2/src/prover/mod.rs new file mode 100644 index 0000000000..c81ebc5672 --- /dev/null +++ b/prover2/src/prover/mod.rs @@ -0,0 +1,232 @@ +use aggregator::MAX_AGG_SNARKS; +use ethers_core::utils::keccak256; +use halo2_proofs::halo2curves::bn256::{Bn256, Fr}; +use snark_verifier::pcs::kzg::{Bdfg21, Kzg}; +use snark_verifier_sdk::{CircuitExt, Snark}; +use tracing::{info, instrument, trace}; + +use crate::{ + prover::config::ProverConfig, + types::{ + layer::ProofLayer, proof::Proof, task::ProvingTask, ProverType, ProverTypeBatch, + ProverTypeBundle, ProverTypeChunk, + }, + util::{gen_rng, read_json, write, write_json, GIT_VERSION}, + ProverError, +}; + +/// Includes the configuration setup related to a [`Prover`]. +pub mod config; + +/// Config parameters for non-native field arithmetics, that will be used to configure the +/// [`FpConfig`][fp_config] chip from halo2-lib. +/// +/// [fp_config]: snark_verifier::loader::halo2::halo2_ecc::fields::fp::FpConfig +pub mod params; + +/// Convenience type for chunk prover. +pub type ChunkProver = Prover; + +/// Convenience type for batch prover. +pub type BatchProver = Prover, false>; + +/// Convenience type for bundle prover. Since bundling of batches is also the final layer, we wish +/// to verify the proof in EVM. +pub type BundleProver = Prover; + +/// A generic prover that is capable of generating proofs for given tasks. +#[derive(Debug)] +pub struct Prover { + /// Config for the prover. + pub config: ProverConfig, +} + +impl Prover { + /// Construct a new prover. + pub fn new(config: ProverConfig) -> Self { + Self { config } + } +} + +impl Prover { + /// Generate a proof for the given task. + #[instrument(name = "Prover::gen_proof", skip(self))] + pub fn gen_proof( + &mut self, + task: Type::Task, + ) -> Result, ProverError> { + info!(name = "gen proof", prover = ?Type::NAME.to_string(), ?task); + + // Early return if the proof for the given task is already available in cache. + let id = task.id(); + + let path_proof = self.config.path_proof(&id); + if let Some(path) = &path_proof { + trace!(name = "read proof from cache", ?path); + if let Ok(proof) = read_json(path) { + trace!(name = "early return cache hit", ?proof); + return Ok(proof); + } + } + + let proof = if EVM_VERIFY { + trace!(name = "gen evm proof", ?task); + self.gen_proof_evm(task)? + } else { + trace!(name = "gen halo2 proof", ?task); + self.gen_proof_halo2(task)? + }; + + // Dump the proof if caching is enabled. + if let Some(path) = &path_proof { + trace!(name = "dump proof", ?path, ?proof); + write_json(path, &proof)?; + } + + info!(name = "gen proof OK", prover = ?Type::NAME.to_string(), ?proof); + Ok(proof) + } + + /// Generate a halo2 proof for the given task. The poseidon hash function is used for + /// fiat-shamir transform on the transcript. + #[instrument(name = "Prover::gen_proof_halo2", skip(self))] + fn gen_proof_halo2( + &mut self, + task: Type::Task, + ) -> Result, ProverError> { + let id = task.id(); + + // Generate SNARKs for all the layers at which the prover operates. + // + // We start from the base layer, i.e. the innermost layer for the prover. + let layer = Type::base_layer()?; + trace!(name = "gen base snark", ?layer); + let (mut snark, aux_data) = self.gen_base_snark(layer, task)?; + trace!(name = "gen base snark OK", ?aux_data); + + // The base layer's SNARK is compressed for every layer of compression. + for layer in Type::compression_layers() { + let kzg_params = self.config.kzg_params(layer)?; + let compression_circuit = Type::build_compression(kzg_params, snark, layer); + trace!(name = "gen compression snark", ?layer); + snark = self.gen_halo2_snark(&id, layer, compression_circuit)?; + trace!(name = "gen compression snark OK", ?layer); + } + + // We have the final compressed SNARK for the proof generation process under the prover. + let outermost_layer = Type::outermost_layer()?; + let pk = self.config.proving_key(outermost_layer)?; + let proof = Proof::new_from_snark(outermost_layer, snark, pk, aux_data)?; + + Ok(proof) + } + + /// Generate an EVM-verifiable proof for the given task. The Keccak256 hash function is used + /// for fiat-shamir transform on the transcript. + #[instrument(name = "Prover::gen_proof_evm", skip(self))] + fn gen_proof_evm( + &mut self, + task: Type::Task, + ) -> Result, ProverError> { + let id = task.id(); + let commit_ref = *GIT_VERSION; + let evm_paths = self.config.path_evm(commit_ref); + + // Generate SNARKs for all the layers at which the prover operates. + // + // We start from the base layer, i.e. the innermost layer for the prover. + let layer = Type::base_layer()?; + trace!(name = "gen base snark", ?layer); + let (mut snark, aux_data) = self.gen_base_snark(layer, task)?; + trace!(name = "gen base snark OK", ?aux_data); + + // The base layer's SNARK is compressed for every layer of compression, except the last + // (final) layer. The final layer of compression is supposed to be EVM-verifiable. + for &layer in Type::compression_layers().iter().rev().skip(1).rev() { + let kzg_params = self.config.kzg_params(layer)?; + let compression_circuit = Type::build_compression(kzg_params, snark, layer); + trace!(name = "gen compression snark", ?layer); + snark = self.gen_halo2_snark(&id, layer, compression_circuit)?; + trace!(name = "gen compression snark OK", ?layer); + } + + // We have the final compressed SNARK for the proof generation process under the prover. + let outermost_layer = Type::outermost_layer()?; + let kzg_params = self.config.kzg_params(outermost_layer)?; + let compression_circuit = Type::build_compression(kzg_params, snark, outermost_layer); + let instances = compression_circuit.instances(); + let num_instances = compression_circuit.num_instance(); + let (kzg_params, pk) = self + .config + .gen_proving_key(outermost_layer, &compression_circuit)?; + let mut rng = gen_rng(); + trace!(name = "gen evm proof", layer = ?outermost_layer); + let raw_proof = snark_verifier_sdk::gen_evm_proof_shplonk( + kzg_params, + pk, + compression_circuit, + instances.clone(), + &mut rng, + ); + trace!(name = "gen evm proof OK", layer = ?outermost_layer); + + // At this point we know the verifying key, KZG setup parameters and the proof. We can + // build the Plonk verifier contract (EVM-verifier). + if let Some((evm_yul_path, evm_deployment_code_path)) = evm_paths { + trace!(name = "gen evm verifier", path = ?evm_yul_path); + let evm_deployment_code = snark_verifier_sdk::gen_evm_verifier::< + Type::CompressionCircuit, + Kzg, + >( + kzg_params, + pk.get_vk(), + num_instances, + Some(evm_yul_path.as_path()), + ); + trace!(name = "write evm verifier deployment code", path = ?evm_deployment_code_path, code_hash = ?keccak256(&evm_deployment_code)); + write(evm_deployment_code_path.as_path(), &evm_deployment_code)?; + } + + let proof = Proof::new_from_raw(outermost_layer, &instances[0], &raw_proof, pk, aux_data); + + Ok(proof) + } + + /// Generates a SNARK for the base circuit of a prover. THe base circuit is generally a circuit + /// with larger number of advice columns while being of lower degree. The SNARK of the base + /// circuit is then compressed using the compression layer to produce a proof that's cheaper to + /// verify. + fn gen_base_snark( + &mut self, + base_layer: ProofLayer, + task: Type::Task, + ) -> Result<(Snark, Type::ProofAuxData), ProverError> { + let id = task.id(); + + // Generate SNARKs for all the layers at which the prover operates. + // + // We start from the base layer, i.e. the innermost layer for the prover. + let (base_circuit, aux_data) = Type::build_base(&task); + let snark = self.gen_halo2_snark(&id, base_layer, base_circuit)?; + + Ok((snark, aux_data)) + } + + /// Generate a SNARK for the given circuit. + fn gen_halo2_snark( + &mut self, + id: &str, + layer: ProofLayer, + circuit: C, + ) -> Result + where + C: CircuitExt, + { + let path = self.config.path_snark(id, layer); + let (kzg_params, pk) = self.config.gen_proving_key(layer, &circuit)?; + let mut rng = gen_rng(); + + snark_verifier_sdk::gen_snark_shplonk(kzg_params, pk, circuit, &mut rng, path) + .map_err(|e| ProverError::GenSnark(id.into(), layer, e)) + } +} diff --git a/prover2/src/prover/params.rs b/prover2/src/prover/params.rs new file mode 100644 index 0000000000..bd3ac51eaf --- /dev/null +++ b/prover2/src/prover/params.rs @@ -0,0 +1,15 @@ +use serde::{Deserialize, Serialize}; +use snark_verifier::loader::halo2::halo2_ecc::fields::fp::FpStrategy; + +/// Parameters to configure the non-native field arithmetic chip. +#[derive(Debug, Serialize, Deserialize)] +pub struct NonNativeParams { + pub strategy: FpStrategy, + pub degree: u32, + pub num_advice: Vec, + pub num_lookup_advice: Vec, + pub num_fixed: usize, + pub lookup_bits: usize, + pub limb_bits: usize, + pub num_limbs: usize, +} diff --git a/prover2/src/types/layer.rs b/prover2/src/types/layer.rs new file mode 100644 index 0000000000..facd3bf17d --- /dev/null +++ b/prover2/src/types/layer.rs @@ -0,0 +1,49 @@ +use serde::{Deserialize, Serialize}; + +/// Various layers in the proof generation process. +#[derive(Clone, Copy, Debug, Serialize, Deserialize, Eq, PartialEq, PartialOrd, Ord, Hash)] +pub enum ProofLayer { + /// The [`SuperCircuit`][super_circuit] (ZkEVM) layer. This is the innermost proof layer. + /// + /// [super_circuit]: zkevm_circuits::super_circuit::SuperCircuit + Layer0, + /// The compression layer on top of layer0. + Layer1, + /// The compression layer on top of layer1. The proof from this layer is the [`Proof`][proof] returned + /// by the [`ChunkProver`][gen_proof]. + /// + /// [proof]: crate::types::proof::Proof + /// [gen_proof]: crate::prover::ChunkProver::gen_proof + Layer2, + /// The batch circuit layer. At this layer, we batch multiple `ChunkProof`s. + Layer3, + /// The compression layer on top of layer3. The proof from this layer is the [`Proof`][proof] returned + /// by the [`BatchProver`][gen_proof]. + /// + /// [proof]: crate::types::proof::Proof + /// [gen_proof]: crate::prover::BatchProver::gen_proof + Layer4, + /// The recursion circuit layer. At this layer, we construct proofs recursively over a previous + /// SNARK from the recursion circuit. + Layer5, + /// The compression layer on top of layer5. The proof from this layer is the [`Proof`][proof] returned + /// by the [`BundleProver`][gen_proof]. + /// + /// [proof]: crate::types::proof::Proof + /// [gen_proof]: crate::prover::BundleProver::gen_proof + Layer6, +} + +impl ToString for ProofLayer { + fn to_string(&self) -> String { + String::from(match self { + Self::Layer0 => "layer0", + Self::Layer1 => "layer1", + Self::Layer2 => "layer2", + Self::Layer3 => "layer3", + Self::Layer4 => "layer4", + Self::Layer5 => "layer5", + Self::Layer6 => "layer6", + }) + } +} diff --git a/prover2/src/types/mod.rs b/prover2/src/types/mod.rs new file mode 100644 index 0000000000..b6ad824516 --- /dev/null +++ b/prover2/src/types/mod.rs @@ -0,0 +1,214 @@ +use aggregator::{BatchCircuit, ChunkInfo, CompressionCircuit}; +use ethers_core::types::H256; +use halo2_proofs::{ + halo2curves::bn256::{Bn256, Fr}, + poly::kzg::commitment::ParamsKZG, +}; +use serde::{de::DeserializeOwned, Deserialize, Serialize}; +use snark_verifier_sdk::{CircuitExt, Snark}; +use zkevm_circuits::super_circuit::params::ScrollSuperCircuit; + +use crate::{ + types::{layer::ProofLayer, task::ProvingTask}, + ProverError, +}; + +pub mod layer; + +pub mod proof; + +pub mod task; +use task::{BatchProvingTask, ChunkProvingTask}; + +/// Defines the behaviour that a prover type must implement. +/// +/// The proof generation process involves the following steps: +/// 1. [`ProverTypeChunk`]: prove the EVM execution trace from a list of blocks ([`layer0`][layer0]) +/// and compress it twice ([`layer1`][layer1] and [`layer2`][layer2]), where the outermost proof +/// (`layer2`) is the `ChunkProof`. +/// 2. [`ProverTypeBatch`]: prove the batching of one or more chunks ([`layer3`][layer3]) +/// and compress it ([`layer4`][layer4]) where this outermost proof (`layer4`) is the `BatchProof`. +/// 3. [`ProverTypeBundle`]: prove the bundling of one or more batches recursively ([`layer5`][layer5]) +/// and compress it ([`layer6`][layer6]) where this outermost proof (`layer6`) is the `BundleProof`. +/// The `BundleProof` is an EVM-verifiable proof. +/// +/// [layer0]: crate::types::layer::ProofLayer::Layer0 +/// [layer1]: crate::types::layer::ProofLayer::Layer1 +/// [layer2]: crate::types::layer::ProofLayer::Layer2 +/// [layer3]: crate::types::layer::ProofLayer::Layer3 +/// [layer4]: crate::types::layer::ProofLayer::Layer4 +/// [layer5]: crate::types::layer::ProofLayer::Layer5 +/// [layer6]: crate::types::layer::ProofLayer::Layer6 +pub trait ProverType: std::fmt::Debug { + /// The name of the prover. + const NAME: &'static str; + + /// The proving task that provides the relevant values required by the prover type to build its + /// base circuit. + type Task: ProvingTask; + + /// The circuit used at the base layer of this prover type. + type BaseCircuit: CircuitExt; + + /// The [`Compression Circuit`][compr_circuit] used to compress the base layer + /// [`SNARK`][snark] one or more times before finally producing the outermost + /// layer's SNARK. + /// + /// [snark]: snark_verifier_sdk::Snark + /// [compr_circuit]: aggregator::CompressionCircuit + type CompressionCircuit: CircuitExt; + + /// The auxiliary data attached to the [`Proof`][proof] from the prover type. + /// For instance, a [`ChunkProver`][chunk_prover] needs to attach the [`ChunkInfo`], + /// which is then used by the [`BatchProver`][batch_prover] to construct its + /// [`BatchProvingTask`]. + /// + /// [proof]: crate::types::proof::Proof + /// [chunk_prover]: crate::prover::ChunkProver + /// [batch_prover]: crate::prover::BatchProver + type ProofAuxData: Serialize + DeserializeOwned + std::fmt::Debug; + + /// The prover supports proof generation at the following layers. + fn layers() -> Vec; + + /// Returns the base layer. The base layer's proof is generated by building the + /// [`BaseCircuit`][base_circuit]. + /// + /// [base_circuit]: Self::BaseCircuit + fn base_layer() -> Result { + Self::layers() + .first() + .ok_or(ProverError::Custom(format!("no layer for {}", Self::NAME))) + .copied() + } + + /// Returns the outermost layer. This is generally the last compression layer of the prover + /// type. + fn outermost_layer() -> Result { + Self::layers() + .last() + .ok_or(ProverError::Custom(format!("no layer for {}", Self::NAME))) + .copied() + } + + /// Returns the subsequent layers after the base layer, i.e. the layers where the previous + /// layer's SNARK is compressed. + fn compression_layers() -> Vec { + Self::layers()[1..].to_vec() + } + + /// Builds the [`BaseCircuit`][base_circuit] given witness in the proving task. + /// + /// [base_circuit]: Self::BaseCircuit + fn build_base(task: &Self::Task) -> (Self::BaseCircuit, Self::ProofAuxData); + + /// Builds the [`CompressionCircuit`][compr_circuit] given the previous layer's SNARK. + /// + /// [compr_circuit]: Self::CompressionCircuit + fn build_compression( + kzg_params: &ParamsKZG, + prev_snark: Snark, + layer: ProofLayer, + ) -> Self::CompressionCircuit; +} + +/// The chunk prover that constructs proofs at [`layer0`][layer0], [`layer1`][layer1] and [`layer2`][layer2]. +/// +/// [layer0]: crate::types::layer::ProofLayer::Layer0 +/// [layer1]: crate::types::layer::ProofLayer::Layer1 +/// [layer2]: crate::types::layer::ProofLayer::Layer2 +#[derive(Default, Debug)] +pub struct ProverTypeChunk; + +/// The batch prover that constructs proofs at [`layer3`][layer3] and [`layer4`][layer4]. +/// +/// [layer3]: crate::types::layer::ProofLayer::Layer3 +/// [layer4]: crate::types::layer::ProofLayer::Layer4 +#[derive(Default, Debug)] +pub struct ProverTypeBatch; + +/// The bundle prover that constructs proofs at [`layer5`][layer5] and [`layer6`][layer6]. +/// +/// [layer5]: crate::types::layer::ProofLayer::Layer5 +/// [layer6]: crate::types::layer::ProofLayer::Layer6 +#[derive(Default, Debug)] +pub struct ProverTypeBundle; + +/// Auxiliary data attached to a [`ChunkProof`][proof] +/// +/// [proof]: crate::types::proof::Proof +#[derive(Serialize, Deserialize, Debug)] +pub struct ChunkProofAuxData { + /// The chunk's information, that is eventually needed by the [`BatchProver`][batch_prover]'s + /// proof generation process. + /// + /// [batch_prover]: crate::prover::BatchProver::gen_proof + pub chunk_info: ChunkInfo, +} + +/// Auxiliary data attached to a [`BatchProof`][proof] +/// +/// [proof]: crate::types::proof::Proof +#[derive(Serialize, Deserialize, Debug)] +pub struct BatchProofAuxData { + /// The hash of the [`BatchHeader`][batch_header] + /// + /// [batch_header]: aggregator::BatchHeader + pub batch_hash: H256, +} + +impl ProverType for ProverTypeChunk { + const NAME: &'static str = "ChunkProver"; + + type Task = ChunkProvingTask; + + type BaseCircuit = ScrollSuperCircuit; + + type CompressionCircuit = CompressionCircuit; + + type ProofAuxData = ChunkProofAuxData; + + fn layers() -> Vec { + vec![ProofLayer::Layer0, ProofLayer::Layer1, ProofLayer::Layer2] + } + + fn build_base(_task: &Self::Task) -> (Self::BaseCircuit, Self::ProofAuxData) { + unimplemented!() + } + + fn build_compression( + _params: &ParamsKZG, + _prev_snark: Snark, + _layer: ProofLayer, + ) -> Self::CompressionCircuit { + unimplemented!() + } +} + +impl ProverType for ProverTypeBatch { + const NAME: &'static str = "BatchProver"; + + type Task = BatchProvingTask; + + type BaseCircuit = BatchCircuit; + + type CompressionCircuit = CompressionCircuit; + + type ProofAuxData = BatchProofAuxData; + + fn layers() -> Vec { + vec![ProofLayer::Layer3, ProofLayer::Layer4] + } + + fn build_base(_task: &Self::Task) -> (Self::BaseCircuit, Self::ProofAuxData) { + unimplemented!() + } + + fn build_compression( + _params: &ParamsKZG, + _prev_snark: Snark, + _layer: ProofLayer, + ) -> Self::CompressionCircuit { + unimplemented!() + } +} diff --git a/prover2/src/types/proof.rs b/prover2/src/types/proof.rs new file mode 100644 index 0000000000..7525ebc4e5 --- /dev/null +++ b/prover2/src/types/proof.rs @@ -0,0 +1,128 @@ +use eth_types::base64; +use halo2_proofs::{ + halo2curves::bn256::{Fr, G1Affine}, + plonk::{Circuit, ProvingKey, VerifyingKey}, + SerdeFormat, +}; +use serde::{Deserialize, Serialize}; +use snark_verifier_sdk::Snark; + +use crate::{ + types::layer::ProofLayer, + util::{deserialize_be, serialize_be, GIT_VERSION}, + ProverError, +}; + +/// Describes an output from a [`Prover`][prover]'s proof generation process when +/// given a [`ProvingTask`][proving_task]. +/// +/// [prover]: crate::prover::Prover +/// [proving_task]: crate::types::task::ProvingTask +#[derive(Serialize, Deserialize, Debug)] +pub struct Proof { + /// Version of the source code (git describe --abbrev=8) used for proof generation. + pub git_version: String, + /// The proof layer. + pub layer: ProofLayer, + /// The raw [`VerificationKey`][vk] for this [`SNARK`][snark] proof. + /// + /// [vk]: halo2_proofs::plonk::VerifyingKey + /// [snark]: snark_verifier_sdk::Snark + #[serde(with = "base64")] + pub vk: Vec, + /// The public instances (flattened bytes) to the SNARK. + #[serde(with = "base64")] + pub instances: Vec, + /// The protocol computed for SNARK. + #[serde(with = "base64")] + pub protocol: Vec, + /// The inner proof. + #[serde(with = "base64")] + pub proof: Vec, + /// Auxiliary data to attach with the proof. This data would generally be required + /// by the next layer's proof generation process. + #[serde(flatten)] + pub aux: Aux, +} + +impl Proof { + /// Construct a new proof given the SNARK for the proof layer and some auxiliary data. + pub fn new_from_snark( + layer: ProofLayer, + snark: Snark, + pk: &ProvingKey, + aux: Aux, + ) -> Result { + let git_version = GIT_VERSION.to_string(); + let vk = pk.get_vk().to_bytes(SerdeFormat::Processed); + let protocol = serde_json::to_vec(&snark.protocol)?; + let instances = snark.instances[0] + .iter() + .flat_map(serialize_be) + .collect::>(); + let proof = snark.proof; + + Ok(Self { + git_version, + layer, + vk, + protocol, + instances, + proof, + aux, + }) + } + + /// Construct a new proof given the raw proof and instances for an EVM-verifiable proof. + pub fn new_from_raw( + layer: ProofLayer, + instances: &[Fr], + proof: &[u8], + pk: &ProvingKey, + aux: Aux, + ) -> Self { + let git_version = GIT_VERSION.to_string(); + let vk = pk.get_vk().to_bytes(SerdeFormat::Processed); + let instances = instances.iter().flat_map(serialize_be).collect::>(); + + Self { + git_version, + layer, + vk, + protocol: vec![], + instances, + proof: proof.to_vec(), + aux, + } + } +} + +impl Proof { + /// Deserialize and return the verifying key. + pub fn verifying_key>(&self) -> Result, ProverError> { + Ok(VerifyingKey::from_bytes::( + &self.vk, + SerdeFormat::Processed, + )?) + } +} + +impl TryInto for Proof { + type Error = serde_json::Error; + + fn try_into(self) -> Result { + let protocol = serde_json::from_slice(&self.protocol)?; + let instances = self + .instances + .chunks_exact(32) + .map(deserialize_be) + .collect::>(); + let proof = self.proof; + + Ok(Snark { + protocol, + instances: vec![instances], + proof, + }) + } +} diff --git a/prover2/src/types/task.rs b/prover2/src/types/task.rs new file mode 100644 index 0000000000..4e38c7aeda --- /dev/null +++ b/prover2/src/types/task.rs @@ -0,0 +1,54 @@ +use serde::{de::DeserializeOwned, Deserialize, Serialize}; + +/// Describes the behaviour to be supported by a [`ProverType`][prover_type] that +/// can be used as an input (or a task) to instruct a [`Prover`][prover] to generate +/// a proof. +/// +/// [prover_type]: crate::types::ProverType +/// [prover]: crate::prover::Prover +pub trait ProvingTask: Serialize + DeserializeOwned + std::fmt::Debug { + /// A unique identifier for the proving task. + fn id(&self) -> String; +} + +/// A [`ProvingTask`] used to build the base circuit, i.e. [`SuperCircuit`][super_circuit], +/// for the [`ChunkProver`][chunk_prover]. +/// +/// [super_circuit]: zkevm_circuits::super_circuit::SuperCircuit +/// [chunk_prover]: crate::types::ProverTypeChunk +#[derive(Debug, Serialize, Deserialize)] +pub struct ChunkProvingTask; + +/// A [`ProvingTask`] used to build the base circuit, i.e. [`BatchCircuit`][batch_circuit], +/// for the [`BatchProver`][batch_prover]. +/// +/// [batch_circuit]: aggregator::BatchCircuit +/// [batch_prover]: crate::types::ProverTypeBatch +#[derive(Debug, Serialize, Deserialize)] +pub struct BatchProvingTask; + +/// A [`ProvingTask`] used to build the base circuit, i.e. [`RecursionCircuit`][recursion_circuit], +/// for the [`BundleProver`][bundle_prover]. +/// +/// [recursion_circuit]: aggregator::RecursionCircuit +/// [bundle_prover]: crate::types::ProverTypeBundle +#[derive(Debug, Serialize, Deserialize)] +pub struct BundleProvingTask; + +impl ProvingTask for ChunkProvingTask { + fn id(&self) -> String { + "chunk".into() + } +} + +impl ProvingTask for BatchProvingTask { + fn id(&self) -> String { + "batch".into() + } +} + +impl ProvingTask for BundleProvingTask { + fn id(&self) -> String { + "bundle".into() + } +} diff --git a/prover2/src/util/dir.rs b/prover2/src/util/dir.rs new file mode 100644 index 0000000000..60c2e6f2c1 --- /dev/null +++ b/prover2/src/util/dir.rs @@ -0,0 +1,69 @@ +use std::{ + env::current_dir, + path::{Path, PathBuf}, +}; + +use crate::{types::layer::ProofLayer, ProverError}; + +/// Test data directory. +pub const TEST_DATA_DIR: &str = "test_data"; + +/// The extension used for JSON files. +pub const JSON_EXT: &str = ".json"; + +/// The config parameters for non native field arithmetics are in a *.config file. +pub const NON_NATIVE_PARAMS_EXT: &str = ".config"; + +/// The config parameters for non native field arithmetics are by default in this directory. +pub const NON_NATIVE_PARAMS_DIR: &str = ".configs"; + +/// The KZG setup parameters are by default in this directory. +pub const KZG_PARAMS_DIR: &str = ".params"; + +/// The directory within cache to store proving tasks in JSON format. +pub const CACHE_PATH_TASKS: &str = "tasks"; + +/// The directory within cache to store SNARKs generated at intermediate proving layers. +pub const CACHE_PATH_SNARKS: &str = "snarks"; + +/// The directory within cache to store proof outputs. +pub const CACHE_PATH_PROOFS: &str = "proofs"; + +/// The directory within cache to store public input data. +pub const CACHE_PATH_PI: &str = "pi"; + +/// The directory within cache to store Verifier contract code. +pub const CACHE_PATH_EVM: &str = "evm"; + +/// The path to the config parameters for a given proof layer. +/// +/// /{layer}.config +pub fn non_native_params_path(dir: &Path, layer: ProofLayer) -> PathBuf { + dir.join(format!("{}{NON_NATIVE_PARAMS_EXT}", layer.to_string())) +} + +/// The path to the KZG params by degree. +/// +/// /params{degree} +pub fn kzg_params_path(dir: &Path, degree: u32) -> PathBuf { + dir.join(format!("params{degree}")) +} + +/// Wrapper functionality for current working directory. +pub fn pwd() -> Result { + Ok(current_dir()?) +} + +/// The default path to find non-native field arithmetic config params. +/// +/// /.config +pub fn default_non_native_params_dir() -> Result { + Ok(pwd()?.join(TEST_DATA_DIR).join(NON_NATIVE_PARAMS_DIR)) +} + +/// The default path to find KZG setup parameters. +/// +/// /.params +pub fn default_kzg_params_dir() -> Result { + Ok(pwd()?.join(TEST_DATA_DIR).join(KZG_PARAMS_DIR)) +} diff --git a/prover2/src/util/env.rs b/prover2/src/util/env.rs new file mode 100644 index 0000000000..7803841f79 --- /dev/null +++ b/prover2/src/util/env.rs @@ -0,0 +1,35 @@ +use std::{env::var, str::FromStr}; + +use crate::{ + ProverError, + ProverError::{EnvVar, Parse}, +}; + +/// Wrapper to read variable from the environment. +pub fn read_env(key: &str) -> Result { + var(key).map_err(|source| EnvVar { + source, + key: key.into(), + }) +} + +/// Read variable from the environment and parse to a generic type. +pub fn read_env_as(key: &str) -> Result +where + T::Err: std::error::Error, +{ + let src = read_env(key)?; + + src.parse::().map_err(|e| Parse { + src, + err: e.to_string(), + }) +} + +/// Read variable from the environment if set, otherwise return the provided default value. +pub fn read_env_or_default(key: &str, default: T) -> T +where + T::Err: std::error::Error, +{ + read_env_as::(key).unwrap_or(default) +} diff --git a/prover2/src/util/io.rs b/prover2/src/util/io.rs new file mode 100644 index 0000000000..ae5d7e87ea --- /dev/null +++ b/prover2/src/util/io.rs @@ -0,0 +1,68 @@ +use std::{ + fs::{self, File}, + io::BufReader, + path::Path, +}; + +use halo2_proofs::{halo2curves::bn256::Bn256, poly::kzg::commitment::ParamsKZG, SerdeFormat}; +use serde::{de::DeserializeOwned, Serialize}; + +use crate::{ + ProverError, + ProverError::{IoReadWrite, ReadWriteJson}, +}; + +/// Wrapper functionality for opening a file. +pub fn open(path: &Path) -> Result { + File::open(path).map_err(|source| IoReadWrite { + source, + path: path.into(), + }) +} + +/// Wrapper functionality for reading bytes from a file. +pub fn read(path: impl AsRef) -> Result, ProverError> { + let path = path.as_ref(); + fs::read(path).map_err(|source| IoReadWrite { + source, + path: path.into(), + }) +} + +/// Wrapper functionality to write bytes to a file. +pub fn write(path: impl AsRef, data: &[u8]) -> Result<(), ProverError> { + let path = path.as_ref(); + fs::write(path, data).map_err(|source| IoReadWrite { + source, + path: path.into(), + }) +} + +/// Wrapper functionality to create a file and write to it. +pub fn create_and_write(path: &Path, data: &[u8]) -> Result<(), ProverError> { + File::create(path)?; + write(path, data) +} + +/// Wrapper functionality for reading a JSON file. +pub fn read_json(path: &Path) -> Result { + let bytes = read(path)?; + serde_json::from_slice(&bytes).map_err(|source| ReadWriteJson { + source, + path: path.into(), + }) +} + +pub fn write_json(path: &Path, data: &T) -> Result<(), ProverError> { + let bytes = serde_json::to_vec(data)?; + create_and_write(path, &bytes) +} + +/// Read KZG setup parameters that are in a custom serde format. +pub fn read_kzg_params(path: &Path) -> Result, ProverError> { + let f = open(path)?; + Ok(ParamsKZG::::read_custom( + &mut BufReader::new(f), + SerdeFormat::RawBytesUnchecked, + )?) +} diff --git a/prover2/src/util/mod.rs b/prover2/src/util/mod.rs new file mode 100644 index 0000000000..9be2eda911 --- /dev/null +++ b/prover2/src/util/mod.rs @@ -0,0 +1,40 @@ +use git_version::git_version; +use once_cell::sync::Lazy; +use rand::{Rng, SeedableRng}; +use rand_xorshift::XorShiftRng; + +mod dir; +pub use dir::{ + default_kzg_params_dir, default_non_native_params_dir, kzg_params_path, non_native_params_path, + CACHE_PATH_EVM, CACHE_PATH_PI, CACHE_PATH_PROOFS, CACHE_PATH_SNARKS, CACHE_PATH_TASKS, + JSON_EXT, +}; + +mod env; +pub use env::read_env_or_default; + +mod io; +pub use io::{read_json, read_kzg_params, write, write_json}; + +mod serde; +pub use serde::{deserialize_be, serialize_be}; + +/// The environment variable to be set to configure custom degree for the super circuit (layer0). +pub const ENV_DEGREE_LAYER0: &str = "SUPER_CIRCUIT_DEGREE"; + +/// The default degree for the super circuit (layer0). +pub const DEFAULT_DEGREE_LAYER0: u32 = 20; + +/// Git version (git describe) of the source code. +pub static GIT_VERSION: Lazy<&str> = Lazy::new(|| { + git_version!(args = ["--abbrev=8", "--always"]) + .split('-') + .last() + .expect("git describe should not fail") +}); + +/// Seed and return a random number generator. +pub fn gen_rng() -> impl Rng + Send { + let seed = [0u8; 16]; + XorShiftRng::from_seed(seed) +} diff --git a/prover2/src/util/serde.rs b/prover2/src/util/serde.rs new file mode 100644 index 0000000000..8caf460bad --- /dev/null +++ b/prover2/src/util/serde.rs @@ -0,0 +1,19 @@ +use halo2_proofs::halo2curves::bn256::Fr; +use snark_verifier::loader::halo2::halo2_ecc::halo2_base::utils::ScalarField; + +/// Serialize a field element [`Fr`] to big-endian bytes. +/// +/// [`Fr`]: halo2_proofs::halo2curves::bn256::Fr +pub fn serialize_be(field: &Fr) -> Vec { + field.to_bytes().into_iter().rev().collect() +} + +/// Deserialize a field element [`Fr`] from big-endian bytes. +/// +/// [`Fr`]: halo2_proofs::halo2curves::bn256::Fr +pub fn deserialize_be(be_bytes: &[u8]) -> Fr { + let mut le_bytes = [0u8; 32]; + le_bytes.copy_from_slice(be_bytes); + le_bytes.reverse(); + Fr::from_bytes_le(&le_bytes) +} diff --git a/prover2/src/verifier.rs b/prover2/src/verifier.rs new file mode 100644 index 0000000000..8b13789179 --- /dev/null +++ b/prover2/src/verifier.rs @@ -0,0 +1 @@ +