diff --git a/halo2_proofs/Cargo.toml b/halo2_proofs/Cargo.toml index 261f0e4f29..de5a92a3f6 100644 --- a/halo2_proofs/Cargo.toml +++ b/halo2_proofs/Cargo.toml @@ -59,6 +59,7 @@ poseidon = { git = 'https://github.com/privacy-scaling-explorations/poseidon.git num-integer = "0.1" num-bigint = { version = "0.4", features = ["rand"] } +crossbeam = "0.8.0" # Developer tooling dependencies plotters = { version = "0.3.0", optional = true } tabbycat = { version = "0.1", features = ["attributes"], optional = true } @@ -66,6 +67,7 @@ log = "0.4.17" # timer ark-std = { version = "0.3.0" } +env_logger = "0.8.0" [dev-dependencies] assert_matches = "1.5" @@ -85,6 +87,7 @@ sanity-checks = [] batch = ["rand_core/getrandom"] shplonk = [] gwc = [] +parallel_syn = [] phase-check = [] profile = ["ark-std/print-trace"] diff --git a/halo2_proofs/src/arithmetic.rs b/halo2_proofs/src/arithmetic.rs index e3fae5f8bb..c06575b549 100644 --- a/halo2_proofs/src/arithmetic.rs +++ b/halo2_proofs/src/arithmetic.rs @@ -206,7 +206,7 @@ fn serial_fft(a: &mut [G], omega: G::Scalar, log_n: u32) { let mut m = 1; for _ in 0..log_n { - let w_m = omega.pow_vartime(&[u64::from(n / (2 * m)), 0, 0, 0]); + let w_m = omega.pow_vartime([u64::from(n / (2 * m)), 0, 0, 0]); let mut k = 0; while k < n { @@ -316,7 +316,7 @@ pub fn generate_twiddle_lookup_table( if is_lut_len_large { let mut twiddle_lut = vec![F::zero(); (1 << log_n) as usize]; parallelize(&mut twiddle_lut, |twiddle_lut, start| { - let mut w_n = omega.pow_vartime(&[start as u64, 0, 0, 0]); + let mut w_n = omega.pow_vartime([start as u64, 0, 0, 0]); for twiddle_lut in twiddle_lut.iter_mut() { *twiddle_lut = w_n; w_n = w_n * omega; @@ -332,18 +332,18 @@ pub fn generate_twiddle_lookup_table( parallelize( &mut twiddle_lut[..low_degree_lut_len], |twiddle_lut, start| { - let mut w_n = omega.pow_vartime(&[start as u64, 0, 0, 0]); + let mut w_n = omega.pow_vartime([start as u64, 0, 0, 0]); for twiddle_lut in twiddle_lut.iter_mut() { *twiddle_lut = w_n; w_n = w_n * omega; } }, ); - let high_degree_omega = omega.pow_vartime(&[(1 << sparse_degree) as u64, 0, 0, 0]); + let high_degree_omega = omega.pow_vartime([(1 << sparse_degree) as u64, 0, 0, 0]); parallelize( &mut twiddle_lut[low_degree_lut_len..], |twiddle_lut, start| { - let mut w_n = high_degree_omega.pow_vartime(&[start as u64, 0, 0, 0]); + let mut w_n = high_degree_omega.pow_vartime([start as u64, 0, 0, 0]); for twiddle_lut in twiddle_lut.iter_mut() { *twiddle_lut = w_n; w_n = w_n * high_degree_omega; @@ -372,7 +372,7 @@ pub fn parallel_fft(a: &mut [G], omega: G::Scalar, log_n: u32) { let twiddle_lut = &*twiddle_lut; for (chunk_idx, tmp) in tmp.chunks_mut(sub_n).enumerate() { scope.spawn(move |_| { - let split_fft_offset = chunk_idx * sub_n >> log_split; + let split_fft_offset = (chunk_idx * sub_n) >> log_split; for (i, tmp) in tmp.chunks_mut(split_m).enumerate() { let split_fft_offset = split_fft_offset + i; split_radix_fft(tmp, a, twiddle_lut, n, split_fft_offset, log_split); @@ -392,7 +392,7 @@ pub fn parallel_fft(a: &mut [G], omega: G::Scalar, log_n: u32) { }); // sub fft - let new_omega = omega.pow_vartime(&[split_m as u64, 0, 0, 0]); + let new_omega = omega.pow_vartime([split_m as u64, 0, 0, 0]); multicore::scope(|scope| { for a in a.chunks_mut(sub_n) { scope.spawn(move |_| { @@ -419,7 +419,7 @@ pub fn parallel_fft(a: &mut [G], omega: G::Scalar, log_n: u32) { /// Convert coefficient bases group elements to lagrange basis by inverse FFT. pub fn g_to_lagrange(g_projective: Vec, k: u32) -> Vec { - let n_inv = C::Scalar::TWO_INV.pow_vartime(&[k as u64, 0, 0, 0]); + let n_inv = C::Scalar::TWO_INV.pow_vartime([k as u64, 0, 0, 0]); let mut omega_inv = C::Scalar::ROOT_OF_UNITY_INV; for _ in k..C::Scalar::S { omega_inv = omega_inv.square(); @@ -464,7 +464,7 @@ pub fn eval_polynomial(poly: &[F], point: F) -> F { { scope.spawn(move |_| { let start = chunk_idx * chunk_size; - out[0] = evaluate(poly, point) * point.pow_vartime(&[start as u64, 0, 0, 0]); + out[0] = evaluate(poly, point) * point.pow_vartime([start as u64, 0, 0, 0]); }); } }); diff --git a/halo2_proofs/src/circuit.rs b/halo2_proofs/src/circuit.rs index 1dc9bd917f..52db772c94 100644 --- a/halo2_proofs/src/circuit.rs +++ b/halo2_proofs/src/circuit.rs @@ -435,6 +435,18 @@ pub trait Layouter { N: Fn() -> NR, NR: Into; + #[cfg(feature = "parallel_syn")] + fn assign_regions( + &mut self, + name: N, + assignments: Vec, + ) -> Result, Error> + where + A: FnMut(Region<'_, F>) -> Result + Send, + AR: Send, + N: Fn() -> NR, + NR: Into; + /// Assign a table region to an absolute row number. /// /// ```ignore @@ -510,6 +522,21 @@ impl<'a, F: Field, L: Layouter + 'a> Layouter for NamespacedLayouter<'a, F self.0.assign_region(name, assignment) } + #[cfg(feature = "parallel_syn")] + fn assign_regions( + &mut self, + name: N, + assignments: Vec, + ) -> Result, Error> + where + A: FnMut(Region<'_, F>) -> Result + Send, + AR: Send, + N: Fn() -> NR, + NR: Into, + { + self.0.assign_regions(name, assignments) + } + fn assign_table(&mut self, name: N, assignment: A) -> Result<(), Error> where A: FnMut(Table<'_, F>) -> Result<(), Error>, diff --git a/halo2_proofs/src/circuit/floor_planner/single_pass.rs b/halo2_proofs/src/circuit/floor_planner/single_pass.rs index 80174a2c51..b95529addb 100644 --- a/halo2_proofs/src/circuit/floor_planner/single_pass.rs +++ b/halo2_proofs/src/circuit/floor_planner/single_pass.rs @@ -2,6 +2,9 @@ use std::cmp; use std::collections::HashMap; use std::fmt; use std::marker::PhantomData; +use std::ops::Range; +use std::sync::{Arc, Mutex}; +use std::time::Instant; use ff::Field; @@ -12,6 +15,7 @@ use crate::{ layouter::{RegionColumn, RegionLayouter, RegionShape, TableLayouter}, Cell, Layouter, Region, RegionIndex, RegionStart, Table, Value, }, + multicore, plonk::{ Advice, Any, Assigned, Assignment, Challenge, Circuit, Column, Error, Fixed, FloorPlanner, Instance, Selector, TableColumn, @@ -33,7 +37,7 @@ impl FloorPlanner for SimpleFloorPlanner { config: C::Config, constants: Vec>, ) -> Result<(), Error> { - let timer = start_timer!(|| format!("SimpleFloorPlanner synthesize")); + let timer = start_timer!(|| ("SimpleFloorPlanner synthesize").to_string()); let layouter = SingleChipLayouter::new(cs, constants)?; let result = circuit.synthesize(config, layouter); end_timer!(timer); @@ -63,7 +67,7 @@ impl<'a, F: Field, CS: Assignment + 'a> fmt::Debug for SingleChipLayouter<'a, } } -impl<'a, F: Field, CS: Assignment> SingleChipLayouter<'a, F, CS> { +impl<'a, F: Field, CS: Assignment + 'a> SingleChipLayouter<'a, F, CS> { /// Creates a new single-chip layouter. pub fn new(cs: &'a mut CS, constants: Vec>) -> Result { let ret = SingleChipLayouter { @@ -76,6 +80,20 @@ impl<'a, F: Field, CS: Assignment> SingleChipLayouter<'a, F, CS> { }; Ok(ret) } + + fn fork(&self, sub_cs: Vec<&'a mut CS>) -> Result, Error> { + Ok(sub_cs + .into_iter() + .map(|sub_cs| Self { + cs: sub_cs, + constants: self.constants.clone(), + regions: self.regions.clone(), + columns: self.columns.clone(), + table_columns: self.table_columns.clone(), + _marker: Default::default(), + }) + .collect::>()) + } } impl<'a, F: Field, CS: Assignment + 'a> Layouter for SingleChipLayouter<'a, F, CS> { @@ -185,6 +203,151 @@ impl<'a, F: Field, CS: Assignment + 'a> Layouter for SingleChipLayouter<'a Ok(result) } + #[cfg(feature = "parallel_syn")] + fn assign_regions( + &mut self, + name: N, + mut assignments: Vec, + ) -> Result, Error> + where + A: FnMut(Region<'_, F>) -> Result + Send, + AR: Send, + N: Fn() -> NR, + NR: Into, + { + let region_index = self.regions.len(); + let region_name: String = name().into(); + // Get region shapes sequentially + let mut ranges = vec![]; + for (i, assignment) in assignments.iter_mut().enumerate() { + // Get shape of the ith sub-region. + let mut shape = RegionShape::new((region_index + i).into()); + let region: &mut dyn RegionLayouter = &mut shape; + assignment(region.into())?; + + let mut region_start = 0; + for column in &shape.columns { + let column_start = self.columns.get(column).cloned().unwrap_or(0); + region_start = cmp::max(region_start, column_start); + } + log::debug!( + "{}_{} start: {}, end: {}", + region_name, + i, + region_start, + region_start + shape.row_count() + ); + self.regions.push(region_start.into()); + ranges.push(region_start..(region_start + shape.row_count())); + + // Update column usage information. + for column in shape.columns.iter() { + self.columns + .insert(*column, region_start + shape.row_count()); + } + } + + // Do actual synthesis of sub-regions in parallel + let cs_fork_time = Instant::now(); + let mut sub_cs = self.cs.fork(&ranges)?; + log::info!( + "CS forked into {} subCS took {:?}", + sub_cs.len(), + cs_fork_time.elapsed() + ); + let ref_sub_cs = sub_cs.iter_mut().collect(); + let sub_layouters = self.fork(ref_sub_cs)?; + let regions_2nd_pass = Instant::now(); + let ret = crossbeam::scope(|scope| { + let mut handles = vec![]; + for (i, (mut assignment, mut sub_layouter)) in assignments + .into_iter() + .zip(sub_layouters.into_iter()) + .enumerate() + { + let region_name = format!("{}_{}", region_name, i); + handles.push(scope.spawn(move |_| { + let sub_region_2nd_pass = Instant::now(); + sub_layouter.cs.enter_region(|| region_name.clone()); + let mut region = + SingleChipLayouterRegion::new(&mut sub_layouter, (region_index + i).into()); + let region_ref: &mut dyn RegionLayouter = &mut region; + let result = assignment(region_ref.into()); + let constant = region.constants.clone(); + sub_layouter.cs.exit_region(); + log::info!( + "region {} 2nd pass synthesis took {:?}", + region_name, + sub_region_2nd_pass.elapsed() + ); + + (result, constant) + })); + } + + handles + .into_iter() + .map(|handle| handle.join().expect("handle.join should never fail")) + .collect::>() + }) + .expect("scope should not fail"); + let cs_merge_time = Instant::now(); + let num_sub_cs = sub_cs.len(); + self.cs.merge(sub_cs)?; + log::info!( + "Merge {} subCS back took {:?}", + num_sub_cs, + cs_merge_time.elapsed() + ); + log::info!( + "{} sub_regions of {} 2nd pass synthesis took {:?}", + ranges.len(), + region_name, + regions_2nd_pass.elapsed() + ); + let (results, constants): (Vec<_>, Vec<_>) = ret.into_iter().unzip(); + + // Check if there are errors in sub-region synthesis + let results = results.into_iter().collect::, Error>>()?; + + // Merge all constants from sub-regions together + let constants_to_assign = constants + .into_iter() + .flat_map(|constant_to_assign| constant_to_assign.into_iter()) + .collect::>(); + + // Assign constants. For the simple floor planner, we assign constants in order in + // the first `constants` column. + if self.constants.is_empty() { + if !constants_to_assign.is_empty() { + return Err(Error::NotEnoughColumnsForConstants); + } + } else { + let constants_column = self.constants[0]; + let next_constant_row = self + .columns + .entry(Column::::from(constants_column).into()) + .or_default(); + for (constant, advice) in constants_to_assign { + self.cs.assign_fixed( + || format!("Constant({:?})", constant.evaluate()), + constants_column, + *next_constant_row, + || Value::known(constant), + )?; + self.cs.copy( + constants_column.into(), + *next_constant_row, + advice.column, + *self.regions[*advice.region_index] + advice.row_offset, + )?; + *next_constant_row += 1; + } + } + + Ok(results) + } + fn assign_table(&mut self, name: N, mut assignment: A) -> Result<(), Error> where A: FnMut(Table<'_, F>) -> Result<(), Error>, diff --git a/halo2_proofs/src/circuit/floor_planner/v1.rs b/halo2_proofs/src/circuit/floor_planner/v1.rs index dbe0a71dc1..0fd28c196f 100644 --- a/halo2_proofs/src/circuit/floor_planner/v1.rs +++ b/halo2_proofs/src/circuit/floor_planner/v1.rs @@ -184,6 +184,20 @@ impl<'p, 'a, F: Field, CS: Assignment + 'a> Layouter for V1Pass<'p, 'a, F, } } + #[cfg(feature = "parallel_syn")] + fn assign_regions( + &mut self, + _name: N, + _assignments: Vec, + ) -> Result, Error> + where + A: FnMut(Region<'_, F>) -> Result, + N: Fn() -> NR, + NR: Into, + { + todo!() + } + fn assign_table(&mut self, name: N, assignment: A) -> Result<(), Error> where A: FnMut(Table<'_, F>) -> Result<(), Error>, diff --git a/halo2_proofs/src/dev.rs b/halo2_proofs/src/dev.rs index cdd1c450b4..f2b95c2c0a 100644 --- a/halo2_proofs/src/dev.rs +++ b/halo2_proofs/src/dev.rs @@ -4,7 +4,8 @@ use std::collections::HashMap; use std::collections::HashSet; use std::fmt; use std::iter; -use std::ops::{Add, Mul, Neg, Range}; +use std::ops::{Add, DerefMut, Mul, Neg, Range}; +use std::sync::Arc; use std::time::{Duration, Instant}; use blake2b_simd::blake2b; @@ -44,16 +45,20 @@ pub use cost::CircuitCost; mod gates; pub use gates::CircuitGates; +use crate::two_dim_vec_to_vec_of_slice; + #[cfg(feature = "dev-graph")] mod graph; +use crate::circuit::Cell; +use crate::helpers::CopyCell; #[cfg(feature = "dev-graph")] #[cfg_attr(docsrs, doc(cfg(feature = "dev-graph")))] pub use graph::{circuit_dot_graph, layout::CircuitLayout}; pub use crate::circuit::value_dev::unwrap_value; -#[derive(Debug)] +#[derive(Clone, Debug)] struct Region { /// The name of the region. Not required to be unique. name: String, @@ -69,6 +74,8 @@ struct Region { /// The cells assigned in this region. We store this as a `Vec` so that if any cells /// are double-assigned, they will be visibly darker. cells: HashMap<(Column, usize), usize>, + /// The copies that need to be enforced in this region. + copies: Vec<(CopyCell, CopyCell)>, } impl Region { @@ -286,7 +293,7 @@ impl Mul for Value { /// )); /// ``` #[derive(Debug)] -pub struct MockProver { +pub struct MockProver<'a, F: Group + Field> { k: u32, n: u32, cs: ConstraintSystem, @@ -298,18 +305,25 @@ pub struct MockProver { current_region: Option, // The fixed cells in the circuit, arranged as [column][row]. - fixed: Vec>>, + fixed_vec: Arc>>>, + fixed: Vec<&'a mut [CellValue]>, // The advice cells in the circuit, arranged as [column][row]. - pub(crate) advice: Vec>>, + pub(crate) advice_vec: Arc>>>, + pub(crate) advice: Vec<&'a mut [CellValue]>, + // This field is used only if the "phase_check" feature is turned on. advice_prev: Vec>>, // The instance cells in the circuit, arranged as [column][row]. instance: Vec>, - selectors: Vec>, + selectors_vec: Arc>>, + selectors: Vec<&'a mut [bool]>, challenges: Vec, - permutation: permutation::keygen::Assembly, + /// For mock prover which is generated from `fork()`, this field is None. + permutation: Option, + + rw_rows: Range, // A range of available rows for assignment and copies. usable_rows: Range, @@ -317,7 +331,7 @@ pub struct MockProver { current_phase: crate::plonk::sealed::Phase, } -impl Assignment for MockProver { +impl<'a, F: Field + Group> Assignment for MockProver<'a, F> { fn enter_region(&mut self, name: N) where NR: Into, @@ -331,6 +345,7 @@ impl Assignment for MockProver { annotations: HashMap::default(), enabled_selectors: HashMap::default(), cells: HashMap::default(), + copies: Vec::new(), }); } @@ -359,6 +374,16 @@ impl Assignment for MockProver { return Err(Error::not_enough_rows_available(self.k)); } + if !self.rw_rows.contains(&row) { + return Err(Error::InvalidRange( + row, + self.current_region + .as_ref() + .map(|region| region.name.clone()) + .unwrap(), + )); + } + // Track that this selector was enabled. We require that all selectors are enabled // inside some region (i.e. no floating selectors). self.current_region @@ -369,7 +394,118 @@ impl Assignment for MockProver { .or_default() .push(row); - self.selectors[selector.0][row] = true; + self.selectors[selector.0][row - self.rw_rows.start] = true; + + Ok(()) + } + + fn fork(&mut self, ranges: &[Range]) -> Result, Error> { + // check ranges are non-overlapping and monotonically increasing + let mut range_start = self.rw_rows.start; + for (i, sub_range) in ranges.iter().enumerate() { + if sub_range.start < range_start { + // TODO: use more precise error type + return Err(Error::Synthesis); + } + if i == ranges.len() - 1 && sub_range.end >= self.rw_rows.end { + return Err(Error::Synthesis); + } + range_start = sub_range.end; + log::debug!( + "subCS_{} rw_rows: {}..{}", + i, + sub_range.start, + sub_range.end + ); + } + + // split self.fixed into several pieces + let fixed_ptrs = self + .fixed + .iter_mut() + .map(|vec| vec.as_mut_ptr()) + .collect::>(); + let selectors_ptrs = self + .selectors + .iter_mut() + .map(|vec| vec.as_mut_ptr()) + .collect::>(); + let advice_ptrs = self + .advice + .iter_mut() + .map(|vec| vec.as_mut_ptr()) + .collect::>(); + + let mut sub_cs = vec![]; + for (_i, sub_range) in ranges.iter().enumerate() { + let fixed = fixed_ptrs + .iter() + .map(|ptr| unsafe { + std::slice::from_raw_parts_mut( + ptr.add(sub_range.start), + sub_range.end - sub_range.start, + ) + }) + .collect::]>>(); + let selectors = selectors_ptrs + .iter() + .map(|ptr| unsafe { + std::slice::from_raw_parts_mut( + ptr.add(sub_range.start), + sub_range.end - sub_range.start, + ) + }) + .collect::>(); + let advice = advice_ptrs + .iter() + .map(|ptr| unsafe { + std::slice::from_raw_parts_mut( + ptr.add(sub_range.start), + sub_range.end - sub_range.start, + ) + }) + .collect::]>>(); + + sub_cs.push(Self { + k: self.k, + n: self.n, + cs: self.cs.clone(), + regions: vec![], + current_region: None, + fixed_vec: self.fixed_vec.clone(), + fixed, + advice_vec: self.advice_vec.clone(), + advice, + advice_prev: self.advice_prev.clone(), + instance: self.instance.clone(), + selectors_vec: self.selectors_vec.clone(), + selectors, + challenges: self.challenges.clone(), + permutation: None, + rw_rows: sub_range.clone(), + usable_rows: self.usable_rows.clone(), + current_phase: self.current_phase, + }); + } + + Ok(sub_cs) + } + + fn merge(&mut self, sub_cs: Vec) -> Result<(), Error> { + for (left, right) in sub_cs + .iter() + .flat_map(|cs| cs.regions.iter()) + .flat_map(|region| region.copies.iter()) + { + self.permutation + .as_mut() + .expect("root cs permutation should be Some") + .copy(left.column, left.row, right.column, right.row)?; + } + + for region in sub_cs.into_iter().map(|cs| cs.regions) { + self.regions.extend_from_slice(®ion[..]) + } Ok(()) } @@ -411,6 +547,16 @@ impl Assignment for MockProver { return Err(Error::not_enough_rows_available(self.k)); } + if !self.rw_rows.contains(&row) { + return Err(Error::InvalidRange( + row, + self.current_region + .as_ref() + .map(|region| region.name.clone()) + .unwrap(), + )); + } + if let Some(region) = self.current_region.as_mut() { region.update_extent(column.into(), row); region @@ -424,16 +570,16 @@ impl Assignment for MockProver { *self .advice .get_mut(column.index()) - .and_then(|v| v.get_mut(row)) + .and_then(|v| v.get_mut(row - self.rw_rows.start)) .ok_or(Error::BoundsFailure)? = assigned; #[cfg(feature = "phase-check")] - if false && self.current_phase.0 > column.column_type().phase.0 { + // if false && self.current_phase.0 > column.column_type().phase.0 { + if false { // Some circuits assign cells more than one times with different values // So this check sometimes can be false alarm - if !self.advice_prev.is_empty() { - if self.advice_prev[column.index()][row] != assigned { - panic!("not same new {assigned:?} old {:?}, column idx {} row {} cur phase {:?} col phase {:?} region {:?}", + if !self.advice_prev.is_empty() && self.advice_prev[column.index()][row] != assigned { + panic!("not same new {assigned:?} old {:?}, column idx {} row {} cur phase {:?} col phase {:?} region {:?}", self.advice_prev[column.index()][row], column.index(), row, @@ -441,7 +587,6 @@ impl Assignment for MockProver { column.column_type().phase, self.current_region ) - } } } @@ -465,6 +610,16 @@ impl Assignment for MockProver { return Err(Error::not_enough_rows_available(self.k)); } + if !self.rw_rows.contains(&row) { + return Err(Error::InvalidRange( + row, + self.current_region + .as_ref() + .map(|region| region.name.clone()) + .unwrap(), + )); + } + if let Some(region) = self.current_region.as_mut() { region.update_extent(column.into(), row); region @@ -474,12 +629,15 @@ impl Assignment for MockProver { .or_default(); } - *self + let fix_cell = self .fixed .get_mut(column.index()) - .and_then(|v| v.get_mut(row)) - .ok_or(Error::BoundsFailure)? = - CellValue::Assigned(to().into_field().evaluate().assign()?); + .and_then(|v| v.get_mut(row - self.rw_rows.start)) + .ok_or(Error::BoundsFailure); + if fix_cell.is_err() { + println!("fix cell is none: {}, row: {}", column.index(), row); + } + *fix_cell? = CellValue::Assigned(to().into_field().evaluate().assign()?); Ok(()) } @@ -495,8 +653,25 @@ impl Assignment for MockProver { return Err(Error::not_enough_rows_available(self.k)); } - self.permutation - .copy(left_column, left_row, right_column, right_row) + match self.permutation.as_mut() { + Some(permutation) => permutation.copy(left_column, left_row, right_column, right_row), + None => { + let left_cell = CopyCell { + column: left_column, + row: left_row, + }; + let right_cell = CopyCell { + column: right_column, + row: right_row, + }; + self.current_region + .as_mut() + .unwrap() + .copies + .push((left_cell, right_cell)); + Ok(()) + } + } } fn fill_from_row( @@ -536,7 +711,7 @@ impl Assignment for MockProver { } } -impl MockProver { +impl<'a, F: FieldExt> MockProver<'a, F> { /// Runs a synthetic keygen-and-prove operation on the given circuit, collecting data /// about the constraints and their assignments. pub fn run>( @@ -571,12 +746,16 @@ impl MockProver { .collect::, _>>()?; // Fixed columns contain no blinding factors. - let fixed = vec![vec![CellValue::Unassigned; n]; cs.num_fixed_columns]; - let selectors = vec![vec![false; n]; cs.num_selectors]; + let fixed_vec = Arc::new(vec![vec![CellValue::Unassigned; n]; cs.num_fixed_columns]); + let fixed = two_dim_vec_to_vec_of_slice!(fixed_vec); + + let selectors_vec = Arc::new(vec![vec![false; n]; cs.num_selectors]); + let selectors = two_dim_vec_to_vec_of_slice!(selectors_vec); + // Advice columns contain blinding factors. let blinding_factors = cs.blinding_factors(); let usable_rows = n - (blinding_factors + 1); - let advice = vec![ + let advice_vec = Arc::new(vec![ { let mut column = vec![CellValue::Unassigned; n]; // Poison unusable rows. @@ -586,7 +765,9 @@ impl MockProver { column }; cs.num_advice_columns - ]; + ]); + let advice = two_dim_vec_to_vec_of_slice!(advice_vec); + let permutation = permutation::keygen::Assembly::new(n, &cs.permutation); let constants = cs.constants.clone(); @@ -601,90 +782,107 @@ impl MockProver { .collect() }; + #[cfg(feature = "phase-check")] + let current_phase = FirstPhase.to_sealed(); + #[cfg(not(feature = "phase-check"))] + let current_phase = crate::plonk::sealed::Phase(cs.max_phase()); + + let mut prover = MockProver { + k, + n: n as u32, + cs, + regions: vec![], + current_region: None, + fixed_vec, + fixed, + advice_vec, + advice, + advice_prev: vec![], + instance, + selectors_vec, + selectors, + challenges: challenges.clone(), + permutation: Some(permutation), + rw_rows: 0..usable_rows, + usable_rows: 0..usable_rows, + current_phase, + }; + #[cfg(feature = "phase-check")] { // check1: phase1 should not assign expr including phase2 challenges // check2: phase2 assigns same phase1 columns with phase1 let mut cur_challenges: Vec = Vec::new(); let mut last_advice: Vec>> = Vec::new(); - for current_phase in cs.phases() { - let mut prover = MockProver { - k, - n: n as u32, - cs: cs.clone(), - regions: vec![], - current_region: None, - fixed: fixed.clone(), - advice: advice.clone(), - advice_prev: last_advice.clone(), - instance: instance.clone(), - selectors: selectors.clone(), - challenges: cur_challenges.clone(), - permutation: permutation.clone(), - usable_rows: 0..usable_rows, - current_phase, - }; + for current_phase in prover.cs.phases() { + prover.current_phase = current_phase; + prover.advice_prev = last_advice; ConcreteCircuit::FloorPlanner::synthesize( &mut prover, circuit, config.clone(), constants.clone(), )?; - for (index, phase) in cs.challenge_phase.iter().enumerate() { + + for (index, phase) in prover.cs.challenge_phase.iter().enumerate() { if current_phase == *phase { debug_assert_eq!(cur_challenges.len(), index); - cur_challenges.push(challenges[index].clone()); + cur_challenges.push(challenges[index]); } } - if !last_advice.is_empty() { + if !prover.advice_prev.is_empty() { let mut err = false; for (idx, advice_values) in prover.advice.iter().enumerate() { - if cs.advice_column_phase[idx].0 < current_phase.0 { - if advice_values != &last_advice[idx] { - log::error!( - "PHASE ERR column{} not same after phase {:?}", - idx, - current_phase - ); - err = true; - } + if prover.cs.advice_column_phase[idx].0 < current_phase.0 + && advice_values != &prover.advice_prev[idx] + { + log::error!( + "PHASE ERR column{} not same after phase {:?}", + idx, + current_phase + ); + err = true; } } if err { panic!("wrong phase assignment"); } } - last_advice = prover.advice; + last_advice = prover.advice_vec.as_ref().clone(); } } - let mut prover = MockProver { - k, - n: n as u32, - cs, - regions: vec![], - current_region: None, - fixed, - advice, - advice_prev: vec![], - instance, - selectors, - challenges: challenges.clone(), - permutation, - usable_rows: 0..usable_rows, - current_phase: ThirdPhase.to_sealed(), - }; - ConcreteCircuit::FloorPlanner::synthesize(&mut prover, circuit, config, constants)?; + #[cfg(not(feature = "phase-check"))] + { + let syn_time = Instant::now(); + ConcreteCircuit::FloorPlanner::synthesize(&mut prover, circuit, config, constants)?; + log::info!("MockProver synthesize took {:?}", syn_time.elapsed()); + } - let (cs, selector_polys) = prover.cs.compress_selectors(prover.selectors.clone()); + let (cs, selector_polys) = prover + .cs + .compress_selectors(prover.selectors_vec.as_ref().clone()); prover.cs = cs; - prover.fixed.extend(selector_polys.into_iter().map(|poly| { - let mut v = vec![CellValue::Unassigned; n]; - for (v, p) in v.iter_mut().zip(&poly[..]) { - *v = CellValue::Assigned(*p); - } - v - })); + Arc::get_mut(&mut prover.fixed_vec) + .expect("get_mut prover.fixed_vec") + .extend(selector_polys.into_iter().map(|poly| { + let mut v = vec![CellValue::Unassigned; n]; + for (v, p) in v.iter_mut().zip(&poly[..]) { + *v = CellValue::Assigned(*p); + } + v + })); + // update prover.fixed as prover.fixed_vec is updated + prover.fixed = unsafe { + let clone = prover.fixed_vec.clone(); + let ptr = Arc::as_ptr(&clone) as *mut Vec>>; + let mut_ref = &mut (*ptr); + mut_ref + .iter_mut() + .map(|vec| vec.as_mut_slice()) + .collect::>() + }; + debug_assert_eq!(Arc::strong_count(&prover.fixed_vec), 1); Ok(prover) } @@ -789,8 +987,8 @@ impl MockProver { move |(poly_index, poly)| match poly.evaluate_lazy( &|scalar| Value::Real(scalar), &|_| panic!("virtual selectors are removed during optimization"), - &util::load(n, row, &self.cs.fixed_queries, &self.fixed), - &util::load(n, row, &self.cs.advice_queries, &self.advice), + &util::load_slice(n, row, &self.cs.fixed_queries, &self.fixed), + &util::load_slice(n, row, &self.cs.advice_queries, &self.advice), &util::load_instance( n, row, @@ -821,8 +1019,18 @@ impl MockProver { cell_values: util::cell_values( gate, poly, - &util::load(n, row, &self.cs.fixed_queries, &self.fixed), - &util::load(n, row, &self.cs.advice_queries, &self.advice), + &util::load_slice( + n, + row, + &self.cs.fixed_queries, + &self.fixed, + ), + &util::load_slice( + n, + row, + &self.cs.advice_queries, + &self.advice, + ), &util::load_instance( n, row, @@ -1002,6 +1210,8 @@ impl MockProver { // Iterate over each column of the permutation self.permutation + .as_ref() + .expect("root cs permutation must be Some") .mapping .iter() .enumerate() @@ -1162,8 +1372,8 @@ impl MockProver { match poly.evaluate_lazy( &|scalar| Value::Real(scalar), &|_| panic!("virtual selectors are removed during optimization"), - &util::load(n, row, &self.cs.fixed_queries, &self.fixed), - &util::load(n, row, &self.cs.advice_queries, &self.advice), + &util::load_slice(n, row, &self.cs.fixed_queries, &self.fixed), + &util::load_slice(n, row, &self.cs.advice_queries, &self.advice), &util::load_instance( n, row, @@ -1194,8 +1404,18 @@ impl MockProver { cell_values: util::cell_values( gate, poly, - &util::load(n, row, &self.cs.fixed_queries, &self.fixed), - &util::load(n, row, &self.cs.advice_queries, &self.advice), + &util::load_slice( + n, + row, + &self.cs.fixed_queries, + &self.fixed, + ), + &util::load_slice( + n, + row, + &self.cs.advice_queries, + &self.advice, + ), &util::load_instance( n, row, @@ -1363,6 +1583,8 @@ impl MockProver { // Iterate over each column of the permutation self.permutation + .as_ref() + .expect("root cs permutation must be Some") .mapping .iter() .enumerate() @@ -1484,12 +1706,12 @@ impl MockProver { /// Returns the list of Fixed Columns used within a MockProver instance and the associated values contained on each Cell. pub fn fixed(&self) -> &Vec>> { - &self.fixed + self.fixed_vec.as_ref() } /// Returns the permutation argument (`Assembly`) used within a MockProver instance. pub fn permutation(&self) -> &Assembly { - &self.permutation + self.permutation.as_ref().unwrap() } } diff --git a/halo2_proofs/src/dev/cost.rs b/halo2_proofs/src/dev/cost.rs index d3043508f5..2a39cc5182 100644 --- a/halo2_proofs/src/dev/cost.rs +++ b/halo2_proofs/src/dev/cost.rs @@ -1,5 +1,6 @@ //! Developer tools for investigating the cost of a circuit. +use std::ops::Range; use std::{ collections::{HashMap, HashSet}, iter, @@ -69,6 +70,14 @@ impl Assignment for Assembly { Ok(()) } + fn fork(&mut self, _ranges: &[Range]) -> Result, Error> { + todo!() + } + + fn merge(&mut self, _sub_cs: Vec) -> Result<(), Error> { + todo!() + } + fn query_instance(&self, _: Column, _: usize) -> Result, Error> { Ok(Value::unknown()) } diff --git a/halo2_proofs/src/dev/failure.rs b/halo2_proofs/src/dev/failure.rs index d5a68c09d8..229cb171f4 100644 --- a/halo2_proofs/src/dev/failure.rs +++ b/halo2_proofs/src/dev/failure.rs @@ -551,8 +551,18 @@ fn render_lookup( let cell_values = input.evaluate( &|_| BTreeMap::default(), &|_| panic!("virtual selectors are removed during optimization"), - &cell_value(&util::load(n, row, &cs.fixed_queries, &prover.fixed)), - &cell_value(&util::load(n, row, &cs.advice_queries, &prover.advice)), + &cell_value(&util::load_slice( + n, + row, + &cs.fixed_queries, + prover.fixed.as_slice(), + )), + &cell_value(&util::load_slice( + n, + row, + &cs.advice_queries, + &prover.advice, + )), &cell_value(&util::load_instance( n, row, diff --git a/halo2_proofs/src/dev/graph.rs b/halo2_proofs/src/dev/graph.rs index 5a43313dea..8482879913 100644 --- a/halo2_proofs/src/dev/graph.rs +++ b/halo2_proofs/src/dev/graph.rs @@ -1,4 +1,5 @@ use ff::Field; +use std::ops::Range; use tabbycat::{AttrList, Edge, GraphBuilder, GraphType, Identity, StmtList}; use crate::{ @@ -99,6 +100,14 @@ impl Assignment for Graph { Ok(()) } + fn fork(&mut self, _ranges: &[Range]) -> Result, Error> { + todo!() + } + + fn merge(&mut self, _sub_cs: Vec) -> Result<(), Error> { + todo!() + } + fn annotate_column(&mut self, _annotation: A, _column: Column) where A: FnOnce() -> AR, diff --git a/halo2_proofs/src/dev/graph/layout.rs b/halo2_proofs/src/dev/graph/layout.rs index 0f2e67a81d..ad989eab10 100644 --- a/halo2_proofs/src/dev/graph/layout.rs +++ b/halo2_proofs/src/dev/graph/layout.rs @@ -181,7 +181,7 @@ impl CircuitLayout { root.draw(&Rectangle::new( [(0, 0), (total_columns, view_bottom)], - &BLACK, + BLACK, ))?; let draw_region = |root: &DrawingArea<_, _>, top_left, bottom_right| { @@ -197,7 +197,7 @@ impl CircuitLayout { [top_left, bottom_right], ShapeStyle::from(&GREEN.mix(0.2)).filled(), ))?; - root.draw(&Rectangle::new([top_left, bottom_right], &BLACK))?; + root.draw(&Rectangle::new([top_left, bottom_right], BLACK))?; Ok(()) }; @@ -432,6 +432,14 @@ impl Assignment for Layout { Ok(()) } + fn fork(&mut self, _ranges: &[Range]) -> Result, Error> { + todo!() + } + + fn merge(&mut self, _sub_cs: Vec) -> Result<(), Error> { + todo!() + } + fn query_instance(&self, _: Column, _: usize) -> Result, Error> { Ok(Value::unknown()) } diff --git a/halo2_proofs/src/dev/util.rs b/halo2_proofs/src/dev/util.rs index a4dbfe5a40..aa1e3ccc42 100644 --- a/halo2_proofs/src/dev/util.rs +++ b/halo2_proofs/src/dev/util.rs @@ -73,6 +73,7 @@ pub(super) fn format_value(v: F) -> String { } } +/* pub(super) fn load<'a, F: FieldExt, T: ColumnType, Q: Into + Copy>( n: i32, row: i32, @@ -85,6 +86,20 @@ pub(super) fn load<'a, F: FieldExt, T: ColumnType, Q: Into + Copy>( cells[column.index()][resolved_row as usize].into() } } +*/ + +pub(super) fn load_slice<'a, F: FieldExt, T: ColumnType, Q: Into + Copy>( + n: i32, + row: i32, + queries: &'a [(Column, Rotation)], + cells: &'a [&mut [CellValue]], +) -> impl Fn(Q) -> Value + 'a { + move |query| { + let (column, at) = &queries[query.into().index]; + let resolved_row = (row + at.0) % n; + cells[column.index()][resolved_row as usize].into() + } +} pub(super) fn load_instance<'a, F: FieldExt, T: ColumnType, Q: Into + Copy>( n: i32, diff --git a/halo2_proofs/src/helpers.rs b/halo2_proofs/src/helpers.rs index 661869f078..ea7adb3382 100644 --- a/halo2_proofs/src/helpers.rs +++ b/halo2_proofs/src/helpers.rs @@ -1,3 +1,4 @@ +use crate::plonk::{Any, Column}; use crate::poly::Polynomial; use ff::Field; use ff::PrimeField; @@ -21,6 +22,13 @@ pub enum SerdeFormat { /// Serialization is the same as `RawBytes`, but no checks are performed. RawBytesUnchecked, } + +#[derive(Clone, Debug)] +pub(crate) struct CopyCell { + pub column: Column, + pub row: usize, +} + // Keep this trait for compatibility with IPA serialization pub(crate) trait CurveRead: CurveAffine { /// Reads a compressed element from the buffer and attempts to parse it @@ -57,6 +65,22 @@ pub(crate) fn base_to_scalar(base: &C::Base) -> C::Scalar { bn_to_field(&bn) } +#[macro_export] +macro_rules! two_dim_vec_to_vec_of_slice { + ($arc_vec:ident) => { + unsafe { + let arc_vec_clone = $arc_vec.clone(); + let ptr = Arc::as_ptr(&arc_vec_clone) as *mut Vec>; + let mut_ref = &mut (*ptr); + + mut_ref + .iter_mut() + .map(|item| item.as_mut_slice()) + .collect::>() + } + }; +} + #[cfg(test)] mod test { use super::*; diff --git a/halo2_proofs/src/lib.rs b/halo2_proofs/src/lib.rs index e577a8c076..0ca61f64e4 100644 --- a/halo2_proofs/src/lib.rs +++ b/halo2_proofs/src/lib.rs @@ -19,7 +19,7 @@ )] #![deny(broken_intra_doc_links)] #![deny(missing_debug_implementations)] -#![deny(unsafe_code)] +// #![deny(unsafe_code)] // Remove this once we update pasta_curves #![allow(unused_imports)] #![allow(clippy::derive_partial_eq_without_eq)] diff --git a/halo2_proofs/src/plonk.rs b/halo2_proofs/src/plonk.rs index 489af783cf..f3857aa42c 100644 --- a/halo2_proofs/src/plonk.rs +++ b/halo2_proofs/src/plonk.rs @@ -79,7 +79,7 @@ where commitment.write(writer, format)?; } self.permutation.write(writer, format)?; - /* + /* // write self.selectors for selector in &self.selectors { // since `selector` is filled with `bool`, we pack them 8 at a time into bytes and then write @@ -119,7 +119,7 @@ where let permutation = permutation::VerifyingKey::read(reader, &cs.permutation, format)?; - /* + /* // read selectors let selectors: Vec> = vec![vec![false; 1 << k]; cs.num_selectors] .into_iter() @@ -163,14 +163,14 @@ impl VerifyingKey { fn bytes_length(&self) -> usize { 8 + (self.fixed_commitments.len() * C::default().to_bytes().as_ref().len()) + self.permutation.bytes_length() - /* - + self.selectors.len() - * (self - .selectors - .get(0) - .map(|selector| selector.len() / 8 + 1) - .unwrap_or(0)) - */ + /* + + self.selectors.len() + * (self + .selectors + .get(0) + .map(|selector| selector.len() / 8 + 1) + .unwrap_or(0)) + */ } fn from_parts( diff --git a/halo2_proofs/src/plonk/circuit.rs b/halo2_proofs/src/plonk/circuit.rs index d591c46cce..f7253b557e 100644 --- a/halo2_proofs/src/plonk/circuit.rs +++ b/halo2_proofs/src/plonk/circuit.rs @@ -2,6 +2,7 @@ use core::cmp::max; use core::ops::{Add, Mul}; use ff::Field; use std::collections::HashMap; +use std::ops::Range; use std::{ convert::TryFrom, ops::{Neg, Sub}, @@ -525,7 +526,7 @@ impl Challenge { /// This trait allows a [`Circuit`] to direct some backend to assign a witness /// for a constraint system. -pub trait Assignment { +pub trait Assignment: Sized + Send { /// Creates a new region and enters into it. /// /// Panics if we are currently in a region (if `exit_region` was not called). @@ -566,6 +567,12 @@ pub trait Assignment { A: FnOnce() -> AR, AR: Into; + /// Fork + fn fork(&mut self, ranges: &[Range]) -> Result, Error>; + + /// Merge + fn merge(&mut self, sub_cs: Vec) -> Result<(), Error>; + /// Queries the cell of an instance column at a particular absolute row. /// /// Returns the cell's value, if known. diff --git a/halo2_proofs/src/plonk/error.rs b/halo2_proofs/src/plonk/error.rs index 33fdae9083..84b14cbd2d 100644 --- a/halo2_proofs/src/plonk/error.rs +++ b/halo2_proofs/src/plonk/error.rs @@ -1,3 +1,4 @@ +use crate::circuit::RegionIndex; use std::cmp; use std::error; use std::fmt; @@ -18,6 +19,8 @@ pub enum Error { ConstraintSystemFailure, /// Out of bounds index passed to a backend BoundsFailure, + /// Out of bounds an subCS is allowed to access(r/w). + InvalidRange(usize, String), /// Opening error Opening, /// Transcript error @@ -60,6 +63,12 @@ impl fmt::Display for Error { Error::InvalidInstances => write!(f, "Provided instances do not match the circuit"), Error::ConstraintSystemFailure => write!(f, "The constraint system is not satisfied"), Error::BoundsFailure => write!(f, "An out-of-bounds index was passed to the backend"), + Error::InvalidRange(row, region_name) => write!( + f, + "the row={} is not in the range that this subCS owns (region name = {})", + row, + region_name, + ), Error::Opening => write!(f, "Multi-opening proof was invalid"), Error::Transcript(e) => write!(f, "Transcript error: {}", e), Error::NotEnoughRowsAvailable { current_k } => write!( diff --git a/halo2_proofs/src/plonk/keygen.rs b/halo2_proofs/src/plonk/keygen.rs index 516665b946..c52c03d50e 100644 --- a/halo2_proofs/src/plonk/keygen.rs +++ b/halo2_proofs/src/plonk/keygen.rs @@ -1,6 +1,7 @@ #![allow(clippy::int_plus_one)] use std::ops::Range; +use std::sync::Arc; use ff::Field; use group::Curve; @@ -14,6 +15,7 @@ use super::{ permutation, Assigned, Challenge, Error, Expression, LagrangeCoeff, Polynomial, ProvingKey, VerifyingKey, }; +use crate::helpers::CopyCell; use crate::{ arithmetic::{parallelize, CurveAffine}, circuit::Value, @@ -22,6 +24,7 @@ use crate::{ commitment::{Blind, Params, MSM}, EvaluationDomain, }, + two_dim_vec_to_vec_of_slice, }; pub(crate) fn create_domain( @@ -47,17 +50,21 @@ where /// Assembly to be used in circuit synthesis. #[derive(Debug)] -struct Assembly { +struct Assembly<'a, F: Field> { k: u32, - fixed: Vec, LagrangeCoeff>>, - permutation: permutation::keygen::Assembly, - selectors: Vec>, + fixed_vec: Arc, LagrangeCoeff>>>, + fixed: Vec<&'a mut [Assigned]>, + permutation: Option, + selectors_vec: Arc>>, + selectors: Vec<&'a mut [bool]>, + rw_rows: Range, + copies: Vec<(CopyCell, CopyCell)>, // A range of available rows for assignment and copies. usable_rows: Range, _marker: std::marker::PhantomData, } -impl Assignment for Assembly { +impl<'a, F: Field> Assignment for Assembly<'a, F> { fn enter_region(&mut self, _: N) where NR: Into, @@ -79,11 +86,93 @@ impl Assignment for Assembly { return Err(Error::not_enough_rows_available(self.k)); } - self.selectors[selector.0][row] = true; + if !self.rw_rows.contains(&row) { + return Err(Error::Synthesis); + } + + self.selectors[selector.0][row - self.rw_rows.start] = true; Ok(()) } + fn fork(&mut self, ranges: &[Range]) -> Result, Error> { + let mut range_start = self.rw_rows.start; + for (i, sub_range) in ranges.iter().enumerate() { + if sub_range.start < range_start { + // TODO: use more precise error type + return Err(Error::Synthesis); + } + if i == ranges.len() - 1 && sub_range.end >= self.rw_rows.end { + return Err(Error::Synthesis); + } + range_start = sub_range.end; + log::debug!( + "subCS_{} rw_rows: {}..{}", + i, + sub_range.start, + sub_range.end + ); + } + + let fixed_ptrs = self + .fixed + .iter_mut() + .map(|vec| vec.as_mut_ptr()) + .collect::>(); + let selectors_ptrs = self + .selectors + .iter_mut() + .map(|vec| vec.as_mut_ptr()) + .collect::>(); + + let mut sub_cs = vec![]; + for sub_range in ranges { + let fixed = fixed_ptrs + .iter() + .map(|ptr| unsafe { + std::slice::from_raw_parts_mut( + ptr.add(sub_range.start), + sub_range.end - sub_range.start, + ) + }) + .collect::]>>(); + let selectors = selectors_ptrs + .iter() + .map(|ptr| unsafe { + std::slice::from_raw_parts_mut( + ptr.add(sub_range.start), + sub_range.end - sub_range.start, + ) + }) + .collect::>(); + + sub_cs.push(Self { + k: 0, + fixed_vec: self.fixed_vec.clone(), + fixed, + permutation: None, + selectors_vec: self.selectors_vec.clone(), + selectors, + rw_rows: sub_range.clone(), + copies: vec![], + usable_rows: self.usable_rows.clone(), + _marker: Default::default(), + }); + } + + Ok(sub_cs) + } + + fn merge(&mut self, sub_cs: Vec) -> Result<(), Error> { + for (left, right) in sub_cs.into_iter().flat_map(|cs| cs.copies.into_iter()) { + self.permutation + .as_mut() + .expect("permutation must be Some") + .copy(left.column, left.row, right.column, right.row)?; + } + Ok(()) + } + fn query_instance(&self, _: Column, row: usize) -> Result, Error> { if !self.usable_rows.contains(&row) { return Err(Error::not_enough_rows_available(self.k)); @@ -127,10 +216,14 @@ impl Assignment for Assembly { return Err(Error::not_enough_rows_available(self.k)); } + if !self.rw_rows.contains(&row) { + return Err(Error::Synthesis); + } + *self .fixed .get_mut(column.index()) - .and_then(|v| v.get_mut(row)) + .and_then(|v| v.get_mut(row - self.rw_rows.start)) .ok_or(Error::BoundsFailure)? = to().into_field().assign()?; Ok(()) @@ -147,8 +240,22 @@ impl Assignment for Assembly { return Err(Error::not_enough_rows_available(self.k)); } - self.permutation - .copy(left_column, left_row, right_column, right_row) + match self.permutation.as_mut() { + None => { + self.copies.push(( + CopyCell { + column: left_column, + row: left_row, + }, + CopyCell { + column: right_column, + row: right_row, + }, + )); + Ok(()) + } + Some(permutation) => permutation.copy(left_column, left_row, right_column, right_row), + } } fn fill_from_row( @@ -215,11 +322,39 @@ where return Err(Error::not_enough_rows_available(params.k())); } + let fixed_vec = Arc::new(vec![domain.empty_lagrange_assigned(); cs.num_fixed_columns]); + let fixed = unsafe { + let fixed_vec_clone = fixed_vec.clone(); + let ptr = Arc::as_ptr(&fixed_vec_clone) as *mut Vec, LagrangeCoeff>>; + let mut_ref = &mut (*ptr); + mut_ref + .iter_mut() + .map(|poly| poly.values.as_mut_slice()) + .collect::>() + }; + + let selectors_vec = Arc::new(vec![vec![false; params.n() as usize]; cs.num_selectors]); + let selectors = unsafe { + let selectors_vec_clone = selectors_vec.clone(); + let ptr = Arc::as_ptr(&selectors_vec_clone) as *mut Vec>; + let mut_ref = &mut (*ptr); + mut_ref + .iter_mut() + .map(|vec| vec.as_mut_slice()) + .collect::>() + }; let mut assembly: Assembly = Assembly { k: params.k(), - fixed: vec![domain.empty_lagrange_assigned(); cs.num_fixed_columns], - permutation: permutation::keygen::Assembly::new(params.n() as usize, &cs.permutation), - selectors: vec![vec![false; params.n() as usize]; cs.num_selectors], + fixed_vec, + fixed, + permutation: Some(permutation::keygen::Assembly::new( + params.n() as usize, + &cs.permutation, + )), + selectors_vec, + selectors, + copies: vec![], + rw_rows: 0..params.n() as usize - (cs.blinding_factors() + 1), usable_rows: 0..params.n() as usize - (cs.blinding_factors() + 1), _marker: std::marker::PhantomData, }; @@ -232,8 +367,13 @@ where cs.constants.clone(), )?; - let mut fixed = batch_invert_assigned(assembly.fixed); - let (cs, selector_polys) = cs.compress_selectors(assembly.selectors.clone()); + debug_assert_eq!(Arc::strong_count(&assembly.fixed_vec), 1); + debug_assert_eq!(Arc::strong_count(&assembly.selectors_vec), 1); + let mut fixed = + batch_invert_assigned(Arc::try_unwrap(assembly.fixed_vec).expect("only one Arc for fixed")); + let (cs, selector_polys) = cs.compress_selectors( + Arc::try_unwrap(assembly.selectors_vec).expect("only one Arc for selectors"), + ); fixed.extend( selector_polys .into_iter() @@ -242,6 +382,8 @@ where let permutation_vk = assembly .permutation + .take() + .expect("permutation must be Some") .build_vk(params, &domain, &cs.permutation); let fixed_commitments = fixed @@ -254,7 +396,7 @@ where fixed_commitments, permutation_vk, cs, -// assembly.selectors, + // assembly.selectors, )) } @@ -302,11 +444,24 @@ where return Err(Error::not_enough_rows_available(params.k())); } + let fixed_vec = Arc::new(vec![domain.empty_lagrange_assigned(); cs.num_fixed_columns]); + let fixed = two_dim_vec_to_vec_of_slice!(fixed_vec); + + let selectors_vec = Arc::new(vec![vec![false; params.n() as usize]; cs.num_selectors]); + let selectors = two_dim_vec_to_vec_of_slice!(selectors_vec); + let mut assembly: Assembly = Assembly { k: params.k(), - fixed: vec![domain.empty_lagrange_assigned(); cs.num_fixed_columns], - permutation: permutation::keygen::Assembly::new(params.n() as usize, &cs.permutation), - selectors: vec![vec![false; params.n() as usize]; cs.num_selectors], + fixed_vec, + fixed, + permutation: Some(permutation::keygen::Assembly::new( + params.n() as usize, + &cs.permutation, + )), + selectors_vec, + selectors, + copies: vec![], + rw_rows: 0..params.n() as usize - (cs.blinding_factors() + 1), usable_rows: 0..params.n() as usize - (cs.blinding_factors() + 1), _marker: std::marker::PhantomData, }; @@ -319,8 +474,13 @@ where cs.constants.clone(), )?; - let mut fixed = batch_invert_assigned(assembly.fixed); - let (cs, selector_polys) = cs.compress_selectors(assembly.selectors.clone()); + debug_assert_eq!(Arc::strong_count(&assembly.fixed_vec), 1); + debug_assert_eq!(Arc::strong_count(&assembly.selectors_vec), 1); + let mut fixed = + batch_invert_assigned(Arc::try_unwrap(assembly.fixed_vec).expect("only one Arc for fixed")); + let (cs, selector_polys) = cs.compress_selectors( + Arc::try_unwrap(assembly.selectors_vec).expect("only one Arc for selectors"), + ); fixed.extend( selector_polys .into_iter() @@ -330,11 +490,12 @@ where let vk = match vk { Some(vk) => vk, None => { - let permutation_vk = - assembly - .permutation - .clone() - .build_vk(params, &domain, &cs.permutation); + let permutation_vk = assembly + .permutation + .as_ref() + .expect("permutation must be Some") + .clone() + .build_vk(params, &domain, &cs.permutation); let fixed_commitments = fixed .iter() @@ -346,7 +507,7 @@ where fixed_commitments, permutation_vk, cs.clone(), -// assembly.selectors.clone(), + // assembly.selectors.clone(), ) } }; @@ -358,6 +519,8 @@ where let permutation_pk = assembly .permutation + .take() + .expect("permutation must be Some") .build_pk(params, &vk.domain, &cs.permutation); // Compute l_0(X) diff --git a/halo2_proofs/src/plonk/permutation/keygen.rs b/halo2_proofs/src/plonk/permutation/keygen.rs index 874213d598..58ba4878cc 100644 --- a/halo2_proofs/src/plonk/permutation/keygen.rs +++ b/halo2_proofs/src/plonk/permutation/keygen.rs @@ -113,7 +113,7 @@ impl Assembly { { let omega = domain.get_omega(); parallelize(&mut omega_powers, |o, start| { - let mut cur = omega.pow_vartime(&[start as u64]); + let mut cur = omega.pow_vartime([start as u64]); for v in o.iter_mut() { *v = cur; cur *= ω @@ -175,7 +175,7 @@ impl Assembly { { let omega = domain.get_omega(); parallelize(&mut omega_powers, |o, start| { - let mut cur = omega.pow_vartime(&[start as u64]); + let mut cur = omega.pow_vartime([start as u64]); for v in o.iter_mut() { *v = cur; cur *= ω diff --git a/halo2_proofs/src/plonk/permutation/prover.rs b/halo2_proofs/src/plonk/permutation/prover.rs index 847d2fbd5f..ad94b02574 100644 --- a/halo2_proofs/src/plonk/permutation/prover.rs +++ b/halo2_proofs/src/plonk/permutation/prover.rs @@ -125,7 +125,7 @@ impl Argument { Any::Instance => instance, }; parallelize(&mut modified_values, |modified_values, start| { - let mut deltaomega = deltaomega * &omega.pow_vartime(&[start as u64, 0, 0, 0]); + let mut deltaomega = deltaomega * &omega.pow_vartime([start as u64, 0, 0, 0]); for (modified_values, value) in modified_values .iter_mut() .zip(values[column.index()][start..].iter()) diff --git a/halo2_proofs/src/plonk/permutation/verifier.rs b/halo2_proofs/src/plonk/permutation/verifier.rs index b892d1720d..a33b210e13 100644 --- a/halo2_proofs/src/plonk/permutation/verifier.rs +++ b/halo2_proofs/src/plonk/permutation/verifier.rs @@ -182,7 +182,7 @@ impl Evaluated { let mut right = set.permutation_product_eval; let mut current_delta = (*beta * &*x) - * &(C::Scalar::DELTA.pow_vartime(&[(chunk_index * chunk_len) as u64])); + * &(C::Scalar::DELTA.pow_vartime([(chunk_index * chunk_len) as u64])); for eval in columns.iter().map(|&column| match column.column_type() { Any::Advice(_) => { advice_evals[vk.cs.get_any_query_index(column, Rotation::cur())] diff --git a/halo2_proofs/src/plonk/prover.rs b/halo2_proofs/src/plonk/prover.rs index 1f50679772..6924625c33 100644 --- a/halo2_proofs/src/plonk/prover.rs +++ b/halo2_proofs/src/plonk/prover.rs @@ -4,8 +4,9 @@ use halo2curves::CurveExt; use rand_core::RngCore; use std::collections::BTreeSet; use std::env::var; -use std::ops::RangeTo; +use std::ops::{Range, RangeTo}; use std::sync::atomic::AtomicUsize; +use std::sync::Arc; use std::time::Instant; use std::{collections::HashMap, iter, mem, sync::atomic::Ordering}; @@ -27,6 +28,7 @@ use crate::{ commitment::{Blind, CommitmentScheme, Params, Prover}, Basis, Coeff, ExtendedLagrangeCoeff, LagrangeCoeff, Polynomial, ProverQuery, }, + two_dim_vec_to_vec_of_slice, }; use crate::{ poly::batch_invert_assigned, @@ -140,9 +142,11 @@ pub fn create_proof< struct WitnessCollection<'a, F: Field> { k: u32, current_phase: sealed::Phase, - advice: Vec, LagrangeCoeff>>, + advice_vec: Arc, LagrangeCoeff>>>, + advice: Vec<&'a mut [Assigned]>, challenges: &'a HashMap, instances: &'a [&'a [F]], + rw_rows: Range, usable_rows: RangeTo, _marker: std::marker::PhantomData, } @@ -170,6 +174,62 @@ pub fn create_proof< Ok(()) } + fn fork(&mut self, ranges: &[Range]) -> Result, Error> { + let mut range_start = self.rw_rows.start; + for (i, sub_range) in ranges.iter().enumerate() { + if sub_range.start < range_start { + return Err(Error::Synthesis); + } + if i == ranges.len() - 1 && sub_range.end >= self.rw_rows.end { + return Err(Error::Synthesis); + } + range_start = sub_range.end; + log::debug!( + "subCS_{} rw_rows: {}..{}", + i, + sub_range.start, + sub_range.end + ); + } + + let advice_ptrs = self + .advice + .iter_mut() + .map(|vec| vec.as_mut_ptr()) + .collect::>(); + + let mut sub_cs = vec![]; + for sub_range in ranges { + let advice = advice_ptrs + .iter() + .map(|ptr| unsafe { + std::slice::from_raw_parts_mut( + ptr.add(sub_range.start), + sub_range.end - sub_range.start, + ) + }) + .collect::]>>(); + + sub_cs.push(Self { + k: 0, + current_phase: self.current_phase, + advice_vec: self.advice_vec.clone(), + advice, + challenges: self.challenges, + instances: self.instances, + rw_rows: sub_range.clone(), + usable_rows: self.usable_rows, + _marker: Default::default(), + }); + } + + Ok(sub_cs) + } + + fn merge(&mut self, _sub_cs: Vec) -> Result<(), Error> { + Ok(()) + } + fn annotate_column(&mut self, _annotation: A, _column: Column) where A: FnOnce() -> AR, @@ -212,10 +272,14 @@ pub fn create_proof< return Err(Error::not_enough_rows_available(self.k)); } + if !self.rw_rows.contains(&row) { + return Err(Error::Synthesis); + } + *self .advice .get_mut(column.index()) - .and_then(|v| v.get_mut(row)) + .and_then(|v| v.get_mut(row - self.rw_rows.start)) .ok_or(Error::BoundsFailure)? = to().into_field().assign()?; Ok(()) @@ -315,10 +379,16 @@ pub fn create_proof< .zip(instances) .enumerate() { + let advice_vec = Arc::new(vec![ + domain.empty_lagrange_assigned(); + meta.num_advice_columns + ]); + let advice_slice = two_dim_vec_to_vec_of_slice!(advice_vec); let mut witness = WitnessCollection { k: params.k(), current_phase, - advice: vec![domain.empty_lagrange_assigned(); meta.num_advice_columns], + advice_vec, + advice: advice_slice, instances, challenges: &challenges, // The prover will not be allowed to assign values to advice @@ -326,6 +396,7 @@ pub fn create_proof< // number of blinding factors and an extra row for use in the // permutation argument. usable_rows: ..unusable_rows_start, + rw_rows: 0..unusable_rows_start, _marker: std::marker::PhantomData, }; @@ -339,23 +410,23 @@ pub fn create_proof< #[cfg(feature = "phase-check")] { - for (idx, advice_col) in witness.advice.iter().enumerate() { - if pk.vk.cs.advice_column_phase[idx].0 < current_phase.0 { - if advice_assignments[circuit_idx][idx].values != advice_col.values { - log::error!( - "advice column {}(at {:?}) changed when {:?}", - idx, - pk.vk.cs.advice_column_phase[idx], - current_phase - ); - } + for (idx, advice_col) in witness.advice_vec.iter().enumerate() { + if pk.vk.cs.advice_column_phase[idx].0 < current_phase.0 + && advice_assignments[circuit_idx][idx].values != advice_col.values + { + log::error!( + "advice column {}(at {:?}) changed when {:?}", + idx, + pk.vk.cs.advice_column_phase[idx], + current_phase + ); } } } let mut advice_values = batch_invert_assigned::( - witness - .advice + Arc::try_unwrap(witness.advice_vec) + .expect("there must only one Arc for advice_vec") .into_iter() .enumerate() .filter_map(|(column_index, advice)| { diff --git a/halo2_proofs/src/poly/domain.rs b/halo2_proofs/src/poly/domain.rs index 2923474112..cd1a0cdc9c 100644 --- a/halo2_proofs/src/poly/domain.rs +++ b/halo2_proofs/src/poly/domain.rs @@ -85,8 +85,8 @@ impl EvaluationDomain { { // Compute the evaluations of t(X) = X^n - 1 in the coset evaluation domain. // We don't have to compute all of them, because it will repeat. - let orig = G::Scalar::ZETA.pow_vartime(&[n as u64, 0, 0, 0]); - let step = extended_omega.pow_vartime(&[n as u64, 0, 0, 0]); + let orig = G::Scalar::ZETA.pow_vartime([n as u64, 0, 0, 0]); + let step = extended_omega.pow_vartime([n as u64, 0, 0, 0]); let mut cur = orig; loop { t_evaluations.push(cur); @@ -510,11 +510,11 @@ impl EvaluationDomain { pub fn rotate_omega(&self, value: G::Scalar, rotation: Rotation) -> G::Scalar { let mut point = value; if rotation.0 >= 0 { - point *= &self.get_omega().pow_vartime(&[rotation.0 as u64]); + point *= &self.get_omega().pow_vartime([rotation.0 as u64]); } else { point *= &self .get_omega_inv() - .pow_vartime(&[(rotation.0 as i64).unsigned_abs()]); + .pow_vartime([(rotation.0 as i64).unsigned_abs()]); } point } diff --git a/halo2_proofs/src/poly/kzg/commitment.rs b/halo2_proofs/src/poly/kzg/commitment.rs index aa86fdc14f..27d188aa5c 100644 --- a/halo2_proofs/src/poly/kzg/commitment.rs +++ b/halo2_proofs/src/poly/kzg/commitment.rs @@ -83,7 +83,7 @@ impl ParamsKZG { let mut g_projective = vec![E::G1::group_zero(); n as usize]; parallelize(&mut g_projective, |g, start| { let mut current_g: E::G1 = g1.into(); - current_g *= s.pow_vartime(&[start as u64]); + current_g *= s.pow_vartime([start as u64]); for g in g.iter_mut() { *g = current_g; current_g *= s; @@ -105,11 +105,11 @@ impl ParamsKZG { } let n_inv = Option::::from(E::Scalar::from(n).invert()) .expect("inversion should be ok for n = 1<, ) -> Result<(), Error> { let cs = StandardPlonk::new(config); + let mut is_first_pass_vec = vec![true; 8]; let _ = cs.public_input(&mut layouter, || Value::known(F::one() + F::one()))?; + let a: Value> = self.a.into(); + let parallel_regions_time = Instant::now(); + #[cfg(feature = "parallel_syn")] + layouter.assign_regions( + || "regions", + (0..8) + .into_iter() + .zip(is_first_pass_vec.chunks_mut(1).into_iter()) + .map(|(_, is_first_pass)| { + |mut region: Region<'_, F>| -> Result<(), Error> { + let n = 1 << 13; + for i in 0..n { + // skip the assign of rows except the last row in the first pass + if is_first_pass[0] && i < n - 1 { + continue; + } + let a0 = + region.assign_advice(|| "config.a", cs.config.a, i, || a)?; + let a1 = + region.assign_advice(|| "config.b", cs.config.b, i, || a)?; + region.assign_advice( + || "config.c", + cs.config.c, + i, + || a.double(), + )?; + + region.assign_fixed( + || "a", + cs.config.sa, + i, + || Value::known(F::one()), + )?; + region.assign_fixed( + || "b", + cs.config.sb, + i, + || Value::known(F::one()), + )?; + region.assign_fixed( + || "c", + cs.config.sc, + i, + || Value::known(F::one()), + )?; + region.assign_fixed( + || "a * b", + cs.config.sm, + i, + || Value::known(F::zero()), + )?; + + region.constrain_equal(a0.cell(), a1.cell())?; + } + is_first_pass[0] = false; + Ok(()) + } + }) + .collect(), + )?; + log::info!( + "parallel_regions assign took {:?}", + parallel_regions_time.elapsed() + ); + for _ in 0..10 { let a: Value> = self.a.into(); let mut a_squared = Value::unknown(); @@ -454,8 +523,12 @@ use std::marker::PhantomData; // Initialize the proving key let vk = keygen_vk(params, &empty_circuit).expect("keygen_vk should not fail"); + log::info!("keygen vk succeed"); - keygen_pk(params, vk, &empty_circuit).expect("keygen_pk should not fail") + let pk = keygen_pk(params, vk, &empty_circuit).expect("keygen_pk should not fail"); + log::info!("keygen pk succeed"); + + pk } fn create_proof< @@ -489,13 +562,6 @@ use std::marker::PhantomData; ) .expect("proof generation should not fail"); - // Check this circuit is satisfied. - let prover = match MockProver::run(K, &circuit, vec![vec![instance]]) { - Ok(prover) => prover, - Err(e) => panic!("{:?}", e), - }; - assert_eq!(prover.verify(), Ok(())); - transcript.finalize() } @@ -538,7 +604,22 @@ use std::marker::PhantomData; use halo2curves::bn256::Bn256; type Scheme = KZGCommitmentScheme; - bad_keys!(Scheme); + // bad_keys!(Scheme); + + let (a, instance, lookup_table) = common!(Scheme); + + let circuit: MyCircuit<::Scalar> = MyCircuit { + a: Value::known(a), + lookup_table, + }; + + // Check this circuit is satisfied. + let prover = match MockProver::run(K, &circuit, vec![vec![instance]]) { + Ok(prover) => prover, + Err(e) => panic!("{:?}", e), + }; + assert_eq!(prover.verify_par(), Ok(())); + log::info!("mock proving succeed!"); let params = ParamsKZG::::new(K); let rng = OsRng; @@ -1021,4 +1102,8 @@ use std::marker::PhantomData; ); } } - + env_logger::init(); + // test_plonk_api_ipa(); + test_plonk_api_gwc(); + // test_plonk_api_shplonk(); +}