feat: add support for 8MiB sector size (#708)

lib/polka-storage-proofs/src/porep/mod.rs

@@ -23,12 +23,17 @@ pub fn generate_random_groth16_parameters(
 ) -> Result<groth16::Parameters<Bls12>, PoRepError> {
     let porep_config = seal_to_config(seal_proof);
     let setup_params = filecoin_proofs::parameters::setup_params(&porep_config)?;
-    let public_params = StackedDrg::<SectorShapeBase, DefaultPieceHasher>::setup(&setup_params)?;
 
-    let circuit = storage_proofs_porep::stacked::StackedCompound::<
-        SectorShapeBase,
-        DefaultPieceHasher,
-    >::blank_circuit(&public_params);
+    let circuit = match seal_proof {
+        RegisteredSealProof::StackedDRG2KiBV1P1 | RegisteredSealProof::StackedDRG8MiBV1 => {
+            let public_params =
+                StackedDrg::<SectorShapeBase, DefaultPieceHasher>::setup(&setup_params)?;
+            storage_proofs_porep::stacked::StackedCompound::<
+                SectorShapeBase,
+                DefaultPieceHasher,
+            >::blank_circuit(&public_params)
+        }
+    };
 
     Ok(groth16::generate_random_parameters::<Bls12, _, _>(
         circuit, &mut OsRng,
@@ -61,14 +66,10 @@ pub enum PoRepError {
 }
 
 fn seal_to_config(seal_proof: RegisteredSealProof) -> filecoin_proofs::PoRepConfig {
-    match seal_proof {
-        RegisteredSealProof::StackedDRG2KiBV1P1 => {
-            // https://github.com/filecoin-project/rust-filecoin-proofs-api/blob/b44e7cecf2a120aa266b6886628e869ba67252af/src/registry.rs#L308
-            let sector_size = 1 << 11;
-            let porep_id = [0u8; 32];
-            let api_version = storage_proofs_core::api_version::ApiVersion::V1_2_0;
-
-            filecoin_proofs::PoRepConfig::new_groth16(sector_size, porep_id, api_version)
-        }
-    }
+    let api_version = storage_proofs_core::api_version::ApiVersion::V1_2_0;
+    filecoin_proofs::PoRepConfig::new_groth16(
+        seal_proof.sector_size().bytes(),
+        seal_proof.porep_id(),
+        api_version,
+    )
 }

lib/polka-storage-proofs/src/porep/sealer.rs

@@ -1,3 +1,4 @@
+use core::marker::PhantomData;
 use std::{fs::File, path::Path};
 
 use bellperson::groth16;
@@ -6,7 +7,8 @@ use filecoin_hashers::Domain;
 use filecoin_proofs::{
     add_piece, as_safe_commitment, parameters::setup_params, DefaultPieceDomain,
     DefaultPieceHasher, PaddedBytesAmount, PoRepConfig, SealCommitPhase1Output,
-    SealPreCommitOutput, SealPreCommitPhase1Output, SectorShapeBase, UnpaddedBytesAmount,
+    SealPreCommitOutput, SealPreCommitPhase1Output, SectorShape2KiB, SectorShape8MiB,
+    UnpaddedBytesAmount,
 };
 use primitives::{
     commitment::{
@@ -66,17 +68,27 @@ where
 
     Ok((piece_padded_file, piece_info))
 }
-pub struct Sealer {
+pub struct Sealer<SectorShape> {
     porep_config: PoRepConfig,
+    _sector_shape: PhantomData<SectorShape>,
 }
 
-impl Sealer {
-    pub fn new(seal_proof: RegisteredSealProof) -> Self {
-        Self {
-            porep_config: seal_to_config(seal_proof),
-        }
+pub fn select_sealer(
+    seal: RegisteredSealProof,
+) -> Sealer<impl filecoin_proofs::MerkleTreeTrait + 'static> {
+    match seal {
+        RegisteredSealProof::StackedDRG2KiBV1P1 => Sealer::<SectorShape2KiB> {
+            porep_config: seal_to_config(seal),
+            _sector_shape: PhantomData,
+        },
+        RegisteredSealProof::StackedDRG8MiBV1 => Sealer::<SectorShape8MiB> {
+            porep_config: seal_to_config(seal),
+            _sector_shape: PhantomData,
+        },
     }
+}
 
+impl<SectorShape: filecoin_proofs::MerkleTreeTrait + 'static> Sealer<SectorShape> {
     /// Adds a Piece and padding to already existing sector file and returns how many bytes were written.
     /// It can return more bytes than the piece size, as it adds padding so a proper Merkle Tree can be created out of the sector.
     /// You need to supply current pieces which are already in the sector, otherwise they'll be overwritten.
@@ -213,7 +225,7 @@ impl Sealer {
             .map(|p| (*p).into())
             .collect::<Vec<filecoin_proofs::PieceInfo>>();
 
-        let p1_output: SealPreCommitPhase1Output<SectorShapeBase> =
+        let p1_output: SealPreCommitPhase1Output<SectorShape> =
             filecoin_proofs::seal_pre_commit_phase1(
                 &self.porep_config,
                 cache_directory,
@@ -275,7 +287,7 @@ impl Sealer {
             .map(filecoin_proofs::PieceInfo::from)
             .collect::<Vec<_>>();
 
-        let scp1: filecoin_proofs::SealCommitPhase1Output<SectorShapeBase> =
+        let scp1: filecoin_proofs::SealCommitPhase1Output<SectorShape> =
             filecoin_proofs::seal_commit_phase1_inner(
                 &self.porep_config,
                 cache_path,
@@ -318,12 +330,12 @@ impl Sealer {
         };
 
         let compound_public_params =
-            <StackedCompound<SectorShapeBase, DefaultPieceHasher> as CompoundProof<
-                StackedDrg<'_, SectorShapeBase, DefaultPieceHasher>,
+            <StackedCompound<SectorShape, DefaultPieceHasher> as CompoundProof<
+                StackedDrg<'_, SectorShape, DefaultPieceHasher>,
                 _,
             >>::setup(&compound_setup_params)?;
 
-        let groth_proofs = StackedCompound::<SectorShapeBase, DefaultPieceHasher>::circuit_proofs(
+        let groth_proofs = StackedCompound::<SectorShape, DefaultPieceHasher>::circuit_proofs(
             &public_inputs,
             vanilla_proofs,
             &compound_public_params.vanilla_params,
@@ -446,7 +458,9 @@ mod test {
     // Biggest possible piece size
     #[case(vec![2048])]
     fn padding_for_sector(#[case] piece_sizes: Vec<usize>) {
-        let sealer = Sealer::new(RegisteredSealProof::StackedDRG2KiBV1P1);
+        use primitives::proofs::RegisteredSealProof;
+
+        let sealer = select_sealer(RegisteredSealProof::StackedDRG2KiBV1P1);
 
         let piece_infos: Vec<(Cursor<Vec<u8>>, PieceInfo)> = piece_sizes
             .into_iter()

lib/polka-storage-proofs/src/post/mod.rs

@@ -4,7 +4,7 @@ use bellperson::groth16;
 use blstrs::Bls12;
 use filecoin_proofs::{
     as_safe_commitment, parameters::window_post_setup_params, PoStType, PrivateReplicaInfo,
-    SectorShapeBase,
+    SectorShape2KiB, SectorShape8MiB, SectorShapeBase,
 };
 use primitives::{proofs::RegisteredPoStProof, sector::SectorNumber};
 use rand::rngs::OsRng;
@@ -28,13 +28,17 @@ pub fn generate_random_groth16_parameters(
 ) -> Result<groth16::Parameters<Bls12>, PoStError> {
     let post_config = seal_to_config(seal_proof);
 
-    let public_params =
-        filecoin_proofs::parameters::window_post_public_params::<SectorShapeBase>(&post_config)?;
-
-    let circuit =
-        storage_proofs_post::fallback::FallbackPoStCompound::<SectorShapeBase>::blank_circuit(
-            &public_params,
-        );
+    let circuit = match seal_proof {
+        RegisteredPoStProof::StackedDRGWindow2KiBV1P1
+        | RegisteredPoStProof::StackedDRGWindow8MiBV1 => {
+            let public_params = filecoin_proofs::parameters::window_post_public_params::<
+                SectorShapeBase,
+            >(&post_config)?;
+            storage_proofs_post::fallback::FallbackPoStCompound::<SectorShapeBase>::blank_circuit(
+                &public_params,
+            )
+        }
+    };
 
     Ok(groth16::generate_random_parameters(circuit, &mut OsRng)?)
 }
@@ -66,25 +70,15 @@ pub fn generate_window_post(
     prover_id: ProverId,
     partition_replicas: Vec<ReplicaInfo>,
 ) -> Result<Vec<groth16::Proof<Bls12>>, PoStError> {
-    type Tree = SectorShapeBase;
-
     let post_config = seal_to_config(proof_type);
-    let mut replicas = BTreeMap::new();
-    for replica in partition_replicas {
-        replicas.insert(
-            storage_proofs_core::sector::SectorId::from(u64::from(replica.sector_id)),
-            PrivateReplicaInfo::<Tree>::new(
-                replica.replica_path,
-                replica.comm_r,
-                replica.cache_path,
-            )?,
-        );
-    }
+
     let randomness_safe = as_safe_commitment(&randomness, "randomness")?;
     let prover_id_safe = as_safe_commitment(&prover_id, "prover_id")?;
 
     let vanilla_params = window_post_setup_params(&post_config);
-    let partitions = get_partitions_for_window_post(replicas.len(), post_config.sector_count);
+    // ASSUMPTION: there are no duplicates in `partition_replicas`, if there are there is a heavy bug upstream.
+    let partitions =
+        get_partitions_for_window_post(partition_replicas.len(), post_config.sector_count);
 
     let sector_count = vanilla_params.sector_count;
     let setup_params = compound_proof::SetupParams {
@@ -93,8 +87,42 @@ pub fn generate_window_post(
         priority: post_config.priority,
     };
 
-    let pub_params: compound_proof::PublicParams<'_, FallbackPoSt<'_, Tree>> =
-        FallbackPoStCompound::setup(&setup_params)?;
+    let (pub_params, replicas) = match proof_type {
+        RegisteredPoStProof::StackedDRGWindow2KiBV1P1 => {
+            let pub_params: compound_proof::PublicParams<'_, FallbackPoSt<'_, SectorShape2KiB>> =
+                FallbackPoStCompound::setup(&setup_params)?;
+
+            let mut replicas = BTreeMap::new();
+            for replica in partition_replicas {
+                replicas.insert(
+                    storage_proofs_core::sector::SectorId::from(u64::from(replica.sector_id)),
+                    PrivateReplicaInfo::<SectorShape2KiB>::new(
+                        replica.replica_path,
+                        replica.comm_r,
+                        replica.cache_path,
+                    )?,
+                );
+            }
+            (pub_params, replicas)
+        }
+        RegisteredPoStProof::StackedDRGWindow8MiBV1 => {
+            let pub_params: compound_proof::PublicParams<'_, FallbackPoSt<'_, SectorShape8MiB>> =
+                FallbackPoStCompound::setup(&setup_params)?;
+
+            let mut replicas = BTreeMap::new();
+            for replica in partition_replicas {
+                replicas.insert(
+                    storage_proofs_core::sector::SectorId::from(u64::from(replica.sector_id)),
+                    PrivateReplicaInfo::<SectorShape8MiB>::new(
+                        replica.replica_path,
+                        replica.comm_r,
+                        replica.cache_path,
+                    )?,
+                );
+            }
+            (pub_params, replicas)
+        }
+    };
 
     let trees: Vec<_> = replicas
         .values()
@@ -127,7 +155,7 @@ pub fn generate_window_post(
         k: None,
     };
 
-    let priv_inputs = fallback::PrivateInputs::<Tree> {
+    let priv_inputs = fallback::PrivateInputs::<_> {
         sectors: &priv_sectors,
     };
 
@@ -151,6 +179,17 @@ fn seal_to_config(seal_proof: RegisteredPoStProof) -> filecoin_proofs::PoStConfi
                 api_version: storage_proofs_core::api_version::ApiVersion::V1_2_0,
             }
         }
+        RegisteredPoStProof::StackedDRGWindow8MiBV1 => {
+            filecoin_proofs::PoStConfig {
+                sector_size: filecoin_proofs::SectorSize(seal_proof.sector_size().bytes()),
+                challenge_count: filecoin_proofs::WINDOW_POST_CHALLENGE_COUNT,
+                // https://github.com/filecoin-project/rust-fil-proofs/blob/266acc39a3ebd6f3d28c6ee335d78e2b7cea06bc/filecoin-proofs/src/constants.rs#L104
+                sector_count: 2,
+                typ: PoStType::Window,
+                priority: true,
+                api_version: storage_proofs_core::api_version::ApiVersion::V1_2_0,
+            }
+        }
     }
 }