diff --git a/Cargo.lock b/Cargo.lock index f3450caad..84cff591a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -13897,6 +13897,7 @@ dependencies = [ "chrono", "cid 0.11.1", "filecoin-hashers", + "filecoin-proofs", "fr32", "hex", "jsonrpsee 0.24.7", diff --git a/examples/big_file_184k.car b/examples/big_file_184k.car new file mode 100644 index 000000000..d20b96081 Binary files /dev/null and b/examples/big_file_184k.car differ diff --git a/lib/polka-storage-proofs/src/porep/mod.rs b/lib/polka-storage-proofs/src/porep/mod.rs index ce251c8ed..9524c6343 100644 --- a/lib/polka-storage-proofs/src/porep/mod.rs +++ b/lib/polka-storage-proofs/src/porep/mod.rs @@ -23,12 +23,17 @@ pub fn generate_random_groth16_parameters( ) -> Result, PoRepError> { let porep_config = seal_to_config(seal_proof); let setup_params = filecoin_proofs::parameters::setup_params(&porep_config)?; - let public_params = StackedDrg::::setup(&setup_params)?; - let circuit = storage_proofs_porep::stacked::StackedCompound::< - SectorShapeBase, - DefaultPieceHasher, - >::blank_circuit(&public_params); + let circuit = match seal_proof { + RegisteredSealProof::StackedDRG2KiBV1P1 | RegisteredSealProof::StackedDRG8MiBV1 => { + let public_params = + StackedDrg::::setup(&setup_params)?; + storage_proofs_porep::stacked::StackedCompound::< + SectorShapeBase, + DefaultPieceHasher, + >::blank_circuit(&public_params) + } + }; Ok(groth16::generate_random_parameters::( circuit, &mut OsRng, @@ -61,14 +66,10 @@ pub enum PoRepError { } fn seal_to_config(seal_proof: RegisteredSealProof) -> filecoin_proofs::PoRepConfig { - match seal_proof { - RegisteredSealProof::StackedDRG2KiBV1P1 => { - // https://github.com/filecoin-project/rust-filecoin-proofs-api/blob/b44e7cecf2a120aa266b6886628e869ba67252af/src/registry.rs#L308 - let sector_size = 1 << 11; - let porep_id = [0u8; 32]; - let api_version = storage_proofs_core::api_version::ApiVersion::V1_2_0; - - filecoin_proofs::PoRepConfig::new_groth16(sector_size, porep_id, api_version) - } - } + let api_version = storage_proofs_core::api_version::ApiVersion::V1_2_0; + filecoin_proofs::PoRepConfig::new_groth16( + seal_proof.sector_size().bytes(), + seal_proof.porep_id(), + api_version, + ) } diff --git a/lib/polka-storage-proofs/src/porep/sealer.rs b/lib/polka-storage-proofs/src/porep/sealer.rs index e6e11a1de..473772f12 100644 --- a/lib/polka-storage-proofs/src/porep/sealer.rs +++ b/lib/polka-storage-proofs/src/porep/sealer.rs @@ -1,3 +1,4 @@ +use core::marker::PhantomData; use std::{fs::File, path::Path}; use bellperson::groth16; @@ -6,7 +7,8 @@ use filecoin_hashers::Domain; use filecoin_proofs::{ add_piece, as_safe_commitment, parameters::setup_params, DefaultPieceDomain, DefaultPieceHasher, PaddedBytesAmount, PoRepConfig, SealCommitPhase1Output, - SealPreCommitOutput, SealPreCommitPhase1Output, SectorShapeBase, UnpaddedBytesAmount, + SealPreCommitOutput, SealPreCommitPhase1Output, SectorShape2KiB, SectorShape8MiB, + UnpaddedBytesAmount, }; use primitives::{ commitment::{ @@ -66,17 +68,27 @@ where Ok((piece_padded_file, piece_info)) } -pub struct Sealer { +pub struct Sealer { porep_config: PoRepConfig, + _sector_shape: PhantomData, } -impl Sealer { - pub fn new(seal_proof: RegisteredSealProof) -> Self { - Self { - porep_config: seal_to_config(seal_proof), - } +pub fn select_sealer( + seal: RegisteredSealProof, +) -> Sealer { + match seal { + RegisteredSealProof::StackedDRG2KiBV1P1 => Sealer:: { + porep_config: seal_to_config(seal), + _sector_shape: PhantomData, + }, + RegisteredSealProof::StackedDRG8MiBV1 => Sealer:: { + porep_config: seal_to_config(seal), + _sector_shape: PhantomData, + }, } +} +impl Sealer { /// Adds a Piece and padding to already existing sector file and returns how many bytes were written. /// It can return more bytes than the piece size, as it adds padding so a proper Merkle Tree can be created out of the sector. /// You need to supply current pieces which are already in the sector, otherwise they'll be overwritten. @@ -213,7 +225,7 @@ impl Sealer { .map(|p| (*p).into()) .collect::>(); - let p1_output: SealPreCommitPhase1Output = + let p1_output: SealPreCommitPhase1Output = filecoin_proofs::seal_pre_commit_phase1( &self.porep_config, cache_directory, @@ -275,7 +287,7 @@ impl Sealer { .map(filecoin_proofs::PieceInfo::from) .collect::>(); - let scp1: filecoin_proofs::SealCommitPhase1Output = + let scp1: filecoin_proofs::SealCommitPhase1Output = filecoin_proofs::seal_commit_phase1_inner( &self.porep_config, cache_path, @@ -318,12 +330,12 @@ impl Sealer { }; let compound_public_params = - as CompoundProof< - StackedDrg<'_, SectorShapeBase, DefaultPieceHasher>, + as CompoundProof< + StackedDrg<'_, SectorShape, DefaultPieceHasher>, _, >>::setup(&compound_setup_params)?; - let groth_proofs = StackedCompound::::circuit_proofs( + let groth_proofs = StackedCompound::::circuit_proofs( &public_inputs, vanilla_proofs, &compound_public_params.vanilla_params, @@ -446,7 +458,9 @@ mod test { // Biggest possible piece size #[case(vec![2048])] fn padding_for_sector(#[case] piece_sizes: Vec) { - let sealer = Sealer::new(RegisteredSealProof::StackedDRG2KiBV1P1); + use primitives::proofs::RegisteredSealProof; + + let sealer = select_sealer(RegisteredSealProof::StackedDRG2KiBV1P1); let piece_infos: Vec<(Cursor>, PieceInfo)> = piece_sizes .into_iter() diff --git a/lib/polka-storage-proofs/src/post/mod.rs b/lib/polka-storage-proofs/src/post/mod.rs index afc49c1bd..6883932ad 100644 --- a/lib/polka-storage-proofs/src/post/mod.rs +++ b/lib/polka-storage-proofs/src/post/mod.rs @@ -4,7 +4,7 @@ use bellperson::groth16; use blstrs::Bls12; use filecoin_proofs::{ as_safe_commitment, parameters::window_post_setup_params, PoStType, PrivateReplicaInfo, - SectorShapeBase, + SectorShape2KiB, SectorShape8MiB, SectorShapeBase, }; use primitives::{proofs::RegisteredPoStProof, sector::SectorNumber}; use rand::rngs::OsRng; @@ -28,13 +28,17 @@ pub fn generate_random_groth16_parameters( ) -> Result, PoStError> { let post_config = seal_to_config(seal_proof); - let public_params = - filecoin_proofs::parameters::window_post_public_params::(&post_config)?; - - let circuit = - storage_proofs_post::fallback::FallbackPoStCompound::::blank_circuit( - &public_params, - ); + let circuit = match seal_proof { + RegisteredPoStProof::StackedDRGWindow2KiBV1P1 + | RegisteredPoStProof::StackedDRGWindow8MiBV1 => { + let public_params = filecoin_proofs::parameters::window_post_public_params::< + SectorShapeBase, + >(&post_config)?; + storage_proofs_post::fallback::FallbackPoStCompound::::blank_circuit( + &public_params, + ) + } + }; Ok(groth16::generate_random_parameters(circuit, &mut OsRng)?) } @@ -66,25 +70,15 @@ pub fn generate_window_post( prover_id: ProverId, partition_replicas: Vec, ) -> Result>, PoStError> { - type Tree = SectorShapeBase; - let post_config = seal_to_config(proof_type); - let mut replicas = BTreeMap::new(); - for replica in partition_replicas { - replicas.insert( - storage_proofs_core::sector::SectorId::from(u64::from(replica.sector_id)), - PrivateReplicaInfo::::new( - replica.replica_path, - replica.comm_r, - replica.cache_path, - )?, - ); - } + let randomness_safe = as_safe_commitment(&randomness, "randomness")?; let prover_id_safe = as_safe_commitment(&prover_id, "prover_id")?; let vanilla_params = window_post_setup_params(&post_config); - let partitions = get_partitions_for_window_post(replicas.len(), post_config.sector_count); + // ASSUMPTION: there are no duplicates in `partition_replicas`, if there are there is a heavy bug upstream. + let partitions = + get_partitions_for_window_post(partition_replicas.len(), post_config.sector_count); let sector_count = vanilla_params.sector_count; let setup_params = compound_proof::SetupParams { @@ -93,8 +87,42 @@ pub fn generate_window_post( priority: post_config.priority, }; - let pub_params: compound_proof::PublicParams<'_, FallbackPoSt<'_, Tree>> = - FallbackPoStCompound::setup(&setup_params)?; + let (pub_params, replicas) = match proof_type { + RegisteredPoStProof::StackedDRGWindow2KiBV1P1 => { + let pub_params: compound_proof::PublicParams<'_, FallbackPoSt<'_, SectorShape2KiB>> = + FallbackPoStCompound::setup(&setup_params)?; + + let mut replicas = BTreeMap::new(); + for replica in partition_replicas { + replicas.insert( + storage_proofs_core::sector::SectorId::from(u64::from(replica.sector_id)), + PrivateReplicaInfo::::new( + replica.replica_path, + replica.comm_r, + replica.cache_path, + )?, + ); + } + (pub_params, replicas) + } + RegisteredPoStProof::StackedDRGWindow8MiBV1 => { + let pub_params: compound_proof::PublicParams<'_, FallbackPoSt<'_, SectorShape8MiB>> = + FallbackPoStCompound::setup(&setup_params)?; + + let mut replicas = BTreeMap::new(); + for replica in partition_replicas { + replicas.insert( + storage_proofs_core::sector::SectorId::from(u64::from(replica.sector_id)), + PrivateReplicaInfo::::new( + replica.replica_path, + replica.comm_r, + replica.cache_path, + )?, + ); + } + (pub_params, replicas) + } + }; let trees: Vec<_> = replicas .values() @@ -127,7 +155,7 @@ pub fn generate_window_post( k: None, }; - let priv_inputs = fallback::PrivateInputs:: { + let priv_inputs = fallback::PrivateInputs::<_> { sectors: &priv_sectors, }; @@ -151,6 +179,17 @@ fn seal_to_config(seal_proof: RegisteredPoStProof) -> filecoin_proofs::PoStConfi api_version: storage_proofs_core::api_version::ApiVersion::V1_2_0, } } + RegisteredPoStProof::StackedDRGWindow8MiBV1 => { + filecoin_proofs::PoStConfig { + sector_size: filecoin_proofs::SectorSize(seal_proof.sector_size().bytes()), + challenge_count: filecoin_proofs::WINDOW_POST_CHALLENGE_COUNT, + // https://github.com/filecoin-project/rust-fil-proofs/blob/266acc39a3ebd6f3d28c6ee335d78e2b7cea06bc/filecoin-proofs/src/constants.rs#L104 + sector_count: 2, + typ: PoStType::Window, + priority: true, + api_version: storage_proofs_core::api_version::ApiVersion::V1_2_0, + } + } } } diff --git a/maat/tests/real_world.rs b/maat/tests/real_world.rs index d187e939b..a53a40263 100644 --- a/maat/tests/real_world.rs +++ b/maat/tests/real_world.rs @@ -1,15 +1,11 @@ use std::{collections::BTreeSet, env, path::Path, sync::Arc, time::Duration}; -use cid::Cid; use codec::Encode; use libp2p::PeerId; use maat::*; use polka_storage_proofs::{porep, post}; -use polka_storage_provider_common::{deadline::Deadline, sector::UnsealedSector}; -use primitives::{ - commitment::{CommP, Commitment}, - sector::{SectorNumber, SectorSize}, -}; +use polka_storage_provider_common::{commp::commp, deadline::Deadline, sector::UnsealedSector}; +use primitives::{proofs::RegisteredPoStProof, sector::SectorNumber}; use storagext::{ clients::ProofsClientExt, multipair::MultiPairSigner, @@ -30,19 +26,17 @@ use zombienet_sdk::NetworkConfigExt; /// Network's collator name. Used for logs and so on. const COLLATOR_NAME: &str = "collator"; -async fn register_storage_provider(client: &storagext::Client, charlie: &Keypair) -where +async fn register_storage_provider( + client: &storagext::Client, + charlie: &Keypair, + post_proof: RegisteredPoStProof, +) where Keypair: subxt::tx::Signer, { let peer_id = PeerId::random(); let result = client - .register_storage_provider( - charlie, - peer_id, - primitives::proofs::RegisteredPoStProof::StackedDRGWindow2KiBV1P1, - true, - ) + .register_storage_provider(charlie, peer_id, post_proof, true) .await .unwrap() .unwrap(); @@ -54,15 +48,11 @@ where let event = event.unwrap(); assert_eq!(event.owner, charlie.account_id().clone().into()); - assert_eq!(event.info.sector_size, SectorSize::_2KiB); - assert_eq!( - event.info.window_post_proof_type, - primitives::proofs::RegisteredPoStProof::StackedDRGWindow2KiBV1P1 - ); + assert_eq!(event.info.sector_size, post_proof.sector_size()); + assert_eq!(event.info.window_post_proof_type, post_proof,); assert_eq!( event.info.window_post_partition_sectors, - primitives::proofs::RegisteredPoStProof::StackedDRGWindow2KiBV1P1 - .window_post_partitions_sector() + post_proof.window_post_partitions_sector() ); } @@ -268,7 +258,7 @@ async fn real_world_use_case() { let workspace_root = env::var("CARGO_MANIFEST_DIR").unwrap(); let data_file_path = Path::new(&workspace_root) .join("..") - .join("examples/test-data-big.car"); + .join("examples/big_file_184k.car"); tracing::info!("loading example file from {:?}", data_file_path); let temp_dir = tempdir().unwrap(); @@ -282,7 +272,7 @@ async fn real_world_use_case() { tracing::info!("generating PoRep parameters..."); // NOTE: it can take 1-2 minutes on slower machines. can be cached someday, but I think it's good enough for now. - let seal_proof = primitives::proofs::RegisteredSealProof::StackedDRG2KiBV1P1; + let seal_proof = primitives::proofs::RegisteredSealProof::StackedDRG8MiBV1; let porep_parameters = porep::generate_random_groth16_parameters(seal_proof).unwrap(); porep_parameters.write(&mut porep_parameters_file).unwrap(); @@ -290,7 +280,7 @@ async fn real_world_use_case() { let porep_mapped_parameters = porep::load_groth16_parameters(porep_parameters_path).unwrap(); tracing::info!("generating PoSt parameters..."); - let post_proof = primitives::proofs::RegisteredPoStProof::StackedDRGWindow2KiBV1P1; + let post_proof = primitives::proofs::RegisteredPoStProof::StackedDRGWindow8MiBV1; let post_parameters = post::generate_random_groth16_parameters(post_proof).unwrap(); post_parameters.write(&mut post_parameters_file).unwrap(); // We need to read it again, as Proof Generating machine requires it in this form and that's the API of bellperson. @@ -309,7 +299,7 @@ async fn real_world_use_case() { let alice_kp = pair_signer_from_str::("//Alice"); let charlie_kp = pair_signer_from_str::("//Charlie"); - register_storage_provider(&client, &charlie_kp).await; + register_storage_provider(&client, &charlie_kp, post_proof).await; // Set PoRep VerifyingKey extrinsic only accepts scale-encoded bytes of Verifying Key in substrate form. let porep_vk = polka_storage_proofs::VerifyingKey::::try_from(porep_parameters.vk) @@ -346,17 +336,18 @@ async fn real_world_use_case() { tracing::debug!("adding {} balance to alice", balance); add_balance(&client, &alice_kp, balance).await; - // Valid piece cid of `examples/test-data-big.car`. - // Calculated with executing `polka-storage-provider-client proofs commp examples/test-data-big.car`. - let piece_cid = - Cid::try_from("baga6ea4seaqbfhdvmk5qygevit25ztjwl7voyikb5k2fqcl2lsuefhaqtukuiii").unwrap(); - let commp = Commitment::::from_cid(&piece_cid).unwrap(); + let (commp, piece_size) = commp(&data_file_path).unwrap(); + tracing::debug!( + "piece_size of {} = {}", + data_file_path.display(), + *piece_size + ); let sector_end_block = 165; // Publish a storage deal let deal = DealProposal { - piece_cid, - piece_size: 2048, + piece_cid: commp.cid(), + piece_size: *piece_size, client: alice_kp.account_id().clone(), provider: charlie_kp.account_id().clone(), label: "My lovely big data".to_string(), diff --git a/pallets/proofs/src/porep/config.rs b/pallets/proofs/src/porep/config.rs index 1bb57014c..9c31b0c75 100644 --- a/pallets/proofs/src/porep/config.rs +++ b/pallets/proofs/src/porep/config.rs @@ -67,7 +67,7 @@ impl Config { pub fn new(seal_proof: RegisteredSealProof) -> Self { let partitions = partitions(seal_proof); Self { - porep_id: porep_id(seal_proof), + porep_id: seal_proof.porep_id(), // PRE-COND: sector size must be divisible by 32 nodes: (seal_proof.sector_size().bytes() / 32) as usize, challenges: InteractiveChallenges::new(partitions, minimum_challenges(seal_proof)), @@ -90,41 +90,12 @@ impl Config { } } -/// References: -/// * -fn nonce(seal_proof: RegisteredSealProof) -> u64 { - #[allow(clippy::match_single_binding)] - match seal_proof { - // If we ever need to change the nonce for any given RegisteredSealProof, match it here. - _ => 0, - } -} - -/// References: -/// * -pub fn porep_id(seal_proof: RegisteredSealProof) -> PoRepID { - let mut porep_id = [0; 32]; - let registered_proof_id = proof_id(seal_proof); - let n = nonce(seal_proof); - - porep_id[0..8].copy_from_slice(®istered_proof_id.to_le_bytes()); - porep_id[8..16].copy_from_slice(&n.to_le_bytes()); - porep_id -} - -/// Reference: -/// * -fn proof_id(seal_proof: RegisteredSealProof) -> u64 { - match seal_proof { - RegisteredSealProof::StackedDRG2KiBV1P1 => 0, - } -} - /// Reference: /// * fn partitions(seal_proof: RegisteredSealProof) -> usize { match seal_proof { RegisteredSealProof::StackedDRG2KiBV1P1 => 1, + RegisteredSealProof::StackedDRG8MiBV1 => 1, } } @@ -133,6 +104,7 @@ fn partitions(seal_proof: RegisteredSealProof) -> usize { fn minimum_challenges(seal_proof: RegisteredSealProof) -> usize { match seal_proof { RegisteredSealProof::StackedDRG2KiBV1P1 => 2, + RegisteredSealProof::StackedDRG8MiBV1 => 2, } } diff --git a/pallets/proofs/src/porep/mod.rs b/pallets/proofs/src/porep/mod.rs index de0c82e31..fb14ce8f3 100644 --- a/pallets/proofs/src/porep/mod.rs +++ b/pallets/proofs/src/porep/mod.rs @@ -254,6 +254,7 @@ fn generate_inclusion_input(challenge: usize) -> Fr { #[cfg(test)] mod tests { + use filecoin_proofs::{MerkleTreeTrait, SectorShape2KiB, SectorShape8MiB}; use primitives::{proofs::RegisteredSealProof, sector::SectorNumber}; use super::{ProofScheme, PublicInputs, Tau}; @@ -270,22 +271,61 @@ mod tests { let comm_d = [15u8; 32]; let comm_r = [151u8; 32]; - let inputs = - ported_generate_public_inputs(&prover_id, sector_id, &ticket, &seed, &comm_d, &comm_r); - let reference_inputs = - reference_generate_public_inputs(prover_id, sector_id, ticket, seed, comm_d, comm_r); + let seal_proof = RegisteredSealProof::StackedDRG2KiBV1P1; + let inputs = ported_generate_public_inputs( + seal_proof, &prover_id, sector_id, &ticket, &seed, &comm_d, &comm_r, + ); + let reference_inputs = reference_generate_public_inputs::( + seal_proof, prover_id, sector_id, ticket, seed, comm_d, comm_r, + ); assert_eq!(reference_inputs.len(), inputs.len()); for index in 0..reference_inputs.len() { // blstrs is based on bls12_381 implementation, so we can compare serialized bytes. assert_eq!( reference_inputs[index].to_bytes_le(), - inputs[index].to_bytes() + inputs[index].to_bytes(), + "failed to compare reference inputs at idx {}", + index + ); + } + } + + #[test] + /// References: + /// * + fn generates_public_inputs_the_same_as_reference_impl_8mib_sector() { + // random numbers, not 0 + let prover_id = [77u8; 32]; + let sector_id = SectorNumber::new(123).unwrap(); + let ticket = [10u8; 32]; + let seed = [10u8; 32]; + let comm_d = [15u8; 32]; + let comm_r = [151u8; 32]; + + let seal_proof = RegisteredSealProof::StackedDRG8MiBV1; + + let inputs = ported_generate_public_inputs( + seal_proof, &prover_id, sector_id, &ticket, &seed, &comm_d, &comm_r, + ); + let reference_inputs = reference_generate_public_inputs::( + seal_proof, prover_id, sector_id, ticket, seed, comm_d, comm_r, + ); + + assert_eq!(reference_inputs.len(), inputs.len()); + for index in 0..reference_inputs.len() { + // blstrs is based on bls12_381 implementation, so we can compare serialized bytes. + assert_eq!( + reference_inputs[index].to_bytes_le(), + inputs[index].to_bytes(), + "failed to compare reference inputs at idx {}", + index ); } } fn ported_generate_public_inputs( + seal_proof: RegisteredSealProof, prover_id: &[u8; 32], sector_id: SectorNumber, ticket: &[u8; 32], @@ -293,7 +333,7 @@ mod tests { comm_d: &[u8; 32], comm_r: &[u8; 32], ) -> Vec { - let proof_scheme = ProofScheme::setup(RegisteredSealProof::StackedDRG2KiBV1P1); + let proof_scheme = ProofScheme::setup(seal_proof); let replica_id = proof_scheme.generate_replica_id(prover_id, sector_id, ticket, comm_d); // `bytes_into_fr_repr_safe` makes sure random values are convertable into Fr let comm_d_fr = @@ -315,7 +355,8 @@ mod tests { .unwrap() } - fn reference_generate_public_inputs( + fn reference_generate_public_inputs( + seal_proof: RegisteredSealProof, prover_id: [u8; 32], sector_id: SectorNumber, ticket: [u8; 32], @@ -323,11 +364,10 @@ mod tests { comm_d: [u8; 32], comm_r: [u8; 32], ) -> Vec { - use filecoin_hashers::{poseidon::PoseidonHasher, sha256::Sha256Hasher}; - use generic_array::typenum::{U0, U8}; + use filecoin_hashers::sha256::Sha256Hasher; use storage_proofs_core::{ api_version::ApiVersion, compound_proof::CompoundProof, drgraph::BASE_DEGREE, - merkle::LCTree, proof::ProofScheme, util::NODE_SIZE, + proof::ProofScheme, util::NODE_SIZE, }; use storage_proofs_porep::stacked::{ generate_replica_id, Challenges, PublicInputs, SetupParams, StackedCompound, @@ -335,14 +375,13 @@ mod tests { }; // https://github.com/filecoin-project/rust-fil-proofs/blob/5a0523ae1ddb73b415ce2fa819367c7989aaf73f/filecoin-proofs/src/constants.rs#L192C28-L192C66 - type SectorShapeBase = LCTree; let setup_params = SetupParams { // https://github.com/filecoin-project/rust-fil-proofs/blob/5a0523ae1ddb73b415ce2fa819367c7989aaf73f/filecoin-proofs/src/constants.rs#L18 - nodes: (1 << 11) / NODE_SIZE, + nodes: (seal_proof.sector_size().bytes() / NODE_SIZE as u64) as usize, degree: BASE_DEGREE, expansion_degree: EXP_DEGREE, // https://github.com/filecoin-project/rust-filecoin-proofs-api/blob/b44e7cecf2a120aa266b6886628e869ba67252af/src/registry.rs#L53 - porep_id: [0u8; 32], + porep_id: seal_proof.porep_id(), // https://github.com/filecoin-project/rust-fil-proofs/blob/5a0523ae1ddb73b415ce2fa819367c7989aaf73f/filecoin-proofs/src/constants.rs#L123 challenges: Challenges::new_interactive(2), // https://github.com/filecoin-project/rust-fil-proofs/blob/5a0523ae1ddb73b415ce2fa819367c7989aaf73f/filecoin-proofs/src/constants.rs#L84 @@ -351,25 +390,20 @@ mod tests { api_features: vec![], }; - let public_params = - StackedDrg::::setup(&setup_params).unwrap(); - let porep_id = [0u8; 32]; + let public_params = StackedDrg::::setup(&setup_params).unwrap(); - let replica_id = generate_replica_id::( + let replica_id = generate_replica_id::<::Hasher, _>( &prover_id, sector_id.into(), &ticket, comm_d, - &porep_id, + &seal_proof.porep_id(), ); let comm_r_safe = fr32::bytes_into_fr_repr_safe(&comm_r).into(); let comm_d_safe = fr32::bytes_into_fr_repr_safe(&comm_d).into(); - let public_inputs = PublicInputs::< - ::Domain, - ::Domain, - > { + let public_inputs = PublicInputs { replica_id, tau: Some(Tau { comm_d: comm_d_safe, @@ -379,7 +413,7 @@ mod tests { k: None, }; - StackedCompound::::generate_public_inputs( + StackedCompound::::generate_public_inputs( &public_inputs, &public_params, None, diff --git a/primitives/src/proofs.rs b/primitives/src/proofs.rs index b0c9f8f59..f9cf9c68c 100644 --- a/primitives/src/proofs.rs +++ b/primitives/src/proofs.rs @@ -53,12 +53,16 @@ pub enum RegisteredSealProof { #[cfg_attr(feature = "clap", clap(name = "2KiB"))] #[cfg_attr(feature = "serde", serde(alias = "2KiB"))] StackedDRG2KiBV1P1, + #[cfg_attr(feature = "clap", clap(name = "8MiB"))] + #[cfg_attr(feature = "serde", serde(alias = "8MiB"))] + StackedDRG8MiBV1, } impl RegisteredSealProof { pub fn sector_size(&self) -> SectorSize { match self { RegisteredSealProof::StackedDRG2KiBV1P1 => SectorSize::_2KiB, + RegisteredSealProof::StackedDRG8MiBV1 => SectorSize::_8MiB, } } @@ -69,6 +73,7 @@ impl RegisteredSealProof { RegisteredSealProof::StackedDRG2KiBV1P1 => { RegisteredPoStProof::StackedDRGWindow2KiBV1P1 } + RegisteredSealProof::StackedDRG8MiBV1 => RegisteredPoStProof::StackedDRGWindow8MiBV1, } } @@ -79,6 +84,39 @@ impl RegisteredSealProof { pub fn proof_size(self) -> usize { match self { RegisteredSealProof::StackedDRG2KiBV1P1 => 192, + RegisteredSealProof::StackedDRG8MiBV1 => 192, + } + } + + /// Byte identifier used to generate the replica. + /// References: + /// * + pub fn porep_id(&self) -> [u8; 32] { + let mut porep_id = [0; 32]; + let registered_proof_id = self.proof_id(); + let n = self.nonce(); + + porep_id[0..8].copy_from_slice(®istered_proof_id.to_le_bytes()); + porep_id[8..16].copy_from_slice(&n.to_le_bytes()); + porep_id + } + + /// References: + /// * + fn nonce(&self) -> u64 { + #[allow(clippy::match_single_binding)] + match self { + // If we ever need to change the nonce for any given RegisteredSealProof, match it here. + _ => 0, + } + } + + /// Reference: + /// * + fn proof_id(&self) -> u64 { + match self { + RegisteredSealProof::StackedDRG2KiBV1P1 => 0, + RegisteredSealProof::StackedDRG8MiBV1 => 1, } } @@ -104,6 +142,9 @@ pub enum RegisteredPoStProof { #[cfg_attr(feature = "clap", clap(name = "2KiB"))] #[cfg_attr(feature = "serde", serde(alias = "2KiB"))] StackedDRGWindow2KiBV1P1, + #[cfg_attr(feature = "clap", clap(name = "8MiB"))] + #[cfg_attr(feature = "serde", serde(alias = "8MiB"))] + StackedDRGWindow8MiBV1, } impl RegisteredPoStProof { @@ -111,6 +152,7 @@ impl RegisteredPoStProof { pub fn sector_size(&self) -> SectorSize { match self { RegisteredPoStProof::StackedDRGWindow2KiBV1P1 => SectorSize::_2KiB, + RegisteredPoStProof::StackedDRGWindow8MiBV1 => SectorSize::_8MiB, } } @@ -120,6 +162,7 @@ impl RegisteredPoStProof { // Resolve to post proof and then compute size from that. match self { RegisteredPoStProof::StackedDRGWindow2KiBV1P1 => 2, + RegisteredPoStProof::StackedDRGWindow8MiBV1 => 2, } } @@ -130,6 +173,7 @@ impl RegisteredPoStProof { pub fn sector_count(&self) -> usize { match self { RegisteredPoStProof::StackedDRGWindow2KiBV1P1 => 2, + RegisteredPoStProof::StackedDRGWindow8MiBV1 => 2, } } diff --git a/storage-provider/client/src/commands/proofs.rs b/storage-provider/client/src/commands/proofs.rs index 130b6d343..30bb3c94b 100644 --- a/storage-provider/client/src/commands/proofs.rs +++ b/storage-provider/client/src/commands/proofs.rs @@ -1,18 +1,13 @@ -use std::{ - fs::File, - io::{BufReader, Write}, - path::PathBuf, - str::FromStr, -}; +use std::{io::Write, path::PathBuf, str::FromStr}; use codec::Encode; use mater::CarV2Reader; use polka_storage_proofs::{ - porep::{self, sealer::Sealer}, + porep::{self, sealer::select_sealer}, post::{self, ReplicaInfo}, ZeroPaddingReader, }; -use polka_storage_provider_common::commp::{calculate_piece_commitment, CommPError}; +use polka_storage_provider_common::commp::{commp, CommPError}; use primitives::{ commitment::{ piece::{PaddedPieceSize, PieceInfo}, @@ -157,20 +152,8 @@ impl ProofsCommand { .map_err(|e| UtilsCommandError::InvalidCARv2(input_path.clone(), e))?; // Calculate the piece commitment. - let source_file = File::open(&input_path)?; - let file_size = source_file.metadata()?.len(); - - let buffered = BufReader::new(source_file); - let padded_piece_size = PaddedPieceSize::from_arbitrary_size(file_size as u64); - let mut zero_padding_reader = ZeroPaddingReader::new(buffered, *padded_piece_size); - - // The calculate_piece_commitment blocks the thread. We could - // use tokio::task::spawn_blocking to avoid this, but in this - // case it doesn't matter because this is the only thing we are - // working on. - let commitment = - calculate_piece_commitment(&mut zero_padding_reader, padded_piece_size) - .map_err(|err| UtilsCommandError::CommPError(err))?; + let (commitment, padded_piece_size) = + commp(&input_path).map_err(|err| UtilsCommandError::CommPError(err))?; let cid = commitment.cid(); // NOTE(@jmg-duarte,09/10/2024): too lazy for proper json @@ -314,7 +297,7 @@ impl ProofsCommand { )?; println!("Creating sector..."); - let sealer = Sealer::new(seal_proof); + let sealer = select_sealer(seal_proof); let piece_infos = sealer .create_sector(vec![(piece_file, piece_info)], unsealed_sector) .map_err(|e| UtilsCommandError::GeneratePoRepError(e))?; diff --git a/storage-provider/common/Cargo.toml b/storage-provider/common/Cargo.toml index 42559d5ef..308319abb 100644 --- a/storage-provider/common/Cargo.toml +++ b/storage-provider/common/Cargo.toml @@ -29,5 +29,8 @@ tokio = { workspace = true, features = ["fs"] } tokio-util = { workspace = true, features = ["rt"] } tracing = { workspace = true } +[dev-dependencies] +filecoin-proofs.workspace = true + [lints] workspace = true diff --git a/storage-provider/common/src/commp.rs b/storage-provider/common/src/commp.rs index b201f773c..737cb0adb 100644 --- a/storage-provider/common/src/commp.rs +++ b/storage-provider/common/src/commp.rs @@ -1,10 +1,15 @@ -use std::io::Read; +use std::{ + fs::File, + io::{BufReader, Read}, + path::Path, +}; use filecoin_hashers::{ sha256::{Sha256Domain, Sha256Hasher}, Domain, }; use fr32::Fr32Reader; +use polka_storage_proofs::ZeroPaddingReader; use primitives::{ commitment::{piece::PaddedPieceSize, CommP, Commitment}, NODE_SIZE, @@ -12,6 +17,23 @@ use primitives::{ use storage_proofs_core::merkle::BinaryMerkleTree; use thiserror::Error; +/// Calculates piece commitment for the file at a path. +/// Assumes the file at path is a CARv2 archive. +pub fn commp>(path: P) -> Result<(Commitment, PaddedPieceSize), CommPError> { + let source_file = File::open(path.as_ref())?; + let file_size = source_file.metadata()?.len(); + + let padded_piece_size = PaddedPieceSize::from_arbitrary_size(file_size as u64); + // how many zeroes we need to add, so after Fr32 padding it'll be a power of two + let padded_with_zeroes = *padded_piece_size.unpadded(); + + let buffered = BufReader::new(source_file); + let mut zero_padding_reader = ZeroPaddingReader::new(buffered, padded_with_zeroes); + + calculate_piece_commitment(&mut zero_padding_reader, padded_piece_size) + .map(|commp| (commp, padded_piece_size)) +} + /// Calculate the piece commitment for a given data source. /// /// Reference — @@ -58,12 +80,36 @@ pub enum CommPError { #[cfg(test)] mod tests { - use std::io::Cursor; + use std::{env, io::Cursor, path::Path}; + use filecoin_proofs::generate_piece_commitment; use polka_storage_proofs::ZeroPaddingReader; use primitives::{commitment::piece::PaddedPieceSize, sector::SectorSize}; use super::calculate_piece_commitment; + use crate::commp::commp; + + #[test] + fn test_filecoin_commp_matches_ours() { + let workspace_root = env::var("CARGO_MANIFEST_DIR").unwrap(); + let path = Path::new(&workspace_root) + .join("../..") + .join("examples/big_file_184k.car"); + + let (commitment, _) = commp(&path).unwrap(); + + let f = std::fs::File::open(&path).unwrap(); + let len = f.metadata().unwrap().len(); + let size = PaddedPieceSize::from_arbitrary_size(len); + + let piece_info = generate_piece_commitment( + ZeroPaddingReader::new(f, *size.unpadded()), + size.unpadded().into(), + ) + .unwrap(); + + assert_eq!(piece_info.commitment, commitment.raw(),); + } #[test] fn test_calculate_piece_commitment() { diff --git a/storage-provider/common/src/sector.rs b/storage-provider/common/src/sector.rs index c7b6d01f9..9edca9e9f 100644 --- a/storage-provider/common/src/sector.rs +++ b/storage-provider/common/src/sector.rs @@ -1,7 +1,7 @@ use std::{path::PathBuf, sync::Arc}; use polka_storage_proofs::porep::{ - sealer::{prepare_piece, BlstrsProof, PreCommitOutput, Sealer, SubstrateProof}, + sealer::{prepare_piece, select_sealer, BlstrsProof, PreCommitOutput, SubstrateProof}, PoRepError, PoRepParameters, }; use primitives::{ @@ -106,7 +106,7 @@ impl UnsealedSector { commitment: Commitment, ) -> Result<(), SectorError> { self.deals.push((deal_id, deal)); - let sealer = Sealer::new(self.seal_proof); + let sealer = select_sealer(self.seal_proof); // would love to use something like scoped spawn blocking let pieces = self.piece_infos.clone(); @@ -138,7 +138,7 @@ impl UnsealedSector { cache_dir_path: PathBuf, sealed_path: PathBuf, ) -> Result { - let sealer: Sealer = Sealer::new(self.seal_proof); + let sealer = select_sealer(self.seal_proof); tokio::fs::create_dir_all(&cache_dir_path).await?; tokio::fs::File::create_new(&sealed_path).await?; @@ -340,7 +340,7 @@ impl PreCommittedSector { // 10 blocks = 1 minute, only testnet const PRECOMMIT_CHALLENGE_DELAY: u64 = 10; - let sealer: Sealer = Sealer::new(self.seal_proof); + let sealer = select_sealer(self.seal_proof); let seal_randomness_height = self.seal_randomness_height; let Some(digest) = xt_client.get_randomness(seal_randomness_height).await? else { @@ -429,6 +429,7 @@ impl PreCommittedSector { } }; + // TODO(@th7nder,28/01/2025): adjust proofs in PoRep for bigger sector sizes // We use sector size 2KiB only at this point, which guarantees to have 1 proof, because it has 1 partition in the config. // That's why `prove_commit` will always generate a 1 proof. let proof: SubstrateProof = proofs[0] diff --git a/storage-provider/server/src/storage.rs b/storage-provider/server/src/storage.rs index 359b440f6..40663edfb 100644 --- a/storage-provider/server/src/storage.rs +++ b/storage-provider/server/src/storage.rs @@ -10,8 +10,7 @@ use axum::{ }; use futures::{TryFutureExt, TryStreamExt}; use mater::Cid; -use polka_storage_proofs::ZeroPaddingReader; -use polka_storage_provider_common::commp::{calculate_piece_commitment, CommPError}; +use polka_storage_provider_common::commp::{commp, CommPError}; use primitives::{commitment::piece::PaddedPieceSize, proofs::RegisteredPoStProof}; use tokio::{ fs::{self, File}, @@ -269,13 +268,7 @@ async fn upload( // is CPU intensive — i.e. blocking — potentially improvement is to move this completely out of // the tokio runtime into an OS thread let piece_commitment_cid = tokio::task::spawn_blocking(move || -> Result<_, CommPError> { - // Yes, we're reloading the file, this requires the std version - let file = std::fs::File::open(&piece_path)?; - let file_size = file.metadata()?.len(); - let piece_size = PaddedPieceSize::from_arbitrary_size(file_size); - let buffered = std::io::BufReader::new(file); - let reader = ZeroPaddingReader::new(buffered, *piece_size); - let piece_commitment = calculate_piece_commitment(reader, piece_size)?; + let (piece_commitment, _) = commp(&piece_path)?; let piece_commitment_cid = piece_commitment.cid(); tracing::debug!(path = %piece_path.display(), commp = %piece_commitment_cid, "calculated piece commitment"); Ok(piece_commitment_cid) @@ -470,12 +463,7 @@ mod delia_endpoints { // Calculate piece commitment let piece_commitment_cid = tokio::task::spawn_blocking(move || -> Result<_, CommPError> { - let file = std::fs::File::open(&file_path)?; - let file_size = file.metadata()?.len(); - let piece_size = PaddedPieceSize::from_arbitrary_size(file_size); - let buffered = std::io::BufReader::new(file); - let reader = ZeroPaddingReader::new(buffered, *piece_size); - let piece_commitment = calculate_piece_commitment(reader, piece_size)?; + let (piece_commitment, _) = commp(&piece_path)?; let piece_commitment_cid = piece_commitment.cid(); tracing::debug!(path = %file_path.display(), commp = %piece_commitment_cid, "calculated piece commitment"); Ok(piece_commitment_cid) diff --git a/storagext/cli/src/cmd/storage_provider.rs b/storagext/cli/src/cmd/storage_provider.rs index a974dc08d..a3d63bc60 100644 --- a/storagext/cli/src/cmd/storage_provider.rs +++ b/storagext/cli/src/cmd/storage_provider.rs @@ -23,6 +23,7 @@ use crate::{missing_keypair_error, operation_takes_a_while, OutputFormat}; fn parse_post_proof(src: &str) -> Result { match src { "2KiB" => Ok(RegisteredPoStProof::StackedDRGWindow2KiBV1P1), + "8MiB" => Ok(RegisteredPoStProof::StackedDRGWindow8MiBV1), unknown => Err(format!("Unknown PoSt Proof type: {}", unknown)), } } diff --git a/storagext/lib/artifacts/metadata.scale b/storagext/lib/artifacts/metadata.scale index 1e1247d1e..d83a4e107 100644 Binary files a/storagext/lib/artifacts/metadata.scale and b/storagext/lib/artifacts/metadata.scale differ