From 7edf8559339783b0bb54ad9ac232c7f22016cc5b Mon Sep 17 00:00:00 2001 From: DrPeterVanNostrand Date: Thu, 3 Mar 2022 18:07:35 +0000 Subject: [PATCH] feat: pasta gpu support --- fil-proofs-param/Cargo.toml | 2 +- fil-proofs-tooling/Cargo.toml | 2 +- filecoin-hashers/Cargo.toml | 10 +- filecoin-hashers/src/blake2s.rs | 25 -- filecoin-hashers/src/poseidon.rs | 31 +- filecoin-hashers/src/poseidon_types.rs | 66 +-- filecoin-hashers/src/sha256.rs | 25 -- filecoin-hashers/src/types.rs | 14 +- filecoin-proofs/Cargo.toml | 2 +- filecoin-proofs/src/api/util.rs | 17 +- fr32/Cargo.toml | 5 +- storage-proofs-core/Cargo.toml | 8 +- storage-proofs-core/src/crypto/sloth.rs | 12 +- storage-proofs-core/src/drgraph.rs | 13 +- storage-proofs-core/src/gadgets/por.rs | 24 +- storage-proofs-core/src/merkle/builders.rs | 8 +- storage-proofs-core/src/merkle/proof.rs | 401 +++++++++++------- storage-proofs-core/src/merkle/tree.rs | 101 ++--- storage-proofs-core/tests/por_circuit.rs | 6 +- storage-proofs-core/tests/por_vanilla.rs | 114 +---- storage-proofs-porep/Cargo.toml | 8 +- storage-proofs-porep/src/drg/vanilla.rs | 20 +- .../src/stacked/circuit/column_proof.rs | 12 +- .../src/stacked/circuit/params.rs | 6 +- .../src/stacked/vanilla/proof.rs | 178 +++++--- storage-proofs-post/Cargo.toml | 8 +- storage-proofs-post/src/fallback/vanilla.rs | 11 +- storage-proofs-update/Cargo.toml | 8 +- storage-proofs-update/src/circuit.rs | 54 +-- storage-proofs-update/src/compound.rs | 18 +- storage-proofs-update/src/constants.rs | 12 +- storage-proofs-update/src/poseidon/circuit.rs | 54 +-- .../src/poseidon/compound.rs | 18 +- storage-proofs-update/src/poseidon/vanilla.rs | 27 +- storage-proofs-update/src/vanilla.rs | 38 +- storage-proofs-update/tests/circuit.rs | 6 +- .../tests/circuit_poseidon.rs | 6 +- storage-proofs-update/tests/common/mod.rs | 25 +- storage-proofs-update/tests/compound.rs | 6 +- 39 files changed, 692 insertions(+), 709 deletions(-) diff --git a/fil-proofs-param/Cargo.toml b/fil-proofs-param/Cargo.toml index 70ddf0cad..ad30d2d27 100644 --- a/fil-proofs-param/Cargo.toml +++ b/fil-proofs-param/Cargo.toml @@ -49,7 +49,7 @@ indicatif = "0.15.0" group = "0.11.0" dialoguer = "0.8.0" clap = "2.33.3" -blstrs = "0.4.0" +blstrs = { version = "0.4.0", features = ["gpu"] } [dependencies.reqwest] version = "0.10" diff --git a/fil-proofs-tooling/Cargo.toml b/fil-proofs-tooling/Cargo.toml index 92d5b2b47..febf854e4 100644 --- a/fil-proofs-tooling/Cargo.toml +++ b/fil-proofs-tooling/Cargo.toml @@ -53,7 +53,7 @@ fdlimit = "0.2.0" dialoguer = "0.8.0" structopt = "0.3.12" humansize = "1.1.0" -blstrs = "0.4.0" +blstrs = { version = "0.4.0", features = ["gpu"] } [features] default = ["opencl", "measurements"] diff --git a/filecoin-hashers/Cargo.toml b/filecoin-hashers/Cargo.toml index bebcbb007..7f810bfa8 100644 --- a/filecoin-hashers/Cargo.toml +++ b/filecoin-hashers/Cargo.toml @@ -10,16 +10,18 @@ readme = "README.md" [dependencies] bellperson = "0.18.0" -blstrs = "0.4.0" +blstrs = { version = "0.4.0", features = ["gpu"] } generic-array = "0.14.4" merkletree = "0.21.0" +ec-gpu = "0.1.0" ff = "0.11.0" anyhow = "1.0.34" serde = "1.0.117" rand = "0.8.0" - -neptune = { version = "5.1.0", optional = true, features = ["arity2", "arity4", "arity8", "arity11", "arity16", "arity24", "arity36"] } -pasta_curves = "0.3.0" +# neptune = { version = "5.1.0", optional = true, features = ["arity2", "arity4", "arity8", "arity11", "arity16", "arity24", "arity36"] } +# pasta_curves = "0.3.0" +neptune = { git = "https://github.com/filecoin-project/neptune", branch = "fr-as-trait", optional = true, features = ["bls", "pasta", "arity2", "arity4", "arity8", "arity11", "arity16", "arity24", "arity36"] } +pasta_curves = { git = "https://github.com/vmx/pasta_curves", branch = "ec-gpu", features = ["gpu"] } lazy_static = { version = "1.4.0", optional = true } blake2s_simd = { version = "0.5.11", optional = true } sha2 = { version = "0.9.2", optional = true } diff --git a/filecoin-hashers/src/blake2s.rs b/filecoin-hashers/src/blake2s.rs index 34c85c9b5..cf14b263e 100644 --- a/filecoin-hashers/src/blake2s.rs +++ b/filecoin-hashers/src/blake2s.rs @@ -80,31 +80,6 @@ impl Into for Blake2sDomain { } } -// Currently, these panics serve as a stopgap to prevent accidental conversions of a Pasta field -// domains to/from a BLS12-381 scalar field domain. -impl From for Blake2sDomain { - fn from(_f: Fr) -> Self { - panic!("cannot convert BLS12-381 scalar into Blake2sDomain") - } -} -#[allow(clippy::from_over_into)] -impl Into for Blake2sDomain { - fn into(self) -> Fr { - panic!("cannot convert Blake2sDomain into BLS12-381 scalar"); - } -} -impl From for Blake2sDomain { - fn from(_f: Fr) -> Self { - panic!("cannot convert BLS12-381 scalar into Blake2sDomain") - } -} -#[allow(clippy::from_over_into)] -impl Into for Blake2sDomain { - fn into(self) -> Fr { - panic!("cannot convert Blake2sDomain into BLS12-381 scalar"); - } -} - impl From<[u8; 32]> for Blake2sDomain { fn from(bytes: [u8; 32]) -> Self { Blake2sDomain { diff --git a/filecoin-hashers/src/poseidon.rs b/filecoin-hashers/src/poseidon.rs index 57d1d30ba..e172ab1e4 100644 --- a/filecoin-hashers/src/poseidon.rs +++ b/filecoin-hashers/src/poseidon.rs @@ -76,31 +76,6 @@ impl Into for PoseidonDomain { } } -// Currently, these panics serve as a stopgap to prevent accidental conversions of a Pasta field -// domains to/from a BLS12-381 scalar field domain. -impl From for PoseidonDomain { - fn from(_f: Fr) -> Self { - panic!("cannot convert BLS12-381 scalar into PoseidonDomain") - } -} -#[allow(clippy::from_over_into)] -impl Into for PoseidonDomain { - fn into(self) -> Fr { - panic!("cannot convert PoseidonDomain into BLS12-381 scalar"); - } -} -impl From for PoseidonDomain { - fn from(_f: Fr) -> Self { - panic!("cannot convert BLS12-381 scalar into PoseidonDomain") - } -} -#[allow(clippy::from_over_into)] -impl Into for PoseidonDomain { - fn into(self) -> Fr { - panic!("cannot convert PoseidonDomain into BLS12-381 scalar"); - } -} - impl From<[u8; 32]> for PoseidonDomain { fn from(bytes: [u8; 32]) -> Self { PoseidonDomain { @@ -377,7 +352,7 @@ impl HashFunction> for PoseidonFunction { Self::hash2_circuit(cs, left, right) } - fn hash_multi_leaf_circuit>( + fn hash_multi_leaf_circuit, CS: ConstraintSystem>( cs: CS, leaves: &[AllocatedNum], _height: usize, @@ -497,7 +472,7 @@ impl HashFunction> for PoseidonFunction { unimplemented!("PoseidonFunction cannot be used within Groth16 circuits") } - fn hash_multi_leaf_circuit>( + fn hash_multi_leaf_circuit, CS: ConstraintSystem>( _cs: CS, _leaves: &[AllocatedNum], _height: usize, @@ -584,7 +559,7 @@ impl HashFunction> for PoseidonFunction { unimplemented!("PoseidonFunction cannot be used within Groth16 circuits") } - fn hash_multi_leaf_circuit>( + fn hash_multi_leaf_circuit, CS: ConstraintSystem>( _cs: CS, _leaves: &[AllocatedNum], _height: usize, diff --git a/filecoin-hashers/src/poseidon_types.rs b/filecoin-hashers/src/poseidon_types.rs index 01401a235..238ec21c4 100644 --- a/filecoin-hashers/src/poseidon_types.rs +++ b/filecoin-hashers/src/poseidon_types.rs @@ -76,63 +76,25 @@ lazy_static! { pub struct FieldArity(PhantomData<(F, A)>) where F: PrimeField, - A: Arity; + A: PoseidonArity; impl typemap::Key for FieldArity where F: PrimeField, - A: Arity, + A: PoseidonArity, { type Value = &'static PoseidonConstants; } -pub trait PoseidonArity: Arity + Send + Sync + Clone + Debug { - #[allow(non_snake_case)] - fn PARAMETERS() -> &'static PoseidonConstants; -} - -impl PoseidonArity for U0 { - fn PARAMETERS() -> &'static PoseidonConstants { - unreachable!("dummy implementation, do not ever call me") - } -} - -impl PoseidonArity for U2 { - fn PARAMETERS() -> &'static PoseidonConstants { - &*POSEIDON_CONSTANTS_2 - } -} - -impl PoseidonArity for U4 { - fn PARAMETERS() -> &'static PoseidonConstants { - &*POSEIDON_CONSTANTS_4 - } -} - -impl PoseidonArity for U8 { - fn PARAMETERS() -> &'static PoseidonConstants { - &*POSEIDON_CONSTANTS_8 - } -} - -impl PoseidonArity for U11 { - fn PARAMETERS() -> &'static PoseidonConstants { - &*POSEIDON_CONSTANTS_11 - } -} - -impl PoseidonArity for U16 { - fn PARAMETERS() -> &'static PoseidonConstants { - &*POSEIDON_CONSTANTS_16 - } -} -impl PoseidonArity for U24 { - fn PARAMETERS() -> &'static PoseidonConstants { - &*POSEIDON_CONSTANTS_24 - } -} -impl PoseidonArity for U36 { - fn PARAMETERS() -> &'static PoseidonConstants { - &*POSEIDON_CONSTANTS_36 - } -} +// A marker trait for arities which are in `POSEIDON_CONSTANTS`; we require that 'PoseidonArity` +// implements `Send + Sync` because those traits are required by `lazy_static`. +pub trait PoseidonArity: Arity + Send + Sync + Clone + Debug {} + +// We must implement `PoseidonArity for U0` because the `U0` arity is used in compound trees +// (each compound tree arity must implement `PoseidonArity`). +impl PoseidonArity for U0 {} +impl PoseidonArity for U2 {} +impl PoseidonArity for U4 {} +impl PoseidonArity for U8 {} +impl PoseidonArity for U11 {} +impl PoseidonArity for PoseidonMDArity {} diff --git a/filecoin-hashers/src/sha256.rs b/filecoin-hashers/src/sha256.rs index b08f6d2e2..289d46373 100644 --- a/filecoin-hashers/src/sha256.rs +++ b/filecoin-hashers/src/sha256.rs @@ -78,31 +78,6 @@ impl Into for Sha256Domain { } } -// Currently, these panics serve as a stopgap to prevent accidental conversions of a Pasta field -// domains to/from a BLS12-381 scalar field domain. -impl From for Sha256Domain { - fn from(_f: Fr) -> Self { - panic!("cannot convert BLS12-381 scalar into Sha256Domain") - } -} -#[allow(clippy::from_over_into)] -impl Into for Sha256Domain { - fn into(self) -> Fr { - panic!("cannot convert Sha256Domain into BLS12-381 scalar"); - } -} -impl From for Sha256Domain { - fn from(_f: Fr) -> Self { - panic!("cannot convert BLS12-381 scalar into Sha256Domain") - } -} -#[allow(clippy::from_over_into)] -impl Into for Sha256Domain { - fn into(self) -> Fr { - panic!("cannot convert Sha256Domain into BLS12-381 scalar"); - } -} - impl From<[u8; 32]> for Sha256Domain { fn from(bytes: [u8; 32]) -> Self { Sha256Domain { diff --git a/filecoin-hashers/src/types.rs b/filecoin-hashers/src/types.rs index 1fda37bc6..19ae32688 100644 --- a/filecoin-hashers/src/types.rs +++ b/filecoin-hashers/src/types.rs @@ -10,6 +10,7 @@ use bellperson::{ ConstraintSystem, SynthesisError, }; use blstrs::Scalar as Fr; +use ec_gpu::GpuField; use ff::{Field, PrimeField}; use merkletree::{ hash::{Algorithm as LightAlgorithm, Hashable as LightHashable}, @@ -28,15 +29,6 @@ pub trait Domain: + Eq + Send + Sync - // TODO (halo): remove once we have Pasta GPU support. - // Currently the `From + Into` trait bounds are used as a stopgap to prevent - // Pasta field domains from being used in GPU code, e.g. currently converting a - // `Sha256Domain` into an `Fr` panics. Remove theses trait bounds once Pasta fields are - // fully supported in `rust-fil-proofs`, e.g. GPU code. - + From - + Into - // Note that `Self::Field` may be `Fr`, in which case the trait bounds - // `From + Into` are redundant. + From + Into + From<[u8; 32]> @@ -45,7 +37,7 @@ pub trait Domain: + Element + StdHash { - type Field: PrimeField; + type Field: PrimeField + GpuField; #[allow(clippy::wrong_self_convention)] fn into_bytes(&self) -> Vec { @@ -110,7 +102,7 @@ pub trait HashFunction: Clone + Debug + Send + Sync + LightAlgorithm< Self::hash_leaf_bits_circuit(cs, &left_bits, &right_bits, height) } - fn hash_multi_leaf_circuit>( + fn hash_multi_leaf_circuit, CS: ConstraintSystem>( cs: CS, leaves: &[AllocatedNum], height: usize, diff --git a/filecoin-proofs/Cargo.toml b/filecoin-proofs/Cargo.toml index 4fa7950d9..40d22d268 100644 --- a/filecoin-proofs/Cargo.toml +++ b/filecoin-proofs/Cargo.toml @@ -42,7 +42,7 @@ group = "0.11.0" byte-slice-cast = "1.0.0" fr32 = { path = "../fr32", version = "^4.0.0", default-features = false } once_cell = "1.8.0" -blstrs = "0.4.0" +blstrs = { version = "0.4.0", features = ["gpu"] } [dev-dependencies] criterion = "0.3" diff --git a/filecoin-proofs/src/api/util.rs b/filecoin-proofs/src/api/util.rs index 8b22f1d83..984a2f2a1 100644 --- a/filecoin-proofs/src/api/util.rs +++ b/filecoin-proofs/src/api/util.rs @@ -1,29 +1,28 @@ use std::mem::size_of; use anyhow::{Context, Result}; -use blstrs::Scalar as Fr; +use ff::PrimeField; use filecoin_hashers::{Domain, Hasher}; -use fr32::{bytes_into_fr, fr_into_bytes}; use merkletree::merkle::{get_merkle_tree_leafs, get_merkle_tree_len}; use storage_proofs_core::merkle::{get_base_tree_count, MerkleTreeTrait}; use typenum::Unsigned; use crate::types::{Commitment, SectorSize}; -pub fn as_safe_commitment>( +pub fn as_safe_commitment>( comm: &[u8; 32], commitment_name: T, -) -> Result { - bytes_into_fr(comm) +) -> Result { + let mut repr = ::Repr::default(); + repr.as_mut().copy_from_slice(comm); + D::Field::from_repr_vartime(repr) .map(Into::into) .with_context(|| format!("Invalid commitment ({})", commitment_name.as_ref(),)) } -pub fn commitment_from_fr(fr: Fr) -> Commitment { +pub fn commitment_from_fr(fr: F) -> Commitment { let mut commitment = [0; 32]; - for (i, b) in fr_into_bytes(&fr).iter().enumerate() { - commitment[i] = *b; - } + commitment.copy_from_slice(fr.to_repr().as_ref()); commitment } diff --git a/fr32/Cargo.toml b/fr32/Cargo.toml index 7686f24d4..68e08c50c 100644 --- a/fr32/Cargo.toml +++ b/fr32/Cargo.toml @@ -14,13 +14,14 @@ byte-slice-cast = "1.0.0" byteorder = "1" ff = "0.11.0" thiserror = "1.0.6" -blstrs = "0.4.0" +blstrs = { version = "0.4.0", features = ["gpu"] } [dev-dependencies] bitvec = "0.17" criterion = "0.3" itertools = "0.9" -pasta_curves = "0.3.0" +# pasta_curves = "0.3.0" +pasta_curves = { git = "https://github.com/vmx/pasta_curves", branch = "ec-gpu", features = ["gpu"] } pretty_assertions = "0.6.1" rand = "0.8" rand_xorshift = "0.3" diff --git a/storage-proofs-core/Cargo.toml b/storage-proofs-core/Cargo.toml index e150d314c..a8bd78441 100644 --- a/storage-proofs-core/Cargo.toml +++ b/storage-proofs-core/Cargo.toml @@ -39,14 +39,15 @@ hex = "0.4.0" generic-array = "0.14.4" anyhow = "1.0.23" thiserror = "1.0.6" -neptune = { version = "5.1.0", features = ["arity2", "arity4", "arity8", "arity11", "arity16", "arity24", "arity36"] } +# neptune = { version = "5.1.0", features = ["arity2", "arity4", "arity8", "arity11", "arity16", "arity24", "arity36"] } +neptune = { git = "https://github.com/filecoin-project/neptune", branch = "wip-fr-as-trait", features = ["bls", "pasta", "arity2", "arity4", "arity8", "arity11", "arity16", "arity24", "arity36"] } cpu-time = { version = "1.0", optional = true } gperftools = { version = "0.2", optional = true } num_cpus = "1.10.1" semver = "0.11.0" fr32 = { path = "../fr32", version = "^4.0.0"} pairing = "0.21" -blstrs = "0.4.0" +blstrs = { version = "0.4.0", features = ["gpu"] } [dev-dependencies] proptest = "0.10" @@ -56,7 +57,8 @@ rand_xorshift = "0.3.0" pretty_assertions = "0.6.1" sha2raw = { path = "../sha2raw", version = "^6.0.0"} filecoin-hashers = { path = "../filecoin-hashers", version = "^6.0.0", default-features = false, features = ["blake2s", "sha256", "poseidon"] } -pasta_curves = "0.3.0" +# pasta_curves = "0.3.0" +pasta_curves = { git = "https://github.com/vmx/pasta_curves", branch = "ec-gpu", features = ["gpu"] } [features] default = ["opencl"] diff --git a/storage-proofs-core/src/crypto/sloth.rs b/storage-proofs-core/src/crypto/sloth.rs index e788393f6..397830d42 100644 --- a/storage-proofs-core/src/crypto/sloth.rs +++ b/storage-proofs-core/src/crypto/sloth.rs @@ -1,22 +1,22 @@ -use blstrs::Scalar as Fr; +use ff::PrimeField; /// Sloth based encoding. #[inline] -pub fn encode(key: &Fr, plaintext: &Fr) -> Fr { - plaintext + key +pub fn encode(key: &F, plaintext: &F) -> F { + *plaintext + *key } /// Sloth based decoding. #[inline] -pub fn decode(key: &Fr, ciphertext: &Fr) -> Fr { - ciphertext - key +pub fn decode(key: &F, ciphertext: &F) -> F { + *ciphertext - *key } #[cfg(test)] mod tests { use super::*; - use ff::PrimeField; + use blstrs::Scalar as Fr; use proptest::{prop_compose, proptest}; // the modulus from `bls12_381::Fr` diff --git a/storage-proofs-core/src/drgraph.rs b/storage-proofs-core/src/drgraph.rs index fe5295252..b5fc96ac4 100644 --- a/storage-proofs-core/src/drgraph.rs +++ b/storage-proofs-core/src/drgraph.rs @@ -3,7 +3,7 @@ use std::fmt::Debug; use std::marker::PhantomData; use anyhow::ensure; -use filecoin_hashers::{Hasher, PoseidonArity}; +use filecoin_hashers::{Domain, Hasher, PoseidonArity}; use fr32::bytes_into_fr_repr_safe; use generic_array::typenum::Unsigned; use merkletree::merkle::get_merkle_tree_row_count; @@ -37,7 +37,10 @@ pub trait Graph: Debug + Clone + PartialEq + Eq { } /// Returns the merkle tree depth. - fn merkle_tree_depth(&self) -> u64 { + fn merkle_tree_depth(&self) -> u64 + where + U: PoseidonArity<::Field>, + { graph_height::(self.size()) as u64 } @@ -365,7 +368,11 @@ mod tests { graph_bucket::>(); } - fn gen_proof(config: Option) { + fn gen_proof(config: Option) + where + H: 'static + Hasher, + U: PoseidonArity<::Field>, + { let leafs = 64; let porep_id = [1; 32]; let g = BucketGraph::::new(leafs, BASE_DEGREE, 0, porep_id, ApiVersion::V1_1_0) diff --git a/storage-proofs-core/src/gadgets/por.rs b/storage-proofs-core/src/gadgets/por.rs index 4c9e1a9e9..7580da102 100644 --- a/storage-proofs-core/src/gadgets/por.rs +++ b/storage-proofs-core/src/gadgets/por.rs @@ -50,9 +50,9 @@ pub struct AuthPath where H: Hasher, H::Domain: Domain, - U: 'static + PoseidonArity, - V: 'static + PoseidonArity, - W: 'static + PoseidonArity, + U: PoseidonArity, + V: PoseidonArity, + W: PoseidonArity, { base: SubPath, sub: SubPath, @@ -63,9 +63,9 @@ impl From>, Option)>> for AuthPath, - U: 'static + PoseidonArity, - V: 'static + PoseidonArity, - W: 'static + PoseidonArity, + U: PoseidonArity, + V: PoseidonArity, + W: PoseidonArity, { fn from(mut base_opts: Vec<(Vec>, Option)>) -> Self { let has_top = W::to_usize() > 0; @@ -130,7 +130,7 @@ struct SubPath where H: Hasher, H::Domain: Domain, - Arity: 'static + PoseidonArity, + Arity: PoseidonArity, { path: Vec>, } @@ -140,7 +140,7 @@ struct PathElement where H: Hasher, H::Domain: Domain, - Arity: 'static + PoseidonArity, + Arity: PoseidonArity, { hashes: Vec>, index: Option, @@ -152,7 +152,7 @@ impl SubPath where H: Hasher, H::Domain: Domain, - Arity: 'static + PoseidonArity, + Arity: PoseidonArity, { fn synthesize>( self, @@ -219,9 +219,9 @@ impl AuthPath where H: Hasher, H::Domain: Domain, - U: PoseidonArity, - V: PoseidonArity, - W: PoseidonArity, + U: PoseidonArity, + V: PoseidonArity, + W: PoseidonArity, { pub fn blank(leaves: usize) -> Self { let has_sub = V::to_usize() > 0; diff --git a/storage-proofs-core/src/merkle/builders.rs b/storage-proofs-core/src/merkle/builders.rs index 28b731164..361fe2eff 100644 --- a/storage-proofs-core/src/merkle/builders.rs +++ b/storage-proofs-core/src/merkle/builders.rs @@ -237,12 +237,16 @@ pub fn create_base_merkle_tree( /// replica path (since the replica file will contain the same data), /// we pass both since we have access from all callers and this avoids /// reading that data from the replica_config here. -pub fn create_base_lcmerkle_tree( +pub fn create_base_lcmerkle_tree( config: StoreConfig, size: usize, data: &[u8], replica_config: &ReplicaConfig, -) -> Result> { +) -> Result> +where + H: Hasher, + BaseTreeArity: PoseidonArity<::Field>, +{ trace!("create_base_lcmerkle_tree called with size {}", size); trace!( "is_merkle_tree_size_valid({}, arity {}) = {}", diff --git a/storage-proofs-core/src/merkle/proof.rs b/storage-proofs-core/src/merkle/proof.rs index 2257ed183..f3cb13d09 100644 --- a/storage-proofs-core/src/merkle/proof.rs +++ b/storage-proofs-core/src/merkle/proof.rs @@ -5,8 +5,7 @@ use std::marker::PhantomData; use std::slice::Iter; use anyhow::{ensure, Result}; -use blstrs::Scalar as Fr; -use filecoin_hashers::{Hasher, PoseidonArity}; +use filecoin_hashers::{Domain, Hasher, PoseidonArity}; use generic_array::typenum::{Unsigned, U0}; use merkletree::hash::Algorithm; use serde::{de::DeserializeOwned, Deserialize, Serialize}; @@ -16,16 +15,22 @@ use crate::drgraph::graph_height; /// Trait to abstract over the concept of Merkle Proof. pub trait MerkleProofTrait: Clone + Serialize + DeserializeOwned + Debug + Sync + Send { type Hasher: Hasher; - type Arity: 'static + PoseidonArity; - type SubTreeArity: 'static + PoseidonArity; - type TopTreeArity: 'static + PoseidonArity; + type Arity: PoseidonArity<<::Domain as Domain>::Field>; + type SubTreeArity: PoseidonArity<<::Domain as Domain>::Field>; + type TopTreeArity: PoseidonArity<<::Domain as Domain>::Field>; /// Try to convert a merkletree proof into this structure. fn try_from_proof( p: merkletree::proof::Proof<::Domain, Self::Arity>, ) -> Result; - fn as_options(&self) -> Vec<(Vec>, Option)> { + #[allow(clippy::type_complexity)] + fn as_options( + &self, + ) -> Vec<( + Vec::Domain as Domain>::Field>>, + Option, + )> { self.path() .iter() .map(|v| { @@ -37,7 +42,16 @@ pub trait MerkleProofTrait: Clone + Serialize + DeserializeOwned + Debug + Sync .collect::>() } - fn into_options_with_leaf(self) -> (Option, Vec<(Vec>, Option)>) { + #[allow(clippy::type_complexity)] + fn into_options_with_leaf( + self, + ) -> ( + Option<<::Domain as Domain>::Field>, + Vec<( + Vec::Domain as Domain>::Field>>, + Option, + )>, + ) { let leaf = self.leaf(); let path = self.path(); ( @@ -52,7 +66,12 @@ pub trait MerkleProofTrait: Clone + Serialize + DeserializeOwned + Debug + Sync .collect::>(), ) } - fn as_pairs(&self) -> Vec<(Vec, usize)> { + fn as_pairs( + &self, + ) -> Vec<( + Vec<<::Domain as Domain>::Field>, + usize, + )> { self.path() .iter() .map(|v| (v.0.iter().copied().map(Into::into).collect(), v.1)) @@ -165,7 +184,11 @@ macro_rules! forward_method { } #[derive(Debug, Default, Clone, Serialize, Deserialize)] -pub struct InclusionPath { +pub struct InclusionPath +where + H: Hasher, + Arity: PoseidonArity<::Field>, +{ #[serde(bound( serialize = "H::Domain: Serialize", deserialize = "H::Domain: Deserialize<'de>" @@ -173,13 +196,21 @@ pub struct InclusionPath { path: Vec>, } -impl From>> for InclusionPath { +impl From>> for InclusionPath +where + H: Hasher, + Arity: PoseidonArity<::Field>, +{ fn from(path: Vec>) -> Self { Self { path } } } -impl InclusionPath { +impl InclusionPath +where + H: Hasher, + Arity: PoseidonArity<::Field>, +{ /// Calculate the root of this path, given the leaf as input. pub fn root(&self, leaf: H::Domain) -> H::Domain { let mut a = H::Function::default(); @@ -215,7 +246,11 @@ impl InclusionPath { } #[derive(Debug, Default, Clone, Serialize, Deserialize)] -pub struct PathElement { +pub struct PathElement +where + H: Hasher, + Arity: PoseidonArity<::Field>, +{ #[serde(bound( serialize = "H::Domain: Serialize", deserialize = "H::Domain: Deserialize<'de>" @@ -230,9 +265,9 @@ pub struct PathElement { #[derive(Debug, Clone, Serialize, Deserialize)] pub struct MerkleProof< H: Hasher, - BaseArity: PoseidonArity, - SubTreeArity: PoseidonArity = U0, - TopTreeArity: PoseidonArity = U0, + BaseArity: PoseidonArity<::Field>, + SubTreeArity: PoseidonArity<::Field> = U0, + TopTreeArity: PoseidonArity<::Field> = U0, > { #[serde(bound( serialize = "H::Domain: Serialize", @@ -241,12 +276,13 @@ pub struct MerkleProof< data: ProofData, } -impl< - H: Hasher, - Arity: 'static + PoseidonArity, - SubTreeArity: 'static + PoseidonArity, - TopTreeArity: 'static + PoseidonArity, - > MerkleProofTrait for MerkleProof +impl MerkleProofTrait + for MerkleProof +where + H: Hasher, + Arity: PoseidonArity<::Field>, + SubTreeArity: PoseidonArity<::Field>, + TopTreeArity: PoseidonArity<::Field>, { type Hasher = H; type Arity = Arity; @@ -296,12 +332,13 @@ impl< } #[derive(Debug, Clone, Serialize, Deserialize)] -enum ProofData< +enum ProofData +where H: Hasher, - BaseArity: PoseidonArity, - SubTreeArity: PoseidonArity, - TopTreeArity: PoseidonArity, -> { + BaseArity: PoseidonArity<::Field>, + SubTreeArity: PoseidonArity<::Field>, + TopTreeArity: PoseidonArity<::Field>, +{ #[serde(bound( serialize = "H::Domain: Serialize", deserialize = "H::Domain: Deserialize<'de>" @@ -320,7 +357,11 @@ enum ProofData< } #[derive(Debug, Default, Clone, Serialize, Deserialize)] -struct SingleProof { +struct SingleProof +where + H: Hasher, + Arity: PoseidonArity<::Field>, +{ /// Root of the merkle tree. #[serde(bound( serialize = "H::Domain: Serialize", @@ -341,14 +382,23 @@ struct SingleProof { path: InclusionPath, } -impl SingleProof { +impl SingleProof +where + H: Hasher, + Arity: PoseidonArity<::Field>, +{ pub fn new(path: InclusionPath, root: H::Domain, leaf: H::Domain) -> Self { SingleProof { root, leaf, path } } } #[derive(Debug, Default, Clone, Serialize, Deserialize)] -struct SubProof { +struct SubProof +where + H: Hasher, + BaseArity: PoseidonArity<::Field>, + SubTreeArity: PoseidonArity<::Field>, +{ #[serde(bound( serialize = "H::Domain: Serialize", deserialize = "H::Domain: Deserialize<'de>" @@ -373,8 +423,11 @@ struct SubProof - SubProof +impl SubProof +where + H: Hasher, + BaseArity: PoseidonArity<::Field>, + SubTreeArity: PoseidonArity<::Field>, { pub fn new( base_proof: InclusionPath, @@ -392,12 +445,13 @@ impl } #[derive(Debug, Default, Clone, Serialize, Deserialize)] -struct TopProof< +struct TopProof +where H: Hasher, - BaseArity: PoseidonArity, - SubTreeArity: PoseidonArity, - TopTreeArity: PoseidonArity, -> { + BaseArity: PoseidonArity<::Field>, + SubTreeArity: PoseidonArity<::Field>, + TopTreeArity: PoseidonArity<::Field>, +{ #[serde(bound( serialize = "H::Domain: Serialize", deserialize = "H::Domain: Deserialize<'de>" @@ -427,12 +481,12 @@ struct TopProof< leaf: H::Domain, } -impl< - H: Hasher, - BaseArity: PoseidonArity, - SubTreeArity: PoseidonArity, - TopTreeArity: PoseidonArity, - > TopProof +impl TopProof +where + H: Hasher, + BaseArity: PoseidonArity<::Field>, + SubTreeArity: PoseidonArity<::Field>, + TopTreeArity: PoseidonArity<::Field>, { pub fn new( base_proof: InclusionPath, @@ -451,12 +505,12 @@ impl< } } -impl< - H: Hasher, - BaseArity: PoseidonArity, - SubTreeArity: PoseidonArity, - TopTreeArity: PoseidonArity, - > MerkleProof +impl MerkleProof +where + H: Hasher, + BaseArity: PoseidonArity<::Field>, + SubTreeArity: PoseidonArity<::Field>, + TopTreeArity: PoseidonArity<::Field>, { pub fn new(n: usize) -> Self { let root = Default::default(); @@ -474,11 +528,16 @@ impl< } /// Converts a merkle_light proof to a SingleProof -fn proof_to_single( +fn proof_to_single( proof: &merkletree::proof::Proof, lemma_start_index: usize, sub_root: Option, -) -> SingleProof { +) -> SingleProof +where + H: Hasher, + Arity: PoseidonArity<::Field>, + TargetArity: PoseidonArity<::Field>, +{ let root = proof.root(); let leaf = if let Some(sub_root) = sub_root { sub_root @@ -493,11 +552,15 @@ fn proof_to_single( /// 'lemma_start_index' is required because sub/top proofs start at /// index 0 and base proofs start at index 1 (skipping the leaf at the /// front) -fn extract_path( +fn extract_path( lemma: &[H::Domain], path: &[usize], lemma_start_index: usize, -) -> InclusionPath { +) -> InclusionPath +where + H: Hasher, + Arity: PoseidonArity<::Field>, +{ let path = lemma[lemma_start_index..lemma.len() - 1] .chunks(Arity::to_usize() - 1) .zip(path.iter()) @@ -511,7 +574,11 @@ fn extract_path( path.into() } -impl SingleProof { +impl SingleProof +where + H: Hasher, + Arity: PoseidonArity<::Field>, +{ fn try_from_proof(p: merkletree::proof::Proof<::Domain, Arity>) -> Result { Ok(proof_to_single(&p, 1, None)) } @@ -545,8 +612,11 @@ impl SingleProof { } } -impl - SubProof +impl SubProof +where + H: Hasher, + Arity: PoseidonArity<::Field>, + SubTreeArity: PoseidonArity<::Field>, { fn try_from_proof(p: merkletree::proof::Proof<::Domain, Arity>) -> Result { ensure!( @@ -607,12 +677,12 @@ impl TopProof +impl TopProof +where + H: Hasher, + Arity: PoseidonArity<::Field>, + SubTreeArity: PoseidonArity<::Field>, + TopTreeArity: PoseidonArity<::Field>, { fn try_from_proof(p: merkletree::proof::Proof<::Domain, Arity>) -> Result { ensure!( @@ -699,6 +769,7 @@ impl< mod tests { use super::*; + use blstrs::Scalar as Fr; use filecoin_hashers::{ blake2s::Blake2sHasher, poseidon::PoseidonHasher, sha256::Sha256Hasher, Domain, }; @@ -745,6 +816,15 @@ mod tests { U0, >, >(); + merklepath::< + MerkleTreeWrapper< + PoseidonHasher, + DiskStore< as Hasher>::Domain>, + U2, + U0, + U0, + >, + >(); } #[test] @@ -758,6 +838,15 @@ mod tests { U0, >, >(); + merklepath::< + MerkleTreeWrapper< + PoseidonHasher, + DiskStore< as Hasher>::Domain>, + U4, + U0, + U0, + >, + >(); } #[test] @@ -771,6 +860,15 @@ mod tests { U0, >, >(); + merklepath::< + MerkleTreeWrapper< + PoseidonHasher, + DiskStore< as Hasher>::Domain>, + U8, + U0, + U0, + >, + >(); } #[test] @@ -784,6 +882,15 @@ mod tests { U0, >, >(); + merklepath::< + MerkleTreeWrapper< + PoseidonHasher, + DiskStore< as Hasher>::Domain>, + U8, + U2, + U0, + >, + >(); } #[test] @@ -797,6 +904,15 @@ mod tests { U0, >, >(); + merklepath::< + MerkleTreeWrapper< + PoseidonHasher, + DiskStore< as Hasher>::Domain>, + U8, + U4, + U0, + >, + >(); } #[test] @@ -810,6 +926,15 @@ mod tests { U2, >, >(); + merklepath::< + MerkleTreeWrapper< + PoseidonHasher, + DiskStore< as Hasher>::Domain>, + U8, + U4, + U2, + >, + >(); } #[test] @@ -823,6 +948,15 @@ mod tests { U0, >, >(); + merklepath::< + MerkleTreeWrapper< + Sha256Hasher, + DiskStore< as Hasher>::Domain>, + U2, + U0, + U0, + >, + >(); } #[test] @@ -836,6 +970,15 @@ mod tests { U0, >, >(); + merklepath::< + MerkleTreeWrapper< + Sha256Hasher, + DiskStore< as Hasher>::Domain>, + U4, + U0, + U0, + >, + >(); } #[test] @@ -849,6 +992,15 @@ mod tests { U0, >, >(); + merklepath::< + MerkleTreeWrapper< + Sha256Hasher, + DiskStore< as Hasher>::Domain>, + U2, + U4, + U0, + >, + >(); } #[test] @@ -862,6 +1014,15 @@ mod tests { U2, >, >(); + merklepath::< + MerkleTreeWrapper< + Sha256Hasher, + DiskStore< as Hasher>::Domain>, + U2, + U4, + U2, + >, + >(); } #[test] @@ -875,6 +1036,15 @@ mod tests { U0, >, >(); + merklepath::< + MerkleTreeWrapper< + Blake2sHasher, + DiskStore< as Hasher>::Domain>, + U2, + U0, + U0, + >, + >(); } #[test] @@ -888,6 +1058,15 @@ mod tests { U0, >, >(); + merklepath::< + MerkleTreeWrapper< + Blake2sHasher, + DiskStore< as Hasher>::Domain>, + U4, + U0, + U0, + >, + >(); } #[test] @@ -901,96 +1080,14 @@ mod tests { U2, >, >(); - } - - #[test] - fn merklepath_poseidon_2_halo() { - type Tree = MerkleTreeWrapper::Domain>, U2, U0, U0>; - merklepath::>>(); - merklepath::>>(); - } - - #[test] - fn merklepath_poseidon_4_halo() { - type Tree = MerkleTreeWrapper::Domain>, U4, U0, U0>; - merklepath::>>(); - merklepath::>>(); - } - - #[test] - fn merklepath_poseidon_8_halo() { - type Tree = MerkleTreeWrapper::Domain>, U8, U0, U0>; - merklepath::>>(); - merklepath::>>(); - } - - #[test] - fn merklepath_poseidon_8_2_halo() { - type Tree = MerkleTreeWrapper::Domain>, U8, U2, U0>; - merklepath::>>(); - merklepath::>>(); - } - - #[test] - fn merklepath_poseidon_8_4_halo() { - type Tree = MerkleTreeWrapper::Domain>, U8, U4, U0>; - merklepath::>>(); - merklepath::>>(); - } - - #[test] - fn merklepath_poseidon_8_4_2_halo() { - type Tree = MerkleTreeWrapper::Domain>, U8, U4, U2>; - merklepath::>>(); - merklepath::>>(); - } - - #[test] - fn merklepath_sha256_2_halo() { - type Tree = MerkleTreeWrapper::Domain>, U2, U0, U0>; - merklepath::>>(); - merklepath::>>(); - } - - #[test] - fn merklepath_sha256_4_halo() { - type Tree = MerkleTreeWrapper::Domain>, U4, U0, U0>; - merklepath::>>(); - merklepath::>>(); - } - - #[test] - fn merklepath_sha256_2_4_halo() { - type Tree = MerkleTreeWrapper::Domain>, U2, U4, U0>; - merklepath::>>(); - merklepath::>>(); - } - - #[test] - fn merklepath_sha256_2_4_2_halo() { - type Tree = MerkleTreeWrapper::Domain>, U2, U4, U2>; - merklepath::>>(); - merklepath::>>(); - } - - #[test] - fn merklepath_blake2s_2_halo() { - type Tree = MerkleTreeWrapper::Domain>, U2, U0, U0>; - merklepath::>>(); - merklepath::>>(); - } - - #[test] - fn merklepath_blake2s_4_halo() { - type Tree = MerkleTreeWrapper::Domain>, U4, U0, U0>; - merklepath::>>(); - merklepath::>>(); - } - - #[test] - fn merklepath_blake2s_8_4_2_halo() { - type Tree = MerkleTreeWrapper::Domain>, U8, U4, U2>; - merklepath::>>(); - merklepath::>>(); + merklepath::< + MerkleTreeWrapper< + Blake2sHasher, + DiskStore< as Hasher>::Domain>, + U8, + U4, + U2, + >, + >(); } } diff --git a/storage-proofs-core/src/merkle/tree.rs b/storage-proofs-core/src/merkle/tree.rs index 970a55c6b..9c62cc869 100644 --- a/storage-proofs-core/src/merkle/tree.rs +++ b/storage-proofs-core/src/merkle/tree.rs @@ -5,7 +5,7 @@ use std::marker::PhantomData; use std::ops::{Deref, DerefMut}; use anyhow::Result; -use filecoin_hashers::{Hasher, PoseidonArity}; +use filecoin_hashers::{Domain, Hasher, PoseidonArity}; use generic_array::typenum::U0; use merkletree::{ hash::Hashable, @@ -18,9 +18,9 @@ use crate::merkle::{LCTree, MerkleProof, MerkleProofTrait}; /// Trait used to abstract over the way Merkle Trees are constructed and stored. pub trait MerkleTreeTrait: Send + Sync + Debug { - type Arity: 'static + PoseidonArity; - type SubTreeArity: 'static + PoseidonArity; - type TopTreeArity: 'static + PoseidonArity; + type Arity: PoseidonArity<<::Domain as Domain>::Field>; + type SubTreeArity: PoseidonArity<<::Domain as Domain>::Field>; + type TopTreeArity: PoseidonArity<<::Domain as Domain>::Field>; type Hasher: 'static + Hasher; type Store: Store<::Domain>; type Proof: MerkleProofTrait< @@ -54,21 +54,21 @@ pub trait MerkleTreeTrait: Send + Sync + Debug { pub struct MerkleTreeWrapper< H: Hasher, S: Store<::Domain>, - U: PoseidonArity, - V: PoseidonArity = U0, - W: PoseidonArity = U0, + U: PoseidonArity<::Field>, + V: PoseidonArity<::Field> = U0, + W: PoseidonArity<::Field> = U0, > { pub inner: MerkleTree<::Domain, ::Function, S, U, V, W>, pub h: PhantomData, } -impl< - H: 'static + Hasher, - S: Store<::Domain>, - U: 'static + PoseidonArity, - V: 'static + PoseidonArity, - W: 'static + PoseidonArity, - > MerkleTreeTrait for MerkleTreeWrapper +impl MerkleTreeTrait for MerkleTreeWrapper +where + H: 'static + Hasher, + S: Store<::Domain>, + U: PoseidonArity<::Field>, + V: PoseidonArity<::Field>, + W: PoseidonArity<::Field>, { type Arity = U; type SubTreeArity = V; @@ -133,14 +133,14 @@ impl< } } -impl< - H: Hasher, - S: Store<::Domain>, - U: PoseidonArity, - V: PoseidonArity, - W: PoseidonArity, - > From::Domain, ::Function, S, U, V, W>> +impl From::Domain, ::Function, S, U, V, W>> for MerkleTreeWrapper +where + H: Hasher, + S: Store<::Domain>, + U: PoseidonArity<::Field>, + V: PoseidonArity<::Field>, + W: PoseidonArity<::Field>, { fn from(tree: MerkleTree<::Domain, ::Function, S, U, V, W>) -> Self { Self { @@ -150,13 +150,13 @@ impl< } } -impl< - H: Hasher, - S: Store<::Domain>, - U: PoseidonArity, - V: PoseidonArity, - W: PoseidonArity, - > MerkleTreeWrapper +impl MerkleTreeWrapper +where + H: Hasher, + S: Store<::Domain>, + U: PoseidonArity<::Field>, + V: PoseidonArity<::Field>, + W: PoseidonArity<::Field>, { pub fn new>(data: I) -> Result { let tree = MerkleTree::new(data)?; @@ -307,13 +307,14 @@ impl< } } -impl< - H: Hasher, - S: Store<::Domain>, - BaseArity: PoseidonArity, - SubTreeArity: PoseidonArity, - TopTreeArity: PoseidonArity, - > Debug for MerkleTreeWrapper +impl Debug + for MerkleTreeWrapper +where + H: Hasher, + S: Store<::Domain>, + BaseArity: PoseidonArity<::Field>, + SubTreeArity: PoseidonArity<::Field>, + TopTreeArity: PoseidonArity<::Field>, { fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { f.debug_struct("MerkleTreeWrapper") @@ -323,13 +324,14 @@ impl< } } -impl< - H: Hasher, - S: Store<::Domain>, - BaseArity: PoseidonArity, - SubTreeArity: PoseidonArity, - TopTreeArity: PoseidonArity, - > Deref for MerkleTreeWrapper +impl Deref + for MerkleTreeWrapper +where + H: Hasher, + S: Store<::Domain>, + BaseArity: PoseidonArity<::Field>, + SubTreeArity: PoseidonArity<::Field>, + TopTreeArity: PoseidonArity<::Field>, { type Target = MerkleTree; @@ -338,13 +340,14 @@ impl< } } -impl< - H: Hasher, - S: Store<::Domain>, - BaseArity: PoseidonArity, - SubTreeArity: PoseidonArity, - TopTreeArity: PoseidonArity, - > DerefMut for MerkleTreeWrapper +impl DerefMut + for MerkleTreeWrapper +where + H: Hasher, + S: Store<::Domain>, + BaseArity: PoseidonArity<::Field>, + SubTreeArity: PoseidonArity<::Field>, + TopTreeArity: PoseidonArity<::Field>, { fn deref_mut(&mut self) -> &mut Self::Target { &mut self.inner diff --git a/storage-proofs-core/tests/por_circuit.rs b/storage-proofs-core/tests/por_circuit.rs index deab11cd2..a863fda78 100644 --- a/storage-proofs-core/tests/por_circuit.rs +++ b/storage-proofs-core/tests/por_circuit.rs @@ -385,9 +385,9 @@ where fn test_por_no_challenge_input(sector_nodes: usize) where - U: PoseidonArity, - V: PoseidonArity, - W: PoseidonArity, + U: PoseidonArity, + V: PoseidonArity, + W: PoseidonArity, { let mut rng = XorShiftRng::from_seed(TEST_SEED); diff --git a/storage-proofs-core/tests/por_vanilla.rs b/storage-proofs-core/tests/por_vanilla.rs index 0650b952f..8645055d7 100644 --- a/storage-proofs-core/tests/por_vanilla.rs +++ b/storage-proofs-core/tests/por_vanilla.rs @@ -24,66 +24,36 @@ type TreeBase = MerkleTreeWrapper::Domain>, U, #[test] fn test_por_poseidon_base_2() { test_por::, U2>>(); + test_por::, U2>>(); } #[test] fn test_por_sha256_base_2() { test_por::, U2>>(); + test_por::, U2>>(); } #[test] fn test_por_blake2s_base_2() { test_por::, U2>>(); -} - -#[test] -fn test_por_poseidon_base_2_halo() { - test_por::, U2>>(); - test_por::, U2>>(); -} - -#[test] -fn test_por_sha256_base_2_halo() { - test_por::, U2>>(); - test_por::, U2>>(); -} - -#[test] -fn test_por_blake2s_base_2_halo() { test_por::, U2>>(); - test_por::, U2>>(); } #[test] fn test_por_poseidon_base_4() { test_por::, U4>>(); + test_por::, U4>>(); } #[test] fn test_por_sha256_base_4() { test_por::, U4>>(); + test_por::, U4>>(); } #[test] fn test_por_blake2s_base_4() { test_por::, U4>>(); -} - -#[test] -fn test_por_poseidon_base_4_halo() { - test_por::, U4>>(); - test_por::, U4>>(); -} - -#[test] -fn test_por_sha256_base_4_halo() { - test_por::, U4>>(); - test_por::, U4>>(); -} - -#[test] -fn test_por_blake2s_base_4_halo() { - test_por::, U4>>(); test_por::, U4>>(); } @@ -128,66 +98,36 @@ fn test_por() { #[test] fn test_por_validates_proof_sha256_base_2() { test_por_validates_proof::, U2>>(); + test_por_validates_proof::, U2>>(); } #[test] fn test_por_validates_proof_blake2s_base_2() { test_por_validates_proof::, U2>>(); + test_por_validates_proof::, U2>>(); } #[test] fn test_por_validates_proof_poseidon_base_2() { test_por_validates_proof::, U2>>(); -} - -#[test] -fn test_por_validates_proof_sha256_base_2_halo() { - test_por_validates_proof::, U2>>(); - test_por_validates_proof::, U2>>(); -} - -#[test] -fn test_por_validates_proof_blake2s_base_2_halo() { - test_por_validates_proof::, U2>>(); - test_por_validates_proof::, U2>>(); -} - -#[test] -fn test_por_validates_proof_poseidon_base_2_halo() { test_por_validates_proof::, U2>>(); - test_por_validates_proof::, U2>>(); } #[test] fn test_por_validates_proof_sha256_base_4() { test_por_validates_proof::, U4>>(); + test_por_validates_proof::, U4>>(); } #[test] fn test_por_validates_proof_blake2s_base_4() { test_por_validates_proof::, U4>>(); + test_por_validates_proof::, U4>>(); } #[test] fn test_por_validates_proof_poseidon_base_4() { test_por_validates_proof::, U4>>(); -} - -#[test] -fn test_por_validates_proof_sha256_base_4_halo() { - test_por_validates_proof::, U4>>(); - test_por_validates_proof::, U4>>(); -} - -#[test] -fn test_por_validates_proof_blake2s_base_4_halo() { - test_por_validates_proof::, U4>>(); - test_por_validates_proof::, U4>>(); -} - -#[test] -fn test_por_validates_proof_poseidon_base_4_halo() { - test_por_validates_proof::, U4>>(); test_por_validates_proof::, U4>>(); } @@ -248,66 +188,36 @@ fn test_por_validates_proof() { #[test] fn test_por_validates_challenge_sha256_base_2() { test_por_validates_challenge::, U2>>(); + test_por_validates_challenge::, U2>>(); } #[test] fn test_por_validates_challenge_blake2s_base_2() { test_por_validates_challenge::, U2>>(); + test_por_validates_challenge::, U2>>(); } #[test] fn test_por_validates_challenge_poseidon_base_2() { test_por_validates_challenge::, U2>>(); -} - -#[test] -fn test_por_validates_challenge_sha256_base_2_halo() { - test_por_validates_challenge::, U2>>(); - test_por_validates_challenge::, U2>>(); -} - -#[test] -fn test_por_validates_challenge_blake2s_base_2_halo() { - test_por_validates_challenge::, U2>>(); - test_por_validates_challenge::, U2>>(); -} - -#[test] -fn test_por_validates_challenge_poseidon_base_2_halo() { test_por_validates_challenge::, U2>>(); - test_por_validates_challenge::, U2>>(); } #[test] fn test_por_validates_challenge_sha256_base_4() { test_por_validates_challenge::, U4>>(); + test_por_validates_challenge::, U4>>(); } #[test] fn test_por_validates_challenge_blake2s_base_4() { test_por_validates_challenge::, U4>>(); + test_por_validates_challenge::, U4>>(); } #[test] fn test_por_validates_challenge_poseidon_base_4() { test_por_validates_challenge::, U4>>(); -} - -#[test] -fn test_por_validates_challenge_sha256_base_4_halo() { - test_por_validates_challenge::, U4>>(); - test_por_validates_challenge::, U4>>(); -} - -#[test] -fn test_por_validates_challenge_blake2s_base_4_halo() { - test_por_validates_challenge::, U4>>(); - test_por_validates_challenge::, U4>>(); -} - -#[test] -fn test_por_validates_challenge_poseidon_base_4_halo() { - test_por_validates_challenge::, U4>>(); test_por_validates_challenge::, U4>>(); } diff --git a/storage-proofs-porep/Cargo.toml b/storage-proofs-porep/Cargo.toml index 6a84a93a9..38d15c853 100644 --- a/storage-proofs-porep/Cargo.toml +++ b/storage-proofs-porep/Cargo.toml @@ -28,7 +28,8 @@ log = "0.4.7" pretty_assertions = "0.6.1" generic-array = "0.14.4" anyhow = "1.0.23" -neptune = { version = "5.1.0", features = ["arity2", "arity4", "arity8", "arity11", "arity16", "arity24", "arity36"] } +# neptune = { version = "5.1.0", features = ["arity2", "arity4", "arity8", "arity11", "arity16", "arity24", "arity36"] } +neptune = { git = "https://github.com/filecoin-project/neptune", branch = "fr-as-trait", features = ["bls", "pasta", "arity2", "arity4", "arity8", "arity11", "arity16", "arity24", "arity36"] } num_cpus = "1.10.1" hex = "0.4.2" bincode = "1.1.2" @@ -42,8 +43,9 @@ fr32 = { path = "../fr32", version = "^4.0.0", default-features = false } yastl = "0.1.2" fil_logger = "0.1" pairing = "0.21" -blstrs = "0.4.0" -pasta_curves = "0.3.0" +blstrs = { version = "0.4.0", features = ["gpu"] } +# pasta_curves = "0.3.0" +pasta_curves = { git = "https://github.com/vmx/pasta_curves", branch = "ec-gpu", features = ["gpu"] } typemap = "0.3.3" [target."cfg(target_arch = \"aarch64\")".dependencies] diff --git a/storage-proofs-porep/src/drg/vanilla.rs b/storage-proofs-porep/src/drg/vanilla.rs index 5793de373..cd5c3d307 100644 --- a/storage-proofs-porep/src/drg/vanilla.rs +++ b/storage-proofs-porep/src/drg/vanilla.rs @@ -134,7 +134,11 @@ where } #[derive(Debug, Clone, Serialize, Deserialize)] -pub struct DataProof { +pub struct DataProof +where + H: Hasher, + U: PoseidonArity<::Field>, +{ #[serde(bound( serialize = "MerkleProof: Serialize", deserialize = "MerkleProof: Deserialize<'de>" @@ -143,7 +147,11 @@ pub struct DataProof { pub data: H::Domain, } -impl DataProof { +impl DataProof +where + H: Hasher, + U: PoseidonArity<::Field>, +{ pub fn new(n: usize) -> Self { DataProof { proof: MerkleProof::new(n), @@ -584,12 +592,16 @@ where /// Creates the encoding key from a `MerkleTree`. /// The algorithm for that is `Blake2s(id | encodedParentNode1 | encodedParentNode1 | ...)`. /// It is only public so that it can be used for benchmarking -pub fn create_key_from_tree( +pub fn create_key_from_tree( id: &H::Domain, node: usize, parents: &[u32], tree: &LCMerkleTree, -) -> Result { +) -> Result +where + H: Hasher, + U: PoseidonArity<::Field>, +{ let mut hasher = Sha256::new(); hasher.update(AsRef::<[u8]>::as_ref(&id)); diff --git a/storage-proofs-porep/src/stacked/circuit/column_proof.rs b/storage-proofs-porep/src/stacked/circuit/column_proof.rs index 7e8da196f..6ba7e7c19 100644 --- a/storage-proofs-porep/src/stacked/circuit/column_proof.rs +++ b/storage-proofs-porep/src/stacked/circuit/column_proof.rs @@ -17,9 +17,9 @@ pub struct ColumnProof where H: Hasher, H::Domain: Domain, - U: 'static + PoseidonArity, - V: 'static + PoseidonArity, - W: 'static + PoseidonArity, + U: PoseidonArity, + V: PoseidonArity, + W: PoseidonArity, { column: Column, inclusion_path: AuthPath, @@ -29,9 +29,9 @@ impl ColumnProof where H: 'static + Hasher, H::Domain: Domain, - U: 'static + PoseidonArity, - V: 'static + PoseidonArity, - W: 'static + PoseidonArity, + U: PoseidonArity, + V: PoseidonArity, + W: PoseidonArity, { /// Create an empty `ColumnProof`, used in `blank_circuit`s. pub fn empty< diff --git a/storage-proofs-porep/src/stacked/circuit/params.rs b/storage-proofs-porep/src/stacked/circuit/params.rs index 00a861371..3fc2ac2a0 100644 --- a/storage-proofs-porep/src/stacked/circuit/params.rs +++ b/storage-proofs-porep/src/stacked/circuit/params.rs @@ -343,9 +343,9 @@ fn enforce_inclusion( where H: 'static + Hasher, H::Domain: Domain, - U: 'static + PoseidonArity, - V: 'static + PoseidonArity, - W: 'static + PoseidonArity, + U: PoseidonArity, + V: PoseidonArity, + W: PoseidonArity, CS: ConstraintSystem, { let root = Root::from_allocated::(root.clone()); diff --git a/storage-proofs-porep/src/stacked/vanilla/proof.rs b/storage-proofs-porep/src/stacked/vanilla/proof.rs index ecc2321f5..3d5baa775 100644 --- a/storage-proofs-porep/src/stacked/vanilla/proof.rs +++ b/storage-proofs-porep/src/stacked/vanilla/proof.rs @@ -9,15 +9,15 @@ use anyhow::Context; use bincode::deserialize; use blstrs::Scalar as Fr; use fdlimit::raise_fd_limit; -use ff::PrimeField; use filecoin_hashers::{poseidon::PoseidonHasher, Domain, HashFunction, Hasher, PoseidonArity}; use generic_array::typenum::{Unsigned, U0, U11, U2, U8}; use lazy_static::lazy_static; -use log::{error, info, trace, warn}; +use log::{error, info, trace}; use merkletree::{ merkle::{get_merkle_tree_len, is_merkle_tree_size_valid}, store::{DiskStore, Store, StoreConfig}, }; +use pasta_curves::{Fp, Fq}; use rayon::prelude::{ IndexedParallelIterator, IntoParallelIterator, ParallelIterator, ParallelSliceMut, }; @@ -38,7 +38,7 @@ use storage_proofs_core::{ use yastl::Pool; use crate::{ - encode::{decode, encode, encode_fr}, + encode::{decode, encode}, stacked::vanilla::{ challenges::LayerChallenges, column::Column, @@ -86,7 +86,7 @@ pub struct LayerState { } pub enum TreeRElementData { - FrList(Vec), + FrList(Vec<<::Domain as Domain>::Field>), ElementList(Vec<::Domain>), } @@ -451,21 +451,25 @@ where // Even if the column builder is enabled, the GPU column builder // only supports Poseidon hashes. pub fn use_gpu_column_builder() -> bool { - // TODO (halo): change `PoseidonHasher` to - // `PoseidonHasher || PoseidonHasher || PoseidonHasher` once `neptune` supports - // Pasta fields in GPU. SETTINGS.use_gpu_column_builder - && TypeId::of::() == TypeId::of::>() + && [ + TypeId::of::>(), + TypeId::of::>(), + TypeId::of::>(), + ] + .contains(&TypeId::of::()) } // Even if the tree builder is enabled, the GPU tree builder // only supports Poseidon hashes. pub fn use_gpu_tree_builder() -> bool { - // TODO (halo): change `PoseidonHasher` to - // `PoseidonHasher || PoseidonHasher || PoseidonHasher` once `neptune` supports - // Pasta fields in GPU. SETTINGS.use_gpu_tree_builder - && TypeId::of::() == TypeId::of::>() + && [ + TypeId::of::>(), + TypeId::of::>(), + TypeId::of::>(), + ] + .contains(&TypeId::of::()) } #[cfg(any(feature = "cuda", feature = "opencl"))] @@ -477,8 +481,8 @@ where labels: &LabelsCache, ) -> Result> where - ColumnArity: 'static + PoseidonArity, - TreeArity: PoseidonArity, + ColumnArity: PoseidonArity<<::Domain as Domain>::Field>, + TreeArity: PoseidonArity<<::Domain as Domain>::Field>, { if Self::use_gpu_column_builder() { Self::generate_tree_c_gpu::( @@ -508,8 +512,8 @@ where labels: &LabelsCache, ) -> Result> where - ColumnArity: 'static + PoseidonArity, - TreeArity: PoseidonArity, + ColumnArity: PoseidonArity<<::Domain as Domain>::Field>, + TreeArity: PoseidonArity<<::Domain as Domain>::Field>, { Self::generate_tree_c_cpu::( layers, @@ -530,15 +534,16 @@ where labels: &LabelsCache, ) -> Result> where - ColumnArity: 'static + PoseidonArity, - TreeArity: PoseidonArity, + ColumnArity: PoseidonArity<<::Domain as Domain>::Field>, + TreeArity: PoseidonArity<<::Domain as Domain>::Field>, { use std::cmp::min; use std::sync::mpsc::sync_channel as channel; use std::sync::{Arc, RwLock}; - use fr32::fr_into_bytes; + use ff::PrimeField; use generic_array::GenericArray; + use log::warn; use neptune::{ batch_hasher::Batcher, column_tree_builder::{ColumnTreeBuilder, ColumnTreeBuilderTrait}, @@ -568,9 +573,15 @@ where let config_count = configs.len(); // Don't move config into closure below. THREAD_POOL.scoped(|s| { // This channel will receive the finished tree data to be written to disk. - let (writer_tx, writer_rx) = channel::<(Vec, Vec)>(0); + let (writer_tx, writer_rx) = channel::<( + Vec<<::Domain as Domain>::Field>, + Vec<<::Domain as Domain>::Field>, + )>(0); s.execute(move || { + let field_size = + std::mem::size_of::<<::Domain as Domain>::Field>(); + for i in 0..config_count { let mut node_index = 0; let builder_tx = builder_tx.clone(); @@ -584,15 +595,13 @@ where chunked_nodes_count, ); - let columns: Vec> = { - use fr32::bytes_into_fr; - + let columns: Vec::Domain as Domain>::Field, + ColumnArity, + >> = { // Allocate layer data array and insert a placeholder for each layer. let mut layer_data: Vec> = - vec![ - vec![0u8; chunked_nodes_count * std::mem::size_of::()]; - layers - ]; + vec![vec![0u8; chunked_nodes_count * field_size]; layers]; // gather all layer data. for (layer_index, mut layer_bytes) in @@ -612,14 +621,18 @@ where .map(|index| { (0..layers) .map(|layer_index| { - bytes_into_fr( - &layer_data[layer_index][std::mem::size_of::() - * index - ..std::mem::size_of::() * (index + 1)], - ) - .expect("Could not create Fr from bytes.") + let mut repr = <<::Domain as + Domain>::Field as PrimeField>::Repr::default(); + repr.as_mut().copy_from_slice( + &layer_data[layer_index][field_size * index..field_size * (index + 1)] + ); + <::Domain as Domain>::Field::from_repr_vartime(repr) + .expect("bytes are invalid field element") }) - .collect::>() + .collect::::Domain as Domain>::Field, + ColumnArity, + >>() }) .collect() }; @@ -655,7 +668,11 @@ where None } }; - let mut column_tree_builder = ColumnTreeBuilder::::new( + let mut column_tree_builder = ColumnTreeBuilder::< + <::Domain as Domain>::Field, + ColumnArity, + TreeArity, + >::new( column_batcher, tree_batcher, nodes_count, @@ -665,7 +682,7 @@ where // Loop until all trees for all configs have been built. for i in 0..config_count { loop { - let (columns, is_final): (Vec>, bool) = + let (columns, is_final): (Vec::Domain as Domain>::Field, ColumnArity>>, bool) = builder_rx.recv().expect("failed to recv columns"); // Just add non-final column batches. @@ -734,7 +751,7 @@ where let store = Arc::new(RwLock::new(tree_c_store)); let batch_size = min(base_data.len(), column_write_batch_size); - let flatten_and_write_store = |data: &Vec, offset| { + let flatten_and_write_store = |data: &Vec<<::Domain as Domain>::Field>, offset| { data.into_par_iter() .chunks(batch_size) .enumerate() @@ -742,7 +759,7 @@ where let mut buf = Vec::with_capacity(batch_size * NODE_SIZE); for fr in fr_elements { - buf.extend(fr_into_bytes(fr)); + buf.extend(fr.to_repr().as_ref()); } store .write() @@ -790,8 +807,8 @@ where labels: &LabelsCache, ) -> Result> where - ColumnArity: PoseidonArity, - TreeArity: PoseidonArity, + ColumnArity: PoseidonArity<<::Domain as Domain>::Field>, + TreeArity: PoseidonArity<<::Domain as Domain>::Field>, { info!("generating tree c using the CPU"); measure_op(Operation::GenerateTreeC, || { @@ -888,18 +905,29 @@ where start: usize, end: usize, ) -> Result> { + use ff::PrimeField; + + use crate::encode::encode_fr; + if Self::use_gpu_tree_builder() { - use fr32::bytes_into_fr; + let field_size = + std::mem::size_of::<<::Domain as Domain>::Field>(); - let mut layer_bytes = vec![0u8; (end - start) * std::mem::size_of::()]; + let mut layer_bytes = vec![0u8; (end - start) * field_size]; source .read_range_into(start, end, &mut layer_bytes) .expect("failed to read layer bytes"); let encoded_data: Vec<_> = layer_bytes .into_par_iter() - .chunks(std::mem::size_of::()) - .map(|chunk| bytes_into_fr(&chunk).expect("Could not create Fr from bytes.")) + .chunks(field_size) + .map(|chunk| { + let mut repr = + <<::Domain as Domain>::Field as PrimeField>::Repr::default(); + repr.as_mut().copy_from_slice(&chunk); + <::Domain as Domain>::Field::from_repr_vartime(repr) + .expect("bytes are invalid field element") + }) .zip( data.expect("failed to unwrap data").as_mut() [(start * NODE_SIZE)..(end * NODE_SIZE)] @@ -910,8 +938,8 @@ where ::Domain::try_from_bytes(data_node_bytes) .expect("try_from_bytes failed"); - let mut encoded_fr: Fr = key; - let data_node_fr: Fr = data_node.into(); + let mut encoded_fr = key; + let data_node_fr: <::Domain as Domain>::Field = data_node.into(); encode_fr(&mut encoded_fr, data_node_fr); let encoded_fr_repr = encoded_fr.to_repr(); data_node_bytes.copy_from_slice(AsRef::<[u8]>::as_ref(&encoded_fr_repr)); @@ -947,7 +975,7 @@ where callback: Option>, ) -> Result> where - TreeArity: PoseidonArity, + TreeArity: PoseidonArity<<::Domain as Domain>::Field>, { let encode_data = match callback { Some(x) => x, @@ -988,7 +1016,7 @@ where callback: Option>, ) -> Result> where - TreeArity: PoseidonArity, + TreeArity: PoseidonArity<<::Domain as Domain>::Field>, { let encode_data = match callback { Some(x) => x, @@ -1017,14 +1045,15 @@ where callback: PrepareTreeRDataCallback, ) -> Result> where - TreeArity: PoseidonArity, + TreeArity: PoseidonArity<<::Domain as Domain>::Field>, { use std::cmp::min; use std::fs::OpenOptions; use std::io::Write; use std::sync::mpsc::sync_channel as channel; - use fr32::fr_into_bytes; + use ff::PrimeField; + use log::warn; use merkletree::merkle::{get_merkle_tree_cache_size, get_merkle_tree_leafs}; use neptune::{ batch_hasher::Batcher, @@ -1042,14 +1071,18 @@ where let max_gpu_tree_batch_size = SETTINGS.max_gpu_tree_batch_size as usize; // This channel will receive batches of leaf nodes and add them to the TreeBuilder. - let (builder_tx, builder_rx) = channel::<(Vec, bool)>(0); + let (builder_tx, builder_rx) = channel::<( + Vec<<::Domain as Domain>::Field>, + bool, + )>(0); let config_count = configs.len(); // Don't move config into closure below. let configs = &configs; let tree_r_last_config = &tree_r_last_config; THREAD_POOL.scoped(|s| { // This channel will receive the finished tree data to be written to disk. - let (writer_tx, writer_rx) = channel::>(0); + let (writer_tx, writer_rx) = + channel::::Domain as Domain>::Field>>(0); s.execute(move || { for i in 0..config_count { @@ -1101,10 +1134,11 @@ where None } }; - let mut tree_builder = TreeBuilder::::new( - batcher, - nodes_count, - tree_r_last_config.rows_to_discard, + let mut tree_builder = TreeBuilder::< + <::Domain as Domain>::Field, + Tree::Arity, + >::new( + batcher, nodes_count, tree_r_last_config.rows_to_discard ) .expect("failed to create TreeBuilder"); @@ -1156,9 +1190,9 @@ where .expect("failed to get merkle tree cache size"); assert_eq!(tree_data_len, cache_size); - let flat_tree_data: Vec<_> = tree_data + let flat_tree_data: Vec = tree_data .into_par_iter() - .flat_map(|el| fr_into_bytes(&el)) + .flat_map(|el| el.to_repr().as_ref().to_vec()) .collect(); // Persist the data to the store based on the current config. @@ -1196,7 +1230,7 @@ where callback: PrepareTreeRDataCallback, ) -> Result> where - TreeArity: PoseidonArity, + TreeArity: PoseidonArity<<::Domain as Domain>::Field>, { let (configs, replica_config) = split_config_and_replica( tree_r_last_config.clone(), @@ -1529,13 +1563,13 @@ where replica_path: PathBuf, ) -> Result> where - TreeArity: PoseidonArity, + TreeArity: PoseidonArity<<::Domain as Domain>::Field>, { use std::fs::OpenOptions; use std::io::Write; - use ff::Field; - use fr32::fr_into_bytes; + use ff::{Field, PrimeField}; + use log::warn; use merkletree::merkle::{get_merkle_tree_cache_size, get_merkle_tree_leafs}; use neptune::{ batch_hasher::Batcher, @@ -1561,15 +1595,19 @@ where None } }; - let mut tree_builder = TreeBuilder::::new( - batcher, - nodes_count, - tree_r_last_config.rows_to_discard, + let mut tree_builder = TreeBuilder::< + <::Domain as Domain>::Field, + Tree::Arity, + >::new( + batcher, nodes_count, tree_r_last_config.rows_to_discard ) .expect("failed to create TreeBuilder"); // Allocate zeros once and reuse. - let zero_leaves: Vec = vec![Fr::zero(); max_gpu_tree_batch_size]; + let zero_leaves = vec![ + <::Domain as Domain>::Field::zero(); + max_gpu_tree_batch_size + ]; for (i, config) in configs.iter().enumerate() { let mut consumed = 0; while consumed < nodes_count { @@ -1607,9 +1645,9 @@ where .expect("failed to get merkle tree cache size"); assert_eq!(tree_data_len, cache_size); - let flat_tree_data: Vec<_> = tree_data + let flat_tree_data: Vec = tree_data .into_par_iter() - .flat_map(|el| fr_into_bytes(&el)) + .flat_map(|el| el.to_repr().as_ref().to_vec()) .collect(); // Persist the data to the store based on the current config. @@ -1664,7 +1702,7 @@ where replica_path: PathBuf, ) -> Result> where - TreeArity: PoseidonArity, + TreeArity: PoseidonArity<<::Domain as Domain>::Field>, { let (configs, replica_config) = split_config_and_replica( tree_r_last_config.clone(), diff --git a/storage-proofs-post/Cargo.toml b/storage-proofs-post/Cargo.toml index e7910d11e..d2a26a0e0 100644 --- a/storage-proofs-post/Cargo.toml +++ b/storage-proofs-post/Cargo.toml @@ -26,17 +26,19 @@ log = "0.4.7" hex = "0.4.0" generic-array = "0.14.4" anyhow = "1.0.23" -neptune = { version = "5.1.0", features = ["arity2", "arity4", "arity8", "arity11", "arity16", "arity24", "arity36"] } +# neptune = { version = "5.1.0", features = ["arity2", "arity4", "arity8", "arity11", "arity16", "arity24", "arity36"] } +neptune = { git = "https://github.com/filecoin-project/neptune", branch = "fr-as-trait", features = ["bls", "pasta", "arity2", "arity4", "arity8", "arity11", "arity16", "arity24", "arity36"] } num_cpus = "1.10.1" fr32 = { path = "../fr32", version = "^4.0.0", default-features = false } -blstrs = "0.4.0" +blstrs = { version = "0.4.0", features = ["gpu"] } [dev-dependencies] tempfile = "3" pretty_assertions = "0.6.1" filecoin-hashers = { path = "../filecoin-hashers", version = "^6.0.0", default-features = false, features = ["poseidon", "sha256", "blake2s"]} rand_xorshift = "0.3.0" -pasta_curves = "0.3.0" +# pasta_curves = "0.3.0" +pasta_curves = { git = "https://github.com/vmx/pasta_curves", branch = "ec-gpu", features = ["gpu"] } [features] default = ["opencl"] diff --git a/storage-proofs-post/src/fallback/vanilla.rs b/storage-proofs-post/src/fallback/vanilla.rs index cf28db29d..5796bdfd9 100644 --- a/storage-proofs-post/src/fallback/vanilla.rs +++ b/storage-proofs-post/src/fallback/vanilla.rs @@ -2,7 +2,6 @@ use std::collections::BTreeSet; use std::marker::PhantomData; use anyhow::ensure; -use blstrs::Scalar as Fr; use byteorder::{ByteOrder, LittleEndian}; use filecoin_hashers::{Domain, HashFunction, Hasher}; use generic_array::typenum::Unsigned; @@ -150,7 +149,15 @@ impl SectorProof

{ .collect() } - pub fn as_options(&self) -> Vec>, Option)>> { + #[allow(clippy::type_complexity)] + pub fn as_options( + &self, + ) -> Vec< + Vec<( + Vec::Domain as Domain>::Field>>, + Option, + )>, + > { self.inclusion_proofs .iter() .map(MerkleProofTrait::as_options) diff --git a/storage-proofs-update/Cargo.toml b/storage-proofs-update/Cargo.toml index fe320af58..886f5d70b 100644 --- a/storage-proofs-update/Cargo.toml +++ b/storage-proofs-update/Cargo.toml @@ -25,12 +25,13 @@ serde = { version = "1.0", features = ["derive"]} serde_json = "1.0" ff = "0.11.0" bellperson = "0.18.0" -blstrs = "0.4.0" +blstrs = { version = "0.4.0", features = ["gpu"] } log = "0.4.7" pretty_assertions = "0.6.1" generic-array = "0.14.4" anyhow = "1.0.23" -neptune = { version = "5.1.0", features = ["arity2", "arity4", "arity8", "arity11", "arity16", "arity24", "arity36"] } +# neptune = { version = "5.1.0", features = ["arity2", "arity4", "arity8", "arity11", "arity16", "arity24", "arity36"] } +neptune = { git = "https://github.com/filecoin-project/neptune", branch = "fr-as-trait", features = ["bls", "pasta", "arity2", "arity4", "arity8", "arity11", "arity16", "arity24", "arity36"] } num_cpus = "1.10.1" hex = "0.4.2" bincode = "1.1.2" @@ -44,7 +45,8 @@ fr32 = { path = "../fr32", version = "^4.0.0", default-features = false } yastl = "0.1.2" fil_logger = "0.1" memmap = "0.7" -pasta_curves = "0.3.0" +# pasta_curves = "0.3.0" +pasta_curves = { git = "https://github.com/vmx/pasta_curves", branch = "ec-gpu", features = ["gpu"] } typemap = "0.3.3" [target."cfg(target_arch = \"aarch64\")".dependencies] diff --git a/storage-proofs-update/src/circuit.rs b/storage-proofs-update/src/circuit.rs index 69ea23d77..26d3d850c 100644 --- a/storage-proofs-update/src/circuit.rs +++ b/storage-proofs-update/src/circuit.rs @@ -99,9 +99,9 @@ impl PublicInputs { #[derive(Clone)] pub struct ChallengeProof where - U: PoseidonArity, - V: PoseidonArity, - W: PoseidonArity, + U: PoseidonArity, + V: PoseidonArity, + W: PoseidonArity, { pub leaf_r_old: Option, pub path_r_old: Vec>>, @@ -114,9 +114,9 @@ where impl From> for ChallengeProof where - U: PoseidonArity, - V: PoseidonArity, - W: PoseidonArity, + U: PoseidonArity, + V: PoseidonArity, + W: PoseidonArity, { fn from(challenge_proof: vanilla::ChallengeProof) -> Self { let vanilla::ChallengeProof { @@ -130,9 +130,9 @@ where impl ChallengeProof where - U: PoseidonArity, - V: PoseidonArity, - W: PoseidonArity, + U: PoseidonArity, + V: PoseidonArity, + W: PoseidonArity, { pub fn from_merkle_proofs( proof_r_old: MerkleProof, U, V, W>, @@ -217,9 +217,9 @@ where #[derive(Clone)] pub struct PrivateInputs where - U: PoseidonArity, - V: PoseidonArity, - W: PoseidonArity, + U: PoseidonArity, + V: PoseidonArity, + W: PoseidonArity, { // CommC created by running SDR-PoRep on the old/un-updated data. pub comm_c: Option, @@ -238,9 +238,9 @@ where impl PrivateInputs where - U: PoseidonArity, - V: PoseidonArity, - W: PoseidonArity, + U: PoseidonArity, + V: PoseidonArity, + W: PoseidonArity, { pub fn new( comm_c: TreeRDomain, @@ -286,9 +286,9 @@ where pub struct EmptySectorUpdateCircuit where - U: PoseidonArity, - V: PoseidonArity, - W: PoseidonArity, + U: PoseidonArity, + V: PoseidonArity, + W: PoseidonArity, { pub pub_params: PublicParams, pub pub_inputs: PublicInputs, @@ -297,18 +297,18 @@ where impl CircuitComponent for EmptySectorUpdateCircuit where - U: PoseidonArity, - V: PoseidonArity, - W: PoseidonArity, + U: PoseidonArity, + V: PoseidonArity, + W: PoseidonArity, { type ComponentPrivateInputs = (); } impl EmptySectorUpdateCircuit where - U: PoseidonArity, - V: PoseidonArity, - W: PoseidonArity, + U: PoseidonArity, + V: PoseidonArity, + W: PoseidonArity, { pub fn blank(pub_params: PublicParams) -> Self { let sector_bytes = (pub_params.sector_nodes as u64) << 5; @@ -329,9 +329,9 @@ where impl Circuit for EmptySectorUpdateCircuit where - U: PoseidonArity, - V: PoseidonArity, - W: PoseidonArity, + U: PoseidonArity, + V: PoseidonArity, + W: PoseidonArity, { fn synthesize>(self, cs: &mut CS) -> Result<(), SynthesisError> { let EmptySectorUpdateCircuit { diff --git a/storage-proofs-update/src/compound.rs b/storage-proofs-update/src/compound.rs index 4ad92de03..19f9814a9 100644 --- a/storage-proofs-update/src/compound.rs +++ b/storage-proofs-update/src/compound.rs @@ -16,9 +16,9 @@ use crate::{ pub struct EmptySectorUpdateCompound where - U: PoseidonArity, - V: PoseidonArity, - W: PoseidonArity, + U: PoseidonArity, + V: PoseidonArity, + W: PoseidonArity, { pub _tree_r: PhantomData<(U, V, W)>, } @@ -26,9 +26,9 @@ where impl CacheableParameters, PublicParams> for EmptySectorUpdateCompound where - U: PoseidonArity, - V: PoseidonArity, - W: PoseidonArity, + U: PoseidonArity, + V: PoseidonArity, + W: PoseidonArity, { fn cache_prefix() -> String { format!("empty-sector-update-{}", TreeR::::display()) @@ -39,9 +39,9 @@ impl<'a, U, V, W> CompoundProof<'a, EmptySectorUpdate, EmptySectorUpdateCircuit> for EmptySectorUpdateCompound where - U: PoseidonArity, - V: PoseidonArity, - W: PoseidonArity, + U: PoseidonArity, + V: PoseidonArity, + W: PoseidonArity, { // Generates a partition circuit's public-inputs. If the `k` argument is `Some` we overwrite // `pub_inputs.k` with the `k` argument's value, otherwise if the `k` argument is `None` we use diff --git a/storage-proofs-update/src/constants.rs b/storage-proofs-update/src/constants.rs index 946ed1cdd..bbbdeb11d 100644 --- a/storage-proofs-update/src/constants.rs +++ b/storage-proofs-update/src/constants.rs @@ -2,9 +2,9 @@ use blstrs::Scalar as Fr; use filecoin_hashers::{ poseidon::{PoseidonDomain, PoseidonHasher}, sha256::{Sha256Domain, Sha256Hasher}, - FieldArity, PoseidonArity, + FieldArity, }; -use generic_array::typenum::{U0, U2, U8}; +use generic_array::typenum::{Unsigned, U0, U2, U8}; use lazy_static::lazy_static; use merkletree::store::DiskStore; use neptune::{ @@ -139,9 +139,11 @@ pub const fn apex_leaf_count(sector_nodes: usize) -> usize { pub fn validate_tree_r_shape(sector_nodes: usize) where - U: PoseidonArity, - V: PoseidonArity, - W: PoseidonArity, + // Use `Unsigned` rather than `PoseidonArity` because `PoseidonArity` requires adding an + // additional type parameter `F: PrimeField` to this function (which we don't need). + U: Unsigned, + V: Unsigned, + W: Unsigned, { let base_arity = U::to_usize(); let sub_arity = V::to_usize(); diff --git a/storage-proofs-update/src/poseidon/circuit.rs b/storage-proofs-update/src/poseidon/circuit.rs index 0e3e023be..985548c46 100644 --- a/storage-proofs-update/src/poseidon/circuit.rs +++ b/storage-proofs-update/src/poseidon/circuit.rs @@ -86,9 +86,9 @@ impl PublicInputs { #[derive(Clone)] pub struct ChallengeProof where - U: PoseidonArity, - V: PoseidonArity, - W: PoseidonArity, + U: PoseidonArity, + V: PoseidonArity, + W: PoseidonArity, { pub leaf_r_old: Option, pub path_r_old: Vec>>, @@ -101,9 +101,9 @@ where impl From> for ChallengeProof where - U: PoseidonArity, - V: PoseidonArity, - W: PoseidonArity, + U: PoseidonArity, + V: PoseidonArity, + W: PoseidonArity, { fn from(challenge_proof: vanilla::ChallengeProof) -> Self { let vanilla::ChallengeProof { @@ -117,9 +117,9 @@ where impl ChallengeProof where - U: PoseidonArity, - V: PoseidonArity, - W: PoseidonArity, + U: PoseidonArity, + V: PoseidonArity, + W: PoseidonArity, { pub fn from_merkle_proofs( proof_r_old: MerkleProof, U, V, W>, @@ -201,9 +201,9 @@ where #[derive(Clone)] pub struct PrivateInputs where - U: PoseidonArity, - V: PoseidonArity, - W: PoseidonArity, + U: PoseidonArity, + V: PoseidonArity, + W: PoseidonArity, { // CommC created by running SDR-PoRep on the old/un-updated data. pub comm_c: Option, @@ -219,9 +219,9 @@ where impl PrivateInputs where - U: PoseidonArity, - V: PoseidonArity, - W: PoseidonArity, + U: PoseidonArity, + V: PoseidonArity, + W: PoseidonArity, { pub fn new( comm_c: TreeRDomain, @@ -259,9 +259,9 @@ where pub struct EmptySectorUpdateCircuit where - U: PoseidonArity, - V: PoseidonArity, - W: PoseidonArity, + U: PoseidonArity, + V: PoseidonArity, + W: PoseidonArity, { pub pub_params: PublicParams, pub pub_inputs: PublicInputs, @@ -270,18 +270,18 @@ where impl CircuitComponent for EmptySectorUpdateCircuit where - U: PoseidonArity, - V: PoseidonArity, - W: PoseidonArity, + U: PoseidonArity, + V: PoseidonArity, + W: PoseidonArity, { type ComponentPrivateInputs = (); } impl EmptySectorUpdateCircuit where - U: PoseidonArity, - V: PoseidonArity, - W: PoseidonArity, + U: PoseidonArity, + V: PoseidonArity, + W: PoseidonArity, { pub fn blank(pub_params: PublicParams) -> Self { let sector_bytes = (pub_params.sector_nodes as u64) << 5; @@ -302,9 +302,9 @@ where impl Circuit for EmptySectorUpdateCircuit where - U: PoseidonArity, - V: PoseidonArity, - W: PoseidonArity, + U: PoseidonArity, + V: PoseidonArity, + W: PoseidonArity, { /// This circuit is NOT AUDITED, USE AT YOUR OWN RISK. fn synthesize>(self, cs: &mut CS) -> Result<(), SynthesisError> { diff --git a/storage-proofs-update/src/poseidon/compound.rs b/storage-proofs-update/src/poseidon/compound.rs index 02cfe8944..bf0c9825d 100644 --- a/storage-proofs-update/src/poseidon/compound.rs +++ b/storage-proofs-update/src/poseidon/compound.rs @@ -20,9 +20,9 @@ use crate::{ pub struct EmptySectorUpdateCompound where - U: PoseidonArity, - V: PoseidonArity, - W: PoseidonArity, + U: PoseidonArity, + V: PoseidonArity, + W: PoseidonArity, { pub _tree_r: PhantomData<(U, V, W)>, } @@ -30,9 +30,9 @@ where impl CacheableParameters, PublicParams> for EmptySectorUpdateCompound where - U: PoseidonArity, - V: PoseidonArity, - W: PoseidonArity, + U: PoseidonArity, + V: PoseidonArity, + W: PoseidonArity, { fn cache_prefix() -> String { format!( @@ -46,9 +46,9 @@ impl<'a, U, V, W> CompoundProof<'a, EmptySectorUpdate, EmptySectorUpdateCircuit> for EmptySectorUpdateCompound where - U: PoseidonArity, - V: PoseidonArity, - W: PoseidonArity, + U: PoseidonArity, + V: PoseidonArity, + W: PoseidonArity, { fn generate_public_inputs( pub_inputs: &PublicInputs, diff --git a/storage-proofs-update/src/poseidon/vanilla.rs b/storage-proofs-update/src/poseidon/vanilla.rs index 2944d342f..448b7c7b6 100644 --- a/storage-proofs-update/src/poseidon/vanilla.rs +++ b/storage-proofs-update/src/poseidon/vanilla.rs @@ -1,7 +1,7 @@ use std::marker::PhantomData; -use ff::PrimeFieldBits; -use filecoin_hashers::{Hasher, PoseidonArity}; +use ff::{PrimeField, PrimeFieldBits}; +use filecoin_hashers::{Domain, Hasher, PoseidonArity}; use serde::{Deserialize, Serialize}; use storage_proofs_core::{error::Result, merkle::MerkleProof, proof::ProofScheme}; @@ -30,10 +30,12 @@ pub struct PublicInputs { #[derive(Clone, Serialize, Deserialize)] pub struct ChallengeProof where + F: PrimeField, TreeRHasher: Hasher, - U: PoseidonArity, - V: PoseidonArity, - W: PoseidonArity, + as Hasher>::Domain: Domain, + U: PoseidonArity, + V: PoseidonArity, + W: PoseidonArity, { #[serde(bound(serialize = "MerkleProof, U, V, W>: Serialize"))] #[serde(bound(deserialize = "MerkleProof, U, V, W>: Deserialize<'de>"))] @@ -49,10 +51,12 @@ where #[derive(Clone, Serialize, Deserialize)] pub struct PartitionProof where + F: PrimeField, TreeRHasher: Hasher, - U: PoseidonArity, - V: PoseidonArity, - W: PoseidonArity, + as Hasher>::Domain: Domain, + U: PoseidonArity, + V: PoseidonArity, + W: PoseidonArity, { #[serde(bound(serialize = "TreeRDomain: Serialize"))] #[serde(bound(deserialize = "TreeRDomain: Deserialize<'de>"))] @@ -72,9 +76,10 @@ impl<'a, F, U, V, W> ProofScheme<'a> for EmptySectorUpdate where F: PrimeFieldBits, TreeRHasher: Hasher>, - U: PoseidonArity, - V: PoseidonArity, - W: PoseidonArity, + TreeRDomain: Domain, + U: PoseidonArity, + V: PoseidonArity, + W: PoseidonArity, { type SetupParams = SetupParams; type PublicParams = PublicParams; diff --git a/storage-proofs-update/src/vanilla.rs b/storage-proofs-update/src/vanilla.rs index d6d5b289d..b3b88ed44 100644 --- a/storage-proofs-update/src/vanilla.rs +++ b/storage-proofs-update/src/vanilla.rs @@ -170,11 +170,14 @@ pub struct PrivateInputs { #[derive(Clone, Serialize, Deserialize)] pub struct ChallengeProof where + F: PrimeField, TreeDHasher: Hasher, TreeRHasher: Hasher, - U: PoseidonArity, - V: PoseidonArity, - W: PoseidonArity, + as Hasher>::Domain: Domain, + as Hasher>::Domain: Domain, + U: PoseidonArity, + V: PoseidonArity, + W: PoseidonArity, { #[serde(bound(serialize = "MerkleProof, U, V, W>: Serialize"))] #[serde(bound(deserialize = "MerkleProof, U, V, W>: Deserialize<'de>"))] @@ -192,9 +195,11 @@ where F: PrimeField, TreeDHasher: Hasher>, TreeRHasher: Hasher>, - U: PoseidonArity, - V: PoseidonArity, - W: PoseidonArity, + TreeDDomain: Domain, + TreeRDomain: Domain, + U: PoseidonArity, + V: PoseidonArity, + W: PoseidonArity, { pub fn verify_merkle_proofs( &self, @@ -219,11 +224,14 @@ where #[derive(Clone, Serialize, Deserialize)] pub struct PartitionProof where + F: PrimeField, TreeDHasher: Hasher, TreeRHasher: Hasher, - U: PoseidonArity, - V: PoseidonArity, - W: PoseidonArity, + as Hasher>::Domain: Domain, + as Hasher>::Domain: Domain, + U: PoseidonArity, + V: PoseidonArity, + W: PoseidonArity, { #[serde(bound(serialize = "TreeRDomain: Serialize"))] #[serde(bound(deserialize = "TreeRDomain: Deserialize<'de>"))] @@ -248,9 +256,9 @@ where TreeRHasher: Hasher>, TreeDDomain: Domain, TreeRDomain: Domain, - U: PoseidonArity, - V: PoseidonArity, - W: PoseidonArity, + U: PoseidonArity, + V: PoseidonArity, + W: PoseidonArity, { type SetupParams = SetupParams; type PublicParams = PublicParams; @@ -554,9 +562,9 @@ where TreeRHasher: Hasher>, TreeDDomain: Domain, TreeRDomain: Domain, - U: PoseidonArity, - V: PoseidonArity, - W: PoseidonArity, + U: PoseidonArity, + V: PoseidonArity, + W: PoseidonArity, { pub fn instantiate_tree_d( tree_d_leafs: usize, diff --git a/storage-proofs-update/tests/circuit.rs b/storage-proofs-update/tests/circuit.rs index a697e857a..b75d1267b 100644 --- a/storage-proofs-update/tests/circuit.rs +++ b/storage-proofs-update/tests/circuit.rs @@ -62,9 +62,9 @@ fn get_apex_leafs(tree_d_new: &TreeD, k: usize) -> Vec { fn test_empty_sector_update_circuit(sector_nodes: usize, constraints_expected: usize) where - U: PoseidonArity, - V: PoseidonArity, - W: PoseidonArity, + U: PoseidonArity, + V: PoseidonArity, + W: PoseidonArity, { validate_tree_r_shape::(sector_nodes); diff --git a/storage-proofs-update/tests/circuit_poseidon.rs b/storage-proofs-update/tests/circuit_poseidon.rs index 287c7f5be..0ce27ee29 100644 --- a/storage-proofs-update/tests/circuit_poseidon.rs +++ b/storage-proofs-update/tests/circuit_poseidon.rs @@ -30,9 +30,9 @@ type TreeRHasher = constants::TreeRHasher; fn test_empty_sector_update_circuit(sector_nodes: usize, constraints_expected: usize) where - U: PoseidonArity, - V: PoseidonArity, - W: PoseidonArity, + U: PoseidonArity, + V: PoseidonArity, + W: PoseidonArity, { validate_tree_r_shape::(sector_nodes); diff --git a/storage-proofs-update/tests/common/mod.rs b/storage-proofs-update/tests/common/mod.rs index aafcc3e8d..3430ab8c4 100644 --- a/storage-proofs-update/tests/common/mod.rs +++ b/storage-proofs-update/tests/common/mod.rs @@ -23,6 +23,7 @@ pub fn create_tree_d_new(labels_d_new: &[TreeDDomain], tmp_dir: &Path) -> where F: PrimeField, TreeDHasher: Hasher>, + TreeDDomain: Domain, { let num_leafs = labels_d_new.len(); let arity = TreeDArity::to_usize(); @@ -47,9 +48,9 @@ where F: PrimeField, TreeRHasher: Hasher>, TreeRDomain: Domain, - U: PoseidonArity, - V: PoseidonArity, - W: PoseidonArity, + U: PoseidonArity, + V: PoseidonArity, + W: PoseidonArity, { create_tree_r(labels_d_new, tmp_dir, "tree-d-new", "data_new") } @@ -63,9 +64,9 @@ where F: PrimeField, TreeRHasher: Hasher>, TreeRDomain: Domain, - U: PoseidonArity, - V: PoseidonArity, - W: PoseidonArity, + U: PoseidonArity, + V: PoseidonArity, + W: PoseidonArity, { create_tree_r(replica_old, tmp_dir, "tree-r-old", "replica_old") } @@ -79,9 +80,9 @@ where F: PrimeField, TreeRHasher: Hasher>, TreeRDomain: Domain, - U: PoseidonArity, - V: PoseidonArity, - W: PoseidonArity, + U: PoseidonArity, + V: PoseidonArity, + W: PoseidonArity, { create_tree_r(replica_new, tmp_dir, "tree-r-new", "replica_new") } @@ -96,9 +97,9 @@ where F: PrimeField, TreeRHasher: Hasher>, TreeRDomain: Domain, - U: PoseidonArity, - V: PoseidonArity, - W: PoseidonArity, + U: PoseidonArity, + V: PoseidonArity, + W: PoseidonArity, { let base_arity = U::to_usize(); diff --git a/storage-proofs-update/tests/compound.rs b/storage-proofs-update/tests/compound.rs index 6042feddf..ac7fa9145 100644 --- a/storage-proofs-update/tests/compound.rs +++ b/storage-proofs-update/tests/compound.rs @@ -37,9 +37,9 @@ type TreeRHasher = constants::TreeRHasher; fn test_empty_sector_update_compound(sector_nodes: usize) where - U: PoseidonArity, - V: PoseidonArity, - W: PoseidonArity, + U: PoseidonArity, + V: PoseidonArity, + W: PoseidonArity, { validate_tree_r_shape::(sector_nodes);