diff --git a/.circleci/config.yml b/.circleci/config.yml index 03d089f1dd..7e923ded01 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -187,10 +187,12 @@ jobs: ulimit -n 20000 ulimit -u 20000 ulimit -n 20000 - cargo +<< pipeline.parameters.nightly-toolchain >> test --all --verbose --release << parameters.cargo-args >> -- --nocapture << parameters.test-args >> - cargo +<< pipeline.parameters.nightly-toolchain >> test test_seal_lifecycle_upgrade --verbose --release << parameters.cargo-args >> -- --nocapture --ignored + # cargo +<< pipeline.parameters.nightly-toolchain >> test --all --verbose --release << parameters.cargo-args >> -- --nocapture << parameters.test-args >> + cargo +<< pipeline.parameters.nightly-toolchain >> test --all --verbose --release test_stacked_porep_prove_verify << parameters.cargo-args >> -- --nocapture << parameters.test-args >> + # cargo +<< pipeline.parameters.nightly-toolchain >> test test_seal_lifecycle_upgrade --verbose --release << parameters.cargo-args >> -- --nocapture --ignored no_output_timeout: 30m environment: + RUST_BACKTRACE: 1 FIL_PROOFS_USE_GPU_COLUMN_BUILDER: true FIL_PROOFS_USE_GPU_TREE_BUILDER: true BELLMAN_CUDA_NVCC_ARGS: --fatbin --gpu-architecture=sm_75 --generate-code=arch=compute_75,code=sm_75 @@ -371,58 +373,58 @@ workflows: jobs: - ensure_groth_parameters_and_keys_linux - cargo_fetch - - rustfmt: - requires: - - cargo_fetch - - clippy: - requires: - - cargo_fetch - - - test_release: - requires: - - cargo_fetch - - ensure_groth_parameters_and_keys_linux - - - test_ignored_release: - name: test_ignored_release_storage_proofs_post - crate: "storage-proofs-post" - requires: - - cargo_fetch - - ensure_groth_parameters_and_keys_linux - - - test_ignored_release: - name: test_ignored_release_storage_proofs_core - crate: "storage-proofs-core" - requires: - - cargo_fetch - - ensure_groth_parameters_and_keys_linux - - - test_ignored_release: - name: test_ignored_release_storage_proofs_porep - crate: "storage-proofs-porep" - requires: - - cargo_fetch - - ensure_groth_parameters_and_keys_linux - - - test_ignored_release: - name: test_ignored_release_filecoin_proofs - crate: "filecoin-proofs" - requires: - - cargo_fetch - - ensure_groth_parameters_and_keys_linux - - - test_gpu_tree_building: - name: test_gpu_tree_building_opencl (regular) - requires: - - cargo_fetch - - ensure_groth_parameters_and_keys_linux - - - test_gpu_tree_building: - name: test_gpu_tree_building_opencl (ignored) - test-args: "--ignored --skip test_seal_lifecycle_upgrade" - requires: - - cargo_fetch - - ensure_groth_parameters_and_keys_linux +# - rustfmt: +# requires: +# - cargo_fetch +# - clippy: +# requires: +# - cargo_fetch +# +# - test_release: +# requires: +# - cargo_fetch +# - ensure_groth_parameters_and_keys_linux +# +# - test_ignored_release: +# name: test_ignored_release_storage_proofs_post +# crate: "storage-proofs-post" +# requires: +# - cargo_fetch +# - ensure_groth_parameters_and_keys_linux +# +# - test_ignored_release: +# name: test_ignored_release_storage_proofs_core +# crate: "storage-proofs-core" +# requires: +# - cargo_fetch +# - ensure_groth_parameters_and_keys_linux +# +# - test_ignored_release: +# name: test_ignored_release_storage_proofs_porep +# crate: "storage-proofs-porep" +# requires: +# - cargo_fetch +# - ensure_groth_parameters_and_keys_linux +# +# - test_ignored_release: +# name: test_ignored_release_filecoin_proofs +# crate: "filecoin-proofs" +# requires: +# - cargo_fetch +# - ensure_groth_parameters_and_keys_linux +# +# - test_gpu_tree_building: +# name: test_gpu_tree_building_opencl (regular) +# requires: +# - cargo_fetch +# - ensure_groth_parameters_and_keys_linux +# +# - test_gpu_tree_building: +# name: test_gpu_tree_building_opencl (ignored) +# test-args: "--ignored --skip test_seal_lifecycle_upgrade" +# requires: +# - cargo_fetch +# - ensure_groth_parameters_and_keys_linux - test_gpu_tree_building: name: test_gpu_tree_building_cuda (regular) @@ -431,96 +433,96 @@ workflows: - cargo_fetch - ensure_groth_parameters_and_keys_linux - - test_gpu_tree_building: - name: test_gpu_tree_building_cuda (ignored) - cargo-args: "--features cuda" - test-args: "--ignored" - requires: - - cargo_fetch - - ensure_groth_parameters_and_keys_linux - - - test_multicore_sdr: - requires: - - cargo_fetch - - ensure_groth_parameters_and_keys_linux - - - test: - name: test_filecoin_proofs - crate: "filecoin-proofs" - requires: - - cargo_fetch - - ensure_groth_parameters_and_keys_linux - - - test: - name: test_storage_proofs_core - crate: "storage-proofs-core" - requires: - - cargo_fetch - - ensure_groth_parameters_and_keys_linux - - - test: - name: test_storage_proofs_post - crate: "storage-proofs-post" - requires: - - cargo_fetch - - ensure_groth_parameters_and_keys_linux - - - test: - name: test_storage_proofs_porep - crate: "storage-proofs-porep" - requires: - - cargo_fetch - - ensure_groth_parameters_and_keys_linux - - - test: - name: test_fil_proofs_tooling - crate: "fil-proofs-tooling" - requires: - - cargo_fetch - - ensure_groth_parameters_and_keys_linux - - - test: - name: test_sha2raw - crate: "sha2raw" - requires: - - cargo_fetch - - ensure_groth_parameters_and_keys_linux - - - test: - name: test_filecoin_hashers - crate: "filecoin-hashers" - requires: - - cargo_fetch - - ensure_groth_parameters_and_keys_linux - - - - test: - name: test_fil_proofs_param - crate: "fil-proofs-param" - requires: - - cargo_fetch - - ensure_groth_parameters_and_keys_linux - - - test_no_gpu: - name: test_no_gpu - requires: - - cargo_fetch - - ensure_groth_parameters_and_keys_linux - - - test_arm_no_gpu: - name: test_arm_no_gpu - requires: - - cargo_fetch - - ensure_groth_parameters_and_keys_linux - - - bench: - requires: - - cargo_fetch - - ensure_groth_parameters_and_keys_linux - - test_darwin - - - test: - name: test_fr32 - crate: "fr32" - requires: - - cargo_fetch +# - test_gpu_tree_building: +# name: test_gpu_tree_building_cuda (ignored) +# cargo-args: "--features cuda" +# test-args: "--ignored" +# requires: +# - cargo_fetch +# - ensure_groth_parameters_and_keys_linux +# +# - test_multicore_sdr: +# requires: +# - cargo_fetch +# - ensure_groth_parameters_and_keys_linux +# +# - test: +# name: test_filecoin_proofs +# crate: "filecoin-proofs" +# requires: +# - cargo_fetch +# - ensure_groth_parameters_and_keys_linux +# +# - test: +# name: test_storage_proofs_core +# crate: "storage-proofs-core" +# requires: +# - cargo_fetch +# - ensure_groth_parameters_and_keys_linux +# +# - test: +# name: test_storage_proofs_post +# crate: "storage-proofs-post" +# requires: +# - cargo_fetch +# - ensure_groth_parameters_and_keys_linux +# +# - test: +# name: test_storage_proofs_porep +# crate: "storage-proofs-porep" +# requires: +# - cargo_fetch +# - ensure_groth_parameters_and_keys_linux +# +# - test: +# name: test_fil_proofs_tooling +# crate: "fil-proofs-tooling" +# requires: +# - cargo_fetch +# - ensure_groth_parameters_and_keys_linux +# +# - test: +# name: test_sha2raw +# crate: "sha2raw" +# requires: +# - cargo_fetch +# - ensure_groth_parameters_and_keys_linux +# +# - test: +# name: test_filecoin_hashers +# crate: "filecoin-hashers" +# requires: +# - cargo_fetch +# - ensure_groth_parameters_and_keys_linux +# +# +# - test: +# name: test_fil_proofs_param +# crate: "fil-proofs-param" +# requires: +# - cargo_fetch +# - ensure_groth_parameters_and_keys_linux +# +# - test_no_gpu: +# name: test_no_gpu +# requires: +# - cargo_fetch +# - ensure_groth_parameters_and_keys_linux +# +# - test_arm_no_gpu: +# name: test_arm_no_gpu +# requires: +# - cargo_fetch +# - ensure_groth_parameters_and_keys_linux +# +# - bench: +# requires: +# - cargo_fetch +# - ensure_groth_parameters_and_keys_linux +# - test_darwin +# +# - test: +# name: test_fr32 +# crate: "fr32" +# requires: +# - cargo_fetch diff --git a/storage-proofs-core/src/merkle/proof.rs b/storage-proofs-core/src/merkle/proof.rs index f3cb13d09a..38149a6395 100644 --- a/storage-proofs-core/src/merkle/proof.rs +++ b/storage-proofs-core/src/merkle/proof.rs @@ -93,9 +93,13 @@ pub trait MerkleProofTrait: Clone + Serialize + DeserializeOwned + Debug + Sync fn validate_data(&self, data: ::Domain) -> bool { if !self.verify() { + println!("\n\tMerkleProof.verify() failure"); return false; } + if self.leaf() != data { + println!("\n\tMerkleProof.verify() failure => self.leaf() != data"); + } self.leaf() == data } diff --git a/storage-proofs-porep/src/stacked/vanilla/column_proof.rs b/storage-proofs-porep/src/stacked/vanilla/column_proof.rs index 184a50e715..06912ae302 100644 --- a/storage-proofs-porep/src/stacked/vanilla/column_proof.rs +++ b/storage-proofs-porep/src/stacked/vanilla/column_proof.rs @@ -48,11 +48,16 @@ impl ColumnProof { challenge: u32, expected_root: &::Domain, ) -> bool { + println!("\n\t ColumnProof.verify => 1"); let c_i = self.column_hash(); + println!("\n\t ColumnProof.verify => 2"); check_eq!(&self.inclusion_proof.root(), expected_root); + println!("\n\t ColumnProof.verify => 3"); check!(self.inclusion_proof.validate_data(c_i.into())); + println!("\n\t ColumnProof.verify => 4"); check!(self.inclusion_proof.validate(challenge as usize)); + println!("\n\t ColumnProof.verify => 5"); true } diff --git a/storage-proofs-porep/src/stacked/vanilla/params.rs b/storage-proofs-porep/src/stacked/vanilla/params.rs index 3fc571c52f..0b5bdadfd1 100644 --- a/storage-proofs-porep/src/stacked/vanilla/params.rs +++ b/storage-proofs-porep/src/stacked/vanilla/params.rs @@ -232,6 +232,8 @@ where ) -> bool { let replica_id = &pub_inputs.replica_id; + println!("\n\tSelf::Proof.verify() => 1"); + check!(challenge < graph.size()); check!(pub_inputs.tau.is_some()); @@ -241,8 +243,13 @@ where check!(self.comm_d_proofs.proves_challenge(challenge)); if let Some(ref tau) = pub_inputs.tau { + // TODO (jake): remove + if self.comm_d_proofs.root() != tau.comm_d { + println!("\n\tSelf::Proof.verify() failure => mismatched comm_d's\n"); + } check_eq!(&self.comm_d_proofs.root(), &tau.comm_d); } else { + println!("\n\tSelf::Proof.verify() failure => tau is None\n"); return false; } @@ -252,19 +259,24 @@ where graph .parents(challenge, &mut parents) .expect("graph parents failure"); // FIXME: error handling + println!("\n\tSelf::Proof.verify() => verifying replica_column_proofs"); check!(self.replica_column_proofs.verify(challenge, &parents)); + println!("\n\tSelf::Proof.verify() => verifying final replica layer"); check!(self.verify_final_replica_layer(challenge)); + println!("\n\tSelf::Proof.verify() => verifying labels"); check!(self.verify_labels(replica_id, &pub_params.layer_challenges)); trace!("verify encoding"); + println!("\n\tSelf::Proof.verify() => verifying encoding proof"); check!(self.encoding_proof.verify( replica_id, &self.comm_r_last_proof.leaf(), &self.comm_d_proofs.leaf() )); + println!("\n\tSelf::Proof.verify() => finished"); true } @@ -325,16 +337,21 @@ pub struct ReplicaColumnProof { impl ReplicaColumnProof { pub fn verify(&self, challenge: usize, parents: &[u32]) -> bool { + println!("\n\t ReplicaColumnProof.verify => 1"); let expected_comm_c = self.c_x.root(); + println!("\n\t ReplicaColumnProof.verify => verifying column proof"); trace!(" verify c_x"); check!(self.c_x.verify(challenge as u32, &expected_comm_c)); + println!("\n\t ReplicaColumnProof.verify => verifying drg-parent column proofs"); trace!(" verify drg_parents"); for (proof, parent) in self.drg_parents.iter().zip(parents.iter()) { check!(proof.verify(*parent, &expected_comm_c)); + println!("\n\t ReplicaColumnProof.verify => verified drg-parent column proof"); } + println!("\n\t ReplicaColumnProof.verify => verifying exp-parent column proofs"); trace!(" verify exp_parents"); for (proof, parent) in self .exp_parents @@ -342,6 +359,7 @@ impl ReplicaColumnProof { .zip(parents.iter().skip(self.drg_parents.len())) { check!(proof.verify(*parent, &expected_comm_c)); + println!("\n\t ReplicaColumnProof.verify => verified exp-parent column proof"); } true diff --git a/storage-proofs-porep/src/stacked/vanilla/proof_scheme.rs b/storage-proofs-porep/src/stacked/vanilla/proof_scheme.rs index f1c724ce14..8b8fe2e5ef 100644 --- a/storage-proofs-porep/src/stacked/vanilla/proof_scheme.rs +++ b/storage-proofs-porep/src/stacked/vanilla/proof_scheme.rs @@ -89,10 +89,13 @@ where let expected_comm_r = if let Some(ref tau) = pub_inputs.tau { &tau.comm_r } else { + println!("\n\tverify_all_partitions failure => tau.is_none()\n"); return Ok(false); }; - let res = partition_proofs.par_iter().enumerate().all(|(k, proofs)| { + // let res = partition_proofs.par_iter().enumerate().all(|(k, proofs)| { + let res = partition_proofs.iter().enumerate().all(|(k, proofs)| { + println!("\n\tverify_all_partitions => partition={}\n", k); trace!( "verifying partition proof {}/{}", k + 1, @@ -107,13 +110,16 @@ where }; if expected_comm_r != &actual_comm_r { + println!("\n\tverify_all_partitions failure => expected_comm_r != actual_comm_r\n"); return false; } let challenges = pub_inputs.challenges(&pub_params.layer_challenges, graph.size(), Some(k)); - proofs.par_iter().enumerate().all(|(i, proof)| { + // proofs.par_iter().enumerate().all(|(i, proof)| { + proofs.iter().enumerate().all(|(i, proof)| { + println!("\n\tverify_all_partitions => challenge proof (i={})\n", i); trace!("verify challenge {}/{}", i + 1, challenges.len()); // Validate for this challenge @@ -121,14 +127,21 @@ where // make sure all proofs have the same comm_c if proof.comm_c() != proofs[0].comm_c() { + println!("\n\tverify_all_partitions => proof.comm_c != proofs[0].comm_c\n"); return false; } // make sure all proofs have the same comm_r_last if proof.comm_r_last() != proofs[0].comm_r_last() { + println!("\n\tverify_all_partitions => proof.comm_r_last != proofs[0].comm_r_last\n"); return false; } - proof.verify(pub_params, pub_inputs, challenge, graph) + // proof.verify(pub_params, pub_inputs, challenge, graph) + let is_valid = proof.verify(pub_params, pub_inputs, challenge, graph); + if !is_valid { + println!("\n\tverify_all_partitions failure => challenge proof invalid\n"); + } + is_valid }) }); diff --git a/storage-proofs-porep/tests/stacked_vanilla.rs b/storage-proofs-porep/tests/stacked_vanilla.rs index be17ac849d..8ff5613a80 100644 --- a/storage-proofs-porep/tests/stacked_vanilla.rs +++ b/storage-proofs-porep/tests/stacked_vanilla.rs @@ -327,6 +327,8 @@ table_tests! { fn test_prove_verify_fixed(n: usize) { let challenges = LayerChallenges::new(DEFAULT_STACKED_LAYERS, 5); + /* + println!("\n\t => Fr"); test_prove_verify::, U8, U0, U0>>(n, challenges.clone()); test_prove_verify::, U8, U2, U0>>(n, challenges.clone()); test_prove_verify::, U8, U8, U2>>(n, challenges.clone()); @@ -350,28 +352,37 @@ fn test_prove_verify_fixed(n: usize) { test_prove_verify::, U8, U0, U0>>(n, challenges.clone()); test_prove_verify::, U8, U2, U0>>(n, challenges.clone()); test_prove_verify::, U8, U8, U2>>(n, challenges.clone()); + */ // Alternate Pasta fields rather than run each test for both fields. + /* + println!("\n => Sha256 Fp"); test_prove_verify::, U8, U0, U0>>(n, challenges.clone()); test_prove_verify::, U8, U2, U0>>(n, challenges.clone()); test_prove_verify::, U8, U8, U2>>(n, challenges.clone()); + println!("\n => Sha256 Fq"); test_prove_verify::, U4, U0, U0>>(n, challenges.clone()); test_prove_verify::, U4, U2, U0>>(n, challenges.clone()); test_prove_verify::, U4, U8, U2>>(n, challenges.clone()); + println!("\n => Blake2s Fp"); test_prove_verify::, U8, U0, U0>>(n, challenges.clone()); test_prove_verify::, U8, U2, U0>>(n, challenges.clone()); test_prove_verify::, U8, U8, U2>>(n, challenges.clone()); + println!("\n => Blake2s Fq"); test_prove_verify::, U4, U0, U0>>(n, challenges.clone()); test_prove_verify::, U4, U2, U0>>(n, challenges.clone()); test_prove_verify::, U4, U8, U2>>(n, challenges.clone()); + */ + println!("\n => Poseidon Fp"); test_prove_verify::, U8, U0, U0>>(n, challenges.clone()); test_prove_verify::, U8, U2, U0>>(n, challenges.clone()); test_prove_verify::, U8, U8, U2>>(n, challenges.clone()); + println!("\n => Poseidon Fq"); test_prove_verify::, U4, U0, U0>>(n, challenges.clone()); test_prove_verify::, U4, U2, U0>>(n, challenges.clone()); test_prove_verify::, U4, U8, U2>>(n, challenges); @@ -468,13 +479,19 @@ where let proofs_are_valid = StackedDrg::verify_all_partitions(&pp, &pub_inputs, all_partition_proofs) - .expect("failed to verify partition proofs"); + // .expect("failed to verify partition proofs"); + .unwrap_or_else(|err| panic!("failed to verify partition proofs:\n{:?}", err)); + + if !proofs_are_valid { + println!("\ntest_prove_verify failure => proofs are invalid, Tree={}\n", Tree::display()); + } // Discard cached MTs that are no longer needed. TemporaryAux::clear_temp(t_aux_orig).expect("t_aux delete failed"); assert!(proofs_are_valid); + cache_dir.close().expect("Failed to remove cache dir"); }