From dbb0193e125b5f25d7bb924275612f7166c1321e Mon Sep 17 00:00:00 2001 From: nemo Date: Fri, 5 Mar 2021 16:38:20 -0500 Subject: [PATCH] feat: split up pc1/pc2 in the window post bench docs: add usage notes in the readme Resolves #1399 --- README.md | 31 +++++++++++++++++++ fil-proofs-tooling/src/bin/benchy/main.rs | 12 ++----- .../src/stacked/vanilla/proof.rs | 27 ++++++++++++++++ 3 files changed, 60 insertions(+), 10 deletions(-) diff --git a/README.md b/README.md index 9e5638608..842ab15f6 100644 --- a/README.md +++ b/README.md @@ -80,6 +80,37 @@ The main benchmarking tool is called `benchy`. `benchy` has several subcommands > cargo run --release --bin benchy -- prodbench ``` +### Window PoSt Bench usages + +The Window PoSt bench can be used a number of ways, some of which are detailed here. + +First, you can run the benchmark and preserve the working directory like this: +``` +cargo run --release --bin benchy -- window-post --size 2KiB --cache window-post-2KiB-dir --preserve-cache +``` + +Then if you want to run the benchmark again to test commit-phase2, you can quickly run it like this: +``` +cargo run --release --bin benchy -- window-post --size 2KiB --skip-precommit-phase1 --skip-precommit-phase2 --skip-commit-phase1 --cache window-post-2KiB-dir +``` + +Alternatively, if you want to test just GPU tree building, you can run it like this: +``` +cargo run --release --bin benchy -- window-post --size 2KiB --skip-precommit-phase1 --skip-commit-phase1 --skip-commit-phase2 --cache window-post-2KiB-dir +``` + +Note that some combinations of arguments will cause destructive changes to your cached directory. For larger benchmark sector sizes, it is recommended that once you create an initial cache, that it be saved to an alternate location in the case that it is corrupted by a test run. For example, the following run sequence will be guaranteed to corrupt your cache: + +``` +# Do NOT run this sequence. For illustrative purposes only: +# Generate clean cache +cargo run --release --bin benchy -- window-post --size 2KiB --cache window-post-2KiB-dir --preserve-cache +# Skip all stages except the first +cargo run --release --bin benchy -- window-post --size 2KiB --skip-precommit-phase2 --skip-commit-phase1 --skip-commit-phase2 --cache broken-cache-dir +``` + +The reason this fails is because new random piece data is generated (rather than loaded from disk from a previous run) in the first step, and then we attempt to use it in later sealing steps using data from previously preserved run. This cannot work. + There is also a bench called `gpu-cpu-test`: ``` diff --git a/fil-proofs-tooling/src/bin/benchy/main.rs b/fil-proofs-tooling/src/bin/benchy/main.rs index f087db9b2..d2e5352bb 100644 --- a/fil-proofs-tooling/src/bin/benchy/main.rs +++ b/fil-proofs-tooling/src/bin/benchy/main.rs @@ -30,7 +30,6 @@ fn main() -> Result<()> { .help("Preserve the directory where cached files are persisted") .takes_value(false), ) - /* .arg( Arg::with_name("skip-precommit-phase1") .long("skip-precommit-phase1") @@ -44,13 +43,6 @@ fn main() -> Result<()> { .required(false) .help("Skip precommit phase 2") .takes_value(false), - )*/ - .arg( - Arg::with_name("skip-precommit") - .long("skip-precommit") - .required(false) - .help("Skip precommit phase 1 & 2") - .takes_value(false), ) .arg( Arg::with_name("skip-commit-phase1") @@ -191,8 +183,8 @@ fn main() -> Result<()> { ("window-post", Some(m)) => { let preserve_cache = m.is_present("preserve-cache"); // For now these options are combined. - let skip_precommit_phase1 = m.is_present("skip-precommit"); - let skip_precommit_phase2 = m.is_present("skip-precommit"); + let skip_precommit_phase1 = m.is_present("skip-precommit-phase1"); + let skip_precommit_phase2 = m.is_present("skip-precommit-phase2"); let skip_commit_phase1 = m.is_present("skip-commit-phase1"); let skip_commit_phase2 = m.is_present("skip-commit-phase2"); let test_resume = m.is_present("test-resume"); diff --git a/storage-proofs-porep/src/stacked/vanilla/proof.rs b/storage-proofs-porep/src/stacked/vanilla/proof.rs index 3822b923a..9f793b420 100644 --- a/storage-proofs-porep/src/stacked/vanilla/proof.rs +++ b/storage-proofs-porep/src/stacked/vanilla/proof.rs @@ -671,6 +671,18 @@ impl<'a, Tree: 'static + MerkleTreeTrait, G: 'static + Hasher> StackedDrg<'a, Tr assert_eq!(tree_len, config.size.expect("config size failure")); // Persist the base and tree data to disk based using the current store config. + let tree_c_store_path = StoreConfig::data_path(&config.path, &config.id); + let tree_c_store_exists = Path::new(&tree_c_store_path).exists(); + trace!( + "tree_c store path {:?} -- exists? {}", + tree_c_store_path, + tree_c_store_exists + ); + if tree_c_store_exists { + std::fs::remove_file(&tree_c_store_path) + .expect("failed to remove tree_c_store_path"); + } + let tree_c_store = DiskStore::<::Domain>::new_with_config( tree_len, @@ -790,6 +802,7 @@ impl<'a, Tree: 'static + MerkleTreeTrait, G: 'static + Hasher> StackedDrg<'a, Tr } assert_eq!(tree_count, trees.len()); + create_disk_tree::< DiskTree, >(configs[0].size.expect("config size failure"), &configs) @@ -1135,6 +1148,20 @@ impl<'a, Tree: 'static + MerkleTreeTrait, G: 'static + Hasher> StackedDrg<'a, Tr i + 1, tree_count ); + + // Remove the tree_r_last store if it exists already + let tree_r_last_store_path = StoreConfig::data_path(&config.path, &config.id); + let tree_r_last_store_exists = Path::new(&tree_r_last_store_path).exists(); + trace!( + "tree_r_last store path {:?} -- exists? {}", + tree_r_last_store_path, + tree_r_last_store_exists + ); + if tree_r_last_store_exists { + std::fs::remove_file(&tree_r_last_store_path) + .expect("failed to remove tree_r_last_store_path"); + } + LCTree::::from_par_iter_with_config( encoded_data, config.clone(),