feat: add fallback option if no t_aux file exists

The only information from the `t_aux` file that is used during Commitphase 1
(C1) is the number of rows that are discarded from TreeC. If that value is
either the default, or set via `FIL_PROOFS_ROWS_TO_DISCARD` accordingly, then
all other parts of `t_aux` can be inferred from the function call.

When `FIL_PROOFS_T_AUX_IMPLICIT_FALLBACK=1` is set, then those inferred values
will be used, in case the `t_aux` file cannot be read or doesn't exist.

README.md

@@ -210,6 +210,12 @@ FIL_PROOFS_VERIFY_PRODUCTION_PARAMS=1
 
 By default, this verification is disabled.
 
+```
+FIL_PROOFS_T_AUX_IMPLICIT_FALLBACK=1
+```
+
+By default, the `t_aux` file must exist after a successful Precommit Phase 2. With enabling this setting, the system will use a `t_aux` based on the default values and passed in parameters, if the `t_aux` file cannot be read.
+
 ## Optimizing for either speed or memory during replication
 
 While replicating and generating the Merkle Trees (MT) for the proof at the same time there will always be a time-memory trade-off to consider, we present here strategies to optimize one at the cost of the other.

filecoin-proofs/src/api/seal.rs

@@ -26,6 +26,7 @@ use storage_proofs_core::{
     parameter_cache::SRS_MAX_PROOFS_TO_AGGREGATE,
     proof::ProofScheme,
     sector::SectorId,
+    settings::SETTINGS,
     util::{default_rows_to_discard, NODE_SIZE},
     Data,
 };
@@ -444,16 +445,35 @@ pub fn seal_commit_phase1_inner<T: AsRef<Path>, Tree: 'static + MerkleTreeTrait>
         deserialize(&p_aux_bytes)
     }?;
 
+    let vanilla_params = setup_params(porep_config)?;
+
     let t_aux = {
         let t_aux_path = cache_path.as_ref().join(CacheKey::TAux.to_string());
-        let t_aux_bytes = fs::read(&t_aux_path)
-            .with_context(|| format!("could not read file t_aux={:?}", t_aux_path))?;
-
-        let mut res: TemporaryAux<_, _> = deserialize(&t_aux_bytes)?;
-
-        // Switch t_aux to the passed in cache_path
-        res.set_cache_path(cache_path);
-        res
+        match fs::read(&t_aux_path) {
+            Ok(t_aux_bytes) => {
+                let mut res: TemporaryAux<_, _> = deserialize(&t_aux_bytes)?;
+
+                // Switch t_aux to the passed in cache_path
+                res.set_cache_path(cache_path);
+                res
+            }
+            Err(error) => {
+                if SETTINGS.t_aux_implicit_fallback {
+                    info!(
+                        "Could not read file t_aux={:?}. Fallback to default values instead.",
+                        t_aux_path
+                    );
+                    TemporaryAux::new(
+                        vanilla_params.nodes,
+                        vanilla_params.layer_challenges.layers(),
+                        cache_path.as_ref().to_path_buf(),
+                    )
+                } else {
+                    return Err(error)
+                        .with_context(|| format!("could not read file t_aux={:?}", t_aux_path));
+                }
+            }
+        }
     };
 
     // Convert TemporaryAux to TemporaryAuxCache, which instantiates all
@@ -489,7 +509,7 @@ pub fn seal_commit_phase1_inner<T: AsRef<Path>, Tree: 'static + MerkleTreeTrait>
     };
 
     let compound_setup_params = compound_proof::SetupParams {
-        vanilla_params: setup_params(porep_config)?,
+        vanilla_params,
         partitions: Some(usize::from(porep_config.partitions)),
         priority: false,
     };

storage-proofs-core/src/settings.rs

@@ -30,6 +30,7 @@ pub struct Settings {
     pub multicore_sdr_producers: usize,
     pub multicore_sdr_producer_stride: u64,
     pub multicore_sdr_lookahead: usize,
+    pub t_aux_implicit_fallback: bool,
 }
 
 impl Default for Settings {
@@ -54,6 +55,7 @@ impl Default for Settings {
             multicore_sdr_producers: 3,
             multicore_sdr_producer_stride: 128,
             multicore_sdr_lookahead: 800,
+            t_aux_implicit_fallback: false,
         }
     }
 }

storage-proofs-porep/src/stacked/vanilla/params.rs

@@ -11,13 +11,14 @@ use fr32::bytes_into_fr_repr_safe;
 use generic_array::typenum::{Unsigned, U2};
 use log::trace;
 use merkletree::{
-    merkle::get_merkle_tree_leafs,
+    merkle::{get_merkle_tree_leafs, get_merkle_tree_len},
     store::{DiskStore, Store, StoreConfig},
 };
 use serde::{Deserialize, Serialize};
 use sha2::{Digest, Sha256};
 use storage_proofs_core::{
     api_version::{ApiFeature, ApiVersion},
+    cache_key::CacheKey,
     drgraph::{Graph, BASE_DEGREE},
     error::Result,
     merkle::{
@@ -26,7 +27,7 @@ use storage_proofs_core::{
         MerkleProofTrait, MerkleTreeTrait,
     },
     parameter_cache::ParameterSetMetadata,
-    util::{data_at_node, NODE_SIZE},
+    util::{data_at_node, default_rows_to_discard, NODE_SIZE},
 };
 
 use crate::stacked::vanilla::{
@@ -838,6 +839,60 @@ impl<Tree: MerkleTreeTrait, G: Hasher> Clone for TemporaryAux<Tree, G> {
 }
 
 impl<Tree: MerkleTreeTrait, G: Hasher> TemporaryAux<Tree, G> {
+    pub fn new(sector_nodes: usize, num_layers: usize, cache_path: PathBuf) -> Self {
+        let labels = (1..=num_layers)
+            .map(|layer| StoreConfig {
+                path: cache_path.clone(),
+                id: CacheKey::label_layer(layer),
+                size: Some(sector_nodes),
+                // The TreeC doesn't discard any rows, hence it can be set to 0.
+                rows_to_discard: 0,
+            })
+            .collect();
+
+        let tree_d_size = get_merkle_tree_len(sector_nodes, BINARY_ARITY)
+            .expect("Tree must have enough leaves and have an arity of power of two");
+        let tree_d_config = StoreConfig {
+            path: cache_path.clone(),
+            id: CacheKey::CommDTree.to_string(),
+            size: Some(tree_d_size),
+            // The TreeD doesn't discard any rows, hence it can be set to 0.
+            rows_to_discard: 0,
+        };
+
+        let tree_size = get_merkle_tree_len(sector_nodes, Tree::Arity::to_usize())
+            .expect("Tree must have enough leaves and have an arity of power of two");
+
+        let tree_r_last_config = StoreConfig {
+            path: cache_path.clone(),
+            id: CacheKey::CommRLastTree.to_string(),
+            size: Some(tree_size),
+            // A default 'rows_to_discard' value will be chosen for tree_r_last, unless the user
+            // overrides this value via the environment setting (FIL_PROOFS_ROWS_TO_DISCARD). If
+            // this value is specified, no checking is done on it and it may result in a broken
+            // configuration. *Use with caution*. It must be noted that if/when this unchecked
+            // value is passed through merkle_light, merkle_light now does a check that does not
+            // allow us to discard more rows than is possible to discard.
+            rows_to_discard: default_rows_to_discard(sector_nodes, Tree::Arity::to_usize()),
+        };
+
+        let tree_c_config = StoreConfig {
+            path: cache_path,
+            id: CacheKey::CommCTree.to_string(),
+            size: Some(tree_size),
+            // The TreeC doesn't discard any rows, hence it can be set to 0.
+            rows_to_discard: 0,
+        };
+
+        Self {
+            labels: Labels::new(labels),
+            tree_d_config,
+            tree_r_last_config,
+            tree_c_config,
+            _g: PhantomData,
+        }
+    }
+
     pub fn set_cache_path<P: AsRef<Path>>(&mut self, cache_path: P) {
         let cp = cache_path.as_ref().to_path_buf();
         for label in self.labels.labels.iter_mut() {