Merge pull request paritytech#1317 from subspace/archived-segment-ref…

…actoring

Archived segment refactoring

crates/pallet-subspace/src/lib.rs

@@ -57,8 +57,8 @@ use sp_runtime::DispatchError;
 use sp_std::collections::btree_map::BTreeMap;
 use sp_std::prelude::*;
 use subspace_core_primitives::{
-    Piece, PublicKey, Randomness, RewardSignature, SectorId, SectorIndex, SegmentHeader,
-    SegmentIndex, SolutionRange, PIECES_IN_SEGMENT,
+    ArchivedHistorySegment, PublicKey, Randomness, RewardSignature, SectorId, SectorIndex,
+    SegmentHeader, SegmentIndex, SolutionRange,
 };
 use subspace_solving::REWARD_SIGNING_CONTEXT;
 use subspace_verification::{
@@ -673,7 +673,7 @@ impl<T: Config> Pallet<T> {
     pub fn total_pieces() -> NonZeroU64 {
         // Chain starts with one segment plotted, even if it is not recorded in the runtime yet
         let number_of_segments = u64::from(SegmentCommitment::<T>::count()).max(1);
-        NonZeroU64::new(number_of_segments * u64::from(PIECES_IN_SEGMENT))
+        NonZeroU64::new(number_of_segments * ArchivedHistorySegment::NUM_PIECES as u64)
             .expect("Neither of multiplied values is zero; qed")
     }
 
@@ -1036,7 +1036,7 @@ impl<T: Config> Pallet<T> {
     pub fn archived_history_size() -> u64 {
         let archived_segments = SegmentCommitment::<T>::count();
 
-        u64::from(archived_segments) * u64::from(PIECES_IN_SEGMENT) * Piece::SIZE as u64
+        u64::from(archived_segments) * ArchivedHistorySegment::SIZE as u64
     }
 
     pub fn chain_constants() -> ChainConstants {

crates/pallet-subspace/src/mock.rs

@@ -37,7 +37,7 @@ use sp_runtime::traits::{Block as BlockT, Header as _, IdentityLookup};
 use sp_runtime::Perbill;
 use std::num::NonZeroU64;
 use std::sync::Once;
-use subspace_archiving::archiver::{ArchivedSegment, Archiver};
+use subspace_archiving::archiver::{Archiver, NewArchivedSegment};
 use subspace_core_primitives::crypto::kzg::{embedded_kzg_settings, Kzg, Witness};
 use subspace_core_primitives::crypto::{blake2b_256_254_hash_to_scalar, kzg, ScalarLegacy};
 use subspace_core_primitives::{
@@ -342,7 +342,7 @@ pub fn create_segment_header(segment_index: SegmentIndex) -> SegmentHeader {
     }
 }
 
-pub fn create_archived_segment() -> ArchivedSegment {
+pub fn create_archived_segment() -> NewArchivedSegment {
     let kzg = Kzg::new(kzg::embedded_kzg_settings());
     let mut archiver = Archiver::new(kzg).unwrap();
 

crates/sc-consensus-subspace-rpc/src/lib.rs

@@ -42,7 +42,7 @@ use sp_runtime::traits::{Block as BlockT, Zero};
 use std::error::Error;
 use std::sync::Arc;
 use std::time::Duration;
-use subspace_archiving::archiver::ArchivedSegment;
+use subspace_archiving::archiver::NewArchivedSegment;
 use subspace_core_primitives::{SegmentCommitment, SegmentHeader, SegmentIndex, Solution};
 use subspace_farmer_components::FarmerProtocolInfo;
 use subspace_networking::libp2p::Multiaddr;
@@ -88,7 +88,7 @@ pub trait SubspaceRpcApi {
     #[subscription(
         name = "subspace_subscribeArchivedSegment" => "subspace_archived_segment",
         unsubscribe = "subspace_unsubscribeArchivedSegment",
-        item = ArchivedSegment,
+        item = NewArchivedSegment,
     )]
     fn subscribe_archived_segment(&self);
 

crates/sc-consensus-subspace/src/archiver.rs

@@ -31,7 +31,7 @@ use sp_objects::ObjectsApi;
 use sp_runtime::generic::SignedBlock;
 use sp_runtime::traits::{Block as BlockT, CheckedSub, Header, NumberFor, One, Zero};
 use std::sync::Arc;
-use subspace_archiving::archiver::{ArchivedSegment, Archiver};
+use subspace_archiving::archiver::{Archiver, NewArchivedSegment};
 use subspace_core_primitives::crypto::kzg::Kzg;
 use subspace_core_primitives::objects::BlockObjectMapping;
 use subspace_core_primitives::{BlockNumber, SegmentHeader};
@@ -176,7 +176,7 @@ where
 {
     confirmation_depth_k: BlockNumber,
     archiver: Archiver,
-    older_archived_segments: Vec<ArchivedSegment>,
+    older_archived_segments: Vec<NewArchivedSegment>,
     best_archived_block: (Block::Hash, NumberFor<Block>),
 }
 
@@ -540,7 +540,7 @@ pub fn start_subspace_archiver<Block, Backend, Client>(
 
 async fn send_archived_segment_notification(
     archived_segment_notification_sender: &SubspaceNotificationSender<ArchivedSegmentNotification>,
-    archived_segment: ArchivedSegment,
+    archived_segment: NewArchivedSegment,
 ) {
     let (acknowledgement_sender, mut acknowledgement_receiver) =
         tracing_unbounded::<()>("subspace_acknowledgement", 100);

crates/sc-consensus-subspace/src/lib.rs

@@ -77,7 +77,7 @@ use std::marker::PhantomData;
 use std::num::NonZeroUsize;
 use std::pin::Pin;
 use std::sync::Arc;
-use subspace_archiving::archiver::{ArchivedSegment, Archiver};
+use subspace_archiving::archiver::{Archiver, NewArchivedSegment};
 use subspace_core_primitives::crypto::kzg;
 use subspace_core_primitives::crypto::kzg::Kzg;
 use subspace_core_primitives::objects::BlockObjectMapping;
@@ -127,7 +127,7 @@ pub struct RewardSigningNotification {
 #[derive(Debug, Clone)]
 pub struct ArchivedSegmentNotification {
     /// Archived segment.
-    pub archived_segment: Arc<ArchivedSegment>,
+    pub archived_segment: Arc<NewArchivedSegment>,
     /// Sender that signified the fact of receiving archived segment by farmer.
     ///
     /// This must be used to send a message or else block import pipeline will get stuck.

crates/sc-consensus-subspace/src/tests.rs

@@ -74,7 +74,9 @@ use subspace_archiving::archiver::Archiver;
 use subspace_core_primitives::crypto::kzg;
 use subspace_core_primitives::crypto::kzg::Kzg;
 use subspace_core_primitives::objects::BlockObjectMapping;
-use subspace_core_primitives::{ChunkSignature, FlatPieces, Piece, PieceIndex, Solution};
+use subspace_core_primitives::{
+    ArchivedHistorySegment, ChunkSignature, FlatPieces, Piece, PieceIndex, Solution,
+};
 use subspace_solving::{create_chunk_signature, REWARD_SIGNING_CONTEXT};
 use substrate_test_runtime::{Block as TestBlock, Hash};
 use tokio::runtime::{Handle, Runtime};
@@ -424,7 +426,7 @@ fn rejects_empty_block() {
     })
 }
 
-fn get_archived_pieces(client: &TestClient) -> Vec<FlatPieces> {
+fn get_archived_segments(client: &TestClient) -> Vec<ArchivedHistorySegment> {
     let kzg = Kzg::new(kzg::embedded_kzg_settings());
     let mut archiver = Archiver::new(kzg).expect("Incorrect parameters for archiver");
 
@@ -505,7 +507,7 @@ async fn run_one_test(mutator: impl Fn(&mut TestHeader, Stage) + Send + Sync + '
             let client = Arc::clone(&client);
 
             move || {
-                let archived_pieces = get_archived_pieces(&client);
+                let archived_pieces = get_archived_segments(&client);
                 archived_pieces_sender.send(archived_pieces).unwrap();
             }
         });

crates/sp-lightclient/src/lib.rs

@@ -36,8 +36,8 @@ use sp_std::cmp::Ordering;
 use sp_std::collections::btree_map::BTreeMap;
 use sp_std::marker::PhantomData;
 use subspace_core_primitives::{
-    BlockWeight, PublicKey, Randomness, RewardSignature, SectorId, SegmentCommitment, SegmentIndex,
-    SolutionRange, PIECES_IN_SEGMENT,
+    ArchivedHistorySegment, BlockWeight, PublicKey, Randomness, RewardSignature, SectorId,
+    SegmentCommitment, SegmentIndex, SolutionRange,
 };
 use subspace_solving::{derive_global_challenge, REWARD_SIGNING_CONTEXT};
 use subspace_verification::{
@@ -666,7 +666,7 @@ impl<Header: HeaderT, Store: Storage<Header>> HeaderImporter<Header, Store> {
                 .ok_or_else(|| ImportError::MissingParent(header.header.hash()))?;
         }
 
-        Ok(segment_commitments_count * u64::from(PIECES_IN_SEGMENT))
+        Ok(segment_commitments_count * ArchivedHistorySegment::NUM_PIECES as u64)
     }
 
     /// Finds a segment commitment mapped against a segment index in the chain with chain_tip as the

crates/sp-lightclient/src/tests.rs

@@ -24,7 +24,7 @@ use sp_runtime::{Digest, DigestItem};
 use std::error::Error;
 use std::io::Cursor;
 use std::num::NonZeroU64;
-use subspace_archiving::archiver::{ArchivedSegment, Archiver};
+use subspace_archiving::archiver::{Archiver, NewArchivedSegment};
 use subspace_core_primitives::crypto::kzg;
 use subspace_core_primitives::crypto::kzg::Kzg;
 use subspace_core_primitives::sector_codec::SectorCodec;
@@ -65,7 +65,7 @@ fn derive_solution_range(
     subspace_core_primitives::bidirectional_distance(local_challenge, audit_chunk) * 2
 }
 
-fn archived_segment(kzg: Kzg) -> ArchivedSegment {
+fn archived_segment(kzg: Kzg) -> NewArchivedSegment {
     // we don't care about the block data
     let mut rng = StdRng::seed_from_u64(0);
     let mut block = vec![0u8; RecordedHistorySegment::SIZE];
@@ -135,7 +135,7 @@ struct ValidHeaderParams<'a> {
 }
 
 struct TestPieceGetter {
-    archived_segment: ArchivedSegment,
+    archived_segment: NewArchivedSegment,
 }
 
 #[async_trait]

crates/subspace-archiving/src/archiver.rs

@@ -33,9 +33,8 @@ use subspace_core_primitives::objects::{
     BlockObject, BlockObjectMapping, PieceObject, PieceObjectMapping,
 };
 use subspace_core_primitives::{
-    ArchivedBlockProgress, Blake2b256Hash, BlockNumber, FlatPieces, LastArchivedBlock, PieceArray,
-    RawRecord, RecordedHistorySegment, SegmentCommitment, SegmentHeader, SegmentIndex,
-    PIECES_IN_SEGMENT,
+    ArchivedBlockProgress, ArchivedHistorySegment, Blake2b256Hash, BlockNumber, LastArchivedBlock,
+    PieceArray, RawRecord, RecordedHistorySegment, SegmentCommitment, SegmentHeader, SegmentIndex,
 };
 use subspace_erasure_coding::ErasureCoding;
 
@@ -171,15 +170,16 @@ pub enum SegmentItem {
     ParentSegmentHeader(SegmentHeader),
 }
 
-/// Archived segment as a combination of segment header hash, segment index and corresponding pieces
+/// Newly archived segment as a combination of segment header hash, segment index and corresponding
+/// archived history segment containing pieces
 #[derive(Debug, Clone, Eq, PartialEq, Decode, Encode)]
 #[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))]
 #[cfg_attr(feature = "serde", serde(rename_all = "camelCase"))]
-pub struct ArchivedSegment {
-    /// Segment header of the segment
+pub struct NewArchivedSegment {
+    /// Segment header
     pub segment_header: SegmentHeader,
-    /// Pieces that correspond to this segment
-    pub pieces: FlatPieces,
+    /// Segment of archived history containing pieces
+    pub pieces: ArchivedHistorySegment,
     /// Mappings for objects stored in corresponding pieces.
     ///
     /// NOTE: Only half (source pieces) will have corresponding mapping item in this `Vec`.
@@ -255,15 +255,15 @@ impl Archiver {
         //  message in `.expect()`
 
         let erasure_coding = ErasureCoding::new(
-            NonZeroUsize::new(PIECES_IN_SEGMENT.ilog2() as usize)
+            NonZeroUsize::new(ArchivedHistorySegment::NUM_PIECES.ilog2() as usize)
                 .expect("Recorded history segment contains at very least one record; qed"),
         )
         .map_err(ArchiverInstantiationError::FailedToInitializeErasureCoding)?;
 
         Ok(Self {
             buffer: VecDeque::default(),
             incremental_record_commitments: IncrementalRecordCommitmentsState::with_capacity(
-                RecordedHistorySegment::RAW_RECORDS,
+                RecordedHistorySegment::NUM_RAW_RECORDS,
             ),
             erasure_coding,
             kzg,
@@ -348,7 +348,7 @@ impl Archiver {
         &mut self,
         bytes: Vec<u8>,
         object_mapping: BlockObjectMapping,
-    ) -> Vec<ArchivedSegment> {
+    ) -> Vec<NewArchivedSegment> {
         // Append new block to the buffer
         self.buffer.push_back(SegmentItem::Block {
             bytes,
@@ -610,11 +610,11 @@ impl Archiver {
     }
 
     // Take segment as an input, apply necessary transformations and produce archived segment
-    fn produce_archived_segment(&mut self, segment: Segment) -> ArchivedSegment {
+    fn produce_archived_segment(&mut self, segment: Segment) -> NewArchivedSegment {
         // Create mappings
         let object_mapping = {
             let mut corrected_object_mapping =
-                vec![PieceObjectMapping::default(); RecordedHistorySegment::RAW_RECORDS];
+                vec![PieceObjectMapping::default(); RecordedHistorySegment::NUM_RAW_RECORDS];
             let Segment::V0 { items } = &segment;
             // `+1` corresponds to enum variant encoding
             let mut base_offset_in_segment = 1;
@@ -677,11 +677,11 @@ impl Archiver {
             // Segment is quite big and no longer necessary
             drop(segment);
 
-            let mut pieces = FlatPieces::new(PIECES_IN_SEGMENT as usize);
+            let mut pieces = ArchivedHistorySegment::default();
 
             // Scratch buffer to avoid re-allocation
             let mut tmp_source_shards_scalars =
-                Vec::<Scalar>::with_capacity(RecordedHistorySegment::RAW_RECORDS);
+                Vec::<Scalar>::with_capacity(RecordedHistorySegment::NUM_RAW_RECORDS);
             // Iterate over the chunks of `Scalar::SAFE_BYTES` bytes of all records
             for record_offset in 0..RawRecord::SIZE / Scalar::SAFE_BYTES {
                 // Collect chunks of each record at the same offset
@@ -785,7 +785,7 @@ impl Archiver {
         self.buffer
             .push_front(SegmentItem::ParentSegmentHeader(segment_header));
 
-        ArchivedSegment {
+        NewArchivedSegment {
             segment_header,
             pieces,
             object_mapping,
@@ -846,7 +846,7 @@ pub fn is_piece_valid(
 
     kzg.verify(
         segment_commitment,
-        PIECES_IN_SEGMENT as usize,
+        ArchivedHistorySegment::NUM_PIECES,
         position,
         &commitment_hash,
         &witness,
@@ -863,7 +863,7 @@ pub fn is_record_commitment_hash_valid(
 ) -> bool {
     kzg.verify(
         commitment,
-        PIECES_IN_SEGMENT as usize,
+        ArchivedHistorySegment::NUM_PIECES,
         position,
         commitment_hash,
         witness,

crates/subspace-archiving/src/piece_reconstructor.rs

@@ -5,9 +5,7 @@ use alloc::vec::Vec;
 use core::num::NonZeroUsize;
 use subspace_core_primitives::crypto::kzg::{Commitment, Kzg, Polynomial};
 use subspace_core_primitives::crypto::{blake2b_256_254_hash_to_scalar, Scalar};
-use subspace_core_primitives::{
-    FlatPieces, Piece, RawRecord, RecordedHistorySegment, PIECES_IN_SEGMENT,
-};
+use subspace_core_primitives::{ArchivedHistorySegment, Piece, RawRecord, RecordedHistorySegment};
 use subspace_erasure_coding::ErasureCoding;
 
 /// Reconstructor-related instantiation error.
@@ -57,7 +55,7 @@ impl PiecesReconstructor {
         //  message in `.expect()`
 
         let erasure_coding = ErasureCoding::new(
-            NonZeroUsize::new(PIECES_IN_SEGMENT.ilog2() as usize)
+            NonZeroUsize::new(ArchivedHistorySegment::NUM_PIECES.ilog2() as usize)
                 .expect("Recorded history segment contains at very least one record; qed"),
         )
         .map_err(ReconstructorInstantiationError::FailedToInitializeErasureCoding)?;
@@ -73,8 +71,8 @@ impl PiecesReconstructor {
     fn reconstruct_shards(
         &self,
         input_pieces: &[Option<Piece>],
-    ) -> Result<(FlatPieces, Polynomial), ReconstructorError> {
-        let mut reconstructed_pieces = FlatPieces::new(PIECES_IN_SEGMENT as usize);
+    ) -> Result<(ArchivedHistorySegment, Polynomial), ReconstructorError> {
+        let mut reconstructed_pieces = ArchivedHistorySegment::default();
 
         if !input_pieces
             .iter()
@@ -105,7 +103,7 @@ impl PiecesReconstructor {
 
             // Scratch buffer to avoid re-allocation
             let mut tmp_shards_scalars =
-                Vec::<Option<Scalar>>::with_capacity(PIECES_IN_SEGMENT as usize);
+                Vec::<Option<Scalar>>::with_capacity(ArchivedHistorySegment::NUM_PIECES);
             // Iterate over the chunks of `Scalar::SAFE_BYTES` bytes of all records
             for record_offset in 0..RawRecord::SIZE / Scalar::SAFE_BYTES {
                 // Collect chunks of each record at the same offset
@@ -145,7 +143,8 @@ impl PiecesReconstructor {
             }
         }
 
-        let mut source_record_commitments = Vec::with_capacity(RecordedHistorySegment::RAW_RECORDS);
+        let mut source_record_commitments =
+            Vec::with_capacity(RecordedHistorySegment::NUM_RAW_RECORDS);
         for (piece, maybe_input_piece) in
             reconstructed_pieces.iter_mut().zip(input_pieces).step_by(2)
         {
@@ -218,7 +217,7 @@ impl PiecesReconstructor {
     pub fn reconstruct_segment(
         &self,
         segment_pieces: &[Option<Piece>],
-    ) -> Result<FlatPieces, ReconstructorError> {
+    ) -> Result<ArchivedHistorySegment, ReconstructorError> {
         let (mut pieces, polynomial) = self.reconstruct_shards(segment_pieces)?;
 
         pieces.iter_mut().enumerate().for_each(|(position, piece)| {
@@ -245,7 +244,7 @@ impl PiecesReconstructor {
     ) -> Result<Piece, ReconstructorError> {
         let (reconstructed_records, polynomial) = self.reconstruct_shards(segment_pieces)?;
 
-        if piece_position >= PIECES_IN_SEGMENT as usize {
+        if piece_position >= ArchivedHistorySegment::NUM_PIECES {
             return Err(ReconstructorError::IncorrectPiecePosition);
         }
 

crates/subspace-archiving/src/reconstructor.rs

@@ -9,8 +9,8 @@ use core::num::NonZeroUsize;
 use parity_scale_codec::Decode;
 use subspace_core_primitives::crypto::Scalar;
 use subspace_core_primitives::{
-    ArchivedBlockProgress, BlockNumber, LastArchivedBlock, Piece, RawRecord,
-    RecordedHistorySegment, SegmentHeader, SegmentIndex, PIECES_IN_SEGMENT,
+    ArchivedBlockProgress, ArchivedHistorySegment, BlockNumber, LastArchivedBlock, Piece,
+    RawRecord, RecordedHistorySegment, SegmentHeader, SegmentIndex,
 };
 use subspace_erasure_coding::ErasureCoding;
 
@@ -77,7 +77,7 @@ impl Reconstructor {
         //  message in `.expect()`
 
         let erasure_coding = ErasureCoding::new(
-            NonZeroUsize::new(PIECES_IN_SEGMENT.ilog2() as usize)
+            NonZeroUsize::new(ArchivedHistorySegment::NUM_PIECES.ilog2() as usize)
                 .expect("Recorded history segment contains at very least one record; qed"),
         )
         .map_err(ReconstructorInstantiationError::FailedToInitializeErasureCoding)?;
@@ -129,7 +129,7 @@ impl Reconstructor {
 
             // Scratch buffer to avoid re-allocation
             let mut tmp_shards_scalars =
-                Vec::<Option<Scalar>>::with_capacity(PIECES_IN_SEGMENT as usize);
+                Vec::<Option<Scalar>>::with_capacity(ArchivedHistorySegment::NUM_PIECES);
             // Iterate over the chunks of `Scalar::SAFE_BYTES` bytes of all records
             for record_offset in 0..RawRecord::SIZE / Scalar::SAFE_BYTES {
                 // Collect chunks of each record at the same offset