diff --git a/crates/pallet-subspace/src/lib.rs b/crates/pallet-subspace/src/lib.rs index 82422a47d051f..f425449f85c62 100644 --- a/crates/pallet-subspace/src/lib.rs +++ b/crates/pallet-subspace/src/lib.rs @@ -57,8 +57,8 @@ use sp_runtime::DispatchError; use sp_std::collections::btree_map::BTreeMap; use sp_std::prelude::*; use subspace_core_primitives::{ - Piece, PublicKey, Randomness, RewardSignature, SectorId, SectorIndex, SegmentHeader, - SegmentIndex, SolutionRange, PIECES_IN_SEGMENT, + ArchivedHistorySegment, PublicKey, Randomness, RewardSignature, SectorId, SectorIndex, + SegmentHeader, SegmentIndex, SolutionRange, }; use subspace_solving::REWARD_SIGNING_CONTEXT; use subspace_verification::{ @@ -673,7 +673,7 @@ impl Pallet { pub fn total_pieces() -> NonZeroU64 { // Chain starts with one segment plotted, even if it is not recorded in the runtime yet let number_of_segments = u64::from(SegmentCommitment::::count()).max(1); - NonZeroU64::new(number_of_segments * u64::from(PIECES_IN_SEGMENT)) + NonZeroU64::new(number_of_segments * ArchivedHistorySegment::NUM_PIECES as u64) .expect("Neither of multiplied values is zero; qed") } @@ -1036,7 +1036,7 @@ impl Pallet { pub fn archived_history_size() -> u64 { let archived_segments = SegmentCommitment::::count(); - u64::from(archived_segments) * u64::from(PIECES_IN_SEGMENT) * Piece::SIZE as u64 + u64::from(archived_segments) * ArchivedHistorySegment::SIZE as u64 } pub fn chain_constants() -> ChainConstants { diff --git a/crates/pallet-subspace/src/mock.rs b/crates/pallet-subspace/src/mock.rs index a4916e3791b37..e4b2ef9359bcc 100644 --- a/crates/pallet-subspace/src/mock.rs +++ b/crates/pallet-subspace/src/mock.rs @@ -37,7 +37,7 @@ use sp_runtime::traits::{Block as BlockT, Header as _, IdentityLookup}; use sp_runtime::Perbill; use std::num::NonZeroU64; use std::sync::Once; -use subspace_archiving::archiver::{ArchivedSegment, Archiver}; +use subspace_archiving::archiver::{Archiver, NewArchivedSegment}; use subspace_core_primitives::crypto::kzg::{embedded_kzg_settings, Kzg, Witness}; use subspace_core_primitives::crypto::{blake2b_256_254_hash_to_scalar, kzg, ScalarLegacy}; use subspace_core_primitives::{ @@ -342,7 +342,7 @@ pub fn create_segment_header(segment_index: SegmentIndex) -> SegmentHeader { } } -pub fn create_archived_segment() -> ArchivedSegment { +pub fn create_archived_segment() -> NewArchivedSegment { let kzg = Kzg::new(kzg::embedded_kzg_settings()); let mut archiver = Archiver::new(kzg).unwrap(); diff --git a/crates/sc-consensus-subspace-rpc/src/lib.rs b/crates/sc-consensus-subspace-rpc/src/lib.rs index e519a4fcaed4f..9445a2d2c7517 100644 --- a/crates/sc-consensus-subspace-rpc/src/lib.rs +++ b/crates/sc-consensus-subspace-rpc/src/lib.rs @@ -42,7 +42,7 @@ use sp_runtime::traits::{Block as BlockT, Zero}; use std::error::Error; use std::sync::Arc; use std::time::Duration; -use subspace_archiving::archiver::ArchivedSegment; +use subspace_archiving::archiver::NewArchivedSegment; use subspace_core_primitives::{SegmentCommitment, SegmentHeader, SegmentIndex, Solution}; use subspace_farmer_components::FarmerProtocolInfo; use subspace_networking::libp2p::Multiaddr; @@ -88,7 +88,7 @@ pub trait SubspaceRpcApi { #[subscription( name = "subspace_subscribeArchivedSegment" => "subspace_archived_segment", unsubscribe = "subspace_unsubscribeArchivedSegment", - item = ArchivedSegment, + item = NewArchivedSegment, )] fn subscribe_archived_segment(&self); diff --git a/crates/sc-consensus-subspace/src/archiver.rs b/crates/sc-consensus-subspace/src/archiver.rs index af7ed4b3aac22..f512188b58908 100644 --- a/crates/sc-consensus-subspace/src/archiver.rs +++ b/crates/sc-consensus-subspace/src/archiver.rs @@ -31,7 +31,7 @@ use sp_objects::ObjectsApi; use sp_runtime::generic::SignedBlock; use sp_runtime::traits::{Block as BlockT, CheckedSub, Header, NumberFor, One, Zero}; use std::sync::Arc; -use subspace_archiving::archiver::{ArchivedSegment, Archiver}; +use subspace_archiving::archiver::{Archiver, NewArchivedSegment}; use subspace_core_primitives::crypto::kzg::Kzg; use subspace_core_primitives::objects::BlockObjectMapping; use subspace_core_primitives::{BlockNumber, SegmentHeader}; @@ -176,7 +176,7 @@ where { confirmation_depth_k: BlockNumber, archiver: Archiver, - older_archived_segments: Vec, + older_archived_segments: Vec, best_archived_block: (Block::Hash, NumberFor), } @@ -540,7 +540,7 @@ pub fn start_subspace_archiver( async fn send_archived_segment_notification( archived_segment_notification_sender: &SubspaceNotificationSender, - archived_segment: ArchivedSegment, + archived_segment: NewArchivedSegment, ) { let (acknowledgement_sender, mut acknowledgement_receiver) = tracing_unbounded::<()>("subspace_acknowledgement", 100); diff --git a/crates/sc-consensus-subspace/src/lib.rs b/crates/sc-consensus-subspace/src/lib.rs index e34ac27cb5225..e7ecd960dd389 100644 --- a/crates/sc-consensus-subspace/src/lib.rs +++ b/crates/sc-consensus-subspace/src/lib.rs @@ -77,7 +77,7 @@ use std::marker::PhantomData; use std::num::NonZeroUsize; use std::pin::Pin; use std::sync::Arc; -use subspace_archiving::archiver::{ArchivedSegment, Archiver}; +use subspace_archiving::archiver::{Archiver, NewArchivedSegment}; use subspace_core_primitives::crypto::kzg; use subspace_core_primitives::crypto::kzg::Kzg; use subspace_core_primitives::objects::BlockObjectMapping; @@ -127,7 +127,7 @@ pub struct RewardSigningNotification { #[derive(Debug, Clone)] pub struct ArchivedSegmentNotification { /// Archived segment. - pub archived_segment: Arc, + pub archived_segment: Arc, /// Sender that signified the fact of receiving archived segment by farmer. /// /// This must be used to send a message or else block import pipeline will get stuck. diff --git a/crates/sc-consensus-subspace/src/tests.rs b/crates/sc-consensus-subspace/src/tests.rs index 921f2195346af..bcb08fc391cb1 100644 --- a/crates/sc-consensus-subspace/src/tests.rs +++ b/crates/sc-consensus-subspace/src/tests.rs @@ -74,7 +74,9 @@ use subspace_archiving::archiver::Archiver; use subspace_core_primitives::crypto::kzg; use subspace_core_primitives::crypto::kzg::Kzg; use subspace_core_primitives::objects::BlockObjectMapping; -use subspace_core_primitives::{ChunkSignature, FlatPieces, Piece, PieceIndex, Solution}; +use subspace_core_primitives::{ + ArchivedHistorySegment, ChunkSignature, FlatPieces, Piece, PieceIndex, Solution, +}; use subspace_solving::{create_chunk_signature, REWARD_SIGNING_CONTEXT}; use substrate_test_runtime::{Block as TestBlock, Hash}; use tokio::runtime::{Handle, Runtime}; @@ -424,7 +426,7 @@ fn rejects_empty_block() { }) } -fn get_archived_pieces(client: &TestClient) -> Vec { +fn get_archived_segments(client: &TestClient) -> Vec { let kzg = Kzg::new(kzg::embedded_kzg_settings()); let mut archiver = Archiver::new(kzg).expect("Incorrect parameters for archiver"); @@ -505,7 +507,7 @@ async fn run_one_test(mutator: impl Fn(&mut TestHeader, Stage) + Send + Sync + ' let client = Arc::clone(&client); move || { - let archived_pieces = get_archived_pieces(&client); + let archived_pieces = get_archived_segments(&client); archived_pieces_sender.send(archived_pieces).unwrap(); } }); diff --git a/crates/sp-lightclient/src/lib.rs b/crates/sp-lightclient/src/lib.rs index 0fcaebe9490d7..bac191f68b526 100644 --- a/crates/sp-lightclient/src/lib.rs +++ b/crates/sp-lightclient/src/lib.rs @@ -36,8 +36,8 @@ use sp_std::cmp::Ordering; use sp_std::collections::btree_map::BTreeMap; use sp_std::marker::PhantomData; use subspace_core_primitives::{ - BlockWeight, PublicKey, Randomness, RewardSignature, SectorId, SegmentCommitment, SegmentIndex, - SolutionRange, PIECES_IN_SEGMENT, + ArchivedHistorySegment, BlockWeight, PublicKey, Randomness, RewardSignature, SectorId, + SegmentCommitment, SegmentIndex, SolutionRange, }; use subspace_solving::{derive_global_challenge, REWARD_SIGNING_CONTEXT}; use subspace_verification::{ @@ -666,7 +666,7 @@ impl> HeaderImporter { .ok_or_else(|| ImportError::MissingParent(header.header.hash()))?; } - Ok(segment_commitments_count * u64::from(PIECES_IN_SEGMENT)) + Ok(segment_commitments_count * ArchivedHistorySegment::NUM_PIECES as u64) } /// Finds a segment commitment mapped against a segment index in the chain with chain_tip as the diff --git a/crates/sp-lightclient/src/tests.rs b/crates/sp-lightclient/src/tests.rs index 0b051862163ba..ee2abf3aa2e6d 100644 --- a/crates/sp-lightclient/src/tests.rs +++ b/crates/sp-lightclient/src/tests.rs @@ -24,7 +24,7 @@ use sp_runtime::{Digest, DigestItem}; use std::error::Error; use std::io::Cursor; use std::num::NonZeroU64; -use subspace_archiving::archiver::{ArchivedSegment, Archiver}; +use subspace_archiving::archiver::{Archiver, NewArchivedSegment}; use subspace_core_primitives::crypto::kzg; use subspace_core_primitives::crypto::kzg::Kzg; use subspace_core_primitives::sector_codec::SectorCodec; @@ -65,7 +65,7 @@ fn derive_solution_range( subspace_core_primitives::bidirectional_distance(local_challenge, audit_chunk) * 2 } -fn archived_segment(kzg: Kzg) -> ArchivedSegment { +fn archived_segment(kzg: Kzg) -> NewArchivedSegment { // we don't care about the block data let mut rng = StdRng::seed_from_u64(0); let mut block = vec![0u8; RecordedHistorySegment::SIZE]; @@ -135,7 +135,7 @@ struct ValidHeaderParams<'a> { } struct TestPieceGetter { - archived_segment: ArchivedSegment, + archived_segment: NewArchivedSegment, } #[async_trait] diff --git a/crates/subspace-archiving/src/archiver.rs b/crates/subspace-archiving/src/archiver.rs index f3cdae08fbed9..efb0d131d9ac4 100644 --- a/crates/subspace-archiving/src/archiver.rs +++ b/crates/subspace-archiving/src/archiver.rs @@ -33,9 +33,8 @@ use subspace_core_primitives::objects::{ BlockObject, BlockObjectMapping, PieceObject, PieceObjectMapping, }; use subspace_core_primitives::{ - ArchivedBlockProgress, Blake2b256Hash, BlockNumber, FlatPieces, LastArchivedBlock, PieceArray, - RawRecord, RecordedHistorySegment, SegmentCommitment, SegmentHeader, SegmentIndex, - PIECES_IN_SEGMENT, + ArchivedBlockProgress, ArchivedHistorySegment, Blake2b256Hash, BlockNumber, LastArchivedBlock, + PieceArray, RawRecord, RecordedHistorySegment, SegmentCommitment, SegmentHeader, SegmentIndex, }; use subspace_erasure_coding::ErasureCoding; @@ -171,15 +170,16 @@ pub enum SegmentItem { ParentSegmentHeader(SegmentHeader), } -/// Archived segment as a combination of segment header hash, segment index and corresponding pieces +/// Newly archived segment as a combination of segment header hash, segment index and corresponding +/// archived history segment containing pieces #[derive(Debug, Clone, Eq, PartialEq, Decode, Encode)] #[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] #[cfg_attr(feature = "serde", serde(rename_all = "camelCase"))] -pub struct ArchivedSegment { - /// Segment header of the segment +pub struct NewArchivedSegment { + /// Segment header pub segment_header: SegmentHeader, - /// Pieces that correspond to this segment - pub pieces: FlatPieces, + /// Segment of archived history containing pieces + pub pieces: ArchivedHistorySegment, /// Mappings for objects stored in corresponding pieces. /// /// NOTE: Only half (source pieces) will have corresponding mapping item in this `Vec`. @@ -255,7 +255,7 @@ impl Archiver { // message in `.expect()` let erasure_coding = ErasureCoding::new( - NonZeroUsize::new(PIECES_IN_SEGMENT.ilog2() as usize) + NonZeroUsize::new(ArchivedHistorySegment::NUM_PIECES.ilog2() as usize) .expect("Recorded history segment contains at very least one record; qed"), ) .map_err(ArchiverInstantiationError::FailedToInitializeErasureCoding)?; @@ -263,7 +263,7 @@ impl Archiver { Ok(Self { buffer: VecDeque::default(), incremental_record_commitments: IncrementalRecordCommitmentsState::with_capacity( - RecordedHistorySegment::RAW_RECORDS, + RecordedHistorySegment::NUM_RAW_RECORDS, ), erasure_coding, kzg, @@ -348,7 +348,7 @@ impl Archiver { &mut self, bytes: Vec, object_mapping: BlockObjectMapping, - ) -> Vec { + ) -> Vec { // Append new block to the buffer self.buffer.push_back(SegmentItem::Block { bytes, @@ -610,11 +610,11 @@ impl Archiver { } // Take segment as an input, apply necessary transformations and produce archived segment - fn produce_archived_segment(&mut self, segment: Segment) -> ArchivedSegment { + fn produce_archived_segment(&mut self, segment: Segment) -> NewArchivedSegment { // Create mappings let object_mapping = { let mut corrected_object_mapping = - vec![PieceObjectMapping::default(); RecordedHistorySegment::RAW_RECORDS]; + vec![PieceObjectMapping::default(); RecordedHistorySegment::NUM_RAW_RECORDS]; let Segment::V0 { items } = &segment; // `+1` corresponds to enum variant encoding let mut base_offset_in_segment = 1; @@ -677,11 +677,11 @@ impl Archiver { // Segment is quite big and no longer necessary drop(segment); - let mut pieces = FlatPieces::new(PIECES_IN_SEGMENT as usize); + let mut pieces = ArchivedHistorySegment::default(); // Scratch buffer to avoid re-allocation let mut tmp_source_shards_scalars = - Vec::::with_capacity(RecordedHistorySegment::RAW_RECORDS); + Vec::::with_capacity(RecordedHistorySegment::NUM_RAW_RECORDS); // Iterate over the chunks of `Scalar::SAFE_BYTES` bytes of all records for record_offset in 0..RawRecord::SIZE / Scalar::SAFE_BYTES { // Collect chunks of each record at the same offset @@ -785,7 +785,7 @@ impl Archiver { self.buffer .push_front(SegmentItem::ParentSegmentHeader(segment_header)); - ArchivedSegment { + NewArchivedSegment { segment_header, pieces, object_mapping, @@ -846,7 +846,7 @@ pub fn is_piece_valid( kzg.verify( segment_commitment, - PIECES_IN_SEGMENT as usize, + ArchivedHistorySegment::NUM_PIECES, position, &commitment_hash, &witness, @@ -863,7 +863,7 @@ pub fn is_record_commitment_hash_valid( ) -> bool { kzg.verify( commitment, - PIECES_IN_SEGMENT as usize, + ArchivedHistorySegment::NUM_PIECES, position, commitment_hash, witness, diff --git a/crates/subspace-archiving/src/piece_reconstructor.rs b/crates/subspace-archiving/src/piece_reconstructor.rs index 4f20c89d11acc..f51f3eefce053 100644 --- a/crates/subspace-archiving/src/piece_reconstructor.rs +++ b/crates/subspace-archiving/src/piece_reconstructor.rs @@ -5,9 +5,7 @@ use alloc::vec::Vec; use core::num::NonZeroUsize; use subspace_core_primitives::crypto::kzg::{Commitment, Kzg, Polynomial}; use subspace_core_primitives::crypto::{blake2b_256_254_hash_to_scalar, Scalar}; -use subspace_core_primitives::{ - FlatPieces, Piece, RawRecord, RecordedHistorySegment, PIECES_IN_SEGMENT, -}; +use subspace_core_primitives::{ArchivedHistorySegment, Piece, RawRecord, RecordedHistorySegment}; use subspace_erasure_coding::ErasureCoding; /// Reconstructor-related instantiation error. @@ -57,7 +55,7 @@ impl PiecesReconstructor { // message in `.expect()` let erasure_coding = ErasureCoding::new( - NonZeroUsize::new(PIECES_IN_SEGMENT.ilog2() as usize) + NonZeroUsize::new(ArchivedHistorySegment::NUM_PIECES.ilog2() as usize) .expect("Recorded history segment contains at very least one record; qed"), ) .map_err(ReconstructorInstantiationError::FailedToInitializeErasureCoding)?; @@ -73,8 +71,8 @@ impl PiecesReconstructor { fn reconstruct_shards( &self, input_pieces: &[Option], - ) -> Result<(FlatPieces, Polynomial), ReconstructorError> { - let mut reconstructed_pieces = FlatPieces::new(PIECES_IN_SEGMENT as usize); + ) -> Result<(ArchivedHistorySegment, Polynomial), ReconstructorError> { + let mut reconstructed_pieces = ArchivedHistorySegment::default(); if !input_pieces .iter() @@ -105,7 +103,7 @@ impl PiecesReconstructor { // Scratch buffer to avoid re-allocation let mut tmp_shards_scalars = - Vec::>::with_capacity(PIECES_IN_SEGMENT as usize); + Vec::>::with_capacity(ArchivedHistorySegment::NUM_PIECES); // Iterate over the chunks of `Scalar::SAFE_BYTES` bytes of all records for record_offset in 0..RawRecord::SIZE / Scalar::SAFE_BYTES { // Collect chunks of each record at the same offset @@ -145,7 +143,8 @@ impl PiecesReconstructor { } } - let mut source_record_commitments = Vec::with_capacity(RecordedHistorySegment::RAW_RECORDS); + let mut source_record_commitments = + Vec::with_capacity(RecordedHistorySegment::NUM_RAW_RECORDS); for (piece, maybe_input_piece) in reconstructed_pieces.iter_mut().zip(input_pieces).step_by(2) { @@ -218,7 +217,7 @@ impl PiecesReconstructor { pub fn reconstruct_segment( &self, segment_pieces: &[Option], - ) -> Result { + ) -> Result { let (mut pieces, polynomial) = self.reconstruct_shards(segment_pieces)?; pieces.iter_mut().enumerate().for_each(|(position, piece)| { @@ -245,7 +244,7 @@ impl PiecesReconstructor { ) -> Result { let (reconstructed_records, polynomial) = self.reconstruct_shards(segment_pieces)?; - if piece_position >= PIECES_IN_SEGMENT as usize { + if piece_position >= ArchivedHistorySegment::NUM_PIECES { return Err(ReconstructorError::IncorrectPiecePosition); } diff --git a/crates/subspace-archiving/src/reconstructor.rs b/crates/subspace-archiving/src/reconstructor.rs index 98f650ebccbff..9f35432d07751 100644 --- a/crates/subspace-archiving/src/reconstructor.rs +++ b/crates/subspace-archiving/src/reconstructor.rs @@ -9,8 +9,8 @@ use core::num::NonZeroUsize; use parity_scale_codec::Decode; use subspace_core_primitives::crypto::Scalar; use subspace_core_primitives::{ - ArchivedBlockProgress, BlockNumber, LastArchivedBlock, Piece, RawRecord, - RecordedHistorySegment, SegmentHeader, SegmentIndex, PIECES_IN_SEGMENT, + ArchivedBlockProgress, ArchivedHistorySegment, BlockNumber, LastArchivedBlock, Piece, + RawRecord, RecordedHistorySegment, SegmentHeader, SegmentIndex, }; use subspace_erasure_coding::ErasureCoding; @@ -77,7 +77,7 @@ impl Reconstructor { // message in `.expect()` let erasure_coding = ErasureCoding::new( - NonZeroUsize::new(PIECES_IN_SEGMENT.ilog2() as usize) + NonZeroUsize::new(ArchivedHistorySegment::NUM_PIECES.ilog2() as usize) .expect("Recorded history segment contains at very least one record; qed"), ) .map_err(ReconstructorInstantiationError::FailedToInitializeErasureCoding)?; @@ -129,7 +129,7 @@ impl Reconstructor { // Scratch buffer to avoid re-allocation let mut tmp_shards_scalars = - Vec::>::with_capacity(PIECES_IN_SEGMENT as usize); + Vec::>::with_capacity(ArchivedHistorySegment::NUM_PIECES); // Iterate over the chunks of `Scalar::SAFE_BYTES` bytes of all records for record_offset in 0..RawRecord::SIZE / Scalar::SAFE_BYTES { // Collect chunks of each record at the same offset diff --git a/crates/subspace-archiving/tests/integration/archiver.rs b/crates/subspace-archiving/tests/integration/archiver.rs index 3f556c166f35d..29b78644c8e27 100644 --- a/crates/subspace-archiving/tests/integration/archiver.rs +++ b/crates/subspace-archiving/tests/integration/archiver.rs @@ -9,8 +9,8 @@ use subspace_core_primitives::crypto::kzg::{embedded_kzg_settings, Commitment, K use subspace_core_primitives::crypto::Scalar; use subspace_core_primitives::objects::{BlockObject, BlockObjectMapping, PieceObject}; use subspace_core_primitives::{ - ArchivedBlockProgress, Blake2b256Hash, LastArchivedBlock, PieceArray, Record, - RecordedHistorySegment, SegmentHeader, SegmentIndex, BLAKE2B_256_HASH_SIZE, PIECES_IN_SEGMENT, + ArchivedBlockProgress, ArchivedHistorySegment, Blake2b256Hash, LastArchivedBlock, PieceArray, + Record, RecordedHistorySegment, SegmentHeader, SegmentIndex, BLAKE2B_256_HASH_SIZE, }; fn extract_data>(data: &[u8], offset: O) -> &[u8] { @@ -137,7 +137,7 @@ fn archiver() { let first_archived_segment = archived_segments.into_iter().next().unwrap(); assert_eq!( first_archived_segment.pieces.len(), - PIECES_IN_SEGMENT as usize + ArchivedHistorySegment::NUM_PIECES ); assert_eq!( first_archived_segment.segment_header.segment_index(), @@ -157,7 +157,7 @@ fn archiver() { assert_eq!( first_archived_segment.object_mapping.len(), - RecordedHistorySegment::RAW_RECORDS + RecordedHistorySegment::NUM_RAW_RECORDS ); // 4 objects fit into the first segment assert_eq!( @@ -219,7 +219,7 @@ fn archiver() { assert_eq!( archived_segments[0].object_mapping.len(), - RecordedHistorySegment::RAW_RECORDS + RecordedHistorySegment::NUM_RAW_RECORDS ); // 1 object fits into the second segment assert_eq!( @@ -232,7 +232,7 @@ fn archiver() { ); assert_eq!( archived_segments[1].object_mapping.len(), - RecordedHistorySegment::RAW_RECORDS + RecordedHistorySegment::NUM_RAW_RECORDS ); // 0 object fits into the second segment assert_eq!( @@ -274,7 +274,10 @@ fn archiver() { let mut previous_segment_header_hash = first_archived_segment.segment_header.hash(); let last_segment_header = archived_segments.iter().last().unwrap().segment_header; for archived_segment in archived_segments { - assert_eq!(archived_segment.pieces.len(), PIECES_IN_SEGMENT as usize); + assert_eq!( + archived_segment.pieces.len(), + ArchivedHistorySegment::NUM_PIECES + ); assert_eq!( archived_segment.segment_header.segment_index(), expected_segment_index diff --git a/crates/subspace-archiving/tests/integration/piece_reconstruction.rs b/crates/subspace-archiving/tests/integration/piece_reconstruction.rs index a88dbe0d0509d..6f38e8e059c34 100644 --- a/crates/subspace-archiving/tests/integration/piece_reconstruction.rs +++ b/crates/subspace-archiving/tests/integration/piece_reconstruction.rs @@ -3,7 +3,7 @@ use subspace_archiving::archiver::Archiver; use subspace_archiving::piece_reconstructor::{PiecesReconstructor, ReconstructorError}; use subspace_core_primitives::crypto::kzg::{embedded_kzg_settings, Kzg}; use subspace_core_primitives::objects::BlockObjectMapping; -use subspace_core_primitives::{FlatPieces, Piece, RecordedHistorySegment, PIECES_IN_SEGMENT}; +use subspace_core_primitives::{ArchivedHistorySegment, FlatPieces, Piece, RecordedHistorySegment}; fn pieces_to_option_of_pieces(pieces: &FlatPieces) -> Vec> { pieces.iter().map(Piece::from).map(Some).collect() @@ -29,20 +29,22 @@ fn segment_reconstruction_works() { let mut maybe_pieces = pieces_to_option_of_pieces(&archived_segments.first().unwrap().pieces); - assert_eq!(maybe_pieces.len(), PIECES_IN_SEGMENT as usize); + assert_eq!(maybe_pieces.len(), ArchivedHistorySegment::NUM_PIECES); // Remove some pieces from the array - for i in 0..PIECES_IN_SEGMENT { - if i > 100 && i < 140 { - maybe_pieces[i as usize] = None; - } - } + maybe_pieces + .iter_mut() + .skip(100) + .take(30) + .for_each(|piece| { + piece.take(); + }); let reconstructor = PiecesReconstructor::new(kzg).unwrap(); let flat_pieces = reconstructor.reconstruct_segment(&maybe_pieces).unwrap(); - assert_eq!(flat_pieces.len(), PIECES_IN_SEGMENT as usize); + assert_eq!(flat_pieces.len(), ArchivedHistorySegment::NUM_PIECES); archived_segments .into_iter() .next() @@ -68,7 +70,7 @@ fn piece_reconstruction_works() { let mut maybe_pieces = pieces_to_option_of_pieces(&archived_segments.first().unwrap().pieces); - assert_eq!(maybe_pieces.len(), PIECES_IN_SEGMENT as usize); + assert_eq!(maybe_pieces.len(), ArchivedHistorySegment::NUM_PIECES); // Remove some pieces from the vector let missing_pieces = maybe_pieces diff --git a/crates/subspace-archiving/tests/integration/reconstructor.rs b/crates/subspace-archiving/tests/integration/reconstructor.rs index df7086f083861..71dad6695316e 100644 --- a/crates/subspace-archiving/tests/integration/reconstructor.rs +++ b/crates/subspace-archiving/tests/integration/reconstructor.rs @@ -6,8 +6,8 @@ use subspace_archiving::reconstructor::{Reconstructor, ReconstructorError}; use subspace_core_primitives::crypto::kzg::{embedded_kzg_settings, Kzg}; use subspace_core_primitives::objects::BlockObjectMapping; use subspace_core_primitives::{ - ArchivedBlockProgress, FlatPieces, LastArchivedBlock, Piece, RecordedHistorySegment, - SegmentIndex, PIECES_IN_SEGMENT, + ArchivedBlockProgress, ArchivedHistorySegment, FlatPieces, LastArchivedBlock, Piece, + RecordedHistorySegment, SegmentIndex, }; fn pieces_to_option_of_pieces(pieces: &FlatPieces) -> Vec> { @@ -275,7 +275,7 @@ fn partial_data() { .source() .map(Piece::from) .map(Some) - .zip(iter::repeat(None).take(RecordedHistorySegment::RAW_RECORDS)) + .zip(iter::repeat(None).take(RecordedHistorySegment::NUM_RAW_RECORDS)) .flat_map(|(a, b)| [a, b]) .collect::>(), ) @@ -290,11 +290,11 @@ fn partial_data() { .unwrap() .add_segment( &iter::repeat(None) - .take(RecordedHistorySegment::RAW_RECORDS) + .take(RecordedHistorySegment::NUM_RAW_RECORDS) .chain( pieces .iter() - .skip(RecordedHistorySegment::RAW_RECORDS) + .skip(RecordedHistorySegment::NUM_RAW_RECORDS) .map(Piece::from) .map(Some), ) @@ -308,9 +308,9 @@ fn partial_data() { { // Mix of data and parity shards let mut pieces = pieces.iter().map(Piece::from).map(Some).collect::>(); - pieces[PIECES_IN_SEGMENT as usize / 4..] + pieces[ArchivedHistorySegment::NUM_PIECES / 4..] .iter_mut() - .take(RecordedHistorySegment::RAW_RECORDS) + .take(RecordedHistorySegment::NUM_RAW_RECORDS) .for_each(|piece| { piece.take(); }); @@ -342,11 +342,11 @@ fn invalid_usage() { &archived_segments[0] .pieces .iter() - .take(RecordedHistorySegment::RAW_RECORDS - 1) + .take(RecordedHistorySegment::NUM_RAW_RECORDS - 1) .map(Piece::from) .map(Some) .chain(iter::repeat(None)) - .take(PIECES_IN_SEGMENT as usize) + .take(ArchivedHistorySegment::NUM_PIECES) .collect::>(), ); @@ -361,7 +361,7 @@ fn invalid_usage() { thread_rng().fill(piece.as_mut()); Some(piece) }) - .take(PIECES_IN_SEGMENT as usize) + .take(ArchivedHistorySegment::NUM_PIECES) .collect::>(), ); diff --git a/crates/subspace-core-primitives/src/lib.rs b/crates/subspace-core-primitives/src/lib.rs index c22257d96aa9c..a95871688939d 100644 --- a/crates/subspace-core-primitives/src/lib.rs +++ b/crates/subspace-core-primitives/src/lib.rs @@ -31,6 +31,7 @@ pub mod crypto; pub mod objects; mod pieces; pub mod sector_codec; +mod segments; #[cfg(test)] mod tests; @@ -39,19 +40,16 @@ extern crate alloc; use crate::crypto::kzg::{Commitment, Witness}; use crate::crypto::{blake2b_256_hash, blake2b_256_hash_with_key, Scalar, ScalarLegacy}; use core::convert::AsRef; -use core::iter::Step; +use core::fmt; use core::num::NonZeroU64; -use core::{fmt, mem}; -use derive_more::{ - Add, AddAssign, Deref, Display, Div, DivAssign, From, Into, Mul, MulAssign, Rem, Sub, SubAssign, -}; +use derive_more::{Add, Deref, Display, Div, From, Into, Mul, Rem, Sub}; use num_traits::{WrappingAdd, WrappingSub}; -use parity_scale_codec::{Decode, Encode, MaxEncodedLen}; +use parity_scale_codec::{Decode, Encode}; pub use pieces::{ - FlatPieces, Piece, PieceArray, RawRecord, Record, RecordWitness, RecordedHistorySegment, - PIECES_IN_SEGMENT, + FlatPieces, Piece, PieceArray, PieceIndex, PieceIndexHash, RawRecord, Record, RecordWitness, }; use scale_info::TypeInfo; +pub use segments::{ArchivedHistorySegment, RecordedHistorySegment, SegmentIndex}; #[cfg(feature = "serde")] use serde::{Deserialize, Serialize}; use uint::static_assertions::const_assert; @@ -97,85 +95,6 @@ pub type SolutionRange = u64; /// The closer solution's tag is to the target, the heavier it is. pub type BlockWeight = u128; -/// Segment index type. -#[derive( - Debug, - Display, - Default, - Copy, - Clone, - Ord, - PartialOrd, - Eq, - PartialEq, - Hash, - Encode, - Decode, - Add, - AddAssign, - Sub, - SubAssign, - Mul, - MulAssign, - Div, - DivAssign, - TypeInfo, - MaxEncodedLen, -)] -#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))] -#[repr(transparent)] -pub struct SegmentIndex(u64); - -impl Step for SegmentIndex { - fn steps_between(start: &Self, end: &Self) -> Option { - u64::steps_between(&start.0, &end.0) - } - - fn forward_checked(start: Self, count: usize) -> Option { - u64::forward_checked(start.0, count).map(Self) - } - - fn backward_checked(start: Self, count: usize) -> Option { - u64::backward_checked(start.0, count).map(Self) - } -} - -impl const From for SegmentIndex { - fn from(original: u64) -> Self { - Self(original) - } -} - -impl const From for u64 { - fn from(original: SegmentIndex) -> Self { - original.0 - } -} - -impl SegmentIndex { - /// Segment index 0. - pub const ZERO: SegmentIndex = SegmentIndex(0); - /// Segment index 1. - pub const ONE: SegmentIndex = SegmentIndex(1); - - /// Get the first piece index in this segment. - pub const fn first_piece_index(&self) -> PieceIndex { - PieceIndex::from(self.0 * PIECES_IN_SEGMENT as u64) - } - - /// Iterator over piece indexes that belong to this segment. - pub fn segment_piece_indexes(&self) -> impl Iterator { - (self.first_piece_index()..).take(PIECES_IN_SEGMENT as usize) - } - - /// Iterator over piece indexes that belong to this segment with source pieces first. - pub fn segment_piece_indexes_source_first(&self) -> impl Iterator { - self.segment_piece_indexes() - .step_by(2) - .chain(self.segment_piece_indexes().skip(1).step_by(2)) - } -} - // TODO: New type /// Segment commitment type. pub type SegmentCommitment = Commitment; @@ -432,105 +351,9 @@ impl SegmentHeader { } } -/// Piece index in consensus -#[derive( - Debug, - Display, - Default, - Copy, - Clone, - Ord, - PartialOrd, - Eq, - PartialEq, - Hash, - Encode, - Decode, - Add, - AddAssign, - Sub, - SubAssign, - Mul, - MulAssign, - Div, - DivAssign, - TypeInfo, - MaxEncodedLen, -)] -#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))] -#[repr(transparent)] -pub struct PieceIndex(u64); - -impl Step for PieceIndex { - fn steps_between(start: &Self, end: &Self) -> Option { - u64::steps_between(&start.0, &end.0) - } - - fn forward_checked(start: Self, count: usize) -> Option { - u64::forward_checked(start.0, count).map(Self) - } - - fn backward_checked(start: Self, count: usize) -> Option { - u64::backward_checked(start.0, count).map(Self) - } -} - -impl const From for PieceIndex { - fn from(original: u64) -> Self { - Self(original) - } -} - -impl const From for u64 { - fn from(original: PieceIndex) -> Self { - original.0 - } -} - -impl PieceIndex { - /// Piece index 0. - pub const ZERO: PieceIndex = PieceIndex(0); - /// Piece index 1. - pub const ONE: PieceIndex = PieceIndex(1); - - /// Convert piece index into bytes. - pub const fn to_bytes(&self) -> [u8; mem::size_of::()] { - self.0.to_le_bytes() - } - - /// Segment index piece index corresponds to - pub const fn segment_index(&self) -> SegmentIndex { - SegmentIndex::from(self.0 / u64::from(PIECES_IN_SEGMENT)) - } - - /// Position of a piece in a segment - pub const fn position(&self) -> u32 { - // Position is statically guaranteed to fit into u32 - (self.0 % u64::from(PIECES_IN_SEGMENT)) as u32 - } -} - /// Sector index in consensus pub type SectorIndex = u64; -/// Hash of `PieceIndex` -#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Hash, Decode, Encode, From, Into)] -#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))] -pub struct PieceIndexHash(Blake2b256Hash); - -impl AsRef<[u8]> for PieceIndexHash { - fn as_ref(&self) -> &[u8] { - self.0.as_ref() - } -} - -impl PieceIndexHash { - /// Constructs `PieceIndexHash` from `PieceIndex` - pub fn from_index(index: PieceIndex) -> Self { - Self(blake2b_256_hash(&index.to_bytes())) - } -} - // TODO: Versioned solution enum /// Farmer solution for slot challenge. #[derive(Clone, Debug, Eq, PartialEq, Encode, Decode, TypeInfo)] @@ -786,14 +609,14 @@ impl From for U256 { } impl From for U256 { - fn from(PieceIndexHash(hash): PieceIndexHash) -> Self { - Self(private_u256::U256::from_big_endian(&hash)) + fn from(hash: PieceIndexHash) -> Self { + Self(private_u256::U256::from_big_endian(hash.as_ref())) } } impl From for PieceIndexHash { fn from(number: U256) -> Self { - Self(number.to_be_bytes()) + Self::from(number.to_be_bytes()) } } diff --git a/crates/subspace-core-primitives/src/pieces.rs b/crates/subspace-core-primitives/src/pieces.rs index 0d86ddda4e2be..6b5e4154ec455 100644 --- a/crates/subspace-core-primitives/src/pieces.rs +++ b/crates/subspace-core-primitives/src/pieces.rs @@ -1,17 +1,22 @@ #[cfg(feature = "serde")] mod serde; -use crate::crypto::Scalar; +use crate::crypto::{blake2b_256_hash, Scalar}; +use crate::segments::{ArchivedHistorySegment, SegmentIndex}; +use crate::Blake2b256Hash; #[cfg(feature = "serde")] use ::serde::{Deserialize, Serialize}; use alloc::boxed::Box; use alloc::vec; use alloc::vec::Vec; use core::array::TryFromSliceError; +use core::iter::Step; use core::mem; -use core::mem::ManuallyDrop; use core::ops::{Deref, DerefMut}; -use derive_more::{AsMut, AsRef, Deref, DerefMut}; +use derive_more::{ + Add, AddAssign, AsMut, AsRef, Deref, DerefMut, Display, Div, DivAssign, From, Into, Mul, + MulAssign, Sub, SubAssign, +}; use parity_scale_codec::{Decode, Encode, Input, MaxEncodedLen}; use scale_info::TypeInfo; @@ -29,78 +34,131 @@ const PIECE_SIZE: usize = 31_744; /// Size of a segment record given the global piece size (in bytes), is guaranteed to be multiple /// of [`Scalar::FULL_BYTES`]. const RECORD_SIZE: usize = Piece::SIZE - RecordCommitment::SIZE - RecordWitness::SIZE; -/// 128 data records and 128 parity records (as a result of erasure coding). -pub const PIECES_IN_SEGMENT: u32 = 256; -/// Raw record contained within recorded history segment before archiving is applied. -/// -/// NOTE: This is a stack-allocated data structure and can cause stack overflow! -#[derive(Debug, Copy, Clone, Eq, PartialEq, Deref, DerefMut)] +/// Piece index in consensus +#[derive( + Debug, + Display, + Default, + Copy, + Clone, + Ord, + PartialOrd, + Eq, + PartialEq, + Hash, + Encode, + Decode, + Add, + AddAssign, + Sub, + SubAssign, + Mul, + MulAssign, + Div, + DivAssign, + TypeInfo, + MaxEncodedLen, +)] +#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))] #[repr(transparent)] -pub struct RawRecord([[u8; Scalar::SAFE_BYTES]; Self::SIZE / Scalar::SAFE_BYTES]); +pub struct PieceIndex(u64); -impl Default for RawRecord { - fn default() -> Self { - Self([Default::default(); Self::SIZE / Scalar::SAFE_BYTES]) +impl Step for PieceIndex { + fn steps_between(start: &Self, end: &Self) -> Option { + u64::steps_between(&start.0, &end.0) + } + + fn forward_checked(start: Self, count: usize) -> Option { + u64::forward_checked(start.0, count).map(Self) + } + + fn backward_checked(start: Self, count: usize) -> Option { + u64::backward_checked(start.0, count).map(Self) } } -impl AsRef<[u8]> for RawRecord { - fn as_ref(&self) -> &[u8] { - self.0.as_slice().flatten() +impl const From for PieceIndex { + fn from(original: u64) -> Self { + Self(original) } } -impl AsMut<[u8]> for RawRecord { - fn as_mut(&mut self) -> &mut [u8] { - self.0.as_mut_slice().flatten_mut() +impl const From for u64 { + fn from(original: PieceIndex) -> Self { + original.0 } } -impl RawRecord { - /// Size of raw record in bytes, is guaranteed to be multiple of [`Scalar::SAFE_BYTES`]. - pub const SIZE: usize = Record::SIZE / Scalar::FULL_BYTES * Scalar::SAFE_BYTES; +impl PieceIndex { + /// Piece index 0. + pub const ZERO: PieceIndex = PieceIndex(0); + /// Piece index 1. + pub const ONE: PieceIndex = PieceIndex(1); + + /// Convert piece index into bytes. + pub const fn to_bytes(&self) -> [u8; mem::size_of::()] { + self.0.to_le_bytes() + } + + /// Segment index piece index corresponds to + pub const fn segment_index(&self) -> SegmentIndex { + SegmentIndex::from(self.0 / ArchivedHistorySegment::NUM_PIECES as u64) + } + + /// Position of a piece in a segment + pub const fn position(&self) -> u32 { + // Position is statically guaranteed to fit into u32 + (self.0 % ArchivedHistorySegment::NUM_PIECES as u64) as u32 + } +} + +/// Hash of `PieceIndex` +#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Hash, Decode, Encode, From, Into)] +#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))] +pub struct PieceIndexHash(Blake2b256Hash); + +impl AsRef<[u8]> for PieceIndexHash { + fn as_ref(&self) -> &[u8] { + self.0.as_ref() + } } -/// Recorded history segment before archiving is applied. +impl PieceIndexHash { + /// Constructs `PieceIndexHash` from `PieceIndex` + pub fn from_index(index: PieceIndex) -> Self { + Self(blake2b_256_hash(&index.to_bytes())) + } +} + +/// Raw record contained within recorded history segment before archiving is applied. /// /// NOTE: This is a stack-allocated data structure and can cause stack overflow! #[derive(Debug, Copy, Clone, Eq, PartialEq, Deref, DerefMut)] #[repr(transparent)] -pub struct RecordedHistorySegment([RawRecord; Self::RAW_RECORDS]); +pub struct RawRecord([[u8; Scalar::SAFE_BYTES]; Self::SIZE / Scalar::SAFE_BYTES]); -impl Default for RecordedHistorySegment { +impl Default for RawRecord { fn default() -> Self { - Self([RawRecord::default(); Self::RAW_RECORDS]) + Self([Default::default(); Self::SIZE / Scalar::SAFE_BYTES]) } } -impl AsRef<[u8]> for RecordedHistorySegment { +impl AsRef<[u8]> for RawRecord { fn as_ref(&self) -> &[u8] { - // SAFETY: Same memory layout due to `#[repr(transparent)]` - let raw_records: &[[u8; RawRecord::SIZE]] = unsafe { mem::transmute(self.0.as_slice()) }; - raw_records.flatten() + self.0.as_slice().flatten() } } -impl AsMut<[u8]> for RecordedHistorySegment { +impl AsMut<[u8]> for RawRecord { fn as_mut(&mut self) -> &mut [u8] { - // SAFETY: Same memory layout due to `#[repr(transparent)]` - let raw_records: &mut [[u8; RawRecord::SIZE]] = - unsafe { mem::transmute(self.0.as_mut_slice()) }; - raw_records.flatten_mut() + self.0.as_mut_slice().flatten_mut() } } -impl RecordedHistorySegment { - /// Size of recorded history segment in bytes. - /// - /// It includes half of the records (just source records) that will later be erasure coded and - /// together with corresponding commitments and witnesses will result in [`PIECES_IN_SEGMENT`] - /// [`Piece`]s of archival history. - pub const SIZE: usize = RawRecord::SIZE * PIECES_IN_SEGMENT as usize / 2; - /// Number of raw records in one segment of recorded history. - pub const RAW_RECORDS: usize = Self::SIZE / RawRecord::SIZE; +impl RawRecord { + /// Size of raw record in bytes, is guaranteed to be a multiple of [`Scalar::SAFE_BYTES`]. + pub const SIZE: usize = Record::SIZE / Scalar::FULL_BYTES * Scalar::SAFE_BYTES; } /// Record contained within a piece. @@ -123,8 +181,8 @@ impl AsMut<[u8]> for Record { } impl Record { - /// Size of a segment record given the global piece size (in bytes), is guaranteed to be - /// multiple of [`Scalar::FULL_BYTES`]. + /// Size of a segment record given the global piece size (in bytes) after erasure coding + /// [`RawRecord`], is guaranteed to be a multiple of [`Scalar::FULL_BYTES`]. pub const SIZE: usize = RECORD_SIZE; /// Get a stream of arrays, each containing safe scalar bytes. @@ -239,7 +297,7 @@ impl Decode for Piece { fn decode(input: &mut I) -> Result { let piece = parity_scale_codec::decode_vec_with_len::(input, Self::SIZE) .map_err(|error| error.chain("Could not decode `Piece.0`"))?; - let mut piece = ManuallyDrop::new(piece); + let mut piece = mem::ManuallyDrop::new(piece); // SAFETY: Original memory is not dropped and guaranteed to be allocated let piece = unsafe { Box::from_raw(piece.as_mut_ptr() as *mut PieceArray) }; Ok(Piece(piece)) @@ -295,7 +353,7 @@ impl AsMut<[u8]> for Piece { } impl Piece { - /// Size of a piece. + /// Size of a piece (in bytes). pub const SIZE: usize = PIECE_SIZE; } diff --git a/crates/subspace-core-primitives/src/segments.rs b/crates/subspace-core-primitives/src/segments.rs new file mode 100644 index 0000000000000..a09b260cd501d --- /dev/null +++ b/crates/subspace-core-primitives/src/segments.rs @@ -0,0 +1,162 @@ +use crate::pieces::{FlatPieces, Piece, PieceIndex, RawRecord}; +use core::iter::Step; +use core::mem; +use derive_more::{ + Add, AddAssign, Deref, DerefMut, Display, Div, DivAssign, Mul, MulAssign, Sub, SubAssign, +}; +use parity_scale_codec::{Decode, Encode, MaxEncodedLen}; +use scale_info::TypeInfo; +#[cfg(feature = "serde")] +use serde::{Deserialize, Serialize}; + +/// Segment index type. +#[derive( + Debug, + Display, + Default, + Copy, + Clone, + Ord, + PartialOrd, + Eq, + PartialEq, + Hash, + Encode, + Decode, + Add, + AddAssign, + Sub, + SubAssign, + Mul, + MulAssign, + Div, + DivAssign, + TypeInfo, + MaxEncodedLen, +)] +#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))] +#[repr(transparent)] +pub struct SegmentIndex(u64); + +impl Step for SegmentIndex { + fn steps_between(start: &Self, end: &Self) -> Option { + u64::steps_between(&start.0, &end.0) + } + + fn forward_checked(start: Self, count: usize) -> Option { + u64::forward_checked(start.0, count).map(Self) + } + + fn backward_checked(start: Self, count: usize) -> Option { + u64::backward_checked(start.0, count).map(Self) + } +} + +impl const From for SegmentIndex { + fn from(original: u64) -> Self { + Self(original) + } +} + +impl const From for u64 { + fn from(original: SegmentIndex) -> Self { + original.0 + } +} + +impl SegmentIndex { + /// Segment index 0. + pub const ZERO: SegmentIndex = SegmentIndex(0); + /// Segment index 1. + pub const ONE: SegmentIndex = SegmentIndex(1); + + /// Get the first piece index in this segment. + pub const fn first_piece_index(&self) -> PieceIndex { + PieceIndex::from(self.0 * ArchivedHistorySegment::NUM_PIECES as u64) + } + + /// Iterator over piece indexes that belong to this segment. + pub fn segment_piece_indexes(&self) -> impl Iterator { + (self.first_piece_index()..).take(ArchivedHistorySegment::NUM_PIECES) + } + + /// Iterator over piece indexes that belong to this segment with source pieces first. + pub fn segment_piece_indexes_source_first(&self) -> impl Iterator { + self.segment_piece_indexes() + .step_by(2) + .chain(self.segment_piece_indexes().skip(1).step_by(2)) + } +} + +/// Recorded history segment before archiving is applied. +/// +/// NOTE: This is a stack-allocated data structure and can cause stack overflow! +#[derive(Debug, Copy, Clone, Eq, PartialEq, Deref, DerefMut)] +#[repr(transparent)] +pub struct RecordedHistorySegment([RawRecord; Self::NUM_RAW_RECORDS]); + +impl Default for RecordedHistorySegment { + fn default() -> Self { + Self([RawRecord::default(); Self::NUM_RAW_RECORDS]) + } +} + +impl AsRef<[u8]> for RecordedHistorySegment { + fn as_ref(&self) -> &[u8] { + // SAFETY: Same memory layout due to `#[repr(transparent)]` + let raw_records: &[[u8; RawRecord::SIZE]] = unsafe { mem::transmute(self.0.as_slice()) }; + raw_records.flatten() + } +} + +impl AsMut<[u8]> for RecordedHistorySegment { + fn as_mut(&mut self) -> &mut [u8] { + // SAFETY: Same memory layout due to `#[repr(transparent)]` + let raw_records: &mut [[u8; RawRecord::SIZE]] = + unsafe { mem::transmute(self.0.as_mut_slice()) }; + raw_records.flatten_mut() + } +} + +impl RecordedHistorySegment { + /// Number of raw records in one segment of recorded history. + pub const NUM_RAW_RECORDS: usize = 128; + /// Erasure coding rate for records during archiving process. + pub const ERASURE_CODING_RATE: (usize, usize) = (1, 2); + /// Size of recorded history segment in bytes. + /// + /// It includes half of the records (just source records) that will later be erasure coded and + /// together with corresponding commitments and witnesses will result in + /// [`ArchivedHistorySegment::NUM_PIECES`] [`Piece`]s of archival history. + pub const SIZE: usize = RawRecord::SIZE * Self::NUM_RAW_RECORDS; +} + +/// Archived history segment after archiving is applied. +#[derive(Debug, Clone, Eq, PartialEq, Deref, DerefMut, Encode, Decode, TypeInfo)] +#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))] +#[repr(transparent)] +pub struct ArchivedHistorySegment(FlatPieces); + +impl Default for ArchivedHistorySegment { + fn default() -> Self { + Self(FlatPieces::new(Self::NUM_PIECES)) + } +} + +impl MaxEncodedLen for ArchivedHistorySegment { + fn max_encoded_len() -> usize { + Self::SIZE + } +} + +impl ArchivedHistorySegment { + /// Number of pieces in one segment of archived history. + pub const NUM_PIECES: usize = RecordedHistorySegment::NUM_RAW_RECORDS + * RecordedHistorySegment::ERASURE_CODING_RATE.1 + / RecordedHistorySegment::ERASURE_CODING_RATE.0; + /// Size of archived history segment in bytes. + /// + /// It includes erasure coded [`crate::pieces::PieceArray`]s (both source and parity) that are + /// composed from [`crate::pieces::Record`]s together with corresponding commitments and witnesses. + pub const SIZE: usize = Piece::SIZE * Self::NUM_PIECES; +} diff --git a/crates/subspace-farmer-components/src/segment_reconstruction.rs b/crates/subspace-farmer-components/src/segment_reconstruction.rs index caf84621465b7..220a8cc49f833 100644 --- a/crates/subspace-farmer-components/src/segment_reconstruction.rs +++ b/crates/subspace-farmer-components/src/segment_reconstruction.rs @@ -33,7 +33,7 @@ pub async fn recover_missing_piece( let semaphore = Semaphore::new(PARALLELISM_LEVEL); let acquired_pieces_counter = AtomicUsize::default(); - let required_pieces_number = RecordedHistorySegment::RAW_RECORDS; + let required_pieces_number = RecordedHistorySegment::NUM_RAW_RECORDS; // This is so we can move references into the future below let semaphore = &semaphore; diff --git a/crates/subspace-farmer/src/node_client.rs b/crates/subspace-farmer/src/node_client.rs index d3560260b7f84..f0a853e0a4721 100644 --- a/crates/subspace-farmer/src/node_client.rs +++ b/crates/subspace-farmer/src/node_client.rs @@ -3,7 +3,7 @@ pub(crate) mod node_rpc_client; use async_trait::async_trait; use futures::Stream; use std::pin::Pin; -use subspace_archiving::archiver::ArchivedSegment; +use subspace_archiving::archiver::NewArchivedSegment; use subspace_core_primitives::{SegmentCommitment, SegmentHeader, SegmentIndex}; use subspace_rpc_primitives::{ FarmerAppInfo, RewardSignatureResponse, RewardSigningInfo, SlotInfo, SolutionResponse, @@ -43,7 +43,7 @@ pub trait NodeClient: Clone + Send + Sync + 'static { /// Subscribe to archived segments async fn subscribe_archived_segments( &self, - ) -> Result + Send + 'static>>, Error>; + ) -> Result + Send + 'static>>, Error>; /// Get segment commitments for the segments async fn segment_commitments( diff --git a/crates/subspace-farmer/src/node_client/node_rpc_client.rs b/crates/subspace-farmer/src/node_client/node_rpc_client.rs index f57322db59102..41aab595fad71 100644 --- a/crates/subspace-farmer/src/node_client/node_rpc_client.rs +++ b/crates/subspace-farmer/src/node_client/node_rpc_client.rs @@ -7,7 +7,7 @@ use jsonrpsee::rpc_params; use jsonrpsee::ws_client::{WsClient, WsClientBuilder}; use std::pin::Pin; use std::sync::Arc; -use subspace_archiving::archiver::ArchivedSegment; +use subspace_archiving::archiver::NewArchivedSegment; use subspace_core_primitives::{SegmentCommitment, SegmentHeader, SegmentIndex}; use subspace_rpc_primitives::{ FarmerAppInfo, RewardSignatureResponse, RewardSigningInfo, SlotInfo, SolutionResponse, @@ -109,7 +109,7 @@ impl NodeClient for NodeRpcClient { async fn subscribe_archived_segments( &self, - ) -> Result + Send + 'static>>, RpcError> { + ) -> Result + Send + 'static>>, RpcError> { let subscription = self .client .subscribe( diff --git a/crates/subspace-farmer/src/ws_rpc_server.rs b/crates/subspace-farmer/src/ws_rpc_server.rs index 4e589eb3a7126..ecbb8d5b85b14 100644 --- a/crates/subspace-farmer/src/ws_rpc_server.rs +++ b/crates/subspace-farmer/src/ws_rpc_server.rs @@ -213,7 +213,7 @@ impl RpcServerImpl { // How much bytes are definitely available starting at `piece_index` and `offset` without // crossing segment boundary let bytes_available_in_segment = { - let data_shards = RecordedHistorySegment::RAW_RECORDS as u32; + let data_shards = RecordedHistorySegment::NUM_RAW_RECORDS as u32; let piece_position = piece_index.position(); // `-2` is because last 2 bytes might contain padding if a piece is the last piece in @@ -400,7 +400,7 @@ impl RpcServerImpl { Vec::::with_capacity((self.pieces_in_segment * self.record_size) as usize); for piece_index in - (segment_index.first_piece_index()..).take(RecordedHistorySegment::RAW_RECORDS) + (segment_index.first_piece_index()..).take(RecordedHistorySegment::NUM_RAW_RECORDS) { let piece = self.read_and_decode_piece(piece_index)?; segment_bytes.extend_from_slice(piece.record().as_ref()); diff --git a/crates/subspace-service/src/dsn.rs b/crates/subspace-service/src/dsn.rs index 57753fb6e4c69..27e6a6f913a46 100644 --- a/crates/subspace-service/src/dsn.rs +++ b/crates/subspace-service/src/dsn.rs @@ -15,7 +15,7 @@ use sp_runtime::traits::Block as BlockT; use std::num::NonZeroUsize; use std::path::PathBuf; use std::sync::Arc; -use subspace_archiving::archiver::ArchivedSegment; +use subspace_archiving::archiver::NewArchivedSegment; use subspace_core_primitives::{PieceIndex, SegmentHeader, SegmentIndex}; use subspace_networking::libp2p::{identity, Multiaddr}; use subspace_networking::utils::pieces::announce_single_piece_index_with_backoff; @@ -279,7 +279,7 @@ pub(crate) async fn publish_pieces( node: &Node, first_piece_index: PieceIndex, segment_index: SegmentIndex, - archived_segment: Arc, + archived_segment: Arc, ) { let pieces_indexes = (first_piece_index..).take(archived_segment.pieces.len()); diff --git a/crates/subspace-service/src/dsn/import_blocks.rs b/crates/subspace-service/src/dsn/import_blocks.rs index 0effe3c32af23..ca0889e01a04e 100644 --- a/crates/subspace-service/src/dsn/import_blocks.rs +++ b/crates/subspace-service/src/dsn/import_blocks.rs @@ -31,7 +31,7 @@ use std::task::Poll; use subspace_archiving::reconstructor::Reconstructor; use subspace_core_primitives::crypto::kzg::{embedded_kzg_settings, Kzg}; use subspace_core_primitives::{ - Piece, RecordedHistorySegment, SegmentHeader, SegmentIndex, PIECES_IN_SEGMENT, + ArchivedHistorySegment, Piece, RecordedHistorySegment, SegmentHeader, SegmentIndex, }; use subspace_networking::utils::piece_provider::{PieceProvider, RetryPolicy}; use subspace_networking::Node; @@ -121,7 +121,7 @@ where for segment_index in (SegmentIndex::ZERO..).take(segments_found) { let pieces_indices = segment_index.segment_piece_indexes_source_first(); - let mut pieces = vec![None::; PIECES_IN_SEGMENT as usize]; + let mut pieces = vec![None::; ArchivedHistorySegment::NUM_PIECES]; let mut pieces_received = 0; for (piece_index, piece) in pieces_indices.zip(pieces.iter_mut()) { @@ -142,7 +142,7 @@ where pieces_received += 1; } - if pieces_received >= RecordedHistorySegment::RAW_RECORDS { + if pieces_received >= RecordedHistorySegment::NUM_RAW_RECORDS { trace!(%segment_index, "Received half of the segment."); break; } diff --git a/crates/subspace-service/src/piece_cache/tests.rs b/crates/subspace-service/src/piece_cache/tests.rs index 02939de126923..1c8862868c1f2 100644 --- a/crates/subspace-service/src/piece_cache/tests.rs +++ b/crates/subspace-service/src/piece_cache/tests.rs @@ -3,7 +3,9 @@ use sc_client_api::AuxStore; use std::cell::RefCell; use std::collections::HashMap; use std::sync::Arc; -use subspace_core_primitives::{FlatPieces, Piece, PieceIndex, PieceIndexHash, PIECES_IN_SEGMENT}; +use subspace_core_primitives::{ + ArchivedHistorySegment, FlatPieces, Piece, PieceIndex, PieceIndexHash, +}; use subspace_networking::libp2p::PeerId; use subspace_networking::utils::multihash::ToMultihash; @@ -46,15 +48,12 @@ impl AuxStore for TestAuxStore { fn basic() { let mut store = PieceCache::new( Arc::new(TestAuxStore::default()), - u64::from(PIECES_IN_SEGMENT) * Piece::SIZE as u64, + ArchivedHistorySegment::SIZE as u64, PeerId::random(), ); store - .add_pieces( - PieceIndex::default(), - &FlatPieces::new(PIECES_IN_SEGMENT as usize), - ) + .add_pieces(PieceIndex::default(), &ArchivedHistorySegment::default()) .unwrap(); let piece_index = PieceIndex::default(); @@ -80,10 +79,7 @@ fn cache_nothing() { let mut store = PieceCache::new(Arc::new(TestAuxStore::default()), 0, PeerId::random()); store - .add_pieces( - PieceIndex::default(), - &FlatPieces::new(PIECES_IN_SEGMENT as usize), - ) + .add_pieces(PieceIndex::default(), &ArchivedHistorySegment::default()) .unwrap(); let piece_index = PieceIndex::default(); diff --git a/test/subspace-test-client/src/lib.rs b/test/subspace-test-client/src/lib.rs index 071872e195978..feed2c1ab4279 100644 --- a/test/subspace-test-client/src/lib.rs +++ b/test/subspace-test-client/src/lib.rs @@ -33,7 +33,7 @@ use std::error::Error; use std::io::Cursor; use std::num::NonZeroU64; use std::sync::Arc; -use subspace_archiving::archiver::ArchivedSegment; +use subspace_archiving::archiver::NewArchivedSegment; use subspace_core_primitives::crypto::kzg; use subspace_core_primitives::crypto::kzg::Kzg; use subspace_core_primitives::objects::BlockObjectMapping; @@ -198,7 +198,7 @@ async fn start_farming( } struct TestPieceGetter { - archived_segment: ArchivedSegment, + archived_segment: NewArchivedSegment, } #[async_trait]