feat!: pack squares densely by implementing ADR013

app/write_square.go

@@ -62,7 +62,7 @@ func finalizeBlobLayout(squareSize uint64, nonreserveStart int, blobTxs []tmprot
 		if removePFBIndexes[tBlob.parsedIndex] {
 			continue
 		}
-		cursor, _ = shares.NextMultipleOfBlobMinSquareSize(cursor, tBlob.sharesUsed, iSS)
+		cursor, _ = shares.NextShareIndex(cursor, tBlob.sharesUsed, iSS)
 		// remove the parsed transaction if it cannot fit into the square
 		if cursor+tBlob.sharesUsed > maxSharesSize {
 			removePFBIndexes[tBlob.parsedIndex] = true

app/write_square_test.go

@@ -83,20 +83,7 @@ func Test_finalizeLayout(t *testing.T) {
 				[]appns.Namespace{ns1, ns1, ns1},
 				[][]int{{1000}, {10000}, {100000}},
 			),
-			expectedIndexes: [][]uint32{
-				// BlobMinSquareSize(2) = 2 so the first blob has to start at the
-				// next multiple of 2 >= 32 which is 32. This blob occupies
-				// shares 32 to 33.
-				{32},
-				// BlobMinSquareSize(20) = 8 so the second blob has to start at
-				// the next multiple of 8 >= 34 which is 40. This blob occupies
-				// shares 40 to 59.
-				{40},
-				// BlobMinSquareSize(199) = 16 so the third blob has to start at
-				// the next multiple of 16 >= 60 which is 64. This blob occupies
-				// shares 64 to 262.
-				{64},
-			},
+			expectedIndexes: [][]uint32{{32}, {35}, {56}},
 		},
 		{
 			squareSize:      32,
@@ -114,29 +101,9 @@ func Test_finalizeLayout(t *testing.T) {
 			blobTxs: generateBlobTxsWithNamespaces(
 				t,
 				[]appns.Namespace{ns1, ns2, ns1},
-				[][]int{{100}, {1000}, {1000}},
+				[][]int{{100}, {900}, {900}}, // 1, 2, 2 shares respectively
 			),
-			expectedIndexes: [][]uint32{{32}, {38}, {34}},
-		},
-		{
-			squareSize:      32,
-			nonreserveStart: 32,
-			blobTxs: generateBlobTxsWithNamespaces(
-				t,
-				[]appns.Namespace{ns1, ns2, ns1},
-				[][]int{{100}, {1000}, {1000}},
-			),
-			expectedIndexes: [][]uint32{{32}, {38}, {34}},
-		},
-		{
-			squareSize:      4,
-			nonreserveStart: 3,
-			blobTxs: generateBlobTxsWithNamespaces(
-				t,
-				[]appns.Namespace{ns1, ns3, ns2},
-				[][]int{{100}, {1000}, {420}},
-			),
-			expectedIndexes: [][]uint32{{3}, {6}, {4}},
+			expectedIndexes: [][]uint32{{32}, {35}, {33}},
 		},
 		{
 			squareSize:      4,
@@ -146,7 +113,7 @@ func Test_finalizeLayout(t *testing.T) {
 				[]appns.Namespace{ns1, ns3, ns3, ns2},
 				[][]int{{100}, {1000, 1000}, {420}},
 			),
-			expectedIndexes: [][]uint32{{4}, {6, 10}, {5}},
+			expectedIndexes: [][]uint32{{4}, {6, 9}, {5}},
 		},
 		{
 			// no blob txs should make it in the square
@@ -159,25 +126,14 @@ func Test_finalizeLayout(t *testing.T) {
 			),
 			expectedIndexes: [][]uint32{},
 		},
-		{
-			// only two blob txs should make it in the square
-			squareSize:      4,
-			nonreserveStart: 4,
-			blobTxs: generateBlobTxsWithNamespaces(
-				t,
-				[]appns.Namespace{ns1, ns2, ns3},
-				[][]int{{1800}, {1800}, {6000}},
-			),
-			expectedIndexes: [][]uint32{{4}, {8}},
-		},
 		{
 			// only one blob tx should make it in the square (after reordering)
 			squareSize:      4,
 			nonreserveStart: 4,
 			blobTxs: generateBlobTxsWithNamespaces(
 				t,
 				[]appns.Namespace{ns3, ns2, ns1},
-				[][]int{{2000}, {2000}, {6000}},
+				[][]int{{2000}, {2000}, {5000}},
 			),
 			expectedIndexes: [][]uint32{{4}},
 		},
@@ -198,19 +154,19 @@ func Test_finalizeLayout(t *testing.T) {
 			blobTxs: generateBlobTxsWithNamespaces(
 				t,
 				[]appns.Namespace{ns1, ns3, ns3, ns1, ns2, ns2},
-				[][]int{{100}, {1400, 1000, 200, 200}, {420}},
+				[][]int{{100}, {1400, 900, 200, 200}, {420}},
 			),
-			expectedIndexes: [][]uint32{{4}, {8, 12, 5, 6}, {7}},
+			expectedIndexes: [][]uint32{{4}, {8, 11, 5, 6}, {7}},
 		},
 		{
 			squareSize:      4,
 			nonreserveStart: 4,
 			blobTxs: generateBlobTxsWithNamespaces(
 				t,
 				[]appns.Namespace{ns1, ns3, ns3, ns1, ns2, ns2},
-				[][]int{{100}, {1000, 1400, 200, 200}, {420}},
+				[][]int{{100}, {900, 1400, 200, 200}, {420}},
 			),
-			expectedIndexes: [][]uint32{{4}, {8, 12, 5, 6}, {7}},
+			expectedIndexes: [][]uint32{{4}, {8, 10, 5, 6}, {7}},
 		},
 	}
 	for i, tt := range tests {

pkg/appconsts/appconsts.go

@@ -72,6 +72,14 @@ const (
 	// data square.
 	MinShareCount = DefaultMinSquareSize * DefaultMinSquareSize
 
+	// SubtreeRootThreshold dictates the threshold for increasing the sub tree
+	// root width for blobs. If the number of subtree roots used to create a
+	// share commitment surpasses this threshold, then the width is increased.
+	// The rationale for this value is described in more detail in ADR013
+	// (./docs/architecture/adr-013).
+	// ADR013 https://github.com/celestiaorg/celestia-app/blob/e905143e8fe138ce6085ae9a5c1af950a2d87638/docs/architecture/adr-013-non-interactive-default-rules-for-zero-padding.md //nolint: lll
+	SubtreeRootThreshold = DefaultMaxSquareSize
+
 	// MaxShareVersion is the maximum value a share version can be.
 	MaxShareVersion = 127
 

pkg/inclusion/paths.go

@@ -14,11 +14,7 @@ type path struct {
 // calculateCommitmentPaths calculates all of the paths to subtree roots needed to
 // create the commitment for a given blob.
 func calculateCommitmentPaths(squareSize, start, blobShareLen int) []path {
-	// TODO: make the non-interactive defaults optional. by calculating the
-	// NextMultipleOfBlobMinSquareSize, we are forcing use of the
-	// non-interactive defaults. If we want to make this optional in the future,
-	// we have to move this next line out of this function.
-	start, _ = shares.NextMultipleOfBlobMinSquareSize(start, blobShareLen, squareSize)
+	start, _ = shares.NextShareIndex(start, blobShareLen, squareSize)
 	startRow, endRow := start/squareSize, (start+blobShareLen-1)/squareSize
 	normalizedStartIndex := start % squareSize
 	normalizedEndIndex := (start + blobShareLen) - endRow*squareSize
@@ -33,11 +29,11 @@ func calculateCommitmentPaths(squareSize, start, blobShareLen int) []path {
 			end = normalizedEndIndex
 		}
 
-		// subTreeRootMaxHeight is the maximum height of a subtree root that was
-		// used to generate the commitment. The height is based on the minimum
-		// square size the blob can fit into. See ADR-008 for more details.
-		subTreeRootMaxHeight := int(math.Log2(float64(shares.MinSquareSize(blobShareLen))))
-		minDepth := maxDepth - subTreeRootMaxHeight
+		// subTreeRootMaxDepth is the maximum height of a subtree root that was
+		// used to generate the commitment. The height is based on the
+		// SubtreeRootThreshold. See ADR-013 for more details.
+		subTreeRootMaxDepth := int(math.Log2(float64(shares.SubTreeWidth(blobShareLen))))
+		minDepth := maxDepth - subTreeRootMaxDepth
 		coords := calculateSubTreeRootCoordinates(maxDepth, minDepth, start, end)
 		for _, c := range coords {
 			paths = append(paths, path{