feat!: square size independent message commitments

.gitignore

@@ -7,5 +7,6 @@ coverage.txt
 .idea
 .vscode
 tools-stamp
-.vscode
+vscode
+.__debug_bin
 profile.out

app/estimate_square_size.go

@@ -57,7 +57,7 @@ func prune(txConf client.TxConfig, txs []*parsedTx, currentShareCount, squareSiz
 		// we ignore the error here, as if there is an error malleating the tx,
 		// then we need to remove it anyway and it will not end up contributing
 		// bytes to the square anyway.
-		_ = txs[i].malleate(txConf, uint64(squareSize))
+		_ = txs[i].malleate(txConf)
 		adjustContigCursor(len(txs[i].malleatedTx) + appconsts.MalleatedTxBytes)
 	}
 
@@ -170,7 +170,7 @@ func rawShareCount(txs []*parsedTx, evd core.EvidenceList) (txShares, evdShares
 		// compensates for the actual size of the message, and in some cases can
 		// result in some wasted square space or picking a square size that is
 		// too large. TODO: improve by making a more accurate estimation formula
-		txBytes += overEstimateMalleatedTxSize(len(pTx.rawTx), len(pTx.msg.Message), len(pTx.msg.MessageShareCommitment))
+		txBytes += overEstimateMalleatedTxSize(len(pTx.rawTx), len(pTx.msg.Message), 1)
 
 		msgSummaries = append(msgSummaries, msgSummary{shares.MsgSharesUsed(int(pTx.msg.MessageSize)), pTx.msg.MessageNamespaceId})
 	}
@@ -217,10 +217,8 @@ func rawShareCount(txs []*parsedTx, evd core.EvidenceList) (txShares, evdShares
 // overEstimateMalleatedTxSize estimates the size of a malleated tx. The formula it uses will always over estimate.
 func overEstimateMalleatedTxSize(txLen, msgLen, sharesCommitments int) int {
 	// the malleated tx uses the original txLen to account for meta data from
-	// the original tx, but removes the message and extra share commitments that
-	// are in the wire message by subtracting msgLen and all extra share
-	// commitments.
-	malleatedTxLen := txLen - msgLen - ((sharesCommitments - 1) * appconsts.ShareCommitmentBytes)
+	// the original tx, but removes the message
+	malleatedTxLen := txLen - msgLen
 	// we need to ensure that the returned number is at least larger than or
 	// equal to the actual number, which is difficult to calculate without
 	// actually malleating the tx

app/estimate_square_size_test.go

@@ -26,7 +26,7 @@ func Test_estimateSquareSize(t *testing.T) {
 	tests := []test{
 		{"empty block minimum square size", 0, 0, 0, appconsts.MinSquareSize},
 		{"full block with only txs", 10000, 0, 0, appconsts.MaxSquareSize},
-		{"random small block square size 2", 0, 1, appconsts.SparseShareContentSize, 2},
+		{"3 tx shares + 2 msg shares = 5 total shares so square size 4", 0, 1, appconsts.SparseShareContentSize, 4},
 		{"random small block square size 4", 0, 1, appconsts.SparseShareContentSize * 10, 4},
 		{"random small block w/ 10 normal txs square size 4", 10, 1, appconsts.SparseShareContentSize, 4},
 		{"random small block square size 16", 0, 4, appconsts.SparseShareContentSize * 8, 16},

app/malleate_txs.go

@@ -30,7 +30,7 @@ func malleateTxs(
 	var trackedMsgs []trackedMessage
 	for i, pTx := range txs {
 		if pTx.msg != nil {
-			err = pTx.malleate(txConf, squareSize)
+			err = pTx.malleate(txConf)
 			if err != nil {
 				txs.remove(i)
 				continue
@@ -83,13 +83,13 @@ func malleateTxs(
 	return processedTxs, msgs, err
 }
 
-func (p *parsedTx) malleate(txConf client.TxConfig, squareSize uint64) error {
+func (p *parsedTx) malleate(txConf client.TxConfig) error {
 	if p.msg == nil || p.tx == nil {
 		return errors.New("can only malleate a tx with a MsgWirePayForData")
 	}
 
 	// parse wire message and create a single message
-	_, unsignedPFD, sig, err := types.ProcessWirePayForData(p.msg, squareSize)
+	_, unsignedPFD, sig, err := types.ProcessWirePayForData(p.msg)
 	if err != nil {
 		return err
 	}

app/test/block_size_test.go

@@ -266,7 +266,6 @@ func generateSignedWirePayForDataTxs(clientCtx client.Context, txConfig client.T
 		msg, err := types.NewWirePayForData(
 			namespace.RandomMessageNamespace(),
 			tmrand.Bytes(thisMessageSize),
-			types.AllSquareSizes(thisMessageSize)...,
 		)
 		if err != nil {
 			return nil, err

app/test/fuzz_abci_test.go

@@ -25,7 +25,7 @@ func TestFuzzPrepareProcessProposal(t *testing.T) {
 	encConf := encoding.MakeConfig(app.ModuleEncodingRegisters...)
 	signer := testutil.GenerateKeyringSigner(t, testAccName)
 	testApp := testutil.SetupTestAppWithGenesisValSet(t)
-	timer := time.After(time.Second * 30)
+	timer := time.After(time.Second * 10)
 	for {
 		select {
 		case <-timer:

app/test/prepare_proposal_test.go

@@ -166,7 +166,7 @@ func generateRawTx(t *testing.T, txConfig client.TxConfig, ns, message []byte, s
 }
 
 func generateSignedWirePayForData(t *testing.T, ns, message []byte, signer *types.KeyringSigner, options []types.TxBuilderOption, ks ...uint64) *types.MsgWirePayForData {
-	msg, err := types.NewWirePayForData(ns, message, ks...)
+	msg, err := types.NewWirePayForData(ns, message)
 	if err != nil {
 		t.Error(err)
 	}

app/test/process_proposal_test.go

@@ -206,7 +206,7 @@ func genRandMsgPayForDataForNamespace(t *testing.T, signer *types.KeyringSigner,
 	_, err := rand.Read(message)
 	require.NoError(t, err)
 
-	commit, err := types.CreateCommitment(squareSize, ns, message)
+	commit, err := types.CreateCommitment(ns, message)
 	require.NoError(t, err)
 
 	pfd := types.MsgPayForData{

app/test_util.go

@@ -123,7 +123,7 @@ func generateRawSendTx(t *testing.T, txConfig client.TxConfig, signer *types.Key
 // generateRawWirePFD creates a tx with a single MsgWirePayForData message using the provided namespace and message
 func generateRawWirePFDTx(t *testing.T, txConfig client.TxConfig, ns, message []byte, signer *types.KeyringSigner, opts ...types.TxBuilderOption) (rawTx []byte) {
 	// create a msg
-	msg := generateSignedWirePayForData(t, ns, message, signer, opts, types.AllSquareSizes(len(message))...)
+	msg := generateSignedWirePayForData(t, ns, message, signer, opts)
 
 	builder := signer.NewTxBuilder(opts...)
 	tx, err := signer.BuildSignedTx(builder, msg)
@@ -136,8 +136,8 @@ func generateRawWirePFDTx(t *testing.T, txConfig client.TxConfig, ns, message []
 	return rawTx
 }
 
-func generateSignedWirePayForData(t *testing.T, ns, message []byte, signer *types.KeyringSigner, options []types.TxBuilderOption, ks ...uint64) *types.MsgWirePayForData {
-	msg, err := types.NewWirePayForData(ns, message, ks...)
+func generateSignedWirePayForData(t *testing.T, ns, message []byte, signer *types.KeyringSigner, options []types.TxBuilderOption) *types.MsgWirePayForData {
+	msg, err := types.NewWirePayForData(ns, message)
 	if err != nil {
 		t.Error(err)
 	}

pkg/appconsts/appconsts.go

@@ -65,11 +65,6 @@ const (
 	// for protobuf
 	MalleatedTxBytes = 32 + 4 + 3
 
-	// ShareCommitmentBytes is the number of bytes used by a protobuf encoded
-	// share commitment. 64 bytes for the signature, 32 bytes for the
-	// commitment, 8 bytes for the uint64, and 4 bytes for the protobuf overhead
-	ShareCommitmentBytes = 64 + 32 + 8 + 4
-
 	// MalleatedTxEstimateBuffer is the "magic" number used to ensure that the
 	// estimate of a malleated transaction is at least as big if not larger than
 	// the actual value. TODO: use a more accurate number

pkg/inclusion/paths.go

@@ -4,6 +4,7 @@ import (
 	"math"
 
 	"github.com/celestiaorg/celestia-app/pkg/shares"
+	"github.com/celestiaorg/celestia-app/x/payment/types"
 )
 
 type path struct {
@@ -32,8 +33,11 @@ func calculateCommitPaths(squareSize, start, msgShareLen int) []path {
 		if i == endRow {
 			end = normalizedEndIndex
 		}
-		coord := calculateSubTreeRootCoordinates(maxDepth, start, end)
-		for _, c := range coord {
+
+		subTreeRootMaxHeight := int(math.Log2(float64(types.MinSquareSize(uint64(msgShareLen)))))
+		minDepth := maxDepth - subTreeRootMaxHeight
+		coords := calculateSubTreeRootCoordinates(maxDepth, minDepth, start, end)
+		for _, c := range coords {
 			paths = append(paths, path{
 				instructions: genSubTreeRootPath(c.depth, uint(c.position)),
 				row:          i,
@@ -94,17 +98,17 @@ func (c coord) climb() coord {
 // canClimbRight uses the current position to calculate the direction of the next
 // climb. Returns true if the next climb is right (if the position (index) is
 // even). please see depth and position example map in docs for coord.
-func (c coord) canClimbRight() bool {
-	return c.position%2 == 0 && c.depth > 0
+func (c coord) canClimbRight(minDepth int) bool {
+	return c.position%2 == 0 && c.depth > minDepth
 }
 
 // calculateSubTreeRootCoordinates generates the sub tree root coordinates of a
 // set of shares for a balanced binary tree of a given depth. It assumes that
 // end does not exceed the range of a tree of the provided depth, and that end
 // >= start. This function works by starting at the first index of the msg and
 // working our way right.
-func calculateSubTreeRootCoordinates(maxDepth, start, end int) []coord {
-	cds := []coord{}
+func calculateSubTreeRootCoordinates(maxDepth, minDepth, start, end int) []coord {
+	coords := []coord{}
 	// leafCursor keeps track of the current leaf that we are starting with when
 	// finding the subtree root for some set. When leafCursor == end, we are
 	// finished calculating sub tree roots
@@ -143,19 +147,19 @@ func calculateSubTreeRootCoordinates(maxDepth, start, end int) []coord {
 		switch {
 		// check if we're finished, if so add the last coord and return
 		case leafCursor+1 == end:
-			cds = append(cds, nodeCursor)
-			return cds
+			coords = append(coords, nodeCursor)
+			return coords
 		// check if we've climbed too high in the tree. if so, add the last
 		// highest node and proceed.
 		case leafCursor+1 > end:
-			cds = append(cds, lastNodeCursor)
+			coords = append(coords, lastNodeCursor)
 			leafCursor = lastLeafCursor + 1
 			reset()
 		// check if can climb right again (only even positions will climb
 		// right). If not, we want to record this coord as it is a subtree
 		// root, then adjust the cursor and proceed.
-		case !nodeCursor.canClimbRight():
-			cds = append(cds, nodeCursor)
+		case !nodeCursor.canClimbRight(minDepth):
+			coords = append(coords, nodeCursor)
 			leafCursor++
 			reset()
 		// proceed to climb higher by incrementing the relevant state and